def rename(path, from_basename, to_basename):
files = glob.glob(path + "/*.tif")
for f in files:
if os.path.isfile(f):
new_filename = get_filename(f).replace(from_basename, to_basename).replace(".", "") + "_3857.tif"
print "Rename:", get_filename(f), "to:", new_filename
os.rename(f, path + "/" + new_filename)
def rename(path, from_basename, to_basename):
files = glob.glob(path + "/*.tif")
for f in files:
if os.path.isfile(f):
new_filename = get_filename(f).replace(from_basename, to_basename) + ".tif"
os.rename(f, path + "/" + new_filename)
def process_metadata(workspace, storage_folder):
folders = glob.glob(storage_folder+ "*")
# for each file
for f in folders:
print "-------------"
print "folder: " + f
# get filename (it's the same as the folder)
filename = get_filename(f)
print "filename: " + f
# get UID of the file
uid = sanitize_name(filename)
print "uid stored layer: " + uid
# get UID of the stored 3857 projection file
uid_3857 = sanitize_name(workspace + ":" + filename.rsplit("_", 1)[0])
print "uid_3857 to search: " + uid_3857
# get path to the file (+.geotiff extention)
path_file = os.path.join(f, filename + ".geotiff")
print "path_file: " + path_file
update_published_layer_with_distribution_uid(uid_3857, uid)
def process_files():
files = glob.glob("/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/*.tif")
output_path = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/storage"
if not os.path.isdir(output_path):
# shutil.rmtree(output_path)
os.mkdir(output_path)
# for each file
for f in files:
# get filename
filename = get_filename(f)
extension = f.split(".")[1]
# if _4326 projection
if filename.endswith("_4326"):
# create the folder
folder_to_move_path = create_folder(output_path, filename)
# move the file to the returned folder path
final_path = os.path.join(folder_to_move_path, filename + "." + extension)
print "Final path: " + str(final_path)
shutil.move(f, final_path)
# rename the file is it's not yet .geotiff
if not final_path.endswith(".geotiff"):
f_geotiff = final_path.replace("."+extension, ".geotiff")
print "Renaming to: " + str(f_geotiff)
os.rename(final_path, f_geotiff)
def process_tifs_warp(input_path, output_path, input_ext='tif'):
if not os.path.exists(output_path):
os.makedirs(output_path)
files = glob.glob(input_path + "/*." + input_ext)
for f in files:
filename = get_filename(f)
output_file_path = output_path + "/" + filename + ".tif"
print output_file_path
cmd = "gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -co TILED=YES -co COMPRESS=DEFLATE '" + f + "' '" + output_file_path + "'"
print cmd
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
output, error = process.communicate()
print output
print error
def process_tifs_translate(input_path, output_path, base_str, replace_str):
if not os.path.exists(output_path):
os.makedirs(output_path)
files = glob.glob(input_path + "/*.tif")
for f in files:
filename = get_filename(f)
output_file_path = output_path + "/" + filename.replace(base_str, replace_str) + ".tif"
print output_file_path
cmd = "gdal_translate -co TILED=YES -co COMPRESS=DEFLATE '" + f + "' '" + output_file_path + "'"
print cmd
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
output, error = process.communicate()
print output
print error
add_gdaladdo(output_file_path)
def calc(basepath, output_path, layers, epsg="3857"):
print "-----Anomaly DPY"
for layer in layers:
# print get_year_by_filename(layer)
year = get_year_by_filename(layer)
data = None
month = get_month_by_filename(layer)
filename = get_filename(layer)
base_filename = get_base_filename(filename)
yearPrev = str(int(year) - 1)
layerPrev = basepath + "/" + base_filename + "_" + yearPrev + month + "_" + epsg + ".tif"
print "Processing: ", layer, layerPrev
try:
print "Reading: ", layer
r = rasterio.open(layer)
r_data = r.read_band(1).astype(float)
print "Reading: ", layerPrev
r_prev = rasterio.open(layerPrev)
r_prev_data = r_prev.read_band(1).astype(float)
if data is None:
data, kwargs = initialize_rasterio_raster(r, rasterio.float32)
nodata = 0
index1 = (r_data != nodata)
index2 = (r_prev_data != nodata)
r_data = index1 * index2 * r_data
r_prev_data = index1 * index2 * r_prev_data
data = r_data - r_prev_data
# writing
output_layer_path = output_path + "/" + filename + ".tif"
print "Writing: ", output_layer_path
with rasterio.open(output_layer_path, 'w', **kwargs) as dst:
dst.write_band(1, data.astype(rasterio.float32))
except Exception, e:
print e
def harvest_raster_folder(path, workspace=None):
# read files
files = glob.glob(os.path.join(path, "*"))
for f in files:
# sanitize file name?
filename = get_filename(f)
extension = get_file_extension(f)
if extension in supported_file:
log.info(filename)
metadata = parse_filename(filename, get_authority(f).upper())
metadata = create_metadata(metadata)
# check if exists a file named like the "file".json ( and in case overwrite the what is used there)
#metadata_file =
print metadata
if extension in metadata_file:
log.info(filename)
def publish(input_folder):
input_files = glob.glob(input_folder + "/*.geotiff")
# filename
for input_file in input_files:
print input_file
filename = get_filename(input_file)
filename, projection_code = filename.split("_")
projection_code = "EPSG:" + projection_code
product = "test_storage"
title = filename.replace("_", " ")
metadata_def = create_metadata(title, product, None, None, projection_code, None, None, True, None, "storage")
try:
print metadata_def
# upload
# data_manager.publish_coveragestore_storage(input_file, metadata_def, False, False, True)
except Exception, e:
print e
def process_tifs(base_path, output_path):
files = glob.glob(base_path + "/*.tif")
for f in files:
filename = get_filename(f)
output_file_path = output_path + "/" + filename + ".tif"
print output_file_path
# "-multi": "",
# "-overwrite": "",
# "-of": "GTiff",
# "-s_srs": "'+proj=sinu +R=6371007.181 +nadgrids=@null +wktext'",
# "-t_srs": "EPSG:3857"
#
# "a_nodata": "-3000",
# "-co": "'TILED=YES'",
# "-co": "'COMPRESS=DEFLATE'"
#
cmd = "gdal_translate -a_nodata -3000 -co TILED=YES -co COMPRESS=DEFLATE '" + f + "' '" + output_file_path + "'"
print cmd
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
output, error = process.communicate()
def export_raster_by_spatial_query(self,
user_json,
distribution_url=None,
distribution_folder=None):
log.info(user_json)
log.info(self.config)
# getting distribution folder
distribution_folder = self._get_distribution_folder(
distribution_folder)
# TODO remove dependency from here?
sq = SpatialQuery(self.config)
vector_filter = user_json["extract_by"]
db_options = vector_filter["options"]
db_datasource = db_options["db"]
layer_code = db_options["layer"]
column_code = db_options["column"]
codes = db_options["codes"]
email_address = None if "email_address" not in user_json else user_json[
"email_address"]
rasters = user_json["raster"]
log.info(rasters)
# create a random tmp folder
zip_folder_id = str(uuid.uuid4()).encode("utf-8")
zip_folder = os.path.join(distribution_folder, zip_folder_id)
os.mkdir(zip_folder)
# create a valid folder name to zip it
output_folder = os.path.join(zip_folder, "layers")
os.mkdir(output_folder)
output_files = []
for raster in rasters:
log.info(raster)
raster_path = get_raster_path(raster)
log.info(raster_path)
# turning relative to absolute path if required
# TODO: handle somehow better (it is used just for test)
if not os.path.isabs(raster_path):
# this is used to normalize relative path used during test
raster_path = os.path.normpath(
os.path.join(os.path.dirname(__file__), raster_path))
log.info(raster_path)
raster_path = os.path.abspath(raster_path)
log.info(raster_path)
srid = get_srid_raster(raster_path)
log.info(srid)
# retrieving bounding box
bbox = sq.query_bbox(db_datasource, layer_code, column_code, codes,
srid)
log.info(bbox)
# create the file on tm folder
db = sq.get_db_instance()
db_connection_string = db.get_connection_string(True)
query = sq.get_query_string_select_all(db_datasource, layer_code,
column_code, codes, "*")
log.info(query)
filepath = crop_raster_on_vector_bbox_and_postgis_db(
raster_path, db_connection_string, query, bbox[0][0],
bbox[0][1], bbox[1][0], bbox[1][1])
# bounding_box = crop_raster_with_bounding_box(raster_path, bbox[0][0], bbox[0][1], bbox[1][0], bbox[1][1])
# move file to distribution tmp folder
path, filename, name = get_filename(filepath, True)
dst_file = os.path.join(output_folder, filename)
move(filepath, dst_file)
# rename file based on uid layer_name (i.e. fenix:trmm_08_2014 -> trmm_08_2014)
output_filename = get_filename(raster_path) + ".tif"
output_file = os.path.join(output_folder, output_filename)
os.rename(dst_file, output_file)
# saving the output file to zip
output_files.append(output_file)
# zip folder or files
# TODO: change and use make_archive
#output_filename = os.path.join(zip_folder, zip_filename)
#make_archive(folder_to_zip, output_filename)
zip_path = zip_files(zip_filename, output_files, zip_folder)
# URL to the resource
if distribution_url is None:
return zip_path
else:
url = distribution_url + zip_folder_id
# send email if email address
self._send_email(url, email_address)
return '{ "url" : "' + url + '"}'
def export_raster_by_spatial_query(self, user_json, distribution_url=None, distribution_folder=None):
log.info(user_json)
log.info(self.config)
# getting distribution folder
distribution_folder = self._get_distribution_folder(distribution_folder)
# TODO remove dependency from here?
sq = SpatialQuery(self.config)
vector_filter = user_json["extract_by"]
db_options = vector_filter["options"]
db_datasource = db_options["db"]
layer_code = db_options["layer"]
column_code = db_options["column"]
codes = db_options["codes"]
email_address = None if "email_address" not in user_json else user_json["email_address"]
rasters = user_json["raster"]
log.info(rasters)
# create a random tmp folder
zip_folder_id = str(uuid.uuid4()).encode("utf-8")
zip_folder = os.path.join(distribution_folder, zip_folder_id)
os.mkdir(zip_folder)
# create a valid folder name to zip it
output_folder = os.path.join(zip_folder, "layers")
os.mkdir(output_folder)
output_files = []
for raster in rasters:
log.info(raster)
raster_path = get_raster_path(raster)
log.info(raster_path)
# turning relative to absolute path if required
# TODO: handle somehow better (it is used just for test)
if not os.path.isabs(raster_path):
# this is used to normalize relative path used during test
raster_path = os.path.normpath(os.path.join(os.path.dirname(__file__), raster_path))
log.info(raster_path)
raster_path = os.path.abspath(raster_path)
log.info(raster_path)
srid = get_srid_raster(raster_path)
log.info(srid)
# retrieving bounding box
bbox = sq.query_bbox(db_datasource, layer_code, column_code, codes, srid)
log.info(bbox)
# create the file on tm folder
db = sq.get_db_instance()
db_connection_string = db.get_connection_string(True)
query = sq.get_query_string_select_all(db_datasource, layer_code, column_code, codes, "*")
log.info(query)
filepath = crop_raster_on_vector_bbox_and_postgis_db(raster_path, db_connection_string, query, bbox[0][0], bbox[0][1], bbox[1][0], bbox[1][1])
# bounding_box = crop_raster_with_bounding_box(raster_path, bbox[0][0], bbox[0][1], bbox[1][0], bbox[1][1])
# move file to distribution tmp folder
path, filename, name = get_filename(filepath, True)
dst_file = os.path.join(output_folder, filename)
move(filepath, dst_file)
# rename file based on uid layer_name (i.e. fenix:trmm_08_2014 -> trmm_08_2014)
output_filename = get_filename(raster_path) + ".tif"
output_file = os.path.join(output_folder, output_filename)
os.rename(dst_file, output_file)
# saving the output file to zip
output_files.append(output_file)
# zip folder or files
# TODO: change and use make_archive
#output_filename = os.path.join(zip_folder, zip_filename)
#make_archive(folder_to_zip, output_filename)
zip_path = zip_files(zip_filename, output_files, zip_folder)
# URL to the resource
if distribution_url is None:
return zip_path
else:
url = distribution_url + zip_folder_id
# send email if email address
self._send_email(url, email_address)
return '{ "url" : "' + url + '"}'
def export_vector_by_spatial_query(self, user_json, distribution_url=None, distribution_folder=None):
vector_filter = user_json["extract_by"]
db_options = vector_filter["options"]
db_datasource = db_options["db"]
layer_code = db_options["layer"]
column_code = db_options["column"]
codes = db_options["codes"]
email_address = None if "email_address" not in user_json else user_json["email_address"]
vectors = user_json["vector"]
# getting distribution folder
distribution_folder = self._get_distribution_folder(distribution_folder)
# TODO remove dependency from here?
sq = SpatialQuery(self.config)
# get file to extract
output_dirs = []
for vector in vectors:
vector_path = get_vector_path(vector)
srid = get_srid_vector(vector_path)
#srid = "'merc'"
log.info(srid)
# get query
query = sq.get_query_string_select_all(db_datasource, layer_code, column_code, codes, "ST_Transform(geom, " + srid + ")")
log.info(query)
db = sq.get_db_instance()
db_connection_string = db.get_connection_string(True)
output_name = get_filename(vector_path)
output_file_path = crop_vector_on_vector_bbox_and_postgis(vector_path, '"' + db_connection_string + '"', query, output_name)
# check if filepath exists
if output_file_path:
output_dirs.append(os.path.dirname(output_file_path))
# create a random tmp folder
zip_folder_id = str(uuid.uuid4()).encode("utf-8")
zip_folder = os.path.join(distribution_folder, zip_folder_id)
os.mkdir(zip_folder)
# create a valid folder name to zip it
output_folder = os.path.join(zip_folder, "layers")
os.mkdir(output_folder)
# move output dirs to distribution folder
for output_dir in output_dirs:
for file in glob.glob(os.path.join(output_dir, "*")):
move(file, output_folder)
# zip the folder
tmp_file = create_tmp_filename()
tmp_zip_path = make_archive(zip_folder, tmp_file)
# TODO: workaround, strangly it cannot be created a zip file in the folder to zip
zip_path = os.path.join(zip_folder, "layers.zip")
os.rename(tmp_zip_path, zip_path)
log.info(zip_path)
# URL to the resource
if distribution_url is None:
return zip_path
else:
url = distribution_url + zip_folder_id
# send email if email address
self._send_email(url, email_address)
return '{ "url" : "' + url + '"}'
def export_vector_by_spatial_query(self,
user_json,
distribution_url=None,
distribution_folder=None):
vector_filter = user_json["extract_by"]
db_options = vector_filter["options"]
db_datasource = db_options["db"]
layer_code = db_options["layer"]
column_code = db_options["column"]
codes = db_options["codes"]
email_address = None if "email_address" not in user_json else user_json[
"email_address"]
vectors = user_json["vector"]
# getting distribution folder
distribution_folder = self._get_distribution_folder(
distribution_folder)
# TODO remove dependency from here?
sq = SpatialQuery(self.config)
# get file to extract
output_dirs = []
for vector in vectors:
vector_path = get_vector_path(vector)
srid = get_srid_vector(vector_path)
#srid = "'merc'"
log.info(srid)
# get query
query = sq.get_query_string_select_all(
db_datasource, layer_code, column_code, codes,
"ST_Transform(geom, " + srid + ")")
log.info(query)
db = sq.get_db_instance()
db_connection_string = db.get_connection_string(True)
output_name = get_filename(vector_path)
output_file_path = crop_vector_on_vector_bbox_and_postgis(
vector_path, '"' + db_connection_string + '"', query,
output_name)
# check if filepath exists
if output_file_path:
output_dirs.append(os.path.dirname(output_file_path))
# create a random tmp folder
zip_folder_id = str(uuid.uuid4()).encode("utf-8")
zip_folder = os.path.join(distribution_folder, zip_folder_id)
os.mkdir(zip_folder)
# create a valid folder name to zip it
output_folder = os.path.join(zip_folder, "layers")
os.mkdir(output_folder)
# move output dirs to distribution folder
for output_dir in output_dirs:
for file in glob.glob(os.path.join(output_dir, "*")):
move(file, output_folder)
# zip the folder
tmp_file = create_tmp_filename()
tmp_zip_path = make_archive(zip_folder, tmp_file)
# TODO: workaround, strangly it cannot be created a zip file in the folder to zip
zip_path = os.path.join(zip_folder, "layers.zip")
os.rename(tmp_zip_path, zip_path)
log.info(zip_path)
# URL to the resource
if distribution_url is None:
return zip_path
else:
url = distribution_url + zip_folder_id
# send email if email address
self._send_email(url, email_address)
return '{ "url" : "' + url + '"}'
def get_base_filename(f):
filename = get_filename(f)
s = filename.split("_")
# return ', '.s[len(s)-3]
return '_'.join(s[:len(s)-2])
def get_date_by_filename(f):
filename = get_filename(f)
s = filename.split("_")
date = s[len(s)-2]
return date
def get_month_by_filename(f):
filename = get_filename(f)
s = filename.split("_")
date = s[len(s)-2]
return date[4:6]
