def crop_by_vector_by_vector(crop_shp_path, input_path, output_name=None): # ogr2ogr -skipfailures -clipsrc afg.shp output.shp G2014_2013_1_mid.shp output_path = None if output_name is None: output_path = create_tmp_filename('shp', 'tmp_shp', 'tmp_shp_' + str(uuid.uuid4()), False) else: # TODO: # get the folder by output_path filename # check if the folder is created, otherwise create it log.warn("TODO: get the folder by output_path filename. check if the folder is created, otherwise create it.") output_path = create_tmp_filename('shp', output_name, 'tmp_shp_' + str(uuid.uuid4()), False) args = [ 'ogr2ogr', '-skipfailures', # TODO: optional overwrite '-overwrite', '-clipsrc', crop_shp_path, output_path, input_path, ] try: cmd = " ".join(args) log.info(cmd) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) p.wait() except Exception, e: log.error(e) raise Exception(e)
def crop_by_vector_from_db(input_file, db_connection_string, query, srcnodata='nodata', dstnodata='nodata'): log.info(query) output_file_gdal_warp = create_tmp_filename('.tif', 'gdal_warp') output_file = create_tmp_filename('.tif', 'output') log.info(input_file) # crop the layer on cutline args = [ "gdalwarp", "-q", "-multi", "-of", "GTiff", "-cutline", db_connection_string, "-csql", query, "-srcnodata", str(srcnodata), "-dstnodata", str(dstnodata), # -crop_to_cutline is needed otherwise the layer is not cropped # TODO: resolve shifting problem # "-crop_to_cutline", # "-dstalpha", input_file, output_file_gdal_warp ] try: log.info(args) #TODO: handle subprocess Error (like that is not taken) output = subprocess.check_output(args) # stdout_value, error = proc.communicate() log.info(output) except Exception, e: raise Exception(e)
def _merge_layers(rows, cols, geotransform, spatialreference, array1, array2, min1, max1, min2, max2, nodata1=None, nodata2=None): path = create_tmp_filename('', ".tif") # find the indexes of the arrays index1 = (array1 > min1) & (array1 <= max1) & (array1 != nodata1) index2 = (array2 > min2) & (array2 <= max2) & (array2 != nodata2) # merge array indexes compound_index = index1 & index2 del index1, index2 # create a new raster output_raster = gdal.GetDriverByName('GTiff').Create( path, rows, cols, 1, gdal.GDT_Int16) # Open the file output_raster.SetGeoTransform(geotransform) srs = SpatialReference(wkt=spatialreference) output_raster.SetProjection(srs.ExportToWkt()) # create raster from the compound_index of the two rasters # TODO: the reshape slows the operation, use matrixes output_raster.GetRasterBand(1).WriteArray( compound_index.reshape(cols, rows)) return path
def crop_raster_with_bounding_box(input_file, minlon, minlat, maxlon, maxlat, srcnodata=None): if srcnodata == None: srcnodata = get_nodata_value(input_file) log.info("crop_raster_with_bounding_box") output_file = create_tmp_filename('.tif', 'gdal_translate_by_bbox') args = [ "gdal_translate", "-a_nodata", str(srcnodata), "-projwin", str(minlat), str(minlon), str(maxlat), str(maxlon), input_file, output_file ] try: log.info(args) proc = subprocess.call(args, stdout=subprocess.PIPE, stderr=None) # proc = subprocess.check_output(args) except Exception, e: raise Exception(e)
def create_shp_from_postgis(db_connection_string, query, output_path=None): ''' Create a shp from a post :param db_connection_string: :param query: :return: ''' # ogr2ogr -f "ESRI Shapefile" /home/vortex/Desktop/export/afg.shp PG:"host=localhost user=fenix dbname=fenix password=Qwaszx" -sql "select * from spatial.gaul0_2015_4326 where adm0_code IN ('1')" # ogr2ogr -f "ESRI Shapefile" output.shp PG:"" -sql "query" if output_path is None: output_path = create_tmp_filename('shp', 'tmp_shp_postgis', 'tmp_shp_postgis_' + str(uuid.uuid4()), False) else: # TODO: # get the folder by output_path filename # check if the folder is created, otherwise create it log.warn("TODO: get the folder by output_path filename. check if the folder is created, otherwise create it.") args = [ 'ogr2ogr', # TODO: optional overwrite '-overwrite', '-f', '"ESRI Shapefile"', output_path, db_connection_string, "-sql", "\"" + query + "\"" ] try: cmd = " ".join(args) log.info(cmd) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) p.wait() except Exception, e: log.error(e) raise Exception(e)
def crop_by_vector_from_db(input_file, db_connection_string, query, srcnodata='nodata', dstnodata='nodata'): log.info(query) output_file_gdal_warp = create_tmp_filename('.tif', 'gdal_warp') output_file = create_tmp_filename('.tif', 'output') log.info(input_file) # crop the layer on cutline args = [ "gdalwarp", "-q", "-multi", "-of", "GTiff", "-cutline", db_connection_string, "-csql", query, "-srcnodata", str(srcnodata), "-dstnodata", str(dstnodata), # -crop_to_cutline is needed otherwise the layer is not cropped # TODO: resolve shifting problem # "-crop_to_cutline", # "-dstalpha", input_file, output_file_gdal_warp ] try: log.info(args) #TODO: handle subprocess Error (like that is not taken) output = subprocess.check_output(args) # stdout_value, error = proc.communicate() log.info(output) except Exception, e: raise Exception(e)
def _merge_layers(rows, cols, geotransform, spatialreference, array1, array2, min1, max1, min2, max2, nodata1=None, nodata2=None): path = create_tmp_filename('', ".tif") # find the indexes of the arrays index1 = (array1 > min1) & (array1 <= max1) & (array1 != nodata1) index2 = (array2 > min2) & (array2 <= max2) & (array2 != nodata2) # merge array indexes compound_index = index1 & index2 del index1, index2 # create a new raster output_raster = gdal.GetDriverByName('GTiff').Create(path, rows, cols, 1, gdal.GDT_Int16) # Open the file output_raster.SetGeoTransform(geotransform) srs = SpatialReference(wkt=spatialreference) output_raster.SetProjection(srs.ExportToWkt()) # create raster from the compound_index of the two rasters # TODO: the reshape slows the operation, use matrixes output_raster.GetRasterBand(1).WriteArray(compound_index.reshape(cols, rows)) return path
def crop_vector_with_bounding_box(input_file_bbox, file_to_crop, output_path=None): # ogr2ogr -f "ESRI Shapefile" output.shp input.shp -clipsrc <x_min> <y_min> <x_max> <y_max> with fiona.open(input_file_bbox) as c: with fiona.open(file_to_crop) as d: if output_path is None: output_path = create_tmp_filename('shp', 'tmp_shp_bbox', 'tmp_shp_bbox_' + str(uuid.uuid4()), False) bounds = c.bounds # s_srs = c.crs['init'] # t_srs = d.crs['init'] # check bounds if bounds[0] == 0.0 and bounds[1] == 0.0 and bounds[2] == 0.0 and bounds[3] == 0.0: msg = "Shapefile " + input_file_bbox + " has 0 invalide size" log.error(msg) raise Exception(msg) # TODO: change bounds coordinate sysmte in case it's needed (i.e. s_srs != t_srs) args = [ 'ogr2ogr', # TODO: optional overwrite '-overwrite', '-f', '"ESRI Shapefile"', output_path, file_to_crop, '-clipsrc', str(bounds[0]), str(bounds[1]), str(bounds[2]), str(bounds[3]), ] try: cmd = " ".join(args) log.info(cmd) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) p.wait() except Exception, e: log.error(e) raise Exception(e)
def crop_raster_with_bounding_box(input_file, minlon, minlat, maxlon, maxlat, srcnodata=None): if srcnodata == None: srcnodata = get_nodata_value(input_file) log.info("crop_raster_with_bounding_box") output_file = create_tmp_filename('.tif', 'gdal_translate_by_bbox') args = [ "gdal_translate", "-a_nodata", str(srcnodata), "-projwin", str(minlat), str(minlon), str(maxlat), str(maxlon), input_file, output_file ] try: log.info(args) proc = subprocess.call(args, stdout=subprocess.PIPE, stderr=None) # proc = subprocess.check_output(args) except Exception, e: raise Exception(e)
def export_vector_by_spatial_query(self, user_json, distribution_url=None, distribution_folder=None): vector_filter = user_json["extract_by"] db_options = vector_filter["options"] db_datasource = db_options["db"] layer_code = db_options["layer"] column_code = db_options["column"] codes = db_options["codes"] email_address = None if "email_address" not in user_json else user_json["email_address"] vectors = user_json["vector"] # getting distribution folder distribution_folder = self._get_distribution_folder(distribution_folder) # TODO remove dependency from here? sq = SpatialQuery(self.config) # get file to extract output_dirs = [] for vector in vectors: vector_path = get_vector_path(vector) srid = get_srid_vector(vector_path) #srid = "'merc'" log.info(srid) # get query query = sq.get_query_string_select_all(db_datasource, layer_code, column_code, codes, "ST_Transform(geom, " + srid + ")") log.info(query) db = sq.get_db_instance() db_connection_string = db.get_connection_string(True) output_name = get_filename(vector_path) output_file_path = crop_vector_on_vector_bbox_and_postgis(vector_path, '"' + db_connection_string + '"', query, output_name) # check if filepath exists if output_file_path: output_dirs.append(os.path.dirname(output_file_path)) # create a random tmp folder zip_folder_id = str(uuid.uuid4()).encode("utf-8") zip_folder = os.path.join(distribution_folder, zip_folder_id) os.mkdir(zip_folder) # create a valid folder name to zip it output_folder = os.path.join(zip_folder, "layers") os.mkdir(output_folder) # move output dirs to distribution folder for output_dir in output_dirs: for file in glob.glob(os.path.join(output_dir, "*")): move(file, output_folder) # zip the folder tmp_file = create_tmp_filename() tmp_zip_path = make_archive(zip_folder, tmp_file) # TODO: workaround, strangly it cannot be created a zip file in the folder to zip zip_path = os.path.join(zip_folder, "layers.zip") os.rename(tmp_zip_path, zip_path) log.info(zip_path) # URL to the resource if distribution_url is None: return zip_path else: url = distribution_url + zip_folder_id # send email if email address self._send_email(url, email_address) return '{ "url" : "' + url + '"}'
def export_vector_by_spatial_query(self, user_json, distribution_url=None, distribution_folder=None): vector_filter = user_json["extract_by"] db_options = vector_filter["options"] db_datasource = db_options["db"] layer_code = db_options["layer"] column_code = db_options["column"] codes = db_options["codes"] email_address = None if "email_address" not in user_json else user_json[ "email_address"] vectors = user_json["vector"] # getting distribution folder distribution_folder = self._get_distribution_folder( distribution_folder) # TODO remove dependency from here? sq = SpatialQuery(self.config) # get file to extract output_dirs = [] for vector in vectors: vector_path = get_vector_path(vector) srid = get_srid_vector(vector_path) #srid = "'merc'" log.info(srid) # get query query = sq.get_query_string_select_all( db_datasource, layer_code, column_code, codes, "ST_Transform(geom, " + srid + ")") log.info(query) db = sq.get_db_instance() db_connection_string = db.get_connection_string(True) output_name = get_filename(vector_path) output_file_path = crop_vector_on_vector_bbox_and_postgis( vector_path, '"' + db_connection_string + '"', query, output_name) # check if filepath exists if output_file_path: output_dirs.append(os.path.dirname(output_file_path)) # create a random tmp folder zip_folder_id = str(uuid.uuid4()).encode("utf-8") zip_folder = os.path.join(distribution_folder, zip_folder_id) os.mkdir(zip_folder) # create a valid folder name to zip it output_folder = os.path.join(zip_folder, "layers") os.mkdir(output_folder) # move output dirs to distribution folder for output_dir in output_dirs: for file in glob.glob(os.path.join(output_dir, "*")): move(file, output_folder) # zip the folder tmp_file = create_tmp_filename() tmp_zip_path = make_archive(zip_folder, tmp_file) # TODO: workaround, strangly it cannot be created a zip file in the folder to zip zip_path = os.path.join(zip_folder, "layers.zip") os.rename(tmp_zip_path, zip_path) log.info(zip_path) # URL to the resource if distribution_url is None: return zip_path else: url = distribution_url + zip_folder_id # send email if email address self._send_email(url, email_address) return '{ "url" : "' + url + '"}'
def _crop_by_vector_database(input_file, query, db_connection_string, minlat, minlon, maxlat, maxlon, srcnodata='nodata', dstnodata='nodata'): log.info(query) output_file_gdal_translate = create_tmp_filename('.tif', 'gdal_translate') output_file_gdal_warp = create_tmp_filename('.tif', 'gdal_warp') output_file = create_tmp_filename('.tif', 'output') args = [ 'gdal_translate', '-projwin', str(minlat), str(minlon), str(maxlat), str(maxlon), input_file, output_file_gdal_translate ] try: log.info(args) #TODO: handle subprocess Error (like that is not taken) proc = subprocess.call(args, stdout=subprocess.PIPE, stderr=None) except: stdout_value = proc.communicate()[0] raise Exception(stdout_value) args = [ 'gdalwarp', "-q", "-multi", "-of", "GTiff", "-cutline", db_connection_string, "-csql", query, "-srcnodata", str(srcnodata), "-dstnodata", str(dstnodata), # -crop_to_cutline is needed otherwise the layer is not cropped # TODO: resolve shifting problem # "-crop_to_cutline", # "-dstalpha", output_file_gdal_translate, output_file_gdal_warp ] try: #log.info(args) #TODO: handle subprocess Error (like that is not taken) proc = subprocess.call(args, stdout=subprocess.PIPE, stderr=None) except: stdout_value = proc.communicate()[0] raise Exception(stdout_value) # TODO: is it useful the third opetation? args = [ 'gdal_translate', "-co", "COMPRESS=DEFLATE", "-a_nodata", str(dstnodata), output_file_gdal_warp, output_file ] try: log.info(args) #TODO: handle subprocess Error (like that is not taken) proc = subprocess.call(args, stdout=subprocess.PIPE, stderr=None) except: stdout_value = proc.communicate()[0] raise Exception(stdout_value) os.remove(output_file_gdal_warp) os.remove(output_file_gdal_translate) if os.path.isfile(output_file): return output_file return None
def _crop_by_vector_database(input_file, query, db_connection_string, minlat, minlon, maxlat, maxlon, srcnodata='nodata', dstnodata='nodata'): log.info(query) output_file_gdal_translate = create_tmp_filename('.tif', 'gdal_translate') output_file_gdal_warp = create_tmp_filename('.tif', 'gdal_warp') output_file = create_tmp_filename('.tif', 'output') args = [ 'gdal_translate', '-projwin', str(minlat), str(minlon), str(maxlat), str(maxlon), input_file, output_file_gdal_translate ] try: log.info(args) #TODO: handle subprocess Error (like that is not taken) proc = subprocess.call(args, stdout=subprocess.PIPE, stderr=None) except: stdout_value = proc.communicate()[0] raise Exception(stdout_value) args = [ 'gdalwarp', "-q", "-multi", "-of", "GTiff", "-cutline", db_connection_string, "-csql", query, "-srcnodata", str(srcnodata), "-dstnodata", str(dstnodata), # -crop_to_cutline is needed otherwise the layer is not cropped # TODO: resolve shifting problem # "-crop_to_cutline", # "-dstalpha", output_file_gdal_translate, output_file_gdal_warp ] try: #log.info(args) #TODO: handle subprocess Error (like that is not taken) proc = subprocess.call(args, stdout=subprocess.PIPE, stderr=None) except: stdout_value = proc.communicate()[0] raise Exception(stdout_value) # TODO: is it useful the third opetation? args = [ 'gdal_translate', "-co", "COMPRESS=DEFLATE", "-a_nodata", str(dstnodata), output_file_gdal_warp, output_file ] try: log.info(args) #TODO: handle subprocess Error (like that is not taken) proc = subprocess.call(args, stdout=subprocess.PIPE, stderr=None) except: stdout_value = proc.communicate()[0] raise Exception(stdout_value) os.remove(output_file_gdal_warp) os.remove(output_file_gdal_translate) if os.path.isfile(output_file): return output_file return None