def preprocess(self): ''' Top of atmosphere is calculated and persisted into a file. Then a cloud mask is created with the given algorithm. ''' solar_zenith = self.get_sensor().parser.get_attribute( rapideye.SOLAR_ZENITH) data_acquisition_date = self.get_sensor().parser.get_attribute( rapideye.ACQUISITION_DATE) solar_azimuth = self.get_sensor().parser.get_attribute( rapideye.SOLAR_AZIMUTH) geotransform = self.get_raster().get_attribute(raster.GEOTRANSFORM) data = self.get_raster().read_data_file_as_array() sun_earth_distance = calculate_distance_sun_earth( data_acquisition_date) top_of_atmosphere_data = calculate_toa_rapideye( calculate_rad_rapideye(data), sun_earth_distance, solar_zenith) top_of_atmosphere_directory = create_filename(get_parent(self.path), 'TOA') create_directory_path(top_of_atmosphere_directory) output_file = create_filename( top_of_atmosphere_directory, get_basename(self.get_files()[2]) + '_toa.tif') # TODO: change [2] in self.get_files()[2] create_raster_from_reference(output_file, top_of_atmosphere_data, self.file_dictionary[_IMAGE], data_type=NumericTypeCodeToGDALTypeCode( numpy.float32)) LOGGER.debug('Top of atmosphere file was created.') cloud_output_file = create_filename( top_of_atmosphere_directory, get_basename(self.get_files()[2]) + '_cloud.tif') if self.algorithm == ANOMALY_DETECTION: LOGGER.debug('Cloud mask by anomaly detection process.') clouds = self.anomaly_detection_cloud_mask(top_of_atmosphere_data, cloud_output_file, solar_zenith, solar_azimuth, geotransform) elif self.algorithm == TIME_SERIES: LOGGER.debug('Cloud mask by reference with time series process.') tile_id = self.get_sensor().get_attribute(TILE_ID) clouds = self.masking_with_time_series(data, cloud_output_file, solar_zenith, solar_azimuth, geotransform, tile_id) create_raster_from_reference(cloud_output_file, clouds, self.file_dictionary[_IMAGE], data_type=NumericTypeCodeToGDALTypeCode( numpy.float32)) LOGGER.info('Cloud mask was created.')
def handle(self, **options): output = options['output'][0] #state = options['state'][0] # for name in get_states_names(): # print name[0] # for footprint in get_rapideye_footprints_from_state(name[0]): # print footprint[0] # # # import time # time.sleep(2) for image_path in options['path']: print image_path basename = '%s_ipcc.tif' % get_basename(image_path) LOGGER.info(basename) target = create_filename(output, basename) #print target start_time = time.time() self.aggregate_by_block(image_path, target, INITIAL_IPCC_2015, FINAL_IPCC_2015) #self.method_by_block(image_path, target) #self.mask_iterating_values(image_path, target, INITIAL_ARRAY, FINAL_ARRAY) LOGGER.info("--- %s seconds ---" % (time.time() - start_time)) LOGGER.info('Dataset was written.')
def get_convex_hull(shape_name, destination_directory): ''' This method will read all objects from a shape file and create a new one with the convex hull of all the geometry points of the first. ''' driver = ogr.GetDriverByName(str('ESRI Shapefile')) shape = driver.Open(shape_name, 0) layer = shape.GetLayer() layer_name = layer.GetName() spatial_reference = layer.GetSpatialRef() prefix = get_basename(shape_name) output_name = create_filename(destination_directory, '%s-hull.shp' % prefix) geometries = ogr.Geometry(ogr.wkbGeometryCollection) for feature in layer: geometries.AddGeometry(feature.GetGeometryRef()) if os.path.exists(output_name): driver.DeleteDataSource(output_name) datasource = driver.CreateDataSource(output_name) out_layer = datasource.CreateLayer(str('states_convexhull'), spatial_reference, geom_type=ogr.wkbPolygon) out_layer.CreateField(ogr.FieldDefn(str('id'), ogr.OFTInteger)) featureDefn = out_layer.GetLayerDefn() feature = ogr.Feature(featureDefn) feature.SetGeometry(geometries.ConvexHull()) feature.SetField(str('id'), 1) out_layer.CreateFeature(feature) shape.Destroy() datasource.Destroy()
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' stack = options['path'][0] output_vector_file = '/Users/agutierrez/%s.gpkg' % get_basename(stack) segmentation = bis.Model() shapes, transform, meta = segmentation.predict(stack) # Vectorize #shapes = features.shapes(segments.astype(np.uint16), transform=transform) start_time = time.time() print 'about to start query' query = persist_database(shapes, meta) print 'done' print time.time() - start_time start_time = time.time() with connection.cursor() as cursor: cursor.execute(query) transaction.commit() print time.time() - start_time
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' output = options['output'][0] models = options['modelname'] model_directory = options['modeldir'][0] pca_model = pca.Model(5) pca_model.load(create_filename(model_directory, 'pca')) for model_name in models: persistence_directory = create_filename(model_directory, model_name) model = load_model(model_name) model_instance = model.Model(persistence_directory) model_instance.load(persistence_directory) block_size = 500 for path in options['path']: image_array = open_handle(path) y_size = image_array.shape[1] x_size = image_array.shape[2] basename = get_basename(path)[:7] warnings.filterwarnings('ignore') final = numpy.zeros((x_size, y_size)) import time start_time = time.time() for i in range(0, y_size, block_size): if i + block_size < y_size: rows = block_size else: rows = y_size - i for j in range(0, x_size, block_size): if j + block_size < x_size: cols = block_size else: cols = x_size - j step = image_array[:, i:i + rows, j:j + cols] step_ravel = step.reshape(10, -1) prediction = model_instance.predict( pca_model.transform(numpy.transpose(step_ravel))) final[i:i + rows, j:j + cols] = prediction.reshape( (rows, cols)) print("--- %s seconds ---" % (time.time() - start_time)) create_directory_path(output) classification = create_filename( output, '%s-%s.tif' % (basename, model_name)) create_raster_from_reference(classification, final.reshape(x_size, y_size), path, data_type=gdal.GDT_Byte, creating_options=['COMPRESS=LZW'])
def get_thumbnail_with_path(self, thumbnail_path): ''' Creates a thumbnail for the scene in true color. ''' from subprocess import call thumnail_directory = create_filename(thumbnail_path, 'thumbnail') create_directory_path(thumnail_directory) filename = self.file_dictionary[_BROWSE] thumbnail = create_filename(thumnail_directory, '%s.jpg' % get_basename(filename)) resize_command = [ '/Library/Frameworks/GDAL.framework/Programs/gdal_translate', filename, '-of', 'JPEG', thumbnail ] call(resize_command)
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' output = options['output'][0] zip = options['path'][0] basename = get_basename(zip) aux_name = create_filename(output, 'aux_%s' % basename) real_name = create_filename(output, basename) with ZipFile(zip, 'r') as unzipped: unzipped.extractall(create_filename(output, 'aux_%s' % basename)) path = create_filename(aux_name, 'trend_tiles') tifs = get_contents_from_folder(path) create_directory_path(real_name) for tif in tifs: source = create_filename(path, tif) target = create_filename(real_name, tif) tile_map(source, target, 4000, 4000, 2, 2) remove_directory(aux_name)
def get_year_from_path(self, path): return get_basename(path)[19:23]
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' shape_name = options['shape'][0] destination = options['dest'][0] paths = options['path'] years = [] for path in paths: years.append(self.get_year_from_path(path)) if os.path.exists(shape_name): LOGGER.info('The file %s was found.' % shape_name) shape_files = split_shape_into_features(shape_name, destination, str('id'), str('nombre'), '__') process_launcher = LocalProcessLauncher() import time start_time = time.time() cover_array = [] tth_array = [] cover_file = 'cover-stats.json' tth_name = 'tth-stats.json' json_directory = create_filename(destination, 'cover_stats') json_file = create_filename(json_directory, cover_file.lower()) tth_file = create_filename(json_directory, tth_name.lower()) for shape_file in shape_files: shape_name = get_basename(shape_file).split('__') anp_id = shape_name[0] anp_name = shape_name[1] basename = '%s.tif' % anp_name dataframe = DataFrame(index=years, columns=[0, 1, 2, 3, 4, 5, 6, 7, 8]) create_directory_path(json_directory) print shape_file for path in paths: #pixel_resolution, dataset = pixel_info(path) year = self.get_year_from_path(path) raster_dir = create_filename( create_filename(destination, 'raster'), year) create_directory_path(raster_dir) raster_file = create_filename(raster_dir, basename) shell_command = 'gdalwarp -ot Byte -co COMPRESS=LZW -cutline %s -crop_to_cutline %s %s' % ( shape_file, path, raster_file) print shell_command print process_launcher.execute(shell_command) raster_array = open_handle(raster_file) ds = gdal.Open(raster_file) geotransform = ds.GetGeoTransform() x_resolution = geotransform[1] y_resolution = geotransform[5] pixel_area = abs(x_resolution * y_resolution) unique_values = numpy.unique(raster_array, return_counts=True) indexes = unique_values[0] counts = unique_values[1] for i in range(len(indexes)): key = indexes[i] dataframe.set_value(year, key, area(int(counts[i]), pixel_area)) dataframe = dataframe.sort_index() columns = list(dataframe.columns.values) index = list(dataframe.index.values) print dataframe cover_array.append( self.dataframe_to_json(dataframe, anp_id, anp_name)) tth_dataframe = DataFrame(columns=columns) for i in range(len(index) - 1): label = '%s-%s' % (index[i], index[i + 1]) tth_column = calculate_thh(dataframe.ix[i], dataframe.ix[i + 1], int(index[i + 1]) - int(index[i])) for j in range(len(tth_column)): tth_dataframe.set_value(label, j, tth_column[j]) tth_dataframe = tth_dataframe.sort_index() print tth_dataframe tth_array.append( self.dataframe_to_json(tth_dataframe, anp_id, anp_name)) self.json_to_file(json_file, cover_array) self.json_to_file(tth_file, tth_array) print("--- %s seconds ---" % (time.time() - start_time))
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' output = options['output'][0] for path in options['path']: bundle = _get_bundle_from_path(path) basename = get_basename(bundle.get_raster_file()) bundle.get_NDVI() ndvi_file = create_filename(output, 'ndvi.tif') red_edge_ndvi_file = create_filename(output, 'red_edge_ndvi.tif') gndvi_file = create_filename(output, 'gndvi.tif') ndre_file = create_filename(output, 'ndre.tif') sovel_file = create_filename(output, 'sovel2.tif') all_file = create_filename(output, '%s_all_features.tif' % basename) print all_file image_array = bundle.get_raster().read_data_file_as_array() ndvi_array = bundle.get_NDVI() ndvi_array[ndvi_array <= -1] = -1 ndvi_array[ndvi_array >= 1] = 1 red_edge_ndvi_array = bundle.get_red_edge_NDVI() red_edge_ndvi_array[red_edge_ndvi_array <= -1] = -1 red_edge_ndvi_array[red_edge_ndvi_array >= 1] = 1 gndvi_array = bundle.get_gndvi() gndvi_array[gndvi_array <= -1] = -1 gndvi_array[gndvi_array >= 1] = 1 ndre_array = bundle.get_ndre() ndre_array[ndre_array <= -1] = -1 ndre_array[ndre_array >= 1] = 1 sobel_filter_array = bundle.get_sobel_filter(sigma=2) #sobel_filter_array[sobel_filter_array<=-1] = -1 #sobel_filter_array[sobel_filter_array>=1] = 1 #create_raster_from_reference(ndvi_file, ndvi_array, bundle.get_raster_file()) #create_raster_from_reference(red_edge_ndvi_file, red_edge_ndvi_array, bundle.get_raster_file()) #create_raster_from_reference(gndvi_file, gndvi_array, bundle.get_raster_file()) #create_raster_from_reference(ndre_file, ndre_array, bundle.get_raster_file()) create_raster_from_reference(sovel_file, sobel_filter_array, bundle.get_raster_file()) all_features = numpy.array([ image_array[0], image_array[1], image_array[2], image_array[3], image_array[4], ndvi_array, red_edge_ndvi_array, gndvi_array, ndre_array, sobel_filter_array ]) print image_array[0].shape print image_array[1].shape print image_array[2].shape print image_array[3].shape print image_array[4].shape print ndvi_array.shape print red_edge_ndvi_array.shape print gndvi_array.shape print ndre_array.shape print sobel_filter_array.shape print all_features.shape create_raster_from_reference(all_file, all_features, bundle.get_raster_file(), creating_options=['BIGTIFF=YES'])
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' output = options['output'][0] models = options['modelname'] model_directory = options['modeldir'][0] region = options['region'][0] start_time = time.time() for path in options['path']: print path scene_bundle = rapideye.Bundle(path) directory = getattr(SETTINGS, 'TEMPORARY') directory_helper = create_filename(directory, 'helper') create_directory_path(directory_helper) categories_file = create_filename(directory, 'categories.json') categories_dictionaty = { 0: "AGRICULTURA DE RIEGO", 1: "AGRICULTURA DE TEMPORAL", 2: "AGUA", 3: "AREAS QUEMADAS", 4: "ASENTAMIENTOS HUMANOS", 5: "BOSQUE CULTIVADO", 6: "BOSQUE DE AYARIN", 7: "BOSQUE DE ENCINO", 8: "BOSQUE DE ENCINO-PINO", 9: "BOSQUE DE GALERIA", 10: "BOSQUE DE MEZQUITE", 11: "BOSQUE DE OYAMEL", 12: "BOSQUE DE PINO", 13: "BOSQUE DE PINO-ENCINO", 14: "BOSQUE INDUCIDO", 15: "BOSQUE MESOFILO DE MONTANA", 16: "DESPROVISTO DE VEGETACION", 17: "INDEFINIDO", 18: "MANGLAR", 19: "MATORRAL SUBTROPICAL", 20: "MEZQUITAL", 21: "NUBES", 22: "PASTIZAL CULTIVADO", 23: "PASTIZAL HALOFILO", 24: "PASTIZAL INDUCIDO", 25: "PASTIZAL NATURAL", 26: "PRADERA DE ALTA MONTANA", 27: "SABANOIDE", 28: "SELVA ALTA PERENNIFOLIA", 29: "SELVA ALTA SUBPERENNIFOLIA", 30: "SELVA BAJA CADUCIFOLIA", 31: "SELVA BAJA ESPINOSA CADUCIFOLIA", 32: "SELVA BAJA SUBCADUCIFOLIA", 33: "SELVA DE GALERIA", 34: "SELVA MEDIANA CADUCIFOLIA", 35: "SELVA MEDIANA SUBCADUCIFOLIA", 36: "SELVA MEDIANA SUBPERENNIFOLIA", 37: "SIN VEGETACION APARENTE", 38: "SOMBRAS", 39: "TULAR", 40: "VEGETACION DE DUNAS COSTERAS", 41: "VEGETACION HALOFILA HIDROFILA", 42: "ZONA URBANA" } basename = get_basename(scene_bundle.get_raster_file()) all_file = create_filename(directory_helper, '%s_all_features.tif' % basename) if not is_file(all_file): scene_bundle.get_feature_array(all_file) filename = get_basename(all_file) if not is_file( create_filename(directory_helper, '%s.shp' % filename)): shell_string = 'docker run --rm -v %s:/data madmex/segment gdal-segment %s.tif -out helper/%s.shp -algo SLIC -region %s' % ( directory, filename, filename, region) launcher = LocalProcessLauncher() LOGGER.debug('Docker command: %s', shell_string) launcher.execute(shell_string) data = read_data_table( create_filename(directory_helper, '%s.shp' % filename)) results = {} for model_name in models: persistence_directory = create_filename( model_directory, model_name) print model_name model = load_model(model_name) model_instance = model.Model(persistence_directory) model_instance.load(persistence_directory) prediction = model_instance.predict(data) results[model_name] = prediction print results create_directory_path(output) write_results( create_filename(directory_helper, '%s.shp' % filename), create_filename(output, '%s_classification.shp' % filename[0:32]), results, categories_dictionaty) LOGGER.info("--- %s seconds ---" % (time.time() - start_time))
def copy_with_dictionary(self, output_directory, dictionary): ''' This method will take a input shape and iterate over its features, creating a new shape file with each one of them. It copies all the fields and the same spatial reference from the original file. The created files are saved in the destination directory using the number of the field given. ''' layer = self.get_layer() layer_name = layer.GetName() spatial_reference = layer.GetSpatialRef() in_feature = layer.GetNextFeature() layer_definition = layer.GetLayerDefn() field_definition = layer_definition.GetFieldDefn(0) column_name = field_definition.GetName() shape_files = [] name = get_basename(self.image_path) create_directory_path(output_directory) output_name = create_filename(output_directory, '%s_mapped.shp' % name) in_layer_definition = layer.GetLayerDefn() data_source = self.driver.CreateDataSource(output_name) out_layer = data_source.CreateLayer(layer_name, spatial_reference, geom_type=ogr.wkbPolygon) while in_feature: in_feature_name = in_feature.GetField(column_name) shape_files.append(output_name) if is_file(output_name): self.driver.DeleteDataSource(output_name) for i in range(0, in_layer_definition.GetFieldCount()): fieldDefn = in_layer_definition.GetFieldDefn(i) out_layer.CreateField(fieldDefn) for i in range(0, in_layer_definition.GetFieldCount()): fieldDefn = in_layer_definition.GetFieldDefn(i) idField = ogr.FieldDefn(str("%sINTEGER" % fieldDefn.GetNameRef()), ogr.OFTInteger) out_layer.CreateField(idField) outLayerDefn = out_layer.GetLayerDefn() geometry = in_feature.GetGeometryRef() out_feature = ogr.Feature(outLayerDefn) out_feature.SetGeometry(geometry) for i in range(0, in_layer_definition.GetFieldCount()): out_feature.SetField(outLayerDefn.GetFieldDefn(i).GetNameRef(), in_feature.GetField(i)) for i in range(0, in_layer_definition.GetFieldCount()): out_feature.SetField(str("%sINTEGER" % in_layer_definition.GetFieldDefn(i).GetNameRef()), str(dictionary[in_feature.GetField(i)])) out_layer.CreateFeature(out_feature) out_feature = None in_feature = None in_feature = layer.GetNextFeature() self.close() return [Data(filename) for filename in shape_files]
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' target_tag = 'DN' start_time_all = time.time() shape_name = options['shape'][0] raster_paths = options['path'] destination = options['dest'] models = options['model'] dataframe_features = None temporary_directory = getattr(SETTINGS, 'TEMPORARY') create_directory_path(temporary_directory) # I read the training data in shape form training_shape = vector.Data(shape_name) training_dataframe = training_shape.to_dataframe() training_path = create_filename(temporary_directory, 'training_raster.tif') categories_file = create_filename(temporary_directory, 'categories.json') training_warped_path = create_filename(temporary_directory, 'training_warped_raster.tif') pixel_size = 0.000462175996292 if not is_file(training_warped_path): training_raster = vector_to_raster( training_shape, training_path, pixel_size, -pixel_size, ['ATTRIBUTE=OBJECTID', 'COMPRESS=LZW']) training_raster_warped = training_raster.reproject( training_warped_path, epgs=32617) else: training_raster_warped = raster.Data(training_warped_path) dem_file = getattr(SETTINGS, 'DEM') dem_raster = raster.Data(dem_file) print dem_raster.get_spatial_reference() print 'reproyecting raster' #dem_raster_warped = dem_raster.reproject(training_warped_path, epgs=32614) #training_raster_warped = raster.Data(training_path) aspect_file = getattr(SETTINGS, 'ASPECT') slope_file = getattr(SETTINGS, 'SLOPE') print dem_file, aspect_file, slope_file for raster_path in raster_paths: scene_bundle = rapideye.Bundle(raster_path) raster_mask = scene_bundle.get_raster() #example_path = create_filename(temporary_directory, 'mask') #create_directory_path(example_path) #raster_to_vector_mask(raster_mask, example_path) print scene_bundle.get_raster_file() basename = get_basename(scene_bundle.get_raster_file()) all_file = create_filename(temporary_directory, '%s_all_features.tif' % basename) # Do not recalculate if the file is already there. if is_file(all_file): features_raster = raster.Data(all_file) else: features_raster = scene_bundle.get_feature_array(all_file) new_df = get_dataframe_from_raster(features_raster, training_raster_warped) if new_df is not None: if dataframe_features is not None: dataframe_features = pandas.concat([ dataframe_features, get_dataframe_from_raster(features_raster, training_raster_warped) ]) else: dataframe_features = get_dataframe_from_raster( features_raster, training_raster_warped) features_size = len(list(dataframe_features)) training_set = dataframe_features.set_index(0).join( training_dataframe.set_index('OBJECTID')) print training_set training_set['target'] = pandas.Categorical.from_array( training_set[target_tag]).labels categories_array = pandas.Categorical.from_array( training_set[target_tag]).categories create_categories_file(categories_file, categories_array) training_set = training_set[training_set['target'] != -1] #features_size includes 0 that is the index of the feature training_set_array = numpy.transpose( numpy.transpose(training_set.as_matrix([range(1, features_size)]))) target_set_array = training_set.pop('target') print training_set_array.shape print target_set_array.shape X_train, X_test, y_train, y_test = train_test_split(training_set_array, target_set_array, train_size=0.8, test_size=0.2) models_directory = create_filename(temporary_directory, 'models') create_directory_path(models_directory) for model_name in models: start_time = time.time() print numpy.unique(y_train) train_model(X_train, X_test, y_train, y_test, models_directory, model_name) print "--- %s seconds training %s model---" % ( (time.time() - start_time), model_name)