def get_thumbnail(self): ''' Creates a thumbnail for the scene in true color. ''' from subprocess import call file_1 = self.file_dictionary[_BASE % (self.get_mission(), 'B1[0-9].TIF')] file_2 = self.file_dictionary[_BASE % (self.get_mission(), 'B2[0-9].TIF')] file_3 = self.file_dictionary[_BASE % (self.get_mission(), 'B3[0-9].TIF')] parent_directory = get_parent(self.path) thumnail_directory = create_filename(parent_directory, 'thumbnail') create_directory_path(thumnail_directory) parent_directory filename = create_filename(thumnail_directory, 'vrt.tif') merge_command = [ '/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt', '-separate', '-o', filename, file_3, file_2, file_1 ] call(merge_command) thumbnail = create_filename(thumnail_directory, 'thumbnail.jpg') resize_command = [ '/Library/Frameworks/GDAL.framework/Programs/gdal_translate', filename, '-of', 'JPEG', '-outsize', '5%', '5%', thumbnail ] call(resize_command)
def save(self, filepath): ''' Persists the trained model to a file. ''' joblib.dump(self.model, create_filename(filepath, '%s.pkl' % self.model_name)) joblib.dump(self.scaler, create_filename(filepath, '%s.pkl' % self.scaler_mlp))
def load(self, filepath): ''' Loads an already train model from a file to perform predictions. ''' self.model = joblib.load( create_filename(filepath, '%s.pkl' % self.model_name)) self.scaler = joblib.load( create_filename(filepath, '%s.pkl' % self.scaler_mlp))
def preprocess(self): ''' Top of atmosphere is calculated and persisted into a file. Then a cloud mask is created with the given algorithm. ''' solar_zenith = self.get_sensor().parser.get_attribute( rapideye.SOLAR_ZENITH) data_acquisition_date = self.get_sensor().parser.get_attribute( rapideye.ACQUISITION_DATE) solar_azimuth = self.get_sensor().parser.get_attribute( rapideye.SOLAR_AZIMUTH) geotransform = self.get_raster().get_attribute(raster.GEOTRANSFORM) data = self.get_raster().read_data_file_as_array() sun_earth_distance = calculate_distance_sun_earth( data_acquisition_date) top_of_atmosphere_data = calculate_toa_rapideye( calculate_rad_rapideye(data), sun_earth_distance, solar_zenith) top_of_atmosphere_directory = create_filename(get_parent(self.path), 'TOA') create_directory_path(top_of_atmosphere_directory) output_file = create_filename( top_of_atmosphere_directory, get_basename(self.get_files()[2]) + '_toa.tif') # TODO: change [2] in self.get_files()[2] create_raster_from_reference(output_file, top_of_atmosphere_data, self.file_dictionary[_IMAGE], data_type=NumericTypeCodeToGDALTypeCode( numpy.float32)) LOGGER.debug('Top of atmosphere file was created.') cloud_output_file = create_filename( top_of_atmosphere_directory, get_basename(self.get_files()[2]) + '_cloud.tif') if self.algorithm == ANOMALY_DETECTION: LOGGER.debug('Cloud mask by anomaly detection process.') clouds = self.anomaly_detection_cloud_mask(top_of_atmosphere_data, cloud_output_file, solar_zenith, solar_azimuth, geotransform) elif self.algorithm == TIME_SERIES: LOGGER.debug('Cloud mask by reference with time series process.') tile_id = self.get_sensor().get_attribute(TILE_ID) clouds = self.masking_with_time_series(data, cloud_output_file, solar_zenith, solar_azimuth, geotransform, tile_id) create_raster_from_reference(cloud_output_file, clouds, self.file_dictionary[_IMAGE], data_type=NumericTypeCodeToGDALTypeCode( numpy.float32)) LOGGER.info('Cloud mask was created.')
def train_model(X_train, X_test, y_train, y_test, output, model_name): model = load_model(model_name) persistence_directory = create_filename(output, model_name) create_directory_path(persistence_directory) model_instance = model.Model(persistence_directory) model_instance.fit(X_train, y_train) model_instance.save(persistence_directory) predicted = model_instance.predict(X_test) model_instance.create_report( y_test, predicted, create_filename(persistence_directory, 'report.txt'))
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' output = options['output'][0] models = options['modelname'] model_directory = options['modeldir'][0] pca_model = pca.Model(5) pca_model.load(create_filename(model_directory, 'pca')) for model_name in models: persistence_directory = create_filename(model_directory, model_name) model = load_model(model_name) model_instance = model.Model(persistence_directory) model_instance.load(persistence_directory) block_size = 500 for path in options['path']: image_array = open_handle(path) y_size = image_array.shape[1] x_size = image_array.shape[2] basename = get_basename(path)[:7] warnings.filterwarnings('ignore') final = numpy.zeros((x_size, y_size)) import time start_time = time.time() for i in range(0, y_size, block_size): if i + block_size < y_size: rows = block_size else: rows = y_size - i for j in range(0, x_size, block_size): if j + block_size < x_size: cols = block_size else: cols = x_size - j step = image_array[:, i:i + rows, j:j + cols] step_ravel = step.reshape(10, -1) prediction = model_instance.predict( pca_model.transform(numpy.transpose(step_ravel))) final[i:i + rows, j:j + cols] = prediction.reshape( (rows, cols)) print("--- %s seconds ---" % (time.time() - start_time)) create_directory_path(output) classification = create_filename( output, '%s-%s.tif' % (basename, model_name)) create_raster_from_reference(classification, final.reshape(x_size, y_size), path, data_type=gdal.GDT_Byte, creating_options=['COMPRESS=LZW'])
def test_create_image(self): import numpy array = numpy.array( [[ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], [ 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1 ], [ 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1 ], [ 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1 ], [ 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1 ], [ 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1 ], [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ]]) red = (1 - array) * 1000 green = (1 - array) * 1000 blue = array * 1000 create_raster( create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'single.tif'), array) create_raster( create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'multi.tif'), numpy.array([red, green, blue]))
def raster_to_vector_mask(raster_object, output_path, no_data=[0]): raster_array = raster_object.read_data_file_as_array() for i in no_data: raster_array[raster_array != i] = 0 raster_array[raster_array != 0] = 1 mask_file = create_filename(output_path, 'mask.tif') create_raster_from_reference(mask_file, raster_array, raster_object.get_file()) mask_raster = raster.Data(mask_file) ds = mask_raster._open_file() rb = ds.GetRasterBand(1) dst_layername = 'POLYGONIZED_STUFF' drv = ogr.GetDriverByName(str('ESRI Shapefile')) dst_ds = drv.CreateDataSource(output_path) dst_layer = dst_ds.CreateLayer(dst_layername, srs = None ) #dst_layer.SetSpatialRef(raster.get_spatial_reference()) gdal.Polygonize(rb, None, dst_layer, -1, [])
def maybe_download_and_extract(target_directory, scene_url): ''' This method will try to download the requested url, if a file with the url name has already been downloaded it will not proceed. ''' filename = scene_url.split('/')[-1] filepath = create_filename(target_directory, filename) if not is_file(filepath): def _progress(count, block_size, total_size): sys.stdout.write('\rDownloading %s %.1f%%' % (filename, float(count * block_size) / float(total_size) * 100.0)) sys.stdout.flush() filepath, _ = urllib.request.urlretrieve(scene_url, filepath, _progress) sys.stdout.write('') sys.stdout.flush() statinfo = os.stat(filepath) logger.info('Successfully downloaded: %s %s bytes' % (filename, statinfo.st_size)) else: logger.info('%s was already downloaded at %s' % (filename, target_directory))
def test_creat_image(self): import numpy filename = '/Users/agutierrez/Development/df/1448114/2015/2015-02-24/l3a/1448114_2015-02-24_RE3_3A_302417.tif' width = get_width(filename) height = get_height(filename) plain = width * height red = numpy.zeros(plain) green = numpy.zeros(plain) blue = numpy.zeros(plain) ired = numpy.zeros(plain) print 'before loop' for i in range(plain): if not i % 2: red[i] = 1 if not i % 3: green[i] = 1 if not i % 5: blue[i] = 1 if not i % 7: ired[i] = 1 print 'loop' final = numpy.array([ numpy.reshape(red, (width, height)), numpy.reshape(green, (width, height)), numpy.reshape(blue, (width, height)), numpy.reshape(ired, (width, height)) ]) print final.shape print 'hello' create_raster_from_reference( create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'sieve.tif'), final, filename)
def split_shape_into_features(shape_name, destination_directory, column, name, sep): ''' This method will take a input shape and iterate over its features, creating a new shape file with each one of them. It copies all the fields and the same spatial reference from the original file. The created files are saved in the destination directory using the number of the field given. ''' driver = ogr.GetDriverByName(str('ESRI Shapefile')) shape = driver.Open(shape_name, 0) layer = shape.GetLayer() layer_name = layer.GetName() spatial_reference = layer.GetSpatialRef() in_feature = layer.GetNextFeature() shape_files = [] while in_feature: encoding = 'utf-8' in_feature_name = in_feature.GetField(column) in_name = create_filename_from_string( in_feature.GetField(name).decode(encoding)) final_path = destination_directory + str(in_feature.GetField(0)) create_directory_path(final_path) output_name = create_filename( final_path, '%s__%s.shp' % (in_feature_name, in_name)) shape_files.append(output_name) if os.path.exists(output_name): driver.DeleteDataSource(output_name) datasource = driver.CreateDataSource(output_name) outLayer = datasource.CreateLayer(layer_name, spatial_reference, geom_type=ogr.wkbPolygon) inLayerDefn = layer.GetLayerDefn() for i in range(0, inLayerDefn.GetFieldCount()): fieldDefn = inLayerDefn.GetFieldDefn(i) #LOGGER.debug(fieldDefn.GetName()) outLayer.CreateField(fieldDefn) outLayerDefn = outLayer.GetLayerDefn() geometry = in_feature.GetGeometryRef() out_feature = ogr.Feature(outLayerDefn) out_feature.SetGeometry(geometry) for i in range(0, outLayerDefn.GetFieldCount()): out_feature.SetField( outLayerDefn.GetFieldDefn(i).GetNameRef(), in_feature.GetField(i)) outLayer.CreateFeature(out_feature) out_feature.Destroy() in_feature.Destroy() in_feature = layer.GetNextFeature() shape.Destroy() datasource.Destroy() return shape_files
def get_convex_hull(shape_name, destination_directory): ''' This method will read all objects from a shape file and create a new one with the convex hull of all the geometry points of the first. ''' driver = ogr.GetDriverByName(str('ESRI Shapefile')) shape = driver.Open(shape_name, 0) layer = shape.GetLayer() layer_name = layer.GetName() spatial_reference = layer.GetSpatialRef() prefix = get_basename(shape_name) output_name = create_filename(destination_directory, '%s-hull.shp' % prefix) geometries = ogr.Geometry(ogr.wkbGeometryCollection) for feature in layer: geometries.AddGeometry(feature.GetGeometryRef()) if os.path.exists(output_name): driver.DeleteDataSource(output_name) datasource = driver.CreateDataSource(output_name) out_layer = datasource.CreateLayer(str('states_convexhull'), spatial_reference, geom_type=ogr.wkbPolygon) out_layer.CreateField(ogr.FieldDefn(str('id'), ogr.OFTInteger)) featureDefn = out_layer.GetLayerDefn() feature = ogr.Feature(featureDefn) feature.SetGeometry(geometries.ConvexHull()) feature.SetField(str('id'), 1) out_layer.CreateFeature(feature) shape.Destroy() datasource.Destroy()
def fit(self, X, y): ''' This method creates the necessary files to train a c5 model with the c5.0 executable. It creates only the names and data files. ''' train_data = numpy.column_stack((X, y)) save_to_file(train_data, create_filename(self.path, '%s.data' % self.model_name)) create_names_file( numpy.unique(y), X.shape[1], create_filename(self.path, '%s.names' % self.model_name)) launcher = LocalProcessLauncher() shell_string = 'docker run --rm -v %s:/results madmex/c5_execution c5.0 -f /results/%s' % ( self.path, self.model_name) LOGGER.debug('Docker command: %s', shell_string) launcher.execute(shell_string)
def handle(self, **options): output = options['output'][0] #state = options['state'][0] # for name in get_states_names(): # print name[0] # for footprint in get_rapideye_footprints_from_state(name[0]): # print footprint[0] # # # import time # time.sleep(2) for image_path in options['path']: print image_path basename = '%s_ipcc.tif' % get_basename(image_path) LOGGER.info(basename) target = create_filename(output, basename) #print target start_time = time.time() self.aggregate_by_block(image_path, target, INITIAL_IPCC_2015, FINAL_IPCC_2015) #self.method_by_block(image_path, target) #self.mask_iterating_values(image_path, target, INITIAL_ARRAY, FINAL_ARRAY) LOGGER.info("--- %s seconds ---" % (time.time() - start_time)) LOGGER.info('Dataset was written.')
def get_thumbnail_with_path(self, thumbnail_path): ''' Creates a thumbnail for the scene in true color. ''' from subprocess import call thumnail_directory = create_filename(thumbnail_path, 'thumbnail') create_directory_path(thumnail_directory) filename = self.file_dictionary[_BROWSE] thumbnail = create_filename(thumnail_directory, '%s.jpg' % get_basename(filename)) resize_command = [ '/Library/Frameworks/GDAL.framework/Programs/gdal_translate', filename, '-of', 'JPEG', thumbnail ] call(resize_command)
def handle(self, *args, **options): LOGGER.info('Hello World!') directory = options['directory'][0] for satellite, filename in FILES.iteritems(): filepath = create_filename(directory, filename) populate_catalog_django(filepath, satellite) print 'Done'
def _look_for_files(self): ''' This method will be called by the constructor to look for files in the given directory. In order for a bundle to be valid, a file for each regular expression must be found. ''' for key in self.file_dictionary.iterkeys(): for name in get_files_from_folder(self.path): if re.match(key, name): self.file_dictionary[key] = create_filename( self.path, name)
def get_information_object(self): row = self.get_sensor().get_attribute(olitirs.ROW) path = self.get_sensor().get_attribute(olitirs.PATH) basename_metadata = get_basename_of_file( self.file_dictionary[_BASE % (self.get_letter(), self.get_mission(), 'MTL.txt')]) information = Information( metadata_path = create_filename(self.get_output_directory(),basename_metadata), grid_id = unicode(path + row), projection = self.get_raster().get_attribute(raster.PROJECTION), cloud_percentage = self.get_sensor().get_attribute(olitirs.CLOUD_COVER), geometry = self.get_raster().get_attribute(raster.FOOTPRINT), elevation_angle = 0.0, resolution = self.get_raster().get_attribute(raster.GEOTRANSFORM)[1] ) return information
def predict(self, X): ''' The c5 model creates a file with the tree extension and it will be loaded by the predict executable. ''' save_to_file_cases( X, create_filename(self.path, '%s.cases' % self.model_name)) local = LocalProcessLauncher() shell_string = 'docker run --rm -v %s:/results madmex/c5_execution predict -f /results/%s' % ( self.path, self.model_name) LOGGER.debug('Docker command: %s', shell_string) output = local.execute(shell_string) by_line = output.split('\n') y = [] for line in by_line: array_predict = line.split() if len(array_predict) == 4: y.append(int(float(array_predict[2]))) return numpy.array(y)
def split(self, output_directory, column=0): ''' This method will take a input shape and iterate over its features, creating a new shape file with each one of them. It copies all the fields and the same spatial reference from the original file. The created files are saved in the destination directory using the number of the field given. ''' layer = self.get_layer() layer_name = layer.GetName() spatial_reference = layer.GetSpatialRef() in_feature = layer.GetNextFeature() layer_definition = layer.GetLayerDefn() field_definition = layer_definition.GetFieldDefn(0) column_name = field_definition.GetName() shape_files = [] create_directory_path(output_directory) in_layer_definition = layer.GetLayerDefn() while in_feature: in_feature_name = in_feature.GetField(column_name) output_name = create_filename(output_directory, '%s.shp' % in_feature_name) shape_files.append(output_name) if is_file(output_name): self.driver.DeleteDataSource(output_name) data_source = self.driver.CreateDataSource(output_name) out_layer = data_source.CreateLayer(layer_name, spatial_reference, geom_type=ogr.wkbPolygon) for i in range(0, in_layer_definition.GetFieldCount()): fieldDefn = in_layer_definition.GetFieldDefn(i) out_layer.CreateField(fieldDefn) outLayerDefn = out_layer.GetLayerDefn() geometry = in_feature.GetGeometryRef() out_feature = ogr.Feature(outLayerDefn) out_feature.SetGeometry(geometry) for i in range(0, outLayerDefn.GetFieldCount()): out_feature.SetField(outLayerDefn.GetFieldDefn(i).GetNameRef(), in_feature.GetField(i)) out_layer.CreateFeature(out_feature) out_feature = None in_feature = None in_feature = layer.GetNextFeature() self.close() return [Data(filename) for filename in shape_files]
def create_shape_from_json(id, json, output_directory): ''' Given a json string containing coordinates, this method creates a shape file. ''' create_directory_path(output_directory) filename = create_filename(output_directory, '%s.shp' % id) shape = Data(filename) if is_file(filename): shape.driver.DeleteDataSource(filename) data_source = shape.driver.CreateDataSource(filename) spatial_reference = osr.SpatialReference() spatial_reference.ImportFromEPSG(4326) layer = data_source.CreateLayer(str('layer'), spatial_reference, geom_type=ogr.wkbPolygon) layer.CreateField(ogr.FieldDefn(str('id'), ogr.OFTString)) feature = ogr.Feature(layer.GetLayerDefn()) feature.SetField(str('id'), str(id)) geometry = ogr.CreateGeometryFromJson(str(json)) feature.SetGeometry(geometry) layer.CreateFeature(feature) shape.close() return shape
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' output = options['output'][0] zip = options['path'][0] basename = get_basename(zip) aux_name = create_filename(output, 'aux_%s' % basename) real_name = create_filename(output, basename) with ZipFile(zip, 'r') as unzipped: unzipped.extractall(create_filename(output, 'aux_%s' % basename)) path = create_filename(aux_name, 'trend_tiles') tifs = get_contents_from_folder(path) create_directory_path(real_name) for tif in tifs: source = create_filename(path, tif) target = create_filename(real_name, tif) tile_map(source, target, 4000, 4000, 2, 2) remove_directory(aux_name)
def handle(self, **options): filepath = options['path'][0] qa_file = get_files_from_folder_filter(filepath, r'.*qa.tif$')[0] print qa_file with rasterio.open(create_filename(filepath, qa_file)) as src: qa_array = src.read() profile = src.profile print profile profile.update( dtype=rasterio.ubyte, compress='lzw', nodata=None) my_values = map(lambda x: bin(x)[2:].zfill(16), numpy.unique(qa_array, return_counts=True)[0]) my_count = numpy.unique(qa_array, return_counts=True)[1] for index in range(len(my_values)): print '%s -> %s' % (my_values[index],my_count[index]) print qa_array.shape cloud_mask = numpy.bitwise_and(qa_array, CLEAR) >> CLEAR_SHIFT cloud_mask_file = create_filename(filepath, 'qa.tif') print numpy.unique(cloud_mask, return_counts=True) print cloud_mask_file print cloud_mask.shape with rasterio.open(cloud_mask_file, 'w', **profile) as dst: dst.write(cloud_mask.astype(rasterio.ubyte)) files_of_interest = get_files_from_folder_filter(filepath, r'.*sr_band[0-9]+.tif$') bands = [] print files_of_interest for filename in files_of_interest: print filename complete_path = create_filename(filepath, filename) with rasterio.open(complete_path) as src: bands.append(src.read()) stack = numpy.array(bands) print stack.shape
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' shape_name = options['shape'][0] destination = options['dest'][0] paths = options['path'] years = [] for path in paths: years.append(self.get_year_from_path(path)) if os.path.exists(shape_name): LOGGER.info('The file %s was found.' % shape_name) shape_files = split_shape_into_features(shape_name, destination, str('id'), str('nombre'), '__') process_launcher = LocalProcessLauncher() import time start_time = time.time() cover_array = [] tth_array = [] cover_file = 'cover-stats.json' tth_name = 'tth-stats.json' json_directory = create_filename(destination, 'cover_stats') json_file = create_filename(json_directory, cover_file.lower()) tth_file = create_filename(json_directory, tth_name.lower()) for shape_file in shape_files: shape_name = get_basename(shape_file).split('__') anp_id = shape_name[0] anp_name = shape_name[1] basename = '%s.tif' % anp_name dataframe = DataFrame(index=years, columns=[0, 1, 2, 3, 4, 5, 6, 7, 8]) create_directory_path(json_directory) print shape_file for path in paths: #pixel_resolution, dataset = pixel_info(path) year = self.get_year_from_path(path) raster_dir = create_filename( create_filename(destination, 'raster'), year) create_directory_path(raster_dir) raster_file = create_filename(raster_dir, basename) shell_command = 'gdalwarp -ot Byte -co COMPRESS=LZW -cutline %s -crop_to_cutline %s %s' % ( shape_file, path, raster_file) print shell_command print process_launcher.execute(shell_command) raster_array = open_handle(raster_file) ds = gdal.Open(raster_file) geotransform = ds.GetGeoTransform() x_resolution = geotransform[1] y_resolution = geotransform[5] pixel_area = abs(x_resolution * y_resolution) unique_values = numpy.unique(raster_array, return_counts=True) indexes = unique_values[0] counts = unique_values[1] for i in range(len(indexes)): key = indexes[i] dataframe.set_value(year, key, area(int(counts[i]), pixel_area)) dataframe = dataframe.sort_index() columns = list(dataframe.columns.values) index = list(dataframe.index.values) print dataframe cover_array.append( self.dataframe_to_json(dataframe, anp_id, anp_name)) tth_dataframe = DataFrame(columns=columns) for i in range(len(index) - 1): label = '%s-%s' % (index[i], index[i + 1]) tth_column = calculate_thh(dataframe.ix[i], dataframe.ix[i + 1], int(index[i + 1]) - int(index[i])) for j in range(len(tth_column)): tth_dataframe.set_value(label, j, tth_column[j]) tth_dataframe = tth_dataframe.sort_index() print tth_dataframe tth_array.append( self.dataframe_to_json(tth_dataframe, anp_id, anp_name)) self.json_to_file(json_file, cover_array) self.json_to_file(tth_file, tth_array) print("--- %s seconds ---" % (time.time() - start_time))
def handle(self, **options): mapgrid = '1449619' acq = get_pair_quality(mapgrid) for image in acq: print image.pk_id print image.pk_id id = options["id"][0] image_path = options["image"][0] reference_path = options["reference"][0] output = options["output"][0] print image_path print reference_path image_bundle = _get_bundle_from_path(image_path) reference_bundle = _get_bundle_from_path(reference_path) #extents = harmonize_images([image_bundle.get_raster(), reference_bundle.get_raster()]) #print extents #print extents['x_offset'] #print extents['y_offset'] shape = image_bundle.get_raster().get_attribute(raster.DATA_SHAPE) invariant_array = numpy.full((shape[0], shape[1]), INV_MASK_VALUE, dtype=np.int) in1 = reference_bundle.get_raster_file() in2 = image_bundle.get_raster_file() in_invar = create_filename(output, 'invariantPixelMask.tif') result = create_filename(output, 'crosscorrelation_next.tif') to_polar = create_filename(output, 'crosscorrelation_polar.tif') create_raster_from_reference(in_invar, invariant_array, image_bundle.get_raster_file(), gdal.GDT_Byte) local = LocalProcessLauncher() volume = '%s:%s' % (output, output) shell_array = [ 'docker', 'run', '--rm', '-v', volume, 'madmex/antares', 'correlation', '-in1', in1, '-in2', in2, '-in_invar', in_invar, '-val_invar', '%s' % INV_MASK_VALUE, '-out', result, '-window_size', '%s' % WINDOW_SIZE, '-max_gap', '%s' % MAX_GAP ] shell_string = ' '.join(shell_array) print shell_string if not is_file(result): log = local.execute(shell_string) crosscorrelation = raster.Data(result, 'GTiff') print crosscorrelation.get_attribute(raster.PROJECTION) print crosscorrelation.get_attribute(raster.GEOTRANSFORM) #tile_map(result, result) correlation_array = crosscorrelation.read_data_file_as_array() band_0 = correlation_array[0, :] band_1 = correlation_array[1, :] phi_band = phi(band_0, band_1) rho_band = rho(band_0, band_1) correlation_array[0, :] = phi_band correlation_array[1, :] = rho_band #create_raster_from_reference(to_polar, correlation_array, result) crosscorrelation_polar = raster.Data(to_polar, 'GTiff') extents = harmonize_images( [crosscorrelation_polar, reference_bundle.get_raster()]) x_offset = extents['x_offset'][1] y_offset = extents['y_offset'][1] x_tile_size = extents['x_range'] y_tile_size = extents['y_range'] aux_name = create_filename(output, 'auxiliar.tif') tile_map(reference_bundle.get_raster_file(), aux_name, x_tile_size, y_tile_size, x_offset, y_offset) aux_array = raster.Data(aux_name, 'GTiff').read_data_file_as_array() crosscorrelation_polar_array = crosscorrelation_polar.read_data_file_as_array( ) stats = calculate_statistics_qa(crosscorrelation_polar_array, aux_array, STAT_CLASSES, STAT_MIN, STAT_MAX, THRESHOLD_COD, THRESHOLD_LOG) desision = calculate_decision(stats['band_1']['histogram'], stats['band_1']['histogram_bins']) print stats quality = QualityAssessment( decision=desision, max=adapt_numpy_float(stats['band_1']['maximum']), min=adapt_numpy_float(stats['band_1']['minimum']), median=adapt_numpy_float(stats['band_1']['median']), mean=adapt_numpy_float(stats['band_1']['mean']), standard_deviation=adapt_numpy_float(stats['band_1']['std']), product_id=1, reference_id=2) persist_quality(quality) print desision
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' random_int = 1 training_template = options['training'][0] target_template = options['target'][0] index_template = options['index'][0] output = options['output'][0] models = options['model'] features = training_template % random_int training = target_template % random_int index = index_template % random_int features_array = open_handle(features) training_array = open_handle(training) index_array = open_handle(index) labels = numpy.unique(index_array) features_flatten = numpy.ravel(features_array).reshape(10, 200 * 200) training_flatten = numpy.ravel(training_array).reshape(200 * 200) array_aux = [] for j in range(features_array.shape[0]): means = scipy.ndimage.measurements.mean(features_array[j, :, :], index_array, labels) print means.shape array_aux.append(means) for j in range(features_array.shape[0]): std = scipy.ndimage.measurements.standard_deviation( features_array[j, :, :], index_array, labels) print std.shape array_aux.append(std) print len(array_aux) features_final = numpy.concatenate([array_aux], axis=0) print features_final.shape label_object = [] for id in labels: values, counts = numpy.unique(training_array[index_array == id], return_counts=True) label_object.append(values[numpy.argmax(counts)]) features_total = features_final training_total = label_object for i in range(1, 53): features = training_template % (i + 1) training = target_template % (i + 1) index = index_template % (i + 1) features_array = open_handle(features) training_array = open_handle(training) index_array = open_handle(index) labels = numpy.unique(index_array) array_aux = [] for j in range(features_array.shape[0]): array_aux.append( scipy.ndimage.measurements.mean(features_array[j, :, :], index_array, labels)) for j in range(features_array.shape[0]): array_aux.append( scipy.ndimage.measurements.standard_deviation( features_array[j, :, :], index_array, labels)) features_final = numpy.concatenate([array_aux], axis=0) print features_final.shape label_object = [] for id in labels: values, counts = numpy.unique( training_array[index_array == id], return_counts=True) #print '********* ', values[numpy.argmax(counts)], counts[numpy.argmax(counts)] label_object.append(values[numpy.argmax(counts)]) print features_total.shape print features_final.shape features_total = numpy.concatenate( (features_total, features_final), axis=1) training_total = numpy.concatenate((training_total, label_object), axis=1) print 'label object', len(label_object) #print scipy.ndimage.measurements.mean(training_array, index_array, labels) features_flatten = numpy.concatenate( (features_flatten, numpy.ravel(features_array).reshape( 10, 200 * 200)), axis=1) training_flatten = numpy.concatenate( (training_flatten, numpy.ravel(training_array).reshape( 200 * 200)), axis=0) # Remove the elements that map to None #mask = training_flatten!=0 #features_flatten = features_flatten[:,mask] #training_flatten = training_flatten[mask] mask = training_total != 0 features_flatten = features_total[:, mask] training_flatten = training_total[mask] print features_flatten.shape print training_flatten.shape X_train, X_test, y_train, y_test = train_test_split( numpy.transpose(features_flatten), training_flatten, train_size=0.8, test_size=0.2) print X_train[0] print y_train[0] unsupervised = pca.Model(5) unsupervised.fit(X_train) ''' X_train = unsupervised.transform(X_train) X_test = unsupervised.transform(X_test) ''' pca_path = create_filename(output, 'pca') create_directory_path(pca_path) unsupervised.save(pca_path) import time for model_name in models: start_time = time.time() train_model(X_train, X_test, y_train, y_test, output, model_name) print "--- %s seconds training %s model---" % ( (time.time() - start_time), model_name)
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' output = options['output'][0] for path in options['path']: bundle = _get_bundle_from_path(path) basename = get_basename(bundle.get_raster_file()) bundle.get_NDVI() ndvi_file = create_filename(output, 'ndvi.tif') red_edge_ndvi_file = create_filename(output, 'red_edge_ndvi.tif') gndvi_file = create_filename(output, 'gndvi.tif') ndre_file = create_filename(output, 'ndre.tif') sovel_file = create_filename(output, 'sovel2.tif') all_file = create_filename(output, '%s_all_features.tif' % basename) print all_file image_array = bundle.get_raster().read_data_file_as_array() ndvi_array = bundle.get_NDVI() ndvi_array[ndvi_array <= -1] = -1 ndvi_array[ndvi_array >= 1] = 1 red_edge_ndvi_array = bundle.get_red_edge_NDVI() red_edge_ndvi_array[red_edge_ndvi_array <= -1] = -1 red_edge_ndvi_array[red_edge_ndvi_array >= 1] = 1 gndvi_array = bundle.get_gndvi() gndvi_array[gndvi_array <= -1] = -1 gndvi_array[gndvi_array >= 1] = 1 ndre_array = bundle.get_ndre() ndre_array[ndre_array <= -1] = -1 ndre_array[ndre_array >= 1] = 1 sobel_filter_array = bundle.get_sobel_filter(sigma=2) #sobel_filter_array[sobel_filter_array<=-1] = -1 #sobel_filter_array[sobel_filter_array>=1] = 1 #create_raster_from_reference(ndvi_file, ndvi_array, bundle.get_raster_file()) #create_raster_from_reference(red_edge_ndvi_file, red_edge_ndvi_array, bundle.get_raster_file()) #create_raster_from_reference(gndvi_file, gndvi_array, bundle.get_raster_file()) #create_raster_from_reference(ndre_file, ndre_array, bundle.get_raster_file()) create_raster_from_reference(sovel_file, sobel_filter_array, bundle.get_raster_file()) all_features = numpy.array([ image_array[0], image_array[1], image_array[2], image_array[3], image_array[4], ndvi_array, red_edge_ndvi_array, gndvi_array, ndre_array, sobel_filter_array ]) print image_array[0].shape print image_array[1].shape print image_array[2].shape print image_array[3].shape print image_array[4].shape print ndvi_array.shape print red_edge_ndvi_array.shape print gndvi_array.shape print ndre_array.shape print sobel_filter_array.shape print all_features.shape create_raster_from_reference(all_file, all_features, bundle.get_raster_file(), creating_options=['BIGTIFF=YES'])
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' output = options['output'][0] models = options['modelname'] model_directory = options['modeldir'][0] region = options['region'][0] start_time = time.time() for path in options['path']: print path scene_bundle = rapideye.Bundle(path) directory = getattr(SETTINGS, 'TEMPORARY') directory_helper = create_filename(directory, 'helper') create_directory_path(directory_helper) categories_file = create_filename(directory, 'categories.json') categories_dictionaty = { 0: "AGRICULTURA DE RIEGO", 1: "AGRICULTURA DE TEMPORAL", 2: "AGUA", 3: "AREAS QUEMADAS", 4: "ASENTAMIENTOS HUMANOS", 5: "BOSQUE CULTIVADO", 6: "BOSQUE DE AYARIN", 7: "BOSQUE DE ENCINO", 8: "BOSQUE DE ENCINO-PINO", 9: "BOSQUE DE GALERIA", 10: "BOSQUE DE MEZQUITE", 11: "BOSQUE DE OYAMEL", 12: "BOSQUE DE PINO", 13: "BOSQUE DE PINO-ENCINO", 14: "BOSQUE INDUCIDO", 15: "BOSQUE MESOFILO DE MONTANA", 16: "DESPROVISTO DE VEGETACION", 17: "INDEFINIDO", 18: "MANGLAR", 19: "MATORRAL SUBTROPICAL", 20: "MEZQUITAL", 21: "NUBES", 22: "PASTIZAL CULTIVADO", 23: "PASTIZAL HALOFILO", 24: "PASTIZAL INDUCIDO", 25: "PASTIZAL NATURAL", 26: "PRADERA DE ALTA MONTANA", 27: "SABANOIDE", 28: "SELVA ALTA PERENNIFOLIA", 29: "SELVA ALTA SUBPERENNIFOLIA", 30: "SELVA BAJA CADUCIFOLIA", 31: "SELVA BAJA ESPINOSA CADUCIFOLIA", 32: "SELVA BAJA SUBCADUCIFOLIA", 33: "SELVA DE GALERIA", 34: "SELVA MEDIANA CADUCIFOLIA", 35: "SELVA MEDIANA SUBCADUCIFOLIA", 36: "SELVA MEDIANA SUBPERENNIFOLIA", 37: "SIN VEGETACION APARENTE", 38: "SOMBRAS", 39: "TULAR", 40: "VEGETACION DE DUNAS COSTERAS", 41: "VEGETACION HALOFILA HIDROFILA", 42: "ZONA URBANA" } basename = get_basename(scene_bundle.get_raster_file()) all_file = create_filename(directory_helper, '%s_all_features.tif' % basename) if not is_file(all_file): scene_bundle.get_feature_array(all_file) filename = get_basename(all_file) if not is_file( create_filename(directory_helper, '%s.shp' % filename)): shell_string = 'docker run --rm -v %s:/data madmex/segment gdal-segment %s.tif -out helper/%s.shp -algo SLIC -region %s' % ( directory, filename, filename, region) launcher = LocalProcessLauncher() LOGGER.debug('Docker command: %s', shell_string) launcher.execute(shell_string) data = read_data_table( create_filename(directory_helper, '%s.shp' % filename)) results = {} for model_name in models: persistence_directory = create_filename( model_directory, model_name) print model_name model = load_model(model_name) model_instance = model.Model(persistence_directory) model_instance.load(persistence_directory) prediction = model_instance.predict(data) results[model_name] = prediction print results create_directory_path(output) write_results( create_filename(directory_helper, '%s.shp' % filename), create_filename(output, '%s_classification.shp' % filename[0:32]), results, categories_dictionaty) LOGGER.info("--- %s seconds ---" % (time.time() - start_time))
def handle(self, **options): ''' This process will call the change detection process from a set of two individual images. It will perform the harmonization and the multivariate alteration detection on the images. It will then perform a maximum correlation factor on them and work with the resulting bands. ''' image_a = options['ima'][0] image_b = options['imb'][0] output_image = options['output'][0] LOGGER.info('Image %s will be compared against image %s. Output will be available' \ ' at %s.', image_a, image_b, output_image) gdal_format = "GTiff" image_a_data_class = raster.Data(image_a, gdal_format) image_b_data_class = raster.Data(image_b, gdal_format) # TODO : remove references to class harmonized harmonized_class = harmonized.Data(image_a_data_class, image_b_data_class) #band1 = image_a_data_class.GetRasterBand(1) #band1 = band1.ReadAsArray(0, 0,image_a_data_class.RasterXSize,image_a_data_class.RasterYSize).astype(float) #print(band1) if harmonized_class: #data_shape_harmonized = harmonized_class.get_attribute(harmonized.DATA_SHAPE) #width, height, bands = data_shape_harmonized #geotransform_harmonized = harmonized_class.get_attribute(raster.GEOTRANSFORM) #projection_harmonized = harmonized_class.get_attribute(harmonized.PROJECTION) image_a_data_array, image_b_data_array = harmonized_class.harmonized_arrays(image_a_data_class, image_b_data_class) imad_class = imad.Transformation([image_a_data_array, image_b_data_array]) imad_class.execute() mad_result = imad_class.output LOGGER.debug('mad_result.shape: %s', mad_result.shape) create_directory_path(getattr(SETTINGS, 'TEST_FOLDER')) mad_output_file = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'mad.tif') create_raster_from_reference(mad_output_file, mad_result,image_a) maf_class = maf.Transformation(imad_class.output) maf_class.execute() maf_result = maf_class.output pdf_file = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'maf_pdf.png') thresholds = calc_threshold_grid(maf_result, pdf_file) class_result = recode_classes_grid(maf_result, thresholds) LOGGER.debug('maf_result.shape: %s', maf_result.shape) LOGGER.debug('class_result.shape: %s', class_result.shape) maf_outputfile = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'maf.tif') class_outputfile = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'class.tif') create_raster_from_reference(maf_outputfile, maf_result, image_a) create_raster_from_reference(class_outputfile, class_result, image_a) print 'Output written in: %s' % mad_output_file print 'Shape is ', imad_class.output.shape
def handle(self, **options): ''' This is the code that does the ingestion. indexes --path /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/Landsat_2000_2008/L5_021_047_2000_022_sr/ --shape /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/AE_LT_new.shp --output ''' path = options['path'][0] for mask_file in os.listdir(path): if mask_file.endswith('_cfmask.tif'): print mask_file basename = mask_file.replace('_cfmask.tif', '') print basename cloud_file_name = mask_file LOGGER.info('Calculating indexes for Landsat scenes.') band_name = basename + '_sr_band%s.tif' final_path = options['output'][0] create_directory_path(final_path) cloud_file = create_filename(path, cloud_file_name) cloud_array = open_handle(cloud_file) cloud_mask = (cloud_array == 4) LOGGER.debug('Recognize sensor depending on file name.') if 'L8' in path: LOGGER.debug('Landsat 8 scene was detected.') blue_file = create_filename(path, band_name % BLUE_L8) green_file = create_filename(path, band_name % GREEN_L8) red_file = create_filename(path, band_name % RED_L8) nir_file = create_filename(path, band_name % NIR_L8) swir_file = create_filename(path, band_name % SWIR_L8) else: LOGGER.debug('Landsat 4, 5 or scene was detected.') blue_file = create_filename(path, band_name % BLUE) green_file = create_filename(path, band_name % GREEN) red_file = create_filename(path, band_name % RED) nir_file = create_filename(path, band_name % NIR) swir_file = create_filename(path, band_name % SWIR) LOGGER.debug('Loading bands of interest.') green_array = open_handle(green_file) red_array = open_handle(red_file) nir_array = open_handle(nir_file) swir_array = open_handle(swir_file) LOGGER.debug('Calculating indexes.') ndvi_array = calculate_index(nir_array, red_array) mndwi_array = calculate_index(green_array, swir_array) ndwig_array = calculate_index(nir_array, swir_array) ndwim_array = calculate_index(green_array, nir_array) LOGGER.debug('Setting cloud mask values to -999.') ndvi_array[cloud_mask] = -999 mndwi_array[cloud_mask] = -999 ndwig_array[cloud_mask] = -999 ndwim_array[cloud_mask] = -999 LOGGER.debug('Creating files for indexes.') ndvi_final_file = create_filename(final_path, basename + '_ndvi.tif') mndwi_final_file = create_filename(final_path, basename + '_mndwi.tif') ndwig_final_file = create_filename(final_path, basename + '_ndwig.tif') ndwim_final_file = create_filename(final_path, basename + '_ndwim.tif') ndvi_clipped_file = create_filename(final_path, basename + '_ndvi_clipped.tif') mndwi_clipped_file = create_filename(final_path, basename + '_mndwi_clipped.tif') ndwig_clipped_file = create_filename(final_path, basename + '_ndwig_clipped.tif') ndwim_clipped_file = create_filename(final_path, basename + '_ndwim_clipped.tif') ndvi_file = create_filename(final_path, basename + '_ndvi_pre.tif') mndwi_file = create_filename(final_path, basename + '_mndwi_pre.tif') ndwig_file = create_filename(final_path, basename + '_ndwig_pre.tif') ndwim_file = create_filename(final_path, basename + '_ndwim_pre.tif') files = [ndvi_file, mndwi_file, ndwig_file, ndwim_file] clipped_files = [ ndvi_clipped_file, mndwi_clipped_file, ndwig_clipped_file, ndwim_clipped_file ] final_files = [ ndvi_final_file, mndwi_final_file, ndwig_final_file, ndwim_final_file ] LOGGER.debug('Writing information to files.') create_raster_from_reference(ndvi_file, ndvi_array, green_file) create_raster_from_reference(mndwi_file, mndwi_array, green_file) create_raster_from_reference(ndwig_file, ndwig_array, green_file) create_raster_from_reference(ndwim_file, ndwim_array, green_file) LOGGER.debug('Deleting arrays to release memory.') del ndvi_array del mndwi_array del ndwig_array del ndwim_array del cloud_array LOGGER.debug('Reference rgb file creation.') rgb_file = create_filename(final_path, basename + '_rgb.tif') merge_command = [ '/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt', '-separate', '-o', rgb_file, red_file, green_file, blue_file ] call(merge_command) shape = options['shape'][0] LOGGER.debug('Cookie cut files using the given shape.') for i in range(4): clip_command = [ '/Library/Frameworks/GDAL.framework/Programs/gdalwarp', '-crop_to_cutline', '-cutline', shape, files[i], clipped_files[i] ] call(clip_command) for i in range(4): aux_array = open_handle(clipped_files[i]) aux_array[(aux_array == 0)] = -9999 create_raster_from_reference(final_files[i], aux_array, ndvi_clipped_file) if not options['debug']: LOGGER.info('Remove auxiliary files.') os.remove(ndvi_clipped_file) os.remove(mndwi_clipped_file) os.remove(ndwig_clipped_file) os.remove(ndwim_clipped_file) os.remove(ndvi_file) os.remove(mndwi_file) os.remove(ndwig_file) os.remove(ndwim_file) print 'Done'