def raster_to_vector_mask(raster_object, output_path, no_data=[0]): raster_array = raster_object.read_data_file_as_array() for i in no_data: raster_array[raster_array != i] = 0 raster_array[raster_array != 0] = 1 mask_file = create_filename(output_path, 'mask.tif') create_raster_from_reference(mask_file, raster_array, raster_object.get_file()) mask_raster = raster.Data(mask_file) ds = mask_raster._open_file() rb = ds.GetRasterBand(1) dst_layername = 'POLYGONIZED_STUFF' drv = ogr.GetDriverByName(str('ESRI Shapefile')) dst_ds = drv.CreateDataSource(output_path) dst_layer = dst_ds.CreateLayer(dst_layername, srs = None ) #dst_layer.SetSpatialRef(raster.get_spatial_reference()) gdal.Polygonize(rb, None, dst_layer, -1, [])
def test_creat_image(self): import numpy filename = '/Users/agutierrez/Development/df/1448114/2015/2015-02-24/l3a/1448114_2015-02-24_RE3_3A_302417.tif' width = get_width(filename) height = get_height(filename) plain = width * height red = numpy.zeros(plain) green = numpy.zeros(plain) blue = numpy.zeros(plain) ired = numpy.zeros(plain) print 'before loop' for i in range(plain): if not i % 2: red[i] = 1 if not i % 3: green[i] = 1 if not i % 5: blue[i] = 1 if not i % 7: ired[i] = 1 print 'loop' final = numpy.array([ numpy.reshape(red, (width, height)), numpy.reshape(green, (width, height)), numpy.reshape(blue, (width, height)), numpy.reshape(ired, (width, height)) ]) print final.shape print 'hello' create_raster_from_reference( create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'sieve.tif'), final, filename)
def test_creat_image(self): import numpy filename = "/Users/agutierrez/Development/df/1448114/2015/2015-02-24/l3a/1448114_2015-02-24_RE3_3A_302417.tif" width = get_width(filename) height = get_height(filename) plain = width * height red = numpy.zeros(plain) green = numpy.zeros(plain) blue = numpy.zeros(plain) ired = numpy.zeros(plain) print "before loop" for i in range(plain): if not i % 2: red[i] = 1 if not i % 3: green[i] = 1 if not i % 5: blue[i] = 1 if not i % 7: ired[i] = 1 print "loop" final = numpy.array( [ numpy.reshape(red, (width, height)), numpy.reshape(green, (width, height)), numpy.reshape(blue, (width, height)), numpy.reshape(ired, (width, height)), ] ) print final.shape print "hello" create_raster_from_reference(create_file_name(getattr(SETTINGS, "TEST_FOLDER"), "sieve.tif"), final, filename)
def get_feature_array(self, outputfile): ''' This method creates an array of features from the bands of this raster. ''' image_array = self.get_raster().read_data_file_as_array() ndvi_array = self.get_NDVI() ndvi_array[ndvi_array <= -1] = -1 ndvi_array[ndvi_array >= 1] = 1 red_edge_ndvi_array = self.get_red_edge_NDVI() red_edge_ndvi_array[red_edge_ndvi_array <= -1] = -1 red_edge_ndvi_array[red_edge_ndvi_array >= 1] = 1 gndvi_array = self.get_gndvi() gndvi_array[gndvi_array <= -1] = -1 gndvi_array[gndvi_array >= 1] = 1 ndre_array = self.get_ndre() ndre_array[ndre_array <= -1] = -1 ndre_array[ndre_array >= 1] = 1 sobel_filter_array = self.get_sobel_filter(sigma=2) all_features = numpy.array([ image_array[0], image_array[1], image_array[2], image_array[3], image_array[4], ndvi_array, red_edge_ndvi_array, gndvi_array, ndre_array, sobel_filter_array ]) create_raster_from_reference(outputfile, all_features, self.get_raster_file(), creating_options=['BIGTIFF=YES']) return raster.Data(outputfile)
def handle(self, **options): paths = options['path'] output = options['output'][0] print output arrays = {} import time for path in paths: start_time = time.time() arrays[path] = open_handle(path) print 'Opened %s' % path print("--- %s seconds ---" % (time.time() - start_time)) size = arrays[paths[0]].shape start_time = time.time() result = numpy.full(size, NO_DATA) mask = numpy.equal(arrays[paths[0]],arrays[paths[1]]) for p in range(2, len(paths)): print 'Image %s ' % p mask = numpy.logical_and(mask, numpy.equal(arrays[paths[0]],arrays[paths[p]])) result[mask] = numpy.array(arrays[paths[0]])[mask] for path in paths: del arrays[path] print("--- %s seconds ---" % (time.time() - start_time)) start_time = time.time() create_raster_from_reference(output, result, paths[0], data_type=gdal.GDT_Byte) print("--- %s seconds ---" % (time.time() - start_time))
def preprocess(self): ''' Top of atmosphere is calculated and persisted into a file. Then a cloud mask is created with the given algorithm. ''' solar_zenith = self.get_sensor().parser.get_attribute( rapideye.SOLAR_ZENITH) data_acquisition_date = self.get_sensor().parser.get_attribute( rapideye.ACQUISITION_DATE) solar_azimuth = self.get_sensor().parser.get_attribute( rapideye.SOLAR_AZIMUTH) geotransform = self.get_raster().get_attribute(raster.GEOTRANSFORM) data = self.get_raster().read_data_file_as_array() sun_earth_distance = calculate_distance_sun_earth( data_acquisition_date) top_of_atmosphere_data = calculate_toa_rapideye( calculate_rad_rapideye(data), sun_earth_distance, solar_zenith) top_of_atmosphere_directory = create_filename(get_parent(self.path), 'TOA') create_directory_path(top_of_atmosphere_directory) output_file = create_filename( top_of_atmosphere_directory, get_basename(self.get_files()[2]) + '_toa.tif') # TODO: change [2] in self.get_files()[2] create_raster_from_reference(output_file, top_of_atmosphere_data, self.file_dictionary[_IMAGE], data_type=NumericTypeCodeToGDALTypeCode( numpy.float32)) LOGGER.debug('Top of atmosphere file was created.') cloud_output_file = create_filename( top_of_atmosphere_directory, get_basename(self.get_files()[2]) + '_cloud.tif') if self.algorithm == ANOMALY_DETECTION: LOGGER.debug('Cloud mask by anomaly detection process.') clouds = self.anomaly_detection_cloud_mask(top_of_atmosphere_data, cloud_output_file, solar_zenith, solar_azimuth, geotransform) elif self.algorithm == TIME_SERIES: LOGGER.debug('Cloud mask by reference with time series process.') tile_id = self.get_sensor().get_attribute(TILE_ID) clouds = self.masking_with_time_series(data, cloud_output_file, solar_zenith, solar_azimuth, geotransform, tile_id) create_raster_from_reference(cloud_output_file, clouds, self.file_dictionary[_IMAGE], data_type=NumericTypeCodeToGDALTypeCode( numpy.float32)) LOGGER.info('Cloud mask was created.')
def method_five(self, path, output): ''' Using numpy utilities, this method mask the desired value. ''' data_array = open_handle(path) data_array[data_array!=MASK] = 0 data_array[data_array==MASK] = 1 create_raster_from_reference(output, data_array, path, gdal.GDT_Byte) del data_array
def method_five(self, path, output): ''' Using numpy utilities, this method mask the desired value. ''' data_array = open_handle(path) data_array[data_array != MASK] = 0 data_array[data_array == MASK] = 1 create_raster_from_reference(output, data_array, path, gdal.GDT_Byte) del data_array
def mask_iterating_values(self, path, output): ''' Iterating over the values in the initial and final arrays this replaces the desired values one by one. ''' data_array = open_handle(path) my_dictionary = dictionary_from_list(INITIAL_ARRAY, FINAL_ARRAY) to_vector_array = replace_in_array(data_array, my_dictionary) create_raster_from_reference(output, to_vector_array, path, gdal.GDT_Byte)
def mask_iterating_values(self, path, output, ini_arr, fin_arr): ''' Iterating over the values in the initial and final arrays this replaces the desired values one by one. ''' data_array = open_handle(path) my_dictionary = dictionary_from_list(ini_arr, fin_arr) to_vector_array = replace_in_array(data_array, my_dictionary) create_raster_from_reference(output, to_vector_array, path, gdal.GDT_Byte)
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' output = options['output'][0] models = options['modelname'] model_directory = options['modeldir'][0] pca_model = pca.Model(5) pca_model.load(create_filename(model_directory, 'pca')) for model_name in models: persistence_directory = create_filename(model_directory, model_name) model = load_model(model_name) model_instance = model.Model(persistence_directory) model_instance.load(persistence_directory) block_size = 500 for path in options['path']: image_array = open_handle(path) y_size = image_array.shape[1] x_size = image_array.shape[2] basename = get_basename(path)[:7] warnings.filterwarnings('ignore') final = numpy.zeros((x_size, y_size)) import time start_time = time.time() for i in range(0, y_size, block_size): if i + block_size < y_size: rows = block_size else: rows = y_size - i for j in range(0, x_size, block_size): if j + block_size < x_size: cols = block_size else: cols = x_size - j step = image_array[:, i:i + rows, j:j + cols] step_ravel = step.reshape(10, -1) prediction = model_instance.predict( pca_model.transform(numpy.transpose(step_ravel))) final[i:i + rows, j:j + cols] = prediction.reshape( (rows, cols)) print("--- %s seconds ---" % (time.time() - start_time)) create_directory_path(output) classification = create_filename( output, '%s-%s.tif' % (basename, model_name)) create_raster_from_reference(classification, final.reshape(x_size, y_size), path, data_type=gdal.GDT_Byte, creating_options=['COMPRESS=LZW'])
def preprocess(self): ''' Top of atmosphere is calculated and persisted into a file. Then a cloud mask is created with the given algorithm. ''' solar_zenith = self.get_sensor().parser.get_attribute(rapideye.SOLAR_ZENITH) data_acquisition_date = self.get_sensor().parser.get_attribute(rapideye.ACQUISITION_DATE) solar_azimuth = self.get_sensor().parser.get_attribute(rapideye.SOLAR_AZIMUTH) geotransform = self.get_raster().get_attribute(raster.GEOTRANSFORM) data = self.get_raster().read_data_file_as_array() sun_earth_distance = calculate_distance_sun_earth(data_acquisition_date) top_of_atmosphere_data = calculate_toa_rapideye(calculate_rad_rapideye(data), sun_earth_distance, solar_zenith) top_of_atmosphere_directory = create_file_name(get_parent(self.path), 'TOA') create_directory_path(top_of_atmosphere_directory) output_file = create_file_name(top_of_atmosphere_directory, get_base_name(self.get_files()[2]) + '_toa.tif') # TODO: change [2] in self.get_files()[2] create_raster_from_reference(output_file, top_of_atmosphere_data, self.file_dictionary[_IMAGE], data_type=NumericTypeCodeToGDALTypeCode(numpy.float32) ) LOGGER.debug('Top of atmosphere file was created.') cloud_output_file = create_file_name(top_of_atmosphere_directory, get_base_name(self.get_files()[2]) + '_cloud.tif') if self.algorithm == ANOMALY_DETECTION: LOGGER.debug('Cloud mask by anomaly detection process.') clouds = self.anomaly_detection_cloud_mask(top_of_atmosphere_data, cloud_output_file, solar_zenith, solar_azimuth, geotransform) elif self.algorithm == TIME_SERIES: LOGGER.debug('Cloud mask by reference with time series process.') tile_id = self.get_sensor().get_attribute(TILE_ID) clouds = self.masking_with_time_series(data, cloud_output_file, solar_zenith, solar_azimuth, geotransform, tile_id) create_raster_from_reference(cloud_output_file, clouds, self.file_dictionary[_IMAGE], data_type=NumericTypeCodeToGDALTypeCode(numpy.float32) ) LOGGER.info('Cloud mask was created.')
def handle(self, **options): paths = options['path'] output = options['output'][0] print output arrays = {} image1 = open_handle(paths[0]) #import time #for path in paths: # start_time = time.time() # arrays[path] = open_handle(path) # print 'Opened %s' % path # print("--- %s seconds ---" % (time.time() - start_time)) size = image1.shape #start_time = time.time() for p in range(2, len(paths)): #print 'Image %s ' % p image2 = open_handle(paths[p]) mask = numpy.logical_and( numpy.equal(image1, open_handle(paths[1])), numpy.equal(image1, image2)) image1[~mask] = NO_DATA #for path in paths: #del arrays[path] #print("--- %s seconds ---" % (time.time() - start_time)) #start_time = time.time() create_raster_from_reference(output, image1y, paths[0], data_type=gdal.GDT_Byte)
def handle(self, **options): mapgrid = '1449619' acq = get_pair_quality(mapgrid) for image in acq: print image.pk_id print image.pk_id id = options["id"][0] image_path = options["image"][0] reference_path = options["reference"][0] output = options["output"][0] print image_path print reference_path image_bundle = _get_bundle_from_path(image_path) reference_bundle = _get_bundle_from_path(reference_path) #extents = harmonize_images([image_bundle.get_raster(), reference_bundle.get_raster()]) #print extents #print extents['x_offset'] #print extents['y_offset'] shape = image_bundle.get_raster().get_attribute(raster.DATA_SHAPE) invariant_array = numpy.full((shape[0], shape[1]), INV_MASK_VALUE, dtype=np.int) in1 = reference_bundle.get_raster_file() in2 = image_bundle.get_raster_file() in_invar = create_filename(output, 'invariantPixelMask.tif') result = create_filename(output, 'crosscorrelation_next.tif') to_polar = create_filename(output, 'crosscorrelation_polar.tif') create_raster_from_reference(in_invar, invariant_array, image_bundle.get_raster_file(), gdal.GDT_Byte) local = LocalProcessLauncher() volume = '%s:%s' % (output, output) shell_array = [ 'docker', 'run', '--rm', '-v', volume, 'madmex/antares', 'correlation', '-in1', in1, '-in2', in2, '-in_invar', in_invar, '-val_invar', '%s' % INV_MASK_VALUE, '-out', result, '-window_size', '%s' % WINDOW_SIZE, '-max_gap', '%s' % MAX_GAP ] shell_string = ' '.join(shell_array) print shell_string if not is_file(result): log = local.execute(shell_string) crosscorrelation = raster.Data(result, 'GTiff') print crosscorrelation.get_attribute(raster.PROJECTION) print crosscorrelation.get_attribute(raster.GEOTRANSFORM) #tile_map(result, result) correlation_array = crosscorrelation.read_data_file_as_array() band_0 = correlation_array[0, :] band_1 = correlation_array[1, :] phi_band = phi(band_0, band_1) rho_band = rho(band_0, band_1) correlation_array[0, :] = phi_band correlation_array[1, :] = rho_band #create_raster_from_reference(to_polar, correlation_array, result) crosscorrelation_polar = raster.Data(to_polar, 'GTiff') extents = harmonize_images( [crosscorrelation_polar, reference_bundle.get_raster()]) x_offset = extents['x_offset'][1] y_offset = extents['y_offset'][1] x_tile_size = extents['x_range'] y_tile_size = extents['y_range'] aux_name = create_filename(output, 'auxiliar.tif') tile_map(reference_bundle.get_raster_file(), aux_name, x_tile_size, y_tile_size, x_offset, y_offset) aux_array = raster.Data(aux_name, 'GTiff').read_data_file_as_array() crosscorrelation_polar_array = crosscorrelation_polar.read_data_file_as_array( ) stats = calculate_statistics_qa(crosscorrelation_polar_array, aux_array, STAT_CLASSES, STAT_MIN, STAT_MAX, THRESHOLD_COD, THRESHOLD_LOG) desision = calculate_decision(stats['band_1']['histogram'], stats['band_1']['histogram_bins']) print stats quality = QualityAssessment( decision=desision, max=adapt_numpy_float(stats['band_1']['maximum']), min=adapt_numpy_float(stats['band_1']['minimum']), median=adapt_numpy_float(stats['band_1']['median']), mean=adapt_numpy_float(stats['band_1']['mean']), standard_deviation=adapt_numpy_float(stats['band_1']['std']), product_id=1, reference_id=2) persist_quality(quality) print desision
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' sum_of_numbers = 0 #path_a = options['a'][0] #path_b = options['b'][0] tiles = ["086W_20N","086W_21N","087W_17N","087W_18N","087W_19N","087W_20N","087W_21N","088W_16N","088W_17N","088W_18N","088W_19N","088W_20N","088W_21N","089W_15N","089W_16N","089W_17N","089W_18N","089W_19N","089W_20N","089W_21N","090W_14N","090W_15N","090W_16N","090W_17N","090W_18N","090W_19N","090W_20N","090W_21N","091W_14N","091W_15N","091W_16N","091W_17N","091W_18N","091W_19N","092W_14N","092W_15N","092W_16N","092W_17N","092W_18N","093W_15N","093W_16N","093W_17N","093W_18N","094W_16N","094W_17N","094W_18N","095W_15N","095W_16N","095W_17N","095W_18N","095W_19N","096W_15N","096W_16N","096W_17N","096W_18N","096W_19N","096W_20N","097W_15N","097W_16N","097W_17N","097W_18N","097W_19N","097W_20N","097W_21N","097W_22N","097W_23N","097W_24N","097W_25N","097W_26N","097W_27N","098W_16N","098W_17N","098W_18N","098W_19N","098W_20N","098W_21N","098W_22N","098W_23N","098W_24N","098W_25N","098W_26N","098W_27N","098W_28N","099W_16N","099W_17N","099W_18N","099W_19N","099W_20N","099W_21N","099W_22N","099W_23N","099W_24N","099W_25N","099W_26N","099W_27N","099W_28N","099W_29N","099W_30N","100W_16N","100W_17N","100W_18N","100W_19N","100W_20N","100W_21N","100W_22N","100W_23N","100W_24N","100W_25N","100W_26N","100W_27N","100W_28N","100W_29N","100W_30N","101W_17N","101W_18N","101W_19N","101W_20N","101W_21N","101W_22N","101W_23N","101W_24N","101W_25N","101W_26N","101W_27N","101W_28N","101W_29N","101W_30N","102W_17N","102W_18N","102W_19N","102W_20N","102W_21N","102W_22N","102W_23N","102W_24N","102W_25N","102W_26N","102W_27N","102W_28N","102W_29N","102W_30N","103W_18N","103W_19N","103W_20N","103W_21N","103W_22N","103W_23N","103W_24N","103W_25N","103W_26N","103W_27N","103W_28N","103W_29N","103W_30N","103W_31N","104W_18N","104W_19N","104W_20N","104W_21N","104W_22N","104W_23N","104W_24N","104W_25N","104W_26N","104W_27N","104W_28N","104W_29N","104W_30N","104W_31N","105W_19N","105W_20N","105W_21N","105W_22N","105W_23N","105W_24N","105W_25N","105W_26N","105W_27N","105W_28N","105W_29N","105W_30N","105W_31N","105W_32N","106W_21N","106W_22N","106W_23N","106W_24N","106W_25N","106W_26N","106W_27N","106W_28N","106W_29N","106W_30N","106W_31N","106W_32N","107W_23N","107W_24N","107W_25N","107W_26N","107W_27N","107W_28N","107W_29N","107W_30N","107W_31N","107W_32N","108W_24N","108W_25N","108W_26N","108W_27N","108W_28N","108W_29N","108W_30N","108W_31N","108W_32N","109W_22N","109W_23N","109W_24N","109W_25N","109W_26N","109W_27N","109W_28N","109W_29N","109W_30N","109W_31N","109W_32N","110W_22N","110W_23N","110W_24N","110W_25N","110W_27N","110W_28N","110W_29N","110W_30N","110W_31N","110W_32N","111W_24N","111W_25N","111W_26N","111W_27N","111W_28N","111W_29N","111W_30N","111W_31N","111W_32N","112W_24N","112W_25N","112W_26N","112W_27N","112W_28N","112W_29N","112W_30N","112W_31N","112W_32N","113W_26N","113W_27N","113W_28N","113W_29N","113W_30N","113W_31N","113W_32N","113W_33N","114W_26N","114W_27N","114W_28N","114W_29N","114W_30N","114W_31N","114W_32N","114W_33N","115W_27N","115W_28N","115W_29N","115W_30N","115W_31N","115W_32N","115W_33N","116W_30N","116W_31N","116W_32N","116W_33N","117W_32N","117W_33N"] print len(tiles) path_1985 = '/Users/agutierrez/Dropbox/Multivariado/classification/classification_1985.tif' path_2009 = '/Users/agutierrez/Dropbox/Multivariado/classification/classification_2009.tif' path_change = '/Users/agutierrez/Dropbox/Multivariado/change.tif' path_change_1985 = '/Users/agutierrez/Dropbox/Multivariado/classification/change_1985.tif' path_change_2009 = '/Users/agutierrez/Dropbox/Multivariado/classification/change_2009.tif' gdal_format = 'GTiff' image_1985 = raster.Data(path_1985, gdal_format) image_2009 = raster.Data(path_2009, gdal_format) image_change = raster.Data(path_change, gdal_format) array_1985 = image_1985.read_data_file_as_array() array_2009 = image_2009.read_data_file_as_array() array_change = image_change.read_data_file_as_array() print array_1985.shape print array_2009.shape print numpy.unique(array_1985, return_counts=True)[0] print numpy.unique(array_1985, return_counts=True)[1] * 30 * 30 * 0.0001 print numpy.unique(array_1985[array_change > .5], return_counts=True) print numpy.unique(array_2009[array_change > .5], return_counts=True) array_1985_masked = array_1985[array_change > .5] array_2009_masked = array_2009[array_change > .5] counts ={'1.0->2.0':0, '1.0->3.0':0, '1.0->4.0':0, '1.0->5.0':0, '2.0->1.0':0, '2.0->3.0':0, '2.0->4.0':0, '2.0->5.0':0, '3.0->1.0':0, '3.0->2.0':0, '3.0->4.0':0, '3.0->5.0':0, '4.0->1.0':0, '4.0->2.0':0, '4.0->3.0':0, '4.0->5.0':0, '5.0->1.0':0, '5.0->2.0':0, '5.0->3.0':0, '5.0->4.0':0} for i in range(len(array_1985_masked)): if not array_1985_masked[i] == array_2009_masked[i]: counts['%s->%s' % (array_1985_masked[i], array_2009_masked[i])] = counts['%s->%s' % (array_1985_masked[i], array_2009_masked[i])] + 30 * 30 * 0.0001 import json print json.dumps(counts, indent=1) array_1985[array_change < .5] = 0 array_2009[array_change < .5] = 0 create_raster_from_reference(path_change_1985, array_1985, path_1985, data_type=gdal.GDT_Byte) create_raster_from_reference(path_change_2009, array_2009, path_1985, data_type=gdal.GDT_Byte) ''' init = 1985 gdal_format = 'GTiff' for tile in tiles: for i in range(30): path_a = '/LUSTRE/MADMEX/tasks/2016_tasks/matt_hansen_forests/tiles/%s/%s_%s.tif' % (tile, tile, init+i) path_b = '/LUSTRE/MADMEX/tasks/2016_tasks/matt_hansen_forests/tiles/%s/%s_%s.tif' % (tile, tile, init+i+1) path_c = '/LUSTRE/MADMEX/tasks/2016_tasks/matt_hansen_forests/tiles_example/%s/%s_%s_%s.tif' % (tile, tile, init+i,init+i+1) directory = '/LUSTRE/MADMEX/tasks/2016_tasks/matt_hansen_forests/tiles_example/%s/' % tile print path_a print path_b print path_c #remove_file(path_c) create_directory_path(directory) if not check_if_file_exists(path_c): image_a = raster.Data(path_a, gdal_format) image_b = raster.Data(path_b, gdal_format) array_a = image_a.read_data_file_as_array() array_b = image_b.read_data_file_as_array() #mask = ((array_a < 30) | (70 < array_a)) & ((array_b < 30) | (70 < array_b)) #array_a[array_a <= 30] = 0 #array_a[30 < array_a] = 1 #array_b[array_b <= 30] = 0 #array_b[30 < array_b] = 1 #diff = array_b - array_a diff = numpy.zeros(numpy.shape(array_a)) upper = 30 lower = 10 diff[(array_b < lower) & (array_a > upper)] = 1 diff[(array_b > upper) & (array_a < lower)] = 2 #diff[mask] = -9999 create_raster_from_reference(path_c, diff, path_a, data_type=gdal.GDT_Byte) else: print 'File exists.' print 'Done %s-%s' % (init+i,init+i) ''' ''' total = {} for state in ['Campeche','Chiapas','Oaxaca','Quintana_Roo','Yucatan']: total_state = {} for where in ['anps', 'corr-anp', 'est-corr-anp']: raster_path = '/LUSTRE/MADMEX/staging/2017_tasks/corredor_biologico/analisis_por_estado-vegetacion/shapes_area_interes_por_estado/%s/%s_%s.tif' % (state, state, where) print raster_path image = raster.Data(raster_path, gdal_format) array = image.read_data_file_as_array() array[array > 0] = 1 counts = numpy.unique(array, return_counts=True) print counts total_state[where] = counts[1][1] total[state] = total_state print total init = 1985 for state in ['Campeche','Chiapas','Oaxaca','Quintana_Roo','Yucatan']: statistics_by_state = [] for where in ['anps', 'corr-anp', 'est-corr-anp']: for i in range(30): mask_path = '/LUSTRE/MADMEX/staging/2017_tasks/corredor_biologico/analisis_por_estado-vegetacion/shapes_area_interes_por_estado/%s/%s_%s.tif' % (state, state, where) mask_image = raster.Data(mask_path, gdal_format) mask_array = mask_image.read_data_file_as_array() path_c = '/LUSTRE/MADMEX/staging/2017_tasks/corredor_biologico/analisis_por_estado-vegetacion/corte_por_area_interes_en_estado/diff-michael-repro/Mexico_TCC_%s_%s_%s_%s.tif' % (state, init+i,init+i+1,where) print path_c image_c = raster.Data(path_c, gdal_format) array_c = image_c.read_data_file_as_array() #array_c = array_c[array_c > -9999] counts = numpy.unique(array_c, return_counts=True) print counts #size_non_zero = len(array_c) #final = numpy.zeros(size_non_zero, dtype=numpy.int) #final[(-40 < array_c) & (array_c < 40)] = 0 #final[40 < array_c] = 1 #final[array_c < -40] = 2 x_resolution = image_c.get_geotransform()[1] y_resolution = image_c.get_geotransform()[5] area = x_resolution * y_resolution stats = {} stats['period'] = '%s-%s' % (init+i,init+i+1) stats['type'] = where stats['resolution'] = area if len(counts[1]) > 2: stats['negative'] = counts[1][0] stats['no-change'] = counts[1][1] stats['positive'] = counts[1][2] else: stats['negative'] = 0 stats['no-change'] = counts[1][0] stats['positive'] = 0 #stats['net'] = (counts[1][1] - counts[1][2]) stats['total'] = total[state][where] statistics_by_state.append(stats) print statistics_by_state stats_path = '/LUSTRE/MADMEX/staging/2017_tasks/corredor_biologico/analisis_por_estado-vegetacion/shapes_area_interes_por_estado/stats/estadisticas-michael-repro-%s.json' % state import json with open(stats_path, 'w') as outfile: json.dump(statistics_by_state, outfile) '''
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' output = options['output'][0] for path in options['path']: bundle = _get_bundle_from_path(path) basename = get_basename(bundle.get_raster_file()) bundle.get_NDVI() ndvi_file = create_filename(output, 'ndvi.tif') red_edge_ndvi_file = create_filename(output, 'red_edge_ndvi.tif') gndvi_file = create_filename(output, 'gndvi.tif') ndre_file = create_filename(output, 'ndre.tif') sovel_file = create_filename(output, 'sovel2.tif') all_file = create_filename(output, '%s_all_features.tif' % basename) print all_file image_array = bundle.get_raster().read_data_file_as_array() ndvi_array = bundle.get_NDVI() ndvi_array[ndvi_array <= -1] = -1 ndvi_array[ndvi_array >= 1] = 1 red_edge_ndvi_array = bundle.get_red_edge_NDVI() red_edge_ndvi_array[red_edge_ndvi_array <= -1] = -1 red_edge_ndvi_array[red_edge_ndvi_array >= 1] = 1 gndvi_array = bundle.get_gndvi() gndvi_array[gndvi_array <= -1] = -1 gndvi_array[gndvi_array >= 1] = 1 ndre_array = bundle.get_ndre() ndre_array[ndre_array <= -1] = -1 ndre_array[ndre_array >= 1] = 1 sobel_filter_array = bundle.get_sobel_filter(sigma=2) #sobel_filter_array[sobel_filter_array<=-1] = -1 #sobel_filter_array[sobel_filter_array>=1] = 1 #create_raster_from_reference(ndvi_file, ndvi_array, bundle.get_raster_file()) #create_raster_from_reference(red_edge_ndvi_file, red_edge_ndvi_array, bundle.get_raster_file()) #create_raster_from_reference(gndvi_file, gndvi_array, bundle.get_raster_file()) #create_raster_from_reference(ndre_file, ndre_array, bundle.get_raster_file()) create_raster_from_reference(sovel_file, sobel_filter_array, bundle.get_raster_file()) all_features = numpy.array([ image_array[0], image_array[1], image_array[2], image_array[3], image_array[4], ndvi_array, red_edge_ndvi_array, gndvi_array, ndre_array, sobel_filter_array ]) print image_array[0].shape print image_array[1].shape print image_array[2].shape print image_array[3].shape print image_array[4].shape print ndvi_array.shape print red_edge_ndvi_array.shape print gndvi_array.shape print ndre_array.shape print sobel_filter_array.shape print all_features.shape create_raster_from_reference(all_file, all_features, bundle.get_raster_file(), creating_options=['BIGTIFF=YES'])
def handle(self, **options): ''' This process will call the change detection process from a set of two individual images. It will perform the harmonization and the multivariate alteration detection on the images. It will then perform a maximum correlation factor on them and work with the resulting bands. ''' image_a = options['ima'][0] image_b = options['imb'][0] output_image = options['output'][0] LOGGER.info('Image %s will be compared against image %s. Output will be available' \ ' at %s.', image_a, image_b, output_image) gdal_format = "GTiff" image_a_data_class = raster.Data(image_a, gdal_format) image_b_data_class = raster.Data(image_b, gdal_format) # TODO : remove references to class harmonized harmonized_class = harmonized.Data(image_a_data_class, image_b_data_class) #band1 = image_a_data_class.GetRasterBand(1) #band1 = band1.ReadAsArray(0, 0,image_a_data_class.RasterXSize,image_a_data_class.RasterYSize).astype(float) #print(band1) if harmonized_class: #data_shape_harmonized = harmonized_class.get_attribute(harmonized.DATA_SHAPE) #width, height, bands = data_shape_harmonized #geotransform_harmonized = harmonized_class.get_attribute(raster.GEOTRANSFORM) #projection_harmonized = harmonized_class.get_attribute(harmonized.PROJECTION) image_a_data_array, image_b_data_array = harmonized_class.harmonized_arrays(image_a_data_class, image_b_data_class) imad_class = imad.Transformation([image_a_data_array, image_b_data_array]) imad_class.execute() mad_result = imad_class.output LOGGER.debug('mad_result.shape: %s', mad_result.shape) create_directory_path(getattr(SETTINGS, 'TEST_FOLDER')) mad_output_file = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'mad.tif') create_raster_from_reference(mad_output_file, mad_result,image_a) maf_class = maf.Transformation(imad_class.output) maf_class.execute() maf_result = maf_class.output pdf_file = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'maf_pdf.png') thresholds = calc_threshold_grid(maf_result, pdf_file) class_result = recode_classes_grid(maf_result, thresholds) LOGGER.debug('maf_result.shape: %s', maf_result.shape) LOGGER.debug('class_result.shape: %s', class_result.shape) maf_outputfile = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'maf.tif') class_outputfile = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'class.tif') create_raster_from_reference(maf_outputfile, maf_result, image_a) create_raster_from_reference(class_outputfile, class_result, image_a) print 'Output written in: %s' % mad_output_file print 'Shape is ', imad_class.output.shape
def handle(self, **options): ''' This is the code that does the ingestion. indexes --path /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/Landsat_2000_2008/L5_021_047_2000_022_sr/ --shape /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/AE_LT_new.shp --output ''' path = options['path'][0] for mask_file in os.listdir(path): if mask_file.endswith('_cfmask.tif'): print mask_file basename = mask_file.replace('_cfmask.tif', '') print basename cloud_file_name = mask_file LOGGER.info('Calculating indexes for Landsat scenes.') band_name = basename + '_sr_band%s.tif' final_path = options['output'][0] create_directory_path(final_path) cloud_file = create_filename(path, cloud_file_name) cloud_array = open_handle(cloud_file) cloud_mask = (cloud_array == 4) LOGGER.debug('Recognize sensor depending on file name.') if 'L8' in path: LOGGER.debug('Landsat 8 scene was detected.') blue_file = create_filename(path, band_name % BLUE_L8) green_file = create_filename(path, band_name % GREEN_L8) red_file = create_filename(path, band_name % RED_L8) nir_file = create_filename(path, band_name % NIR_L8) swir_file = create_filename(path, band_name % SWIR_L8) else: LOGGER.debug('Landsat 4, 5 or scene was detected.') blue_file = create_filename(path, band_name % BLUE) green_file = create_filename(path, band_name % GREEN) red_file = create_filename(path, band_name % RED) nir_file = create_filename(path, band_name % NIR) swir_file = create_filename(path, band_name % SWIR) LOGGER.debug('Loading bands of interest.') green_array = open_handle(green_file) red_array = open_handle(red_file) nir_array = open_handle(nir_file) swir_array = open_handle(swir_file) LOGGER.debug('Calculating indexes.') ndvi_array = calculate_index(nir_array, red_array) mndwi_array = calculate_index(green_array, swir_array) ndwig_array = calculate_index(nir_array, swir_array) ndwim_array = calculate_index(green_array, nir_array) LOGGER.debug('Setting cloud mask values to -999.') ndvi_array[cloud_mask] = -999 mndwi_array[cloud_mask] = -999 ndwig_array[cloud_mask] = -999 ndwim_array[cloud_mask] = -999 LOGGER.debug('Creating files for indexes.') ndvi_final_file = create_filename(final_path, basename + '_ndvi.tif') mndwi_final_file = create_filename(final_path, basename + '_mndwi.tif') ndwig_final_file = create_filename(final_path, basename + '_ndwig.tif') ndwim_final_file = create_filename(final_path, basename + '_ndwim.tif') ndvi_clipped_file = create_filename(final_path, basename + '_ndvi_clipped.tif') mndwi_clipped_file = create_filename(final_path, basename + '_mndwi_clipped.tif') ndwig_clipped_file = create_filename(final_path, basename + '_ndwig_clipped.tif') ndwim_clipped_file = create_filename(final_path, basename + '_ndwim_clipped.tif') ndvi_file = create_filename(final_path, basename + '_ndvi_pre.tif') mndwi_file = create_filename(final_path, basename + '_mndwi_pre.tif') ndwig_file = create_filename(final_path, basename + '_ndwig_pre.tif') ndwim_file = create_filename(final_path, basename + '_ndwim_pre.tif') files = [ndvi_file, mndwi_file, ndwig_file, ndwim_file] clipped_files = [ ndvi_clipped_file, mndwi_clipped_file, ndwig_clipped_file, ndwim_clipped_file ] final_files = [ ndvi_final_file, mndwi_final_file, ndwig_final_file, ndwim_final_file ] LOGGER.debug('Writing information to files.') create_raster_from_reference(ndvi_file, ndvi_array, green_file) create_raster_from_reference(mndwi_file, mndwi_array, green_file) create_raster_from_reference(ndwig_file, ndwig_array, green_file) create_raster_from_reference(ndwim_file, ndwim_array, green_file) LOGGER.debug('Deleting arrays to release memory.') del ndvi_array del mndwi_array del ndwig_array del ndwim_array del cloud_array LOGGER.debug('Reference rgb file creation.') rgb_file = create_filename(final_path, basename + '_rgb.tif') merge_command = [ '/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt', '-separate', '-o', rgb_file, red_file, green_file, blue_file ] call(merge_command) shape = options['shape'][0] LOGGER.debug('Cookie cut files using the given shape.') for i in range(4): clip_command = [ '/Library/Frameworks/GDAL.framework/Programs/gdalwarp', '-crop_to_cutline', '-cutline', shape, files[i], clipped_files[i] ] call(clip_command) for i in range(4): aux_array = open_handle(clipped_files[i]) aux_array[(aux_array == 0)] = -9999 create_raster_from_reference(final_files[i], aux_array, ndvi_clipped_file) if not options['debug']: LOGGER.info('Remove auxiliary files.') os.remove(ndvi_clipped_file) os.remove(mndwi_clipped_file) os.remove(ndwig_clipped_file) os.remove(ndwim_clipped_file) os.remove(ndvi_file) os.remove(mndwi_file) os.remove(ndwig_file) os.remove(ndwim_file) print 'Done'
def handle(self, **options): """ This is the code that does the ingestion. indexes --path /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/Landsat_2000_2008/L5_021_047_2000_022_sr/ --shape /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/AE_LT_new.shp --output """ path = options["path"][0] for mask_file in os.listdir(path): if mask_file.endswith("_cfmask.tif"): print mask_file basename = mask_file.replace("_cfmask.tif", "") print basename cloud_file_name = mask_file LOGGER.info("Calculating indexes for Landsat scenes.") band_name = basename + "_sr_band%s.tif" final_path = options["output"][0] create_directory_path(final_path) cloud_file = create_file_name(path, cloud_file_name) cloud_array = open_handle(cloud_file) cloud_mask = cloud_array == 4 LOGGER.debug("Recognize sensor depending on file name.") if "L8" in path: LOGGER.debug("Landsat 8 scene was detected.") blue_file = create_file_name(path, band_name % BLUE_L8) green_file = create_file_name(path, band_name % GREEN_L8) red_file = create_file_name(path, band_name % RED_L8) nir_file = create_file_name(path, band_name % NIR_L8) swir_file = create_file_name(path, band_name % SWIR_L8) else: LOGGER.debug("Landsat 4, 5 or scene was detected.") blue_file = create_file_name(path, band_name % BLUE) green_file = create_file_name(path, band_name % GREEN) red_file = create_file_name(path, band_name % RED) nir_file = create_file_name(path, band_name % NIR) swir_file = create_file_name(path, band_name % SWIR) LOGGER.debug("Loading bands of interest.") green_array = open_handle(green_file) red_array = open_handle(red_file) nir_array = open_handle(nir_file) swir_array = open_handle(swir_file) LOGGER.debug("Calculating indexes.") ndvi_array = calculate_index(nir_array, red_array) mndwi_array = calculate_index(green_array, swir_array) ndwig_array = calculate_index(nir_array, swir_array) ndwim_array = calculate_index(green_array, nir_array) LOGGER.debug("Setting cloud mask values to -999.") ndvi_array[cloud_mask] = -999 mndwi_array[cloud_mask] = -999 ndwig_array[cloud_mask] = -999 ndwim_array[cloud_mask] = -999 LOGGER.debug("Creating files for indexes.") ndvi_final_file = create_file_name(final_path, basename + "_ndvi.tif") mndwi_final_file = create_file_name(final_path, basename + "_mndwi.tif") ndwig_final_file = create_file_name(final_path, basename + "_ndwig.tif") ndwim_final_file = create_file_name(final_path, basename + "_ndwim.tif") ndvi_clipped_file = create_file_name(final_path, basename + "_ndvi_clipped.tif") mndwi_clipped_file = create_file_name(final_path, basename + "_mndwi_clipped.tif") ndwig_clipped_file = create_file_name(final_path, basename + "_ndwig_clipped.tif") ndwim_clipped_file = create_file_name(final_path, basename + "_ndwim_clipped.tif") ndvi_file = create_file_name(final_path, basename + "_ndvi_pre.tif") mndwi_file = create_file_name(final_path, basename + "_mndwi_pre.tif") ndwig_file = create_file_name(final_path, basename + "_ndwig_pre.tif") ndwim_file = create_file_name(final_path, basename + "_ndwim_pre.tif") files = [ndvi_file, mndwi_file, ndwig_file, ndwim_file] clipped_files = [ndvi_clipped_file, mndwi_clipped_file, ndwig_clipped_file, ndwim_clipped_file] final_files = [ndvi_final_file, mndwi_final_file, ndwig_final_file, ndwim_final_file] LOGGER.debug("Writing information to files.") create_raster_from_reference(ndvi_file, ndvi_array, green_file) create_raster_from_reference(mndwi_file, mndwi_array, green_file) create_raster_from_reference(ndwig_file, ndwig_array, green_file) create_raster_from_reference(ndwim_file, ndwim_array, green_file) LOGGER.debug("Deleting arrays to release memory.") del ndvi_array del mndwi_array del ndwig_array del ndwim_array del cloud_array LOGGER.debug("Reference rgb file creation.") rgb_file = create_file_name(final_path, basename + "_rgb.tif") merge_command = [ "/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt", "-separate", "-o", rgb_file, red_file, green_file, blue_file, ] call(merge_command) shape = options["shape"][0] LOGGER.debug("Cookie cut files using the given shape.") for i in range(4): clip_command = [ "/Library/Frameworks/GDAL.framework/Programs/gdalwarp", "-crop_to_cutline", "-cutline", shape, files[i], clipped_files[i], ] call(clip_command) for i in range(4): aux_array = open_handle(clipped_files[i]) aux_array[(aux_array == 0)] = -9999 create_raster_from_reference(final_files[i], aux_array, ndvi_clipped_file) if not options["debug"]: LOGGER.info("Remove auxiliary files.") os.remove(ndvi_clipped_file) os.remove(mndwi_clipped_file) os.remove(ndwig_clipped_file) os.remove(ndwim_clipped_file) os.remove(ndvi_file) os.remove(mndwi_file) os.remove(ndwig_file) os.remove(ndwim_file) print "Done"
clouds = self.masking_with_time_series(data, cloud_output_file, solar_zenith, solar_azimuth, geotransform, tile_id) create_raster_from_reference(cloud_output_file, clouds, self.file_dictionary[_IMAGE], data_type=NumericTypeCodeToGDALTypeCode(numpy.float32) ) LOGGER.info('Cloud mask was created.') if __name__ == '__main__': #path = '/Users/amaury/Documents/rapideye/df/1447813/2012/2012-03-14/l3a' #bundle = Bundle(path) #print bundle.get_files() #print bundle.can_identify() image = "/Users/agutierrez/Documents/rapideye/df/1447813/2012/2012-03-14/l3a/2012-03-14T182106_RE2_3A-NAC_11040070_149070.tif" gdal_format = "GTiff" data_class = raster.Data(image, gdal_format) array = data_class.read_data_file_as_array() width, height, bands = data_class.get_attribute(raster.DATA_SHAPE) feature_bands = numpy.zeros([2, width, height]) from madmex.processing.raster import calculate_ndvi feature_bands[0, :, :] = calculate_ndvi(array[4, :, :], array[2, :, :]) feature_bands[1, :, :] = calculate_ndvi(array[3, :, :], array[2, :, :]) out1 = get_parent(image) + 'result_ndvi_1.tif' out2 = get_parent(image) + 'result_ndvi_2.tif' create_raster_from_reference(out1, feature_bands[0, :, :], image) create_raster_from_reference(out2, feature_bands[1, :, :], image) print 'Done'