def get_thumbnail_with_path(self, thumbnail_path): ''' Creates a thumbnail for the scene in true color. ''' from subprocess import call thumnail_directory = create_file_name(thumbnail_path, 'thumbnail') create_directory_path(thumnail_directory) filename = self.file_dictionary[_BROWSE] thumbnail = create_file_name(thumnail_directory, '%s.jpg' % get_base_name(filename)) resize_command = ['/Library/Frameworks/GDAL.framework/Programs/gdal_translate', filename, '-of', 'JPEG', thumbnail] call(resize_command)
def test_creat_image(self): import numpy filename = "/Users/agutierrez/Development/df/1448114/2015/2015-02-24/l3a/1448114_2015-02-24_RE3_3A_302417.tif" width = get_width(filename) height = get_height(filename) plain = width * height red = numpy.zeros(plain) green = numpy.zeros(plain) blue = numpy.zeros(plain) ired = numpy.zeros(plain) print "before loop" for i in range(plain): if not i % 2: red[i] = 1 if not i % 3: green[i] = 1 if not i % 5: blue[i] = 1 if not i % 7: ired[i] = 1 print "loop" final = numpy.array( [ numpy.reshape(red, (width, height)), numpy.reshape(green, (width, height)), numpy.reshape(blue, (width, height)), numpy.reshape(ired, (width, height)), ] ) print final.shape print "hello" create_raster_from_reference(create_file_name(getattr(SETTINGS, "TEST_FOLDER"), "sieve.tif"), final, filename)
def get_thumbnail(self): ''' Creates a thumbnail for the scene in true color. ''' from subprocess import call file_1 = self.file_dictionary[_BASE % (self.get_mission(), 'B1[0-9].TIF')] file_2 = self.file_dictionary[_BASE % (self.get_mission(), 'B2[0-9].TIF')] file_3 = self.file_dictionary[_BASE % (self.get_mission(), 'B3[0-9].TIF')] parent_directory = get_parent(self.path) thumnail_directory = create_file_name(parent_directory, 'thumbnail') create_directory_path(thumnail_directory) parent_directory filename = create_file_name(thumnail_directory, 'vrt.tif') merge_command = ['/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt', '-separate', '-o', filename, file_3, file_2, file_1] call(merge_command) thumbnail = create_file_name(thumnail_directory, 'thumbnail.jpg') resize_command = ['/Library/Frameworks/GDAL.framework/Programs/gdal_translate', filename, '-of', 'JPEG', '-outsize', '5%', '5%', thumbnail] call(resize_command)
def _look_for_files(self): ''' This method will be called by the constructor to look for files in the given directory. In order for a bundle to be valid, a file for each regular expression must be found. ''' for key in self.file_dictionary.iterkeys(): for name in get_files_from_folder(self.path): if re.match(key, name): self.file_dictionary[key] = create_file_name(self.path, name)
def preprocess(self): ''' Top of atmosphere is calculated and persisted into a file. Then a cloud mask is created with the given algorithm. ''' solar_zenith = self.get_sensor().parser.get_attribute(rapideye.SOLAR_ZENITH) data_acquisition_date = self.get_sensor().parser.get_attribute(rapideye.ACQUISITION_DATE) solar_azimuth = self.get_sensor().parser.get_attribute(rapideye.SOLAR_AZIMUTH) geotransform = self.get_raster().get_attribute(raster.GEOTRANSFORM) data = self.get_raster().read_data_file_as_array() sun_earth_distance = calculate_distance_sun_earth(data_acquisition_date) top_of_atmosphere_data = calculate_toa_rapideye(calculate_rad_rapideye(data), sun_earth_distance, solar_zenith) top_of_atmosphere_directory = create_file_name(get_parent(self.path), 'TOA') create_directory_path(top_of_atmosphere_directory) output_file = create_file_name(top_of_atmosphere_directory, get_base_name(self.get_files()[2]) + '_toa.tif') # TODO: change [2] in self.get_files()[2] create_raster_from_reference(output_file, top_of_atmosphere_data, self.file_dictionary[_IMAGE], data_type=NumericTypeCodeToGDALTypeCode(numpy.float32) ) LOGGER.debug('Top of atmosphere file was created.') cloud_output_file = create_file_name(top_of_atmosphere_directory, get_base_name(self.get_files()[2]) + '_cloud.tif') if self.algorithm == ANOMALY_DETECTION: LOGGER.debug('Cloud mask by anomaly detection process.') clouds = self.anomaly_detection_cloud_mask(top_of_atmosphere_data, cloud_output_file, solar_zenith, solar_azimuth, geotransform) elif self.algorithm == TIME_SERIES: LOGGER.debug('Cloud mask by reference with time series process.') tile_id = self.get_sensor().get_attribute(TILE_ID) clouds = self.masking_with_time_series(data, cloud_output_file, solar_zenith, solar_azimuth, geotransform, tile_id) create_raster_from_reference(cloud_output_file, clouds, self.file_dictionary[_IMAGE], data_type=NumericTypeCodeToGDALTypeCode(numpy.float32) ) LOGGER.info('Cloud mask was created.')
def handle(self, **options): output = options['output'][0] for image_path in options['path']: print image_path basename = '%s.tif' % get_base_name(image_path) print basename target = create_file_name(output, basename) print target start_time = time.time() #self.method_by_block(image_path, target) self.mask_iterating_values(image_path, target) print("--- %s seconds ---" % (time.time() - start_time)) print 'Dataset was written.'
def test_create_image(self): import numpy array = numpy.array( [ [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1], [1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1], [1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1], [1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1], [1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], ] ) red = (1 - array) * 1000 green = (1 - array) * 1000 blue = array * 1000 create_raster(create_file_name(getattr(SETTINGS, "TEST_FOLDER"), "single.tif"), array) create_raster(create_file_name(getattr(SETTINGS, "TEST_FOLDER"), "multi.tif"), numpy.array([red, green, blue]))
def split_shape_into_features(shape_name, destination_directory, column): ''' This method will take a input shape and iterate over its features, creating a new shape file with each one of them. It copies all the fields and the same spatial reference from the original file. The created files are saved in the destination directory using the number of the field given. ''' driver = ogr.GetDriverByName(str('ESRI Shapefile')) shape = driver.Open(shape_name, 0) layer = shape.GetLayer() layer_name = layer.GetName() spatial_reference = layer.GetSpatialRef() in_feature = layer.GetNextFeature() while in_feature: encoding = 'utf-8' in_feature_name = create_filename_from_string(in_feature.GetField(column).decode(encoding)) final_path = destination_directory + str(in_feature.GetField(0)) create_directory_path(final_path) output_name = create_file_name(final_path, in_feature_name + '.shp') if os.path.exists(output_name): driver.DeleteDataSource(output_name) datasource = driver.CreateDataSource(output_name) outLayer = datasource.CreateLayer(layer_name, spatial_reference, geom_type=ogr.wkbPolygon) inLayerDefn = layer.GetLayerDefn() for i in range(0, inLayerDefn.GetFieldCount()): fieldDefn = inLayerDefn.GetFieldDefn(i) LOGGER.debug(fieldDefn.GetName()) outLayer.CreateField(fieldDefn) outLayerDefn = outLayer.GetLayerDefn() geometry = in_feature.GetGeometryRef() out_feature = ogr.Feature(outLayerDefn) out_feature.SetGeometry(geometry) for i in range(0, outLayerDefn.GetFieldCount()): out_feature.SetField(outLayerDefn.GetFieldDefn(i).GetNameRef(), in_feature.GetField(i)) outLayer.CreateFeature(out_feature) out_feature.Destroy() in_feature.Destroy() in_feature = layer.GetNextFeature() shape.Destroy() datasource.Destroy()
def get_information_object(self): row = self.get_sensor().get_attribute(olitirs.ROW) path = self.get_sensor().get_attribute(olitirs.PATH) basename_metadata = get_basename_of_file( self.file_dictionary[_BASE % (self.get_letter(), self.get_mission(), "MTL.txt")] ) information = Information( metadata_path=create_file_name(self.get_output_directory(), basename_metadata), grid_id=unicode(path + row), projection=self.get_raster().get_attribute(raster.PROJECTION), cloud_percentage=self.get_sensor().get_attribute(olitirs.CLOUD_COVER), geometry=self.get_raster().get_attribute(raster.FOOTPRINT), elevation_angle=0.0, resolution=self.get_raster().get_attribute(raster.GEOTRANSFORM)[1], ) return information
def handle(self, **options): ''' This is the code that does the ingestion. ''' interest_band = 1 for image_path in options['path']: print image_path ds = gdal.Open(image_path) bands = ds.RasterCount geotransform = ds.GetGeoTransform() x_resolution = geotransform[1] y_resolution = geotransform[5] pixel_area = abs(x_resolution * y_resolution) array = numpy.array(ds.GetRasterBand(interest_band).ReadAsArray()) print numpy.unique(array) flat = numpy.ravel(array) length = len(flat) parent = get_parent(image_path) basename = '%s.txt' % get_base_name(image_path) target = create_file_name(parent, basename) count = 1 values = {} progress = 0 for value in flat: count = count + 1 if not values.get(value): values[value] = 1 else: values[value] = values[value] + 1 if count % 1000000 == 0: aux = progress progress = math.floor(100 * count / float(length)) if not aux == progress: print str(int(progress)) + '%\r' added = self.add_up(INITIAL_ARRAY, values) area = self.transform_to_area(added, pixel_area) with open(target, "a") as f: json.dump(area, f)
def handle(self, **options): """ This process will call the change detection process from a set of two individual images. It will perform the harmonization and the multivariate alteration detection on the images. It will then perform a maximum correlation factor on them and work with the resulting bands. """ image_a = options["ima"][0] image_b = options["imb"][0] output_image = options["output"][0] LOGGER.info( "Image %s will be compared against image %s. Output will be available" " at %s.", image_a, image_b, output_image, ) gdal_format = "GTiff" image_a_data_class = raster.Data(image_a, gdal_format) image_b_data_class = raster.Data(image_b, gdal_format) # TODO : remove references to class harmonized harmonized_class = harmonized.Data(image_a_data_class, image_b_data_class) if harmonized_class: data_shape_harmonized = harmonized_class.get_attribute(harmonized.DATA_SHAPE) width, height, bands = data_shape_harmonized geotransform_harmonized = harmonized_class.get_attribute(harmonized.GEOTRANSFORM) projection_harmonized = harmonized_class.get_attribute(harmonized.PROJECTION) image_a_data_array, image_b_data_array = harmonized_class.harmonized_arrays( image_a_data_class, image_b_data_class ) imad_class = imad.Transformation([image_a_data_array, image_b_data_array]) imad_class.execute() maf_class = maf.Transformation(imad_class.output) maf_class.execute() mad_result = imad_class.output maf_result = maf_class.output pdf_file = create_file_name(getattr(SETTINGS, "TEST_FOLDER"), "maf_pdf.png") thresholds = calc_threshold_grid(maf_result, pdf_file) class_result = recode_classes_grid(maf_result, thresholds) LOGGER.debug("mad_result.shape: %s", mad_result.shape) LOGGER.debug("maf_result.shape: %s", maf_result.shape) LOGGER.debug("class_result.shape: %s", class_result.shape) mad_output_file = create_file_name(getattr(SETTINGS, "TEST_FOLDER"), "mad.tif") maf_outputfile = create_file_name(getattr(SETTINGS, "TEST_FOLDER"), "maf.tif") class_outputfile = create_file_name(getattr(SETTINGS, "TEST_FOLDER"), "class.tif") create_raster(mad_output_file, mad_result, geotransform_harmonized, projection_harmonized) create_raster(maf_outputfile, maf_result, geotransform_harmonized, projection_harmonized) create_raster(class_outputfile, class_result, geotransform_harmonized, projection_harmonized) print "Output written in: %s" % mad_output_file print "Shape is ", imad_class.output.shape
def handle(self, **options): """ This is the code that does the ingestion. indexes --path /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/Landsat_2000_2008/L5_021_047_2000_022_sr/ --shape /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/AE_LT_new.shp --output """ path = options["path"][0] for mask_file in os.listdir(path): if mask_file.endswith("_cfmask.tif"): print mask_file basename = mask_file.replace("_cfmask.tif", "") print basename cloud_file_name = mask_file LOGGER.info("Calculating indexes for Landsat scenes.") band_name = basename + "_sr_band%s.tif" final_path = options["output"][0] create_directory_path(final_path) cloud_file = create_file_name(path, cloud_file_name) cloud_array = open_handle(cloud_file) cloud_mask = cloud_array == 4 LOGGER.debug("Recognize sensor depending on file name.") if "L8" in path: LOGGER.debug("Landsat 8 scene was detected.") blue_file = create_file_name(path, band_name % BLUE_L8) green_file = create_file_name(path, band_name % GREEN_L8) red_file = create_file_name(path, band_name % RED_L8) nir_file = create_file_name(path, band_name % NIR_L8) swir_file = create_file_name(path, band_name % SWIR_L8) else: LOGGER.debug("Landsat 4, 5 or scene was detected.") blue_file = create_file_name(path, band_name % BLUE) green_file = create_file_name(path, band_name % GREEN) red_file = create_file_name(path, band_name % RED) nir_file = create_file_name(path, band_name % NIR) swir_file = create_file_name(path, band_name % SWIR) LOGGER.debug("Loading bands of interest.") green_array = open_handle(green_file) red_array = open_handle(red_file) nir_array = open_handle(nir_file) swir_array = open_handle(swir_file) LOGGER.debug("Calculating indexes.") ndvi_array = calculate_index(nir_array, red_array) mndwi_array = calculate_index(green_array, swir_array) ndwig_array = calculate_index(nir_array, swir_array) ndwim_array = calculate_index(green_array, nir_array) LOGGER.debug("Setting cloud mask values to -999.") ndvi_array[cloud_mask] = -999 mndwi_array[cloud_mask] = -999 ndwig_array[cloud_mask] = -999 ndwim_array[cloud_mask] = -999 LOGGER.debug("Creating files for indexes.") ndvi_final_file = create_file_name(final_path, basename + "_ndvi.tif") mndwi_final_file = create_file_name(final_path, basename + "_mndwi.tif") ndwig_final_file = create_file_name(final_path, basename + "_ndwig.tif") ndwim_final_file = create_file_name(final_path, basename + "_ndwim.tif") ndvi_clipped_file = create_file_name(final_path, basename + "_ndvi_clipped.tif") mndwi_clipped_file = create_file_name(final_path, basename + "_mndwi_clipped.tif") ndwig_clipped_file = create_file_name(final_path, basename + "_ndwig_clipped.tif") ndwim_clipped_file = create_file_name(final_path, basename + "_ndwim_clipped.tif") ndvi_file = create_file_name(final_path, basename + "_ndvi_pre.tif") mndwi_file = create_file_name(final_path, basename + "_mndwi_pre.tif") ndwig_file = create_file_name(final_path, basename + "_ndwig_pre.tif") ndwim_file = create_file_name(final_path, basename + "_ndwim_pre.tif") files = [ndvi_file, mndwi_file, ndwig_file, ndwim_file] clipped_files = [ndvi_clipped_file, mndwi_clipped_file, ndwig_clipped_file, ndwim_clipped_file] final_files = [ndvi_final_file, mndwi_final_file, ndwig_final_file, ndwim_final_file] LOGGER.debug("Writing information to files.") create_raster_from_reference(ndvi_file, ndvi_array, green_file) create_raster_from_reference(mndwi_file, mndwi_array, green_file) create_raster_from_reference(ndwig_file, ndwig_array, green_file) create_raster_from_reference(ndwim_file, ndwim_array, green_file) LOGGER.debug("Deleting arrays to release memory.") del ndvi_array del mndwi_array del ndwig_array del ndwim_array del cloud_array LOGGER.debug("Reference rgb file creation.") rgb_file = create_file_name(final_path, basename + "_rgb.tif") merge_command = [ "/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt", "-separate", "-o", rgb_file, red_file, green_file, blue_file, ] call(merge_command) shape = options["shape"][0] LOGGER.debug("Cookie cut files using the given shape.") for i in range(4): clip_command = [ "/Library/Frameworks/GDAL.framework/Programs/gdalwarp", "-crop_to_cutline", "-cutline", shape, files[i], clipped_files[i], ] call(clip_command) for i in range(4): aux_array = open_handle(clipped_files[i]) aux_array[(aux_array == 0)] = -9999 create_raster_from_reference(final_files[i], aux_array, ndvi_clipped_file) if not options["debug"]: LOGGER.info("Remove auxiliary files.") os.remove(ndvi_clipped_file) os.remove(mndwi_clipped_file) os.remove(ndwig_clipped_file) os.remove(ndwim_clipped_file) os.remove(ndvi_file) os.remove(mndwi_file) os.remove(ndwig_file) os.remove(ndwim_file) print "Done"