def get_thumbnail_with_path(self, thumbnail_path):
'''
Creates a thumbnail for the scene in true color.
'''
from subprocess import call
thumnail_directory = create_file_name(thumbnail_path, 'thumbnail')
create_directory_path(thumnail_directory)
filename = self.file_dictionary[_BROWSE]
thumbnail = create_file_name(thumnail_directory, '%s.jpg' % get_base_name(filename))
resize_command = ['/Library/Frameworks/GDAL.framework/Programs/gdal_translate', filename, '-of', 'JPEG', thumbnail]
call(resize_command)
def test_creat_image(self):
import numpy
filename = "/Users/agutierrez/Development/df/1448114/2015/2015-02-24/l3a/1448114_2015-02-24_RE3_3A_302417.tif"
width = get_width(filename)
height = get_height(filename)
plain = width * height
red = numpy.zeros(plain)
green = numpy.zeros(plain)
blue = numpy.zeros(plain)
ired = numpy.zeros(plain)
print "before loop"
for i in range(plain):
if not i % 2:
red[i] = 1
if not i % 3:
green[i] = 1
if not i % 5:
blue[i] = 1
if not i % 7:
ired[i] = 1
print "loop"
final = numpy.array(
[
numpy.reshape(red, (width, height)),
numpy.reshape(green, (width, height)),
numpy.reshape(blue, (width, height)),
numpy.reshape(ired, (width, height)),
]
)
print final.shape
print "hello"
create_raster_from_reference(create_file_name(getattr(SETTINGS, "TEST_FOLDER"), "sieve.tif"), final, filename)
def get_thumbnail(self):
'''
Creates a thumbnail for the scene in true color.
'''
from subprocess import call
file_1 = self.file_dictionary[_BASE % (self.get_mission(), 'B1[0-9].TIF')]
file_2 = self.file_dictionary[_BASE % (self.get_mission(), 'B2[0-9].TIF')]
file_3 = self.file_dictionary[_BASE % (self.get_mission(), 'B3[0-9].TIF')]
parent_directory = get_parent(self.path)
thumnail_directory = create_file_name(parent_directory, 'thumbnail')
create_directory_path(thumnail_directory)
parent_directory
filename = create_file_name(thumnail_directory, 'vrt.tif')
merge_command = ['/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt', '-separate', '-o', filename, file_3, file_2, file_1]
call(merge_command)
thumbnail = create_file_name(thumnail_directory, 'thumbnail.jpg')
resize_command = ['/Library/Frameworks/GDAL.framework/Programs/gdal_translate', filename, '-of', 'JPEG', '-outsize', '5%', '5%', thumbnail]
call(resize_command)
def _look_for_files(self):
'''
This method will be called by the constructor to look for files in the
given directory. In order for a bundle to be valid, a file for each
regular expression must be found.
'''
for key in self.file_dictionary.iterkeys():
for name in get_files_from_folder(self.path):
if re.match(key, name):
self.file_dictionary[key] = create_file_name(self.path, name)
def preprocess(self):
'''
Top of atmosphere is calculated and persisted into a file. Then a cloud
mask is created with the given algorithm.
'''
solar_zenith = self.get_sensor().parser.get_attribute(rapideye.SOLAR_ZENITH)
data_acquisition_date = self.get_sensor().parser.get_attribute(rapideye.ACQUISITION_DATE)
solar_azimuth = self.get_sensor().parser.get_attribute(rapideye.SOLAR_AZIMUTH)
geotransform = self.get_raster().get_attribute(raster.GEOTRANSFORM)
data = self.get_raster().read_data_file_as_array()
sun_earth_distance = calculate_distance_sun_earth(data_acquisition_date)
top_of_atmosphere_data = calculate_toa_rapideye(calculate_rad_rapideye(data), sun_earth_distance, solar_zenith)
top_of_atmosphere_directory = create_file_name(get_parent(self.path), 'TOA')
create_directory_path(top_of_atmosphere_directory)
output_file = create_file_name(top_of_atmosphere_directory, get_base_name(self.get_files()[2]) + '_toa.tif') # TODO: change [2] in self.get_files()[2]
create_raster_from_reference(output_file,
top_of_atmosphere_data,
self.file_dictionary[_IMAGE],
data_type=NumericTypeCodeToGDALTypeCode(numpy.float32)
)
LOGGER.debug('Top of atmosphere file was created.')
cloud_output_file = create_file_name(top_of_atmosphere_directory, get_base_name(self.get_files()[2]) + '_cloud.tif')
if self.algorithm == ANOMALY_DETECTION:
LOGGER.debug('Cloud mask by anomaly detection process.')
clouds = self.anomaly_detection_cloud_mask(top_of_atmosphere_data, cloud_output_file, solar_zenith, solar_azimuth, geotransform)
elif self.algorithm == TIME_SERIES:
LOGGER.debug('Cloud mask by reference with time series process.')
tile_id = self.get_sensor().get_attribute(TILE_ID)
clouds = self.masking_with_time_series(data, cloud_output_file, solar_zenith, solar_azimuth, geotransform, tile_id)
create_raster_from_reference(cloud_output_file,
clouds,
self.file_dictionary[_IMAGE],
data_type=NumericTypeCodeToGDALTypeCode(numpy.float32)
)
LOGGER.info('Cloud mask was created.')
def handle(self, **options):
output = options['output'][0]
for image_path in options['path']:
print image_path
basename = '%s.tif' % get_base_name(image_path)
print basename
target = create_file_name(output, basename)
print target
start_time = time.time()
#self.method_by_block(image_path, target)
self.mask_iterating_values(image_path, target)
print("--- %s seconds ---" % (time.time() - start_time))
print 'Dataset was written.'
def test_create_image(self):
import numpy
array = numpy.array(
[
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
[1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1],
[1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1],
[1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1],
[1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1],
[1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
]
)
red = (1 - array) * 1000
green = (1 - array) * 1000
blue = array * 1000
create_raster(create_file_name(getattr(SETTINGS, "TEST_FOLDER"), "single.tif"), array)
create_raster(create_file_name(getattr(SETTINGS, "TEST_FOLDER"), "multi.tif"), numpy.array([red, green, blue]))
def split_shape_into_features(shape_name, destination_directory, column):
'''
This method will take a input shape and iterate over its features, creating
a new shape file with each one of them. It copies all the fields and the
same spatial reference from the original file. The created files are saved
in the destination directory using the number of the field given.
'''
driver = ogr.GetDriverByName(str('ESRI Shapefile'))
shape = driver.Open(shape_name, 0)
layer = shape.GetLayer()
layer_name = layer.GetName()
spatial_reference = layer.GetSpatialRef()
in_feature = layer.GetNextFeature()
while in_feature:
encoding = 'utf-8'
in_feature_name = create_filename_from_string(in_feature.GetField(column).decode(encoding))
final_path = destination_directory + str(in_feature.GetField(0))
create_directory_path(final_path)
output_name = create_file_name(final_path, in_feature_name + '.shp')
if os.path.exists(output_name):
driver.DeleteDataSource(output_name)
datasource = driver.CreateDataSource(output_name)
outLayer = datasource.CreateLayer(layer_name, spatial_reference, geom_type=ogr.wkbPolygon)
inLayerDefn = layer.GetLayerDefn()
for i in range(0, inLayerDefn.GetFieldCount()):
fieldDefn = inLayerDefn.GetFieldDefn(i)
LOGGER.debug(fieldDefn.GetName())
outLayer.CreateField(fieldDefn)
outLayerDefn = outLayer.GetLayerDefn()
geometry = in_feature.GetGeometryRef()
out_feature = ogr.Feature(outLayerDefn)
out_feature.SetGeometry(geometry)
for i in range(0, outLayerDefn.GetFieldCount()):
out_feature.SetField(outLayerDefn.GetFieldDefn(i).GetNameRef(), in_feature.GetField(i))
outLayer.CreateFeature(out_feature)
out_feature.Destroy()
in_feature.Destroy()
in_feature = layer.GetNextFeature()
shape.Destroy()
datasource.Destroy()
def get_information_object(self):
row = self.get_sensor().get_attribute(olitirs.ROW)
path = self.get_sensor().get_attribute(olitirs.PATH)
basename_metadata = get_basename_of_file(
self.file_dictionary[_BASE % (self.get_letter(), self.get_mission(), "MTL.txt")]
)
information = Information(
metadata_path=create_file_name(self.get_output_directory(), basename_metadata),
grid_id=unicode(path + row),
projection=self.get_raster().get_attribute(raster.PROJECTION),
cloud_percentage=self.get_sensor().get_attribute(olitirs.CLOUD_COVER),
geometry=self.get_raster().get_attribute(raster.FOOTPRINT),
elevation_angle=0.0,
resolution=self.get_raster().get_attribute(raster.GEOTRANSFORM)[1],
)
return information
def handle(self, **options):
'''
This is the code that does the ingestion.
'''
interest_band = 1
for image_path in options['path']:
print image_path
ds = gdal.Open(image_path)
bands = ds.RasterCount
geotransform = ds.GetGeoTransform()
x_resolution = geotransform[1]
y_resolution = geotransform[5]
pixel_area = abs(x_resolution * y_resolution)
array = numpy.array(ds.GetRasterBand(interest_band).ReadAsArray())
print numpy.unique(array)
flat = numpy.ravel(array)
length = len(flat)
parent = get_parent(image_path)
basename = '%s.txt' % get_base_name(image_path)
target = create_file_name(parent, basename)
count = 1
values = {}
progress = 0
for value in flat:
count = count + 1
if not values.get(value):
values[value] = 1
else:
values[value] = values[value] + 1
if count % 1000000 == 0:
aux = progress
progress = math.floor(100 * count / float(length))
if not aux == progress:
print str(int(progress)) + '%\r'
added = self.add_up(INITIAL_ARRAY, values)
area = self.transform_to_area(added, pixel_area)
with open(target, "a") as f:
json.dump(area, f)
def handle(self, **options):
"""
This process will call the change detection process from a set of two
individual images. It will perform the harmonization and the multivariate
alteration detection on the images. It will then perform a maximum
correlation factor on them and work with the resulting bands.
"""
image_a = options["ima"][0]
image_b = options["imb"][0]
output_image = options["output"][0]
LOGGER.info(
"Image %s will be compared against image %s. Output will be available" " at %s.",
image_a,
image_b,
output_image,
)
gdal_format = "GTiff"
image_a_data_class = raster.Data(image_a, gdal_format)
image_b_data_class = raster.Data(image_b, gdal_format)
# TODO : remove references to class harmonized
harmonized_class = harmonized.Data(image_a_data_class, image_b_data_class)
if harmonized_class:
data_shape_harmonized = harmonized_class.get_attribute(harmonized.DATA_SHAPE)
width, height, bands = data_shape_harmonized
geotransform_harmonized = harmonized_class.get_attribute(harmonized.GEOTRANSFORM)
projection_harmonized = harmonized_class.get_attribute(harmonized.PROJECTION)
image_a_data_array, image_b_data_array = harmonized_class.harmonized_arrays(
image_a_data_class, image_b_data_class
)
imad_class = imad.Transformation([image_a_data_array, image_b_data_array])
imad_class.execute()
maf_class = maf.Transformation(imad_class.output)
maf_class.execute()
mad_result = imad_class.output
maf_result = maf_class.output
pdf_file = create_file_name(getattr(SETTINGS, "TEST_FOLDER"), "maf_pdf.png")
thresholds = calc_threshold_grid(maf_result, pdf_file)
class_result = recode_classes_grid(maf_result, thresholds)
LOGGER.debug("mad_result.shape: %s", mad_result.shape)
LOGGER.debug("maf_result.shape: %s", maf_result.shape)
LOGGER.debug("class_result.shape: %s", class_result.shape)
mad_output_file = create_file_name(getattr(SETTINGS, "TEST_FOLDER"), "mad.tif")
maf_outputfile = create_file_name(getattr(SETTINGS, "TEST_FOLDER"), "maf.tif")
class_outputfile = create_file_name(getattr(SETTINGS, "TEST_FOLDER"), "class.tif")
create_raster(mad_output_file, mad_result, geotransform_harmonized, projection_harmonized)
create_raster(maf_outputfile, maf_result, geotransform_harmonized, projection_harmonized)
create_raster(class_outputfile, class_result, geotransform_harmonized, projection_harmonized)
print "Output written in: %s" % mad_output_file
print "Shape is ", imad_class.output.shape
def handle(self, **options):
"""
This is the code that does the ingestion.
indexes --path /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/Landsat_2000_2008/L5_021_047_2000_022_sr/ --shape /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/AE_LT_new.shp --output
"""
path = options["path"][0]
for mask_file in os.listdir(path):
if mask_file.endswith("_cfmask.tif"):
print mask_file
basename = mask_file.replace("_cfmask.tif", "")
print basename
cloud_file_name = mask_file
LOGGER.info("Calculating indexes for Landsat scenes.")
band_name = basename + "_sr_band%s.tif"
final_path = options["output"][0]
create_directory_path(final_path)
cloud_file = create_file_name(path, cloud_file_name)
cloud_array = open_handle(cloud_file)
cloud_mask = cloud_array == 4
LOGGER.debug("Recognize sensor depending on file name.")
if "L8" in path:
LOGGER.debug("Landsat 8 scene was detected.")
blue_file = create_file_name(path, band_name % BLUE_L8)
green_file = create_file_name(path, band_name % GREEN_L8)
red_file = create_file_name(path, band_name % RED_L8)
nir_file = create_file_name(path, band_name % NIR_L8)
swir_file = create_file_name(path, band_name % SWIR_L8)
else:
LOGGER.debug("Landsat 4, 5 or scene was detected.")
blue_file = create_file_name(path, band_name % BLUE)
green_file = create_file_name(path, band_name % GREEN)
red_file = create_file_name(path, band_name % RED)
nir_file = create_file_name(path, band_name % NIR)
swir_file = create_file_name(path, band_name % SWIR)
LOGGER.debug("Loading bands of interest.")
green_array = open_handle(green_file)
red_array = open_handle(red_file)
nir_array = open_handle(nir_file)
swir_array = open_handle(swir_file)
LOGGER.debug("Calculating indexes.")
ndvi_array = calculate_index(nir_array, red_array)
mndwi_array = calculate_index(green_array, swir_array)
ndwig_array = calculate_index(nir_array, swir_array)
ndwim_array = calculate_index(green_array, nir_array)
LOGGER.debug("Setting cloud mask values to -999.")
ndvi_array[cloud_mask] = -999
mndwi_array[cloud_mask] = -999
ndwig_array[cloud_mask] = -999
ndwim_array[cloud_mask] = -999
LOGGER.debug("Creating files for indexes.")
ndvi_final_file = create_file_name(final_path, basename + "_ndvi.tif")
mndwi_final_file = create_file_name(final_path, basename + "_mndwi.tif")
ndwig_final_file = create_file_name(final_path, basename + "_ndwig.tif")
ndwim_final_file = create_file_name(final_path, basename + "_ndwim.tif")
ndvi_clipped_file = create_file_name(final_path, basename + "_ndvi_clipped.tif")
mndwi_clipped_file = create_file_name(final_path, basename + "_mndwi_clipped.tif")
ndwig_clipped_file = create_file_name(final_path, basename + "_ndwig_clipped.tif")
ndwim_clipped_file = create_file_name(final_path, basename + "_ndwim_clipped.tif")
ndvi_file = create_file_name(final_path, basename + "_ndvi_pre.tif")
mndwi_file = create_file_name(final_path, basename + "_mndwi_pre.tif")
ndwig_file = create_file_name(final_path, basename + "_ndwig_pre.tif")
ndwim_file = create_file_name(final_path, basename + "_ndwim_pre.tif")
files = [ndvi_file, mndwi_file, ndwig_file, ndwim_file]
clipped_files = [ndvi_clipped_file, mndwi_clipped_file, ndwig_clipped_file, ndwim_clipped_file]
final_files = [ndvi_final_file, mndwi_final_file, ndwig_final_file, ndwim_final_file]
LOGGER.debug("Writing information to files.")
create_raster_from_reference(ndvi_file, ndvi_array, green_file)
create_raster_from_reference(mndwi_file, mndwi_array, green_file)
create_raster_from_reference(ndwig_file, ndwig_array, green_file)
create_raster_from_reference(ndwim_file, ndwim_array, green_file)
LOGGER.debug("Deleting arrays to release memory.")
del ndvi_array
del mndwi_array
del ndwig_array
del ndwim_array
del cloud_array
LOGGER.debug("Reference rgb file creation.")
rgb_file = create_file_name(final_path, basename + "_rgb.tif")
merge_command = [
"/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt",
"-separate",
"-o",
rgb_file,
red_file,
green_file,
blue_file,
]
call(merge_command)
shape = options["shape"][0]
LOGGER.debug("Cookie cut files using the given shape.")
for i in range(4):
clip_command = [
"/Library/Frameworks/GDAL.framework/Programs/gdalwarp",
"-crop_to_cutline",
"-cutline",
shape,
files[i],
clipped_files[i],
]
call(clip_command)
for i in range(4):
aux_array = open_handle(clipped_files[i])
aux_array[(aux_array == 0)] = -9999
create_raster_from_reference(final_files[i], aux_array, ndvi_clipped_file)
if not options["debug"]:
LOGGER.info("Remove auxiliary files.")
os.remove(ndvi_clipped_file)
os.remove(mndwi_clipped_file)
os.remove(ndwig_clipped_file)
os.remove(ndwim_clipped_file)
os.remove(ndvi_file)
os.remove(mndwi_file)
os.remove(ndwig_file)
os.remove(ndwim_file)
print "Done"
