def persist_bundle(bundle, keep=False):
'''
This function persist a bundle in both the database and the file system. It
is responsibility of the bundle to provide information about which files
should be persisted in the file system, and to build the database object
that will be inserted in the database. The database is configured using the
session maker in the connection module.
In order to achieve its purpose, this method creates a list of the actions
to perform. Once the list is fully populated, it calls the act method for
each element in the list. If any of the actions in the list fails, a
rollback is performed in all of them, the result is the same state as before
the method was called.
'''
destination = bundle.get_output_directory()
create_directory_path(destination)
actions = []
session = SESSION_MAKER()
try:
if not session.query(Product).filter(
Product.product_path ==
bundle.get_database_object().product_path).count():
if not keep:
LOGGER.debug(
'This process will move the files to a new destination.')
for file_name in bundle.get_files():
actions.append(
filesystem.InsertAction(file_name, destination))
else:
LOGGER.debug(
'This process will keep the original path for the files.')
actions.append(
database.InsertAction(bundle.get_database_object(), session))
import madmex.mapper.bundle.rapideye as re
if isinstance(bundle, re.Bundle):
actions.append(
database.InsertAction(bundle.get_features_object(),
session))
def do_result(action):
'''
Lambda function to perform an action and return the result.
'''
action.act()
return action.success
if not reduce(lambda x, y: x and y, map(do_result, actions)):
LOGGER.debug('Some action went wrong at persistence process, '
'rollback will be performed.')
for action in actions:
action.undo()
else:
LOGGER.info('Ingestion was successful.')
else:
LOGGER.info(
'An instance of this object already exist in the database.')
except Exception:
LOGGER.error('Not expected error at persistence.driver')
raise
finally:
session.close()
def test_imad_pair_images(self):
'''
Perform an imad transformation with two images
'''
from madmex.transformation import imad
from madmex.mapper.data import harmonized
from madmex.mapper.data import raster
image1 = '/LUSTRE/MADMEX/eodata/rapideye/1147524/2012/2012-10-18/l3a/2012-10-18T191005_RE3_3A-NAC_11137283_149747.tif'
image2 = '/LUSTRE/MADMEX/eodata/rapideye/1147524/2013/2013-09-09/l3a/1147524_2013-09-09_RE5_3A_175826.tif'
#image2 = '/LUSTRE/MADMEX/eodata/spot/556297/2010/2010-01-26/1a/556_297_260110_SP5.img'
gdal_format = "GTiff"
image1_data_class =raster.Data(image1, gdal_format)
image2_data_class = raster.Data(image2, gdal_format)
harmonized_class = harmonized.Data(image1_data_class, image2_data_class)
if harmonized_class:
data_shape_harmonized = harmonized_class.get_attribute(harmonized.DATA_SHAPE)
width, height, bands = data_shape_harmonized
geotransform_harmonized = harmonized_class.get_attribute(harmonized.GEOTRANSFORM)
projection_harmonized = harmonized_class.get_attribute(harmonized.PROJECTION)
image1_data_array, image2_data_array = harmonized_class.harmonized_arrays(image1_data_class, image2_data_class)
imad_class = imad.Transformation([image1_data_array, image2_data_array])
imad_class.execute()
output = os.path.join(os.path.expanduser('~'),'test_imad_pair_images')
create_directory_path(output)
output+= '/result_mad.tif'
mad_image = harmonized_class.create_from_reference(output, width, height, (bands+1), geotransform_harmonized, projection_harmonized)
harmonized_class.write_raster(mad_image, imad_class.output)
print 'corrlist'
print imad_class.outcorrlist
def split_shape_into_features(shape_name, destination_directory, column, name,
sep):
'''
This method will take a input shape and iterate over its features, creating
a new shape file with each one of them. It copies all the fields and the
same spatial reference from the original file. The created files are saved
in the destination directory using the number of the field given.
'''
driver = ogr.GetDriverByName(str('ESRI Shapefile'))
shape = driver.Open(shape_name, 0)
layer = shape.GetLayer()
layer_name = layer.GetName()
spatial_reference = layer.GetSpatialRef()
in_feature = layer.GetNextFeature()
shape_files = []
while in_feature:
encoding = 'utf-8'
in_feature_name = in_feature.GetField(column)
in_name = create_filename_from_string(
in_feature.GetField(name).decode(encoding))
final_path = destination_directory + str(in_feature.GetField(0))
create_directory_path(final_path)
output_name = create_filename(
final_path, '%s__%s.shp' % (in_feature_name, in_name))
shape_files.append(output_name)
if os.path.exists(output_name):
driver.DeleteDataSource(output_name)
datasource = driver.CreateDataSource(output_name)
outLayer = datasource.CreateLayer(layer_name,
spatial_reference,
geom_type=ogr.wkbPolygon)
inLayerDefn = layer.GetLayerDefn()
for i in range(0, inLayerDefn.GetFieldCount()):
fieldDefn = inLayerDefn.GetFieldDefn(i)
#LOGGER.debug(fieldDefn.GetName())
outLayer.CreateField(fieldDefn)
outLayerDefn = outLayer.GetLayerDefn()
geometry = in_feature.GetGeometryRef()
out_feature = ogr.Feature(outLayerDefn)
out_feature.SetGeometry(geometry)
for i in range(0, outLayerDefn.GetFieldCount()):
out_feature.SetField(
outLayerDefn.GetFieldDefn(i).GetNameRef(),
in_feature.GetField(i))
outLayer.CreateFeature(out_feature)
out_feature.Destroy()
in_feature.Destroy()
in_feature = layer.GetNextFeature()
shape.Destroy()
datasource.Destroy()
return shape_files
def get_thumbnail(self):
'''
Creates a thumbnail for the scene in true color.
'''
from subprocess import call
file_1 = self.file_dictionary[_BASE %
(self.get_mission(), 'B1[0-9].TIF')]
file_2 = self.file_dictionary[_BASE %
(self.get_mission(), 'B2[0-9].TIF')]
file_3 = self.file_dictionary[_BASE %
(self.get_mission(), 'B3[0-9].TIF')]
parent_directory = get_parent(self.path)
thumnail_directory = create_filename(parent_directory, 'thumbnail')
create_directory_path(thumnail_directory)
parent_directory
filename = create_filename(thumnail_directory, 'vrt.tif')
merge_command = [
'/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt',
'-separate', '-o', filename, file_3, file_2, file_1
]
call(merge_command)
thumbnail = create_filename(thumnail_directory, 'thumbnail.jpg')
resize_command = [
'/Library/Frameworks/GDAL.framework/Programs/gdal_translate',
filename, '-of', 'JPEG', '-outsize', '5%', '5%', thumbnail
]
call(resize_command)
def preprocess(self):
'''
Top of atmosphere is calculated and persisted into a file. Then a cloud
mask is created with the given algorithm.
'''
solar_zenith = self.get_sensor().parser.get_attribute(
rapideye.SOLAR_ZENITH)
data_acquisition_date = self.get_sensor().parser.get_attribute(
rapideye.ACQUISITION_DATE)
solar_azimuth = self.get_sensor().parser.get_attribute(
rapideye.SOLAR_AZIMUTH)
geotransform = self.get_raster().get_attribute(raster.GEOTRANSFORM)
data = self.get_raster().read_data_file_as_array()
sun_earth_distance = calculate_distance_sun_earth(
data_acquisition_date)
top_of_atmosphere_data = calculate_toa_rapideye(
calculate_rad_rapideye(data), sun_earth_distance, solar_zenith)
top_of_atmosphere_directory = create_filename(get_parent(self.path),
'TOA')
create_directory_path(top_of_atmosphere_directory)
output_file = create_filename(
top_of_atmosphere_directory,
get_basename(self.get_files()[2]) +
'_toa.tif') # TODO: change [2] in self.get_files()[2]
create_raster_from_reference(output_file,
top_of_atmosphere_data,
self.file_dictionary[_IMAGE],
data_type=NumericTypeCodeToGDALTypeCode(
numpy.float32))
LOGGER.debug('Top of atmosphere file was created.')
cloud_output_file = create_filename(
top_of_atmosphere_directory,
get_basename(self.get_files()[2]) + '_cloud.tif')
if self.algorithm == ANOMALY_DETECTION:
LOGGER.debug('Cloud mask by anomaly detection process.')
clouds = self.anomaly_detection_cloud_mask(top_of_atmosphere_data,
cloud_output_file,
solar_zenith,
solar_azimuth,
geotransform)
elif self.algorithm == TIME_SERIES:
LOGGER.debug('Cloud mask by reference with time series process.')
tile_id = self.get_sensor().get_attribute(TILE_ID)
clouds = self.masking_with_time_series(data, cloud_output_file,
solar_zenith, solar_azimuth,
geotransform, tile_id)
create_raster_from_reference(cloud_output_file,
clouds,
self.file_dictionary[_IMAGE],
data_type=NumericTypeCodeToGDALTypeCode(
numpy.float32))
LOGGER.info('Cloud mask was created.')
def train_model(X_train, X_test, y_train, y_test, output, model_name):
model = load_model(model_name)
persistence_directory = create_filename(output, model_name)
create_directory_path(persistence_directory)
model_instance = model.Model(persistence_directory)
model_instance.fit(X_train, y_train)
model_instance.save(persistence_directory)
predicted = model_instance.predict(X_test)
model_instance.create_report(
y_test, predicted, create_filename(persistence_directory,
'report.txt'))
def handle(self, **options):
'''
In this example command, the values that come from the user input are
added up and the result is printed in the screen.
'''
output = options['output'][0]
models = options['modelname']
model_directory = options['modeldir'][0]
pca_model = pca.Model(5)
pca_model.load(create_filename(model_directory, 'pca'))
for model_name in models:
persistence_directory = create_filename(model_directory,
model_name)
model = load_model(model_name)
model_instance = model.Model(persistence_directory)
model_instance.load(persistence_directory)
block_size = 500
for path in options['path']:
image_array = open_handle(path)
y_size = image_array.shape[1]
x_size = image_array.shape[2]
basename = get_basename(path)[:7]
warnings.filterwarnings('ignore')
final = numpy.zeros((x_size, y_size))
import time
start_time = time.time()
for i in range(0, y_size, block_size):
if i + block_size < y_size:
rows = block_size
else:
rows = y_size - i
for j in range(0, x_size, block_size):
if j + block_size < x_size:
cols = block_size
else:
cols = x_size - j
step = image_array[:, i:i + rows, j:j + cols]
step_ravel = step.reshape(10, -1)
prediction = model_instance.predict(
pca_model.transform(numpy.transpose(step_ravel)))
final[i:i + rows, j:j + cols] = prediction.reshape(
(rows, cols))
print("--- %s seconds ---" % (time.time() - start_time))
create_directory_path(output)
classification = create_filename(
output, '%s-%s.tif' % (basename, model_name))
create_raster_from_reference(classification,
final.reshape(x_size, y_size),
path,
data_type=gdal.GDT_Byte,
creating_options=['COMPRESS=LZW'])
def get_thumbnail_with_path(self, thumbnail_path):
'''
Creates a thumbnail for the scene in true color.
'''
from subprocess import call
thumnail_directory = create_file_name(thumbnail_path, 'thumbnail')
create_directory_path(thumnail_directory)
filename = self.file_dictionary[_BROWSE]
thumbnail = create_file_name(thumnail_directory, '%s.jpg' % get_base_name(filename))
resize_command = ['/Library/Frameworks/GDAL.framework/Programs/gdal_translate', filename, '-of', 'JPEG', thumbnail]
call(resize_command)
def persist_bundle(bundle, keep=False):
"""
This function persist a bundle in both the database and the file system. It
is responsibility of the bundle to provide information about which files
should be persisted in the file system, and to build the database object
that will be inserted in the database. The database is configured using the
session maker in the connection module.
In order to achieve its purpose, this method creates a list of the actions
to perform. Once the list is fully populated, it calls the act method for
each element in the list. If any of the actions in the list fails, a
rollback is performed in all of them, the result is the same state as before
the method was called.
"""
destination = bundle.get_output_directory()
create_directory_path(destination)
actions = []
session = SESSION_MAKER()
try:
if not session.query(Product).filter(Product.product_path == bundle.get_database_object().product_path).count():
if not keep:
LOGGER.debug("This process will move the files to a new destination.")
for file_name in bundle.get_files():
actions.append(filesystem.InsertAction(file_name, destination))
else:
LOGGER.debug("This process will keep the original path for the files.")
actions.append(database.InsertAction(bundle.get_database_object(), session))
actions.append(database.InsertAction(bundle.get_information_object(), session))
actions.append(database.InsertAction(bundle.get_features_object(), session))
def do_result(action):
"""
Lambda function to perform an action and return the result.
"""
action.act()
return action.success
if not reduce(lambda x, y: x and y, map(do_result, actions)):
LOGGER.debug("Some action went wrong at persistence process, " "rollback will be performed.")
for action in actions:
action.undo()
else:
LOGGER.info("Ingestion was successful.")
else:
LOGGER.info("An instance of this object already exist in the database.")
except Exception:
LOGGER.error("Not expected error at persistence.driver")
raise
finally:
session.close()
def split_shape_into_features(shape_name, destination_directory, column):
'''
This method will take a input shape and iterate over its features, creating
a new shape file with each one of them. It copies all the fields and the
same spatial reference from the original file. The created files are saved
in the destination directory using the number of the field given.
'''
driver = ogr.GetDriverByName(str('ESRI Shapefile'))
shape = driver.Open(shape_name, 0)
layer = shape.GetLayer()
layer_name = layer.GetName()
spatial_reference = layer.GetSpatialRef()
in_feature = layer.GetNextFeature()
while in_feature:
encoding = 'utf-8'
in_feature_name = create_filename_from_string(in_feature.GetField(column).decode(encoding))
final_path = destination_directory + str(in_feature.GetField(0))
create_directory_path(final_path)
output_name = create_file_name(final_path, in_feature_name + '.shp')
if os.path.exists(output_name):
driver.DeleteDataSource(output_name)
datasource = driver.CreateDataSource(output_name)
outLayer = datasource.CreateLayer(layer_name, spatial_reference, geom_type=ogr.wkbPolygon)
inLayerDefn = layer.GetLayerDefn()
for i in range(0, inLayerDefn.GetFieldCount()):
fieldDefn = inLayerDefn.GetFieldDefn(i)
LOGGER.debug(fieldDefn.GetName())
outLayer.CreateField(fieldDefn)
outLayerDefn = outLayer.GetLayerDefn()
geometry = in_feature.GetGeometryRef()
out_feature = ogr.Feature(outLayerDefn)
out_feature.SetGeometry(geometry)
for i in range(0, outLayerDefn.GetFieldCount()):
out_feature.SetField(outLayerDefn.GetFieldDefn(i).GetNameRef(), in_feature.GetField(i))
outLayer.CreateFeature(out_feature)
out_feature.Destroy()
in_feature.Destroy()
in_feature = layer.GetNextFeature()
shape.Destroy()
datasource.Destroy()
def intersect(self, output_directory, geometry):
layer = self.get_layer()
layer_name = layer.GetName()
spatial_reference = layer.GetSpatialRef()
inSpatialRef = osr.SpatialReference()
inSpatialRef.ImportFromEPSG(4326)
coordTransform = osr.CoordinateTransformation(inSpatialRef, spatial_reference)
geometry.Transform(coordTransform)
in_feature = layer.GetNextFeature()
layer_definition = layer.GetLayerDefn()
field_definition = layer_definition.GetFieldDefn(0)
column_name = field_definition.GetName()
shape_files = []
create_directory_path(output_directory)
in_layer_definition = layer.GetLayerDefn()
output_name = output_directory
print output_name
if is_file(output_name):
self.driver.DeleteDataSource(output_name)
data_source = self.driver.CreateDataSource(output_name)
out_layer = data_source.CreateLayer(layer_name, spatial_reference, geom_type=ogr.wkbPolygon)
for i in range(0, in_layer_definition.GetFieldCount()):
fieldDefn = in_layer_definition.GetFieldDefn(i)
out_layer.CreateField(fieldDefn)
while in_feature:
outLayerDefn = out_layer.GetLayerDefn()
feature_geometry = in_feature.GetGeometryRef()
out_feature = ogr.Feature(outLayerDefn)
print geometry.ExportToWkt()
print feature_geometry.ExportToWkt()
if geometry.Intersect(feature_geometry):
print 'It intersects!!!!'
intersection = geometry.Intersection(feature_geometry)
out_feature.SetGeometry(intersection)
for i in range(0, outLayerDefn.GetFieldCount()):
out_feature.SetField(outLayerDefn.GetFieldDefn(i).GetNameRef(), in_feature.GetField(i))
out_layer.CreateFeature(out_feature)
out_feature = None
in_feature = None
in_feature = layer.GetNextFeature()
self.close()
return Data(output_name)
def get_thumbnail_with_path(self, thumbnail_path):
'''
Creates a thumbnail for the scene in true color.
'''
from subprocess import call
thumnail_directory = create_filename(thumbnail_path, 'thumbnail')
create_directory_path(thumnail_directory)
filename = self.file_dictionary[_BROWSE]
thumbnail = create_filename(thumnail_directory,
'%s.jpg' % get_basename(filename))
resize_command = [
'/Library/Frameworks/GDAL.framework/Programs/gdal_translate',
filename, '-of', 'JPEG', thumbnail
]
call(resize_command)
def get_thumbnail(self):
'''
Creates a thumbnail for the scene in true color.
'''
from subprocess import call
file_1 = self.file_dictionary[_BASE % (self.get_mission(), 'B1[0-9].TIF')]
file_2 = self.file_dictionary[_BASE % (self.get_mission(), 'B2[0-9].TIF')]
file_3 = self.file_dictionary[_BASE % (self.get_mission(), 'B3[0-9].TIF')]
parent_directory = get_parent(self.path)
thumnail_directory = create_file_name(parent_directory, 'thumbnail')
create_directory_path(thumnail_directory)
parent_directory
filename = create_file_name(thumnail_directory, 'vrt.tif')
merge_command = ['/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt', '-separate', '-o', filename, file_3, file_2, file_1]
call(merge_command)
thumbnail = create_file_name(thumnail_directory, 'thumbnail.jpg')
resize_command = ['/Library/Frameworks/GDAL.framework/Programs/gdal_translate', filename, '-of', 'JPEG', '-outsize', '5%', '5%', thumbnail]
call(resize_command)
def handle(self, **options):
'''
In this example command, the values that come from the user input are
added up and the result is printed in the screen.
'''
output = options['output'][0]
zip = options['path'][0]
basename = get_basename(zip)
aux_name = create_filename(output, 'aux_%s' % basename)
real_name = create_filename(output, basename)
with ZipFile(zip, 'r') as unzipped:
unzipped.extractall(create_filename(output, 'aux_%s' % basename))
path = create_filename(aux_name, 'trend_tiles')
tifs = get_contents_from_folder(path)
create_directory_path(real_name)
for tif in tifs:
source = create_filename(path, tif)
target = create_filename(real_name, tif)
tile_map(source, target, 4000, 4000, 2, 2)
remove_directory(aux_name)
def split(self, output_directory, column=0):
'''
This method will take a input shape and iterate over its features, creating
a new shape file with each one of them. It copies all the fields and the
same spatial reference from the original file. The created files are saved
in the destination directory using the number of the field given.
'''
layer = self.get_layer()
layer_name = layer.GetName()
spatial_reference = layer.GetSpatialRef()
in_feature = layer.GetNextFeature()
layer_definition = layer.GetLayerDefn()
field_definition = layer_definition.GetFieldDefn(0)
column_name = field_definition.GetName()
shape_files = []
create_directory_path(output_directory)
in_layer_definition = layer.GetLayerDefn()
while in_feature:
in_feature_name = in_feature.GetField(column_name)
output_name = create_filename(output_directory, '%s.shp' % in_feature_name)
shape_files.append(output_name)
if is_file(output_name):
self.driver.DeleteDataSource(output_name)
data_source = self.driver.CreateDataSource(output_name)
out_layer = data_source.CreateLayer(layer_name, spatial_reference, geom_type=ogr.wkbPolygon)
for i in range(0, in_layer_definition.GetFieldCount()):
fieldDefn = in_layer_definition.GetFieldDefn(i)
out_layer.CreateField(fieldDefn)
outLayerDefn = out_layer.GetLayerDefn()
geometry = in_feature.GetGeometryRef()
out_feature = ogr.Feature(outLayerDefn)
out_feature.SetGeometry(geometry)
for i in range(0, outLayerDefn.GetFieldCount()):
out_feature.SetField(outLayerDefn.GetFieldDefn(i).GetNameRef(), in_feature.GetField(i))
out_layer.CreateFeature(out_feature)
out_feature = None
in_feature = None
in_feature = layer.GetNextFeature()
self.close()
return [Data(filename) for filename in shape_files]
def create_shape_from_json(id, json, output_directory):
'''
Given a json string containing coordinates, this method creates a shape file.
'''
create_directory_path(output_directory)
filename = create_filename(output_directory, '%s.shp' % id)
shape = Data(filename)
if is_file(filename):
shape.driver.DeleteDataSource(filename)
data_source = shape.driver.CreateDataSource(filename)
spatial_reference = osr.SpatialReference()
spatial_reference.ImportFromEPSG(4326)
layer = data_source.CreateLayer(str('layer'), spatial_reference, geom_type=ogr.wkbPolygon)
layer.CreateField(ogr.FieldDefn(str('id'), ogr.OFTString))
feature = ogr.Feature(layer.GetLayerDefn())
feature.SetField(str('id'), str(id))
geometry = ogr.CreateGeometryFromJson(str(json))
feature.SetGeometry(geometry)
layer.CreateFeature(feature)
shape.close()
return shape
def preprocess(self):
'''
Top of atmosphere is calculated and persisted into a file. Then a cloud
mask is created with the given algorithm.
'''
solar_zenith = self.get_sensor().parser.get_attribute(rapideye.SOLAR_ZENITH)
data_acquisition_date = self.get_sensor().parser.get_attribute(rapideye.ACQUISITION_DATE)
solar_azimuth = self.get_sensor().parser.get_attribute(rapideye.SOLAR_AZIMUTH)
geotransform = self.get_raster().get_attribute(raster.GEOTRANSFORM)
data = self.get_raster().read_data_file_as_array()
sun_earth_distance = calculate_distance_sun_earth(data_acquisition_date)
top_of_atmosphere_data = calculate_toa_rapideye(calculate_rad_rapideye(data), sun_earth_distance, solar_zenith)
top_of_atmosphere_directory = create_file_name(get_parent(self.path), 'TOA')
create_directory_path(top_of_atmosphere_directory)
output_file = create_file_name(top_of_atmosphere_directory, get_base_name(self.get_files()[2]) + '_toa.tif') # TODO: change [2] in self.get_files()[2]
create_raster_from_reference(output_file,
top_of_atmosphere_data,
self.file_dictionary[_IMAGE],
data_type=NumericTypeCodeToGDALTypeCode(numpy.float32)
)
LOGGER.debug('Top of atmosphere file was created.')
cloud_output_file = create_file_name(top_of_atmosphere_directory, get_base_name(self.get_files()[2]) + '_cloud.tif')
if self.algorithm == ANOMALY_DETECTION:
LOGGER.debug('Cloud mask by anomaly detection process.')
clouds = self.anomaly_detection_cloud_mask(top_of_atmosphere_data, cloud_output_file, solar_zenith, solar_azimuth, geotransform)
elif self.algorithm == TIME_SERIES:
LOGGER.debug('Cloud mask by reference with time series process.')
tile_id = self.get_sensor().get_attribute(TILE_ID)
clouds = self.masking_with_time_series(data, cloud_output_file, solar_zenith, solar_azimuth, geotransform, tile_id)
create_raster_from_reference(cloud_output_file,
clouds,
self.file_dictionary[_IMAGE],
data_type=NumericTypeCodeToGDALTypeCode(numpy.float32)
)
LOGGER.info('Cloud mask was created.')
def test_imad_pair_images(self):
'''
Perform an imad transformation with two images
'''
from madmex.transformation import imad
from madmex.mapper.data import harmonized
from madmex.mapper.data import raster
image1 = '/LUSTRE/MADMEX/eodata/rapideye/1147524/2012/2012-10-18/l3a/2012-10-18T191005_RE3_3A-NAC_11137283_149747.tif'
image2 = '/LUSTRE/MADMEX/eodata/rapideye/1147524/2013/2013-09-09/l3a/1147524_2013-09-09_RE5_3A_175826.tif'
#image2 = '/LUSTRE/MADMEX/eodata/spot/556297/2010/2010-01-26/1a/556_297_260110_SP5.img'
gdal_format = "GTiff"
image1_data_class = raster.Data(image1, gdal_format)
image2_data_class = raster.Data(image2, gdal_format)
harmonized_class = harmonized.Data(image1_data_class,
image2_data_class)
if harmonized_class:
data_shape_harmonized = harmonized_class.get_attribute(
harmonized.DATA_SHAPE)
width, height, bands = data_shape_harmonized
geotransform_harmonized = harmonized_class.get_attribute(
harmonized.GEOTRANSFORM)
projection_harmonized = harmonized_class.get_attribute(
harmonized.PROJECTION)
image1_data_array, image2_data_array = harmonized_class.harmonized_arrays(
image1_data_class, image2_data_class)
imad_class = imad.Transformation(
[image1_data_array, image2_data_array])
imad_class.execute()
output = os.path.join(os.path.expanduser('~'),
'test_imad_pair_images')
create_directory_path(output)
output += '/result_mad.tif'
mad_image = harmonized_class.create_from_reference(
output, width, height, (bands + 1), geotransform_harmonized,
projection_harmonized)
harmonized_class.write_raster(mad_image, imad_class.output)
print 'corrlist'
print imad_class.outcorrlist
def handle(self, **options):
'''
This process will call the change detection process from a set of two
individual images. It will perform the harmonization and the multivariate
alteration detection on the images. It will then perform a maximum
correlation factor on them and work with the resulting bands.
'''
image_a = options['ima'][0]
image_b = options['imb'][0]
output_image = options['output'][0]
LOGGER.info('Image %s will be compared against image %s. Output will be available' \
' at %s.', image_a, image_b, output_image)
gdal_format = "GTiff"
image_a_data_class = raster.Data(image_a, gdal_format)
image_b_data_class = raster.Data(image_b, gdal_format)
# TODO : remove references to class harmonized
harmonized_class = harmonized.Data(image_a_data_class, image_b_data_class)
#band1 = image_a_data_class.GetRasterBand(1)
#band1 = band1.ReadAsArray(0, 0,image_a_data_class.RasterXSize,image_a_data_class.RasterYSize).astype(float)
#print(band1)
if harmonized_class:
#data_shape_harmonized = harmonized_class.get_attribute(harmonized.DATA_SHAPE)
#width, height, bands = data_shape_harmonized
#geotransform_harmonized = harmonized_class.get_attribute(raster.GEOTRANSFORM)
#projection_harmonized = harmonized_class.get_attribute(harmonized.PROJECTION)
image_a_data_array, image_b_data_array = harmonized_class.harmonized_arrays(image_a_data_class, image_b_data_class)
imad_class = imad.Transformation([image_a_data_array, image_b_data_array])
imad_class.execute()
mad_result = imad_class.output
LOGGER.debug('mad_result.shape: %s', mad_result.shape)
create_directory_path(getattr(SETTINGS, 'TEST_FOLDER'))
mad_output_file = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'mad.tif')
create_raster_from_reference(mad_output_file, mad_result,image_a)
maf_class = maf.Transformation(imad_class.output)
maf_class.execute()
maf_result = maf_class.output
pdf_file = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'maf_pdf.png')
thresholds = calc_threshold_grid(maf_result, pdf_file)
class_result = recode_classes_grid(maf_result, thresholds)
LOGGER.debug('maf_result.shape: %s', maf_result.shape)
LOGGER.debug('class_result.shape: %s', class_result.shape)
maf_outputfile = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'maf.tif')
class_outputfile = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'class.tif')
create_raster_from_reference(maf_outputfile, maf_result, image_a)
create_raster_from_reference(class_outputfile, class_result, image_a)
print 'Output written in: %s' % mad_output_file
print 'Shape is ', imad_class.output.shape
def handle(self, **options):
"""
This is the code that does the ingestion.
indexes --path /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/Landsat_2000_2008/L5_021_047_2000_022_sr/ --shape /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/AE_LT_new.shp --output
"""
path = options["path"][0]
for mask_file in os.listdir(path):
if mask_file.endswith("_cfmask.tif"):
print mask_file
basename = mask_file.replace("_cfmask.tif", "")
print basename
cloud_file_name = mask_file
LOGGER.info("Calculating indexes for Landsat scenes.")
band_name = basename + "_sr_band%s.tif"
final_path = options["output"][0]
create_directory_path(final_path)
cloud_file = create_file_name(path, cloud_file_name)
cloud_array = open_handle(cloud_file)
cloud_mask = cloud_array == 4
LOGGER.debug("Recognize sensor depending on file name.")
if "L8" in path:
LOGGER.debug("Landsat 8 scene was detected.")
blue_file = create_file_name(path, band_name % BLUE_L8)
green_file = create_file_name(path, band_name % GREEN_L8)
red_file = create_file_name(path, band_name % RED_L8)
nir_file = create_file_name(path, band_name % NIR_L8)
swir_file = create_file_name(path, band_name % SWIR_L8)
else:
LOGGER.debug("Landsat 4, 5 or scene was detected.")
blue_file = create_file_name(path, band_name % BLUE)
green_file = create_file_name(path, band_name % GREEN)
red_file = create_file_name(path, band_name % RED)
nir_file = create_file_name(path, band_name % NIR)
swir_file = create_file_name(path, band_name % SWIR)
LOGGER.debug("Loading bands of interest.")
green_array = open_handle(green_file)
red_array = open_handle(red_file)
nir_array = open_handle(nir_file)
swir_array = open_handle(swir_file)
LOGGER.debug("Calculating indexes.")
ndvi_array = calculate_index(nir_array, red_array)
mndwi_array = calculate_index(green_array, swir_array)
ndwig_array = calculate_index(nir_array, swir_array)
ndwim_array = calculate_index(green_array, nir_array)
LOGGER.debug("Setting cloud mask values to -999.")
ndvi_array[cloud_mask] = -999
mndwi_array[cloud_mask] = -999
ndwig_array[cloud_mask] = -999
ndwim_array[cloud_mask] = -999
LOGGER.debug("Creating files for indexes.")
ndvi_final_file = create_file_name(final_path, basename + "_ndvi.tif")
mndwi_final_file = create_file_name(final_path, basename + "_mndwi.tif")
ndwig_final_file = create_file_name(final_path, basename + "_ndwig.tif")
ndwim_final_file = create_file_name(final_path, basename + "_ndwim.tif")
ndvi_clipped_file = create_file_name(final_path, basename + "_ndvi_clipped.tif")
mndwi_clipped_file = create_file_name(final_path, basename + "_mndwi_clipped.tif")
ndwig_clipped_file = create_file_name(final_path, basename + "_ndwig_clipped.tif")
ndwim_clipped_file = create_file_name(final_path, basename + "_ndwim_clipped.tif")
ndvi_file = create_file_name(final_path, basename + "_ndvi_pre.tif")
mndwi_file = create_file_name(final_path, basename + "_mndwi_pre.tif")
ndwig_file = create_file_name(final_path, basename + "_ndwig_pre.tif")
ndwim_file = create_file_name(final_path, basename + "_ndwim_pre.tif")
files = [ndvi_file, mndwi_file, ndwig_file, ndwim_file]
clipped_files = [ndvi_clipped_file, mndwi_clipped_file, ndwig_clipped_file, ndwim_clipped_file]
final_files = [ndvi_final_file, mndwi_final_file, ndwig_final_file, ndwim_final_file]
LOGGER.debug("Writing information to files.")
create_raster_from_reference(ndvi_file, ndvi_array, green_file)
create_raster_from_reference(mndwi_file, mndwi_array, green_file)
create_raster_from_reference(ndwig_file, ndwig_array, green_file)
create_raster_from_reference(ndwim_file, ndwim_array, green_file)
LOGGER.debug("Deleting arrays to release memory.")
del ndvi_array
del mndwi_array
del ndwig_array
del ndwim_array
del cloud_array
LOGGER.debug("Reference rgb file creation.")
rgb_file = create_file_name(final_path, basename + "_rgb.tif")
merge_command = [
"/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt",
"-separate",
"-o",
rgb_file,
red_file,
green_file,
blue_file,
]
call(merge_command)
shape = options["shape"][0]
LOGGER.debug("Cookie cut files using the given shape.")
for i in range(4):
clip_command = [
"/Library/Frameworks/GDAL.framework/Programs/gdalwarp",
"-crop_to_cutline",
"-cutline",
shape,
files[i],
clipped_files[i],
]
call(clip_command)
for i in range(4):
aux_array = open_handle(clipped_files[i])
aux_array[(aux_array == 0)] = -9999
create_raster_from_reference(final_files[i], aux_array, ndvi_clipped_file)
if not options["debug"]:
LOGGER.info("Remove auxiliary files.")
os.remove(ndvi_clipped_file)
os.remove(mndwi_clipped_file)
os.remove(ndwig_clipped_file)
os.remove(ndwim_clipped_file)
os.remove(ndvi_file)
os.remove(mndwi_file)
os.remove(ndwig_file)
os.remove(ndwim_file)
print "Done"
def handle(self, **options):
'''
In this example command, the values that come from the user input are
added up and the result is printed in the screen.
'''
shape_name = options['shape'][0]
destination = options['dest'][0]
paths = options['path']
years = []
for path in paths:
years.append(self.get_year_from_path(path))
if os.path.exists(shape_name):
LOGGER.info('The file %s was found.' % shape_name)
shape_files = split_shape_into_features(shape_name, destination,
str('id'), str('nombre'), '__')
process_launcher = LocalProcessLauncher()
import time
start_time = time.time()
cover_array = []
tth_array = []
cover_file = 'cover-stats.json'
tth_name = 'tth-stats.json'
json_directory = create_filename(destination, 'cover_stats')
json_file = create_filename(json_directory, cover_file.lower())
tth_file = create_filename(json_directory, tth_name.lower())
for shape_file in shape_files:
shape_name = get_basename(shape_file).split('__')
anp_id = shape_name[0]
anp_name = shape_name[1]
basename = '%s.tif' % anp_name
dataframe = DataFrame(index=years,
columns=[0, 1, 2, 3, 4, 5, 6, 7, 8])
create_directory_path(json_directory)
print shape_file
for path in paths:
#pixel_resolution, dataset = pixel_info(path)
year = self.get_year_from_path(path)
raster_dir = create_filename(
create_filename(destination, 'raster'), year)
create_directory_path(raster_dir)
raster_file = create_filename(raster_dir, basename)
shell_command = 'gdalwarp -ot Byte -co COMPRESS=LZW -cutline %s -crop_to_cutline %s %s' % (
shape_file, path, raster_file)
print shell_command
print process_launcher.execute(shell_command)
raster_array = open_handle(raster_file)
ds = gdal.Open(raster_file)
geotransform = ds.GetGeoTransform()
x_resolution = geotransform[1]
y_resolution = geotransform[5]
pixel_area = abs(x_resolution * y_resolution)
unique_values = numpy.unique(raster_array, return_counts=True)
indexes = unique_values[0]
counts = unique_values[1]
for i in range(len(indexes)):
key = indexes[i]
dataframe.set_value(year, key,
area(int(counts[i]), pixel_area))
dataframe = dataframe.sort_index()
columns = list(dataframe.columns.values)
index = list(dataframe.index.values)
print dataframe
cover_array.append(
self.dataframe_to_json(dataframe, anp_id, anp_name))
tth_dataframe = DataFrame(columns=columns)
for i in range(len(index) - 1):
label = '%s-%s' % (index[i], index[i + 1])
tth_column = calculate_thh(dataframe.ix[i],
dataframe.ix[i + 1],
int(index[i + 1]) - int(index[i]))
for j in range(len(tth_column)):
tth_dataframe.set_value(label, j, tth_column[j])
tth_dataframe = tth_dataframe.sort_index()
print tth_dataframe
tth_array.append(
self.dataframe_to_json(tth_dataframe, anp_id, anp_name))
self.json_to_file(json_file, cover_array)
self.json_to_file(tth_file, tth_array)
print("--- %s seconds ---" % (time.time() - start_time))
def handle(self, **options):
image_to_be_classified = options['image'][0]
#landmask_path = options['landmask_path'][0]
outlier = options['outlier'][0]
folder_results = getattr(SETTINGS, 'BIG_FOLDER')
shutil.copy(image_to_be_classified, folder_results)
image_to_be_classified = folder_results + get_basename_of_file(image_to_be_classified)
landmask_path = getattr(SETTINGS, 'LANDMASK_PATH')
image_for_segmentation = '/results/' + get_basename_of_file(image_to_be_classified)
LOGGER.info('Starting segmentation with: %s' % image_for_segmentation)
val_t = 50
val_s = 0.7
val_c = 0.3
val_xt = 40
val_rows = 625
val_tile = True
val_mp = True
folder_and_bind_segmentation = getattr(SETTINGS, 'FOLDER_SEGMENTATION')
folder_and_bind_license = getattr(SETTINGS, 'FOLDER_SEGMENTATION_LICENSE')
folder_and_bind_image = getattr(SETTINGS, 'BIG_FOLDER_HOST')
LOGGER.info('starting segmentation')
command = 'run_container'
hosts_from_command = get_host_from_command(command)
LOGGER.info('The command to be executed is %s in the host %s' % (command, hosts_from_command[0].hostname))
remote = RemoteProcessLauncher(hosts_from_command[0])
arguments = 'docker run --rm -v ' + folder_and_bind_segmentation + ' -v ' + folder_and_bind_license + ' -v ' + folder_and_bind_image + ' madmex/segmentation python /segmentation/segment.py ' + image_for_segmentation
arguments+= ' -t ' + str(val_t) + ' -s ' + str(val_s) + ' -c ' + str(val_c) + ' --tile ' + str(val_tile) + ' --mp ' + str(val_mp) + ' --xt ' + str(val_xt) + ' --rows ' + str(val_rows)
remote.execute(arguments)
LOGGER.info('Finished segmentation')
image_segmentation_file = image_to_be_classified + '_' + str(val_t) + '_' + ''.join(str(val_s).split('.'))+ '_' + ''.join(str(val_c).split('.')) + '.tif'
LOGGER.info('Starting vectorization of segmentation file: %s' % image_segmentation_file)
image_segmentation_shp_file = image_segmentation_file + '.shp'
vectorize_raster(image_segmentation_file, 1, image_segmentation_shp_file, 'objects', 'id')
LOGGER.info('Finished vectorization: %s' % image_segmentation_shp_file)
gdal_format = 'GTiff'
image_to_be_classified_class = raster.Data(image_to_be_classified, gdal_format)
width, height, bands = image_to_be_classified_class.get_attribute(raster.DATA_SHAPE)
LOGGER.info('Identifying landmask %s' % landmask_path)
bundle = _get_bundle_from_path(landmask_path)
if bundle:
LOGGER.info('Directory %s is a %s bundle', landmask_path, bundle.get_name())
LOGGER.info('Rasterizing vector shape')
options_to_create = new_options_for_create_raster_from_reference(image_to_be_classified_class.metadata, raster.DATA_SHAPE, (width,height, 1), {})
dataset_landmask_rasterized = create_raster_tiff_from_reference(image_to_be_classified_class.metadata, '', None, options_to_create)
bundle.rasterize(dataset_landmask_rasterized, [1], [1]) #the rasterized process changes the dataset
options_to_create = new_options_for_create_raster_from_reference(image_to_be_classified_class.metadata, raster.GDAL_CREATE_OPTIONS, ['COMPRESS=LZW'], {})
image = folder_results + 'landmask_rasterized.tif'
create_raster_tiff_from_reference(image_to_be_classified_class.metadata, image, dataset_landmask_rasterized.ReadAsArray(), options_to_create)
LOGGER.info('Finished rasterizing vector shape')
LOGGER.info('Polygonizing the landmask rasterized')
landmask_folder = folder_results + 'landmask_from_rasterize/'
layer_landmask = 'landmask'
landmask_file = landmask_folder + layer_landmask + '.shp'
layer_landmask = 'landmask'
vectorize_raster(folder_results + 'landmask_rasterized.tif', 1, landmask_folder, layer_landmask, 'id')
LOGGER.info('Folder of polygon: %s' % landmask_folder)
image_segmentation_file_class = raster.Data(image_segmentation_file, gdal_format)
LOGGER.info('Reading array of %s' % image_for_segmentation)
array_sg_raster = image_segmentation_file_class.read_data_file_as_array()
unique_labels_for_objects = numpy.unique(array_sg_raster)
LOGGER.info('Calculating zonal stats for :%s' % image_to_be_classified)
LOGGER.info('Reading array of %s' % image_to_be_classified)
array_image_to_be_classified = image_to_be_classified_class.read_data_file_as_array()
array_zonal_statistics = calculate_zonal_statistics(array_image_to_be_classified, array_sg_raster, unique_labels_for_objects)
LOGGER.info('finished zonal statistics')
array_zonal_statistics_labeled = append_labels_to_array(array_zonal_statistics, unique_labels_for_objects)
LOGGER.info('Shape of array of zonal statistics labeled %s %s' % (array_zonal_statistics_labeled.shape[0], array_zonal_statistics_labeled.shape[1]))
LOGGER.info('Building data frame')
dataframe_zonal_statistics = create_names_of_dataframe_from_filename(build_dataframe_from_array(array_zonal_statistics_labeled.T), array_zonal_statistics_labeled.shape[0], get_basename_of_file(image_to_be_classified))
LOGGER.info('Filling NaN with zeros')
dataframe_zonal_statistics = dataframe_zonal_statistics.fillna(0)
file_name = folder_results + 'dataframe_zonal_statistics'
dataframe_zonal_statistics.to_csv(file_name, sep='\t', encoding='utf-8', index = False)
LOGGER.info('Working with the training data')
training_data_file = getattr(SETTINGS, 'TRAINING_DATA')
LOGGER.info('Clipping training_data_file: %s with: %s' % (training_data_file, landmask_file))
training_data_file_clipped = folder_results + get_basename_of_file(training_data_file) + '_cropped_subprocess_call.tif'
command = [
'gdalwarp', '-cutline', landmask_file,
'-crop_to_cutline', '-of', 'GTiff','-co', 'compress=lzw', '-co', 'tiled=yes', training_data_file, training_data_file_clipped
]
subprocess.call(command)
LOGGER.info('Finished clipping of training data file')
LOGGER.info('Starting warping of file: %s according to %s ' % (training_data_file_clipped, image_segmentation_file))
dataset_warped_training_data_file = warp_raster_from_reference(training_data_file_clipped, image_segmentation_file_class.data_file, None)
LOGGER.info('Starting resizing of array of training file: %s' % training_data_file_clipped)
array_resized_and_warped_training_data_file = get_array_resized_from_reference_dataset(dataset_warped_training_data_file, image_segmentation_file_class.data_file)
import gdal
training_data_file_resized_and_warped = folder_results + get_basename_of_file(training_data_file) + '_resized_and_warped.tif'
options_to_create = new_options_for_create_raster_from_reference(image_to_be_classified_class.metadata, raster.GDAL_CREATE_OPTIONS, ['TILED=YES', 'COMPRESS=LZW', 'INTERLEAVE=BAND'], {})
create_raster_tiff_from_reference(image_to_be_classified_class.metadata, training_data_file_resized_and_warped, array_resized_and_warped_training_data_file, options_to_create, data_type = gdal.GDT_Int32)
LOGGER.info('Starting resampling')
array_training_data_resampled = resample_numpy_array(array_resized_and_warped_training_data_file, width, height, interpolation = 'nearest')
training_data_file_resampled = folder_results + get_basename_of_file(training_data_file) + '_resampled_from_resized_and_warped.tif'
create_raster_tiff_from_reference(image_to_be_classified_class.metadata, training_data_file_resampled, array_training_data_resampled, options_to_create, data_type = gdal.GDT_Int32)
LOGGER.info('Calculating zonal histograms for file: %s according to: %s' % (training_data_file_resampled, image_segmentation_file))
unique_classes = numpy.unique(array_training_data_resampled)
array_of_distribution_of_classes_per_object_segmentation = calculate_zonal_histograms(array_training_data_resampled, unique_classes, array_sg_raster, unique_labels_for_objects)
LOGGER.info('Shape of zonal histogram: %s %s' % (array_of_distribution_of_classes_per_object_segmentation.shape[0], array_of_distribution_of_classes_per_object_segmentation.shape[1]))
array_training_data_resampled = None
LOGGER.info('Getting objects that have a class of at least .75 proportion within zonal histogram')
dataframe_of_objects_for_training_data = get_objects_by_relative_proportion_from_raster_as_dataframe(array_of_distribution_of_classes_per_object_segmentation, unique_labels_for_objects, unique_classes, ["id", "given"], 0.75)
file_name = folder_results + 'dataframe_of_objects_for_training_data'
dataframe_of_objects_for_training_data.to_csv(file_name, sep='\t', encoding='utf-8', index = False)
LOGGER.info('Number of rows and columns of dataframe of pure objects of training data %s %s' % (len(dataframe_of_objects_for_training_data.index), len(dataframe_of_objects_for_training_data.columns) ))
array_of_distribution_of_classes_per_object_segmentation = None
dataframe_of_objects_for_training_data = None
dataframe_of_objects_for_training_data = pandas.read_csv(file_name, sep='\t')
LOGGER.info('Joining dataframe of dataframe zonal statistics and dataframe of objects of training data')
dataframe_all_joined_classified = join_dataframes_by_column_name([dataframe_zonal_statistics, dataframe_of_objects_for_training_data], 'id')
LOGGER.info('Number of rows and columns of dataframe joined %s %s' % (len(dataframe_all_joined_classified.index), len(dataframe_all_joined_classified.columns) ))
if outlier == 'True':
LOGGER.info('Starting outlier elimination with dataframe of zonal statistics and dataframe of pure objects of training data')
LOGGER.info('Starting principal component analysis')
array_reduced_pca = reduce_dimensionality(dataframe_all_joined_classified, .95, ['id', 'given'])
LOGGER.info('Shape of reduced array of zonal statistics and pure objects of training data by pca: %s %s' %(array_reduced_pca.shape[0], array_reduced_pca.shape[1]) )
labels_of_objects_reduced_dataframe = dataframe_all_joined_classified['id'].values
LOGGER.info('Appending labels')
array_reduced_pca_labeled = append_labels_to_array(array_reduced_pca.T, labels_of_objects_reduced_dataframe)
LOGGER.info('Shape of array reduced by pca and labeled: %s %s' %(array_reduced_pca_labeled.shape[0], array_reduced_pca_labeled.shape[1]))
LOGGER.info('Building data frame')
dataframe_reduced_pca_file = folder_results + 'dataframe_joined_for_zonal_statistics_and_pure_objects_of_training_data_reduced_by_pca'
dataframe_reduced_pca = create_names_of_dataframe_from_filename(build_dataframe_from_array(array_reduced_pca_labeled.T), array_reduced_pca_labeled.shape[0], get_basename_of_file(dataframe_reduced_pca_file))
dataframe_reduced_pca.to_csv(dataframe_reduced_pca_file, sep=',', encoding='utf-8', index = False)
LOGGER.info('Starting with elimination of outliers')
LOGGER.info('Joining reduced dataframe by pca with object ids and dataframe of pure objects of training data')
dataframe_reduced_pca_with_classes= join_dataframes_by_column_name([dataframe_reduced_pca, dataframe_of_objects_for_training_data], 'id')
LOGGER.info('Number of rows and columns of dataframe joined: (%s,%s)' %(len(dataframe_reduced_pca_with_classes.index), len(dataframe_reduced_pca_with_classes.columns)))
dataframe_reduced_pca_with_classes.to_csv(dataframe_reduced_pca_file + 'classes', sep = ',', encoding = 'utf8', index = False)
unique_classes = numpy.unique(dataframe_of_objects_for_training_data['given'].values)
object_ids_outlier_elimination = outlier_elimination_for_dataframe(dataframe_reduced_pca_with_classes, 'id', 'given', 'id', 3, unique_classes, 0.15)
object_ids_outlier_elimination_file = folder_results + 'dataframe_object_ids_outlier_elimination'
object_ids_outlier_elimination.to_csv(object_ids_outlier_elimination_file, sep = ',', encoding = 'utf-8', index = False)
LOGGER.info('Joining all dataframes according to ids of outlier elimination ')
dataframe_all_joined_classified = join_dataframes_by_column_name([object_ids_outlier_elimination, dataframe_all_joined_classified], 'id')
LOGGER.info('Number of rows and columns of dataframe joined classified: (%s,%s)' %(len(dataframe_all_joined_classified.index), len(dataframe_all_joined_classified.columns)))
dataframe_zonal_statistics['given'] = '?'
LOGGER.info('Number of rows and columns of dataframe for classifying: (%s,%s)' %(len(dataframe_zonal_statistics.index), len(dataframe_zonal_statistics.columns)))
index_of_objects_not_id_zero = dataframe_zonal_statistics['id'] > 0
dataframe_all_joined_for_classifying = dataframe_zonal_statistics[index_of_objects_not_id_zero]
LOGGER.info('Number of rows and columns of dataframe for classifying after removing object with id zero: (%s,%s)' %(len(dataframe_all_joined_for_classifying.index), len(dataframe_all_joined_for_classifying.columns)))
LOGGER.info('Generating data file')
dataframe_all_joined_classified_file = folder_results + 'C5.data'
dataframe_all_joined_classified.to_csv(dataframe_all_joined_classified_file, sep = ',', encoding = 'utf-8', index = False, header = False)
LOGGER.info('Generating cases file')
dataframe_all_joined_for_classifying_file = folder_results + 'C5.cases'
dataframe_all_joined_for_classifying.to_csv(dataframe_all_joined_for_classifying_file, sep = ',', encoding = 'utf-8', index = False, header = False)
LOGGER.info('Generating names file')
unique_classes = numpy.unique(dataframe_all_joined_classified['given'].values)
name_namesfile = folder_results + 'C5.names'
generate_namesfile(dataframe_all_joined_classified.columns, unique_classes,name_namesfile, 'id', 'given')
command = 'run_container'
hosts_from_command = get_host_from_command(command)
LOGGER.info('The command to be executed is %s in the host %s' % (command, hosts_from_command[0].hostname))
remote = RemoteProcessLauncher(hosts_from_command[0])
folder_and_bind_c5 = getattr(SETTINGS, 'BIG_FOLDER_HOST')
arguments = 'docker run --rm -v ' + folder_and_bind_c5 + ' madmex/c5_execution ' + 'c5.0 -b -f /results/C5'
LOGGER.info('Beginning C5')
remote.execute(arguments)
LOGGER.info('Begining predict')
arguments = 'docker run --rm -v ' + folder_and_bind_c5 + ' madmex/c5_execution ' + 'predict -f /results/C5'
remote = RemoteProcessLauncher(hosts_from_command[0])
output = remote.execute(arguments, True)
LOGGER.info('Writing C5 result to csv')
C5_result = write_C5_result_to_csv(output, folder_results)
LOGGER.info('Using result of C5: %s for generating land cover shapefile and raster image' % C5_result)
LOGGER.info('Using result of C5 for generating land cover shapefile and raster image')
C5_result = folder_results + 'C5_result.csv'
dataframe_c5_result = pandas.read_csv(C5_result)
FORMAT = 'ESRI Shapefile'
image_segmentation_shp_class = vector.Data(image_segmentation_shp_file, FORMAT)
LOGGER.info('Joining dataframe %s to %s' %(C5_result, image_segmentation_shp_file))
dataframe_joined_shp_segmentation_and_c5_result = join_C5_dataframe_and_shape(image_segmentation_shp_class, 'id', dataframe_c5_result, 'id')
LOGGER.info('Number of rows and columns of dataframe joined: (%s,%s)' %(len(dataframe_joined_shp_segmentation_and_c5_result.index), len(dataframe_joined_shp_segmentation_and_c5_result.columns)))
dataframe_joined_shp_segmentation_and_c5_result_file = folder_results + 'dataframe_joined_shp_segmentation_and_c5_result.csv'
LOGGER.info('Writing csv of join between c5 result and segmentation shape: %s' % dataframe_joined_shp_segmentation_and_c5_result_file)
dataframe_joined_shp_segmentation_and_c5_result.to_csv(dataframe_joined_shp_segmentation_and_c5_result_file, sep =',', encoding = 'utf8', index = False)
LOGGER.info('Writing C5 result joined with segmentation shape to shapefile')
segmentation_and_c5_result_file_vectorized_folder = folder_results + 'segmentation_and_c5_result_vectorized/'
create_directory_path(segmentation_and_c5_result_file_vectorized_folder)
sql = "SELECT a.id, a.predicted, a.confidence, st_geomfromtext(a.geom," + image_segmentation_shp_class.srid+ ") as geometry "
sql+= "from dataframe_joined_shp_segmentation_and_c5_result a"
shp_result = segmentation_and_c5_result_file_vectorized_folder + '/C5_result_joined_segmentation_shape.shp'
command = [
'ogr2ogr', shp_result,
dataframe_joined_shp_segmentation_and_c5_result_file,
'-dialect', 'sqlite', '-sql', sql
]
subprocess.call(command)
LOGGER.info('Rasterizing segmentation and c5 result shape of folder %s' % segmentation_and_c5_result_file_vectorized_folder)
LOGGER.info('Identifying segmentation and c5 shape folder %s' % segmentation_and_c5_result_file_vectorized_folder)
bundle = _get_bundle_from_path(segmentation_and_c5_result_file_vectorized_folder)
if bundle:
LOGGER.info('Directory %s is a %s bundle', segmentation_and_c5_result_file_vectorized_folder, bundle.get_name())
LOGGER.info('Rasterizing vector shape to get land cover tif')
options_to_create = new_options_for_create_raster_from_reference(image_to_be_classified_class.metadata, raster.DATA_SHAPE, (width,height, 1), {})
dataset_shape_sg_and_c5_rasterized = create_raster_tiff_from_reference(image_to_be_classified_class.metadata, '', None, options_to_create)
bundle.rasterize(dataset_shape_sg_and_c5_rasterized, [1], None, ["ATTRIBUTE=predicted" ]) #the rasterized process changes the dataset
options_to_create = new_options_for_create_raster_from_reference(image_to_be_classified_class.metadata, raster.GDAL_CREATE_OPTIONS, ['COMPRESS=LZW'], {})
image =folder_results + 'madmex_lcc_prueba.tif'
create_raster_tiff_from_reference(image_to_be_classified_class.metadata, image, dataset_shape_sg_and_c5_rasterized.ReadAsArray(), options_to_create, data_type = gdal.GDT_Int32)
LOGGER.info('Finished rasterizing vector shape')
LOGGER.info('Rasterizing vector shape to get confidence tif')
bundle.rasterize(dataset_shape_sg_and_c5_rasterized, [1], None, ["ATTRIBUTE=confidence" ])
image =folder_results + 'madmex_lcc_confidence_prueba.tif'
create_raster_tiff_from_reference(image_to_be_classified_class.metadata, image, dataset_shape_sg_and_c5_rasterized.ReadAsArray(), options_to_create)
LOGGER.info('Finished rasterizing vector shape')
LOGGER.info('Finished workflow classification :)')
def handle(self, **options):
'''
In this example command, the values that come from the user input are
added up and the result is printed in the screen.
'''
target_tag = 'DN'
start_time_all = time.time()
shape_name = options['shape'][0]
raster_paths = options['path']
destination = options['dest']
models = options['model']
dataframe_features = None
temporary_directory = getattr(SETTINGS, 'TEMPORARY')
create_directory_path(temporary_directory)
# I read the training data in shape form
training_shape = vector.Data(shape_name)
training_dataframe = training_shape.to_dataframe()
training_path = create_filename(temporary_directory,
'training_raster.tif')
categories_file = create_filename(temporary_directory,
'categories.json')
training_warped_path = create_filename(temporary_directory,
'training_warped_raster.tif')
pixel_size = 0.000462175996292
if not is_file(training_warped_path):
training_raster = vector_to_raster(
training_shape, training_path, pixel_size, -pixel_size,
['ATTRIBUTE=OBJECTID', 'COMPRESS=LZW'])
training_raster_warped = training_raster.reproject(
training_warped_path, epgs=32617)
else:
training_raster_warped = raster.Data(training_warped_path)
dem_file = getattr(SETTINGS, 'DEM')
dem_raster = raster.Data(dem_file)
print dem_raster.get_spatial_reference()
print 'reproyecting raster'
#dem_raster_warped = dem_raster.reproject(training_warped_path, epgs=32614)
#training_raster_warped = raster.Data(training_path)
aspect_file = getattr(SETTINGS, 'ASPECT')
slope_file = getattr(SETTINGS, 'SLOPE')
print dem_file, aspect_file, slope_file
for raster_path in raster_paths:
scene_bundle = rapideye.Bundle(raster_path)
raster_mask = scene_bundle.get_raster()
#example_path = create_filename(temporary_directory, 'mask')
#create_directory_path(example_path)
#raster_to_vector_mask(raster_mask, example_path)
print scene_bundle.get_raster_file()
basename = get_basename(scene_bundle.get_raster_file())
all_file = create_filename(temporary_directory,
'%s_all_features.tif' % basename)
# Do not recalculate if the file is already there.
if is_file(all_file):
features_raster = raster.Data(all_file)
else:
features_raster = scene_bundle.get_feature_array(all_file)
new_df = get_dataframe_from_raster(features_raster,
training_raster_warped)
if new_df is not None:
if dataframe_features is not None:
dataframe_features = pandas.concat([
dataframe_features,
get_dataframe_from_raster(features_raster,
training_raster_warped)
])
else:
dataframe_features = get_dataframe_from_raster(
features_raster, training_raster_warped)
features_size = len(list(dataframe_features))
training_set = dataframe_features.set_index(0).join(
training_dataframe.set_index('OBJECTID'))
print training_set
training_set['target'] = pandas.Categorical.from_array(
training_set[target_tag]).labels
categories_array = pandas.Categorical.from_array(
training_set[target_tag]).categories
create_categories_file(categories_file, categories_array)
training_set = training_set[training_set['target'] != -1]
#features_size includes 0 that is the index of the feature
training_set_array = numpy.transpose(
numpy.transpose(training_set.as_matrix([range(1, features_size)])))
target_set_array = training_set.pop('target')
print training_set_array.shape
print target_set_array.shape
X_train, X_test, y_train, y_test = train_test_split(training_set_array,
target_set_array,
train_size=0.8,
test_size=0.2)
models_directory = create_filename(temporary_directory, 'models')
create_directory_path(models_directory)
for model_name in models:
start_time = time.time()
print numpy.unique(y_train)
train_model(X_train, X_test, y_train, y_test, models_directory,
model_name)
print "--- %s seconds training %s model---" % (
(time.time() - start_time), model_name)
def handle(self, **options):
'''
In this example command, the values that come from the user input are
added up and the result is printed in the screen.
'''
output = options['output'][0]
models = options['modelname']
model_directory = options['modeldir'][0]
region = options['region'][0]
start_time = time.time()
for path in options['path']:
print path
scene_bundle = rapideye.Bundle(path)
directory = getattr(SETTINGS, 'TEMPORARY')
directory_helper = create_filename(directory, 'helper')
create_directory_path(directory_helper)
categories_file = create_filename(directory, 'categories.json')
categories_dictionaty = {
0: "AGRICULTURA DE RIEGO",
1: "AGRICULTURA DE TEMPORAL",
2: "AGUA",
3: "AREAS QUEMADAS",
4: "ASENTAMIENTOS HUMANOS",
5: "BOSQUE CULTIVADO",
6: "BOSQUE DE AYARIN",
7: "BOSQUE DE ENCINO",
8: "BOSQUE DE ENCINO-PINO",
9: "BOSQUE DE GALERIA",
10: "BOSQUE DE MEZQUITE",
11: "BOSQUE DE OYAMEL",
12: "BOSQUE DE PINO",
13: "BOSQUE DE PINO-ENCINO",
14: "BOSQUE INDUCIDO",
15: "BOSQUE MESOFILO DE MONTANA",
16: "DESPROVISTO DE VEGETACION",
17: "INDEFINIDO",
18: "MANGLAR",
19: "MATORRAL SUBTROPICAL",
20: "MEZQUITAL",
21: "NUBES",
22: "PASTIZAL CULTIVADO",
23: "PASTIZAL HALOFILO",
24: "PASTIZAL INDUCIDO",
25: "PASTIZAL NATURAL",
26: "PRADERA DE ALTA MONTANA",
27: "SABANOIDE",
28: "SELVA ALTA PERENNIFOLIA",
29: "SELVA ALTA SUBPERENNIFOLIA",
30: "SELVA BAJA CADUCIFOLIA",
31: "SELVA BAJA ESPINOSA CADUCIFOLIA",
32: "SELVA BAJA SUBCADUCIFOLIA",
33: "SELVA DE GALERIA",
34: "SELVA MEDIANA CADUCIFOLIA",
35: "SELVA MEDIANA SUBCADUCIFOLIA",
36: "SELVA MEDIANA SUBPERENNIFOLIA",
37: "SIN VEGETACION APARENTE",
38: "SOMBRAS",
39: "TULAR",
40: "VEGETACION DE DUNAS COSTERAS",
41: "VEGETACION HALOFILA HIDROFILA",
42: "ZONA URBANA"
}
basename = get_basename(scene_bundle.get_raster_file())
all_file = create_filename(directory_helper,
'%s_all_features.tif' % basename)
if not is_file(all_file):
scene_bundle.get_feature_array(all_file)
filename = get_basename(all_file)
if not is_file(
create_filename(directory_helper, '%s.shp' % filename)):
shell_string = 'docker run --rm -v %s:/data madmex/segment gdal-segment %s.tif -out helper/%s.shp -algo SLIC -region %s' % (
directory, filename, filename, region)
launcher = LocalProcessLauncher()
LOGGER.debug('Docker command: %s', shell_string)
launcher.execute(shell_string)
data = read_data_table(
create_filename(directory_helper, '%s.shp' % filename))
results = {}
for model_name in models:
persistence_directory = create_filename(
model_directory, model_name)
print model_name
model = load_model(model_name)
model_instance = model.Model(persistence_directory)
model_instance.load(persistence_directory)
prediction = model_instance.predict(data)
results[model_name] = prediction
print results
create_directory_path(output)
write_results(
create_filename(directory_helper, '%s.shp' % filename),
create_filename(output,
'%s_classification.shp' % filename[0:32]),
results, categories_dictionaty)
LOGGER.info("--- %s seconds ---" % (time.time() - start_time))
def handle(self, **options):
'''
In this example command, the values that come from the user input are
added up and the result is printed in the screen.
'''
random_int = 1
training_template = options['training'][0]
target_template = options['target'][0]
index_template = options['index'][0]
output = options['output'][0]
models = options['model']
features = training_template % random_int
training = target_template % random_int
index = index_template % random_int
features_array = open_handle(features)
training_array = open_handle(training)
index_array = open_handle(index)
labels = numpy.unique(index_array)
features_flatten = numpy.ravel(features_array).reshape(10, 200 * 200)
training_flatten = numpy.ravel(training_array).reshape(200 * 200)
array_aux = []
for j in range(features_array.shape[0]):
means = scipy.ndimage.measurements.mean(features_array[j, :, :],
index_array, labels)
print means.shape
array_aux.append(means)
for j in range(features_array.shape[0]):
std = scipy.ndimage.measurements.standard_deviation(
features_array[j, :, :], index_array, labels)
print std.shape
array_aux.append(std)
print len(array_aux)
features_final = numpy.concatenate([array_aux], axis=0)
print features_final.shape
label_object = []
for id in labels:
values, counts = numpy.unique(training_array[index_array == id],
return_counts=True)
label_object.append(values[numpy.argmax(counts)])
features_total = features_final
training_total = label_object
for i in range(1, 53):
features = training_template % (i + 1)
training = target_template % (i + 1)
index = index_template % (i + 1)
features_array = open_handle(features)
training_array = open_handle(training)
index_array = open_handle(index)
labels = numpy.unique(index_array)
array_aux = []
for j in range(features_array.shape[0]):
array_aux.append(
scipy.ndimage.measurements.mean(features_array[j, :, :],
index_array, labels))
for j in range(features_array.shape[0]):
array_aux.append(
scipy.ndimage.measurements.standard_deviation(
features_array[j, :, :], index_array, labels))
features_final = numpy.concatenate([array_aux], axis=0)
print features_final.shape
label_object = []
for id in labels:
values, counts = numpy.unique(
training_array[index_array == id], return_counts=True)
#print '********* ', values[numpy.argmax(counts)], counts[numpy.argmax(counts)]
label_object.append(values[numpy.argmax(counts)])
print features_total.shape
print features_final.shape
features_total = numpy.concatenate(
(features_total, features_final), axis=1)
training_total = numpy.concatenate((training_total, label_object),
axis=1)
print 'label object', len(label_object)
#print scipy.ndimage.measurements.mean(training_array, index_array, labels)
features_flatten = numpy.concatenate(
(features_flatten, numpy.ravel(features_array).reshape(
10, 200 * 200)),
axis=1)
training_flatten = numpy.concatenate(
(training_flatten, numpy.ravel(training_array).reshape(
200 * 200)),
axis=0)
# Remove the elements that map to None
#mask = training_flatten!=0
#features_flatten = features_flatten[:,mask]
#training_flatten = training_flatten[mask]
mask = training_total != 0
features_flatten = features_total[:, mask]
training_flatten = training_total[mask]
print features_flatten.shape
print training_flatten.shape
X_train, X_test, y_train, y_test = train_test_split(
numpy.transpose(features_flatten),
training_flatten,
train_size=0.8,
test_size=0.2)
print X_train[0]
print y_train[0]
unsupervised = pca.Model(5)
unsupervised.fit(X_train)
'''
X_train = unsupervised.transform(X_train)
X_test = unsupervised.transform(X_test)
'''
pca_path = create_filename(output, 'pca')
create_directory_path(pca_path)
unsupervised.save(pca_path)
import time
for model_name in models:
start_time = time.time()
train_model(X_train, X_test, y_train, y_test, output, model_name)
print "--- %s seconds training %s model---" % (
(time.time() - start_time), model_name)
def handle(self, **options):
'''
This is the code that does the ingestion.
indexes --path /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/Landsat_2000_2008/L5_021_047_2000_022_sr/ --shape /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/AE_LT_new.shp --output
'''
path = options['path'][0]
for mask_file in os.listdir(path):
if mask_file.endswith('_cfmask.tif'):
print mask_file
basename = mask_file.replace('_cfmask.tif', '')
print basename
cloud_file_name = mask_file
LOGGER.info('Calculating indexes for Landsat scenes.')
band_name = basename + '_sr_band%s.tif'
final_path = options['output'][0]
create_directory_path(final_path)
cloud_file = create_filename(path, cloud_file_name)
cloud_array = open_handle(cloud_file)
cloud_mask = (cloud_array == 4)
LOGGER.debug('Recognize sensor depending on file name.')
if 'L8' in path:
LOGGER.debug('Landsat 8 scene was detected.')
blue_file = create_filename(path, band_name % BLUE_L8)
green_file = create_filename(path, band_name % GREEN_L8)
red_file = create_filename(path, band_name % RED_L8)
nir_file = create_filename(path, band_name % NIR_L8)
swir_file = create_filename(path, band_name % SWIR_L8)
else:
LOGGER.debug('Landsat 4, 5 or scene was detected.')
blue_file = create_filename(path, band_name % BLUE)
green_file = create_filename(path, band_name % GREEN)
red_file = create_filename(path, band_name % RED)
nir_file = create_filename(path, band_name % NIR)
swir_file = create_filename(path, band_name % SWIR)
LOGGER.debug('Loading bands of interest.')
green_array = open_handle(green_file)
red_array = open_handle(red_file)
nir_array = open_handle(nir_file)
swir_array = open_handle(swir_file)
LOGGER.debug('Calculating indexes.')
ndvi_array = calculate_index(nir_array, red_array)
mndwi_array = calculate_index(green_array, swir_array)
ndwig_array = calculate_index(nir_array, swir_array)
ndwim_array = calculate_index(green_array, nir_array)
LOGGER.debug('Setting cloud mask values to -999.')
ndvi_array[cloud_mask] = -999
mndwi_array[cloud_mask] = -999
ndwig_array[cloud_mask] = -999
ndwim_array[cloud_mask] = -999
LOGGER.debug('Creating files for indexes.')
ndvi_final_file = create_filename(final_path, basename + '_ndvi.tif')
mndwi_final_file = create_filename(final_path, basename + '_mndwi.tif')
ndwig_final_file = create_filename(final_path, basename + '_ndwig.tif')
ndwim_final_file = create_filename(final_path, basename + '_ndwim.tif')
ndvi_clipped_file = create_filename(final_path,
basename + '_ndvi_clipped.tif')
mndwi_clipped_file = create_filename(final_path,
basename + '_mndwi_clipped.tif')
ndwig_clipped_file = create_filename(final_path,
basename + '_ndwig_clipped.tif')
ndwim_clipped_file = create_filename(final_path,
basename + '_ndwim_clipped.tif')
ndvi_file = create_filename(final_path, basename + '_ndvi_pre.tif')
mndwi_file = create_filename(final_path, basename + '_mndwi_pre.tif')
ndwig_file = create_filename(final_path, basename + '_ndwig_pre.tif')
ndwim_file = create_filename(final_path, basename + '_ndwim_pre.tif')
files = [ndvi_file, mndwi_file, ndwig_file, ndwim_file]
clipped_files = [
ndvi_clipped_file, mndwi_clipped_file, ndwig_clipped_file,
ndwim_clipped_file
]
final_files = [
ndvi_final_file, mndwi_final_file, ndwig_final_file,
ndwim_final_file
]
LOGGER.debug('Writing information to files.')
create_raster_from_reference(ndvi_file, ndvi_array, green_file)
create_raster_from_reference(mndwi_file, mndwi_array, green_file)
create_raster_from_reference(ndwig_file, ndwig_array, green_file)
create_raster_from_reference(ndwim_file, ndwim_array, green_file)
LOGGER.debug('Deleting arrays to release memory.')
del ndvi_array
del mndwi_array
del ndwig_array
del ndwim_array
del cloud_array
LOGGER.debug('Reference rgb file creation.')
rgb_file = create_filename(final_path, basename + '_rgb.tif')
merge_command = [
'/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt',
'-separate', '-o', rgb_file, red_file, green_file, blue_file
]
call(merge_command)
shape = options['shape'][0]
LOGGER.debug('Cookie cut files using the given shape.')
for i in range(4):
clip_command = [
'/Library/Frameworks/GDAL.framework/Programs/gdalwarp',
'-crop_to_cutline', '-cutline', shape, files[i],
clipped_files[i]
]
call(clip_command)
for i in range(4):
aux_array = open_handle(clipped_files[i])
aux_array[(aux_array == 0)] = -9999
create_raster_from_reference(final_files[i], aux_array,
ndvi_clipped_file)
if not options['debug']:
LOGGER.info('Remove auxiliary files.')
os.remove(ndvi_clipped_file)
os.remove(mndwi_clipped_file)
os.remove(ndwig_clipped_file)
os.remove(ndwim_clipped_file)
os.remove(ndvi_file)
os.remove(mndwi_file)
os.remove(ndwig_file)
os.remove(ndwim_file)
print 'Done'