def get_thumbnail(self):
'''
Creates a thumbnail for the scene in true color.
'''
from subprocess import call
file_1 = self.file_dictionary[_BASE %
(self.get_mission(), 'B1[0-9].TIF')]
file_2 = self.file_dictionary[_BASE %
(self.get_mission(), 'B2[0-9].TIF')]
file_3 = self.file_dictionary[_BASE %
(self.get_mission(), 'B3[0-9].TIF')]
parent_directory = get_parent(self.path)
thumnail_directory = create_filename(parent_directory, 'thumbnail')
create_directory_path(thumnail_directory)
parent_directory
filename = create_filename(thumnail_directory, 'vrt.tif')
merge_command = [
'/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt',
'-separate', '-o', filename, file_3, file_2, file_1
]
call(merge_command)
thumbnail = create_filename(thumnail_directory, 'thumbnail.jpg')
resize_command = [
'/Library/Frameworks/GDAL.framework/Programs/gdal_translate',
filename, '-of', 'JPEG', '-outsize', '5%', '5%', thumbnail
]
call(resize_command)
def save(self, filepath):
'''
Persists the trained model to a file.
'''
joblib.dump(self.model,
create_filename(filepath, '%s.pkl' % self.model_name))
joblib.dump(self.scaler,
create_filename(filepath, '%s.pkl' % self.scaler_mlp))
def load(self, filepath):
'''
Loads an already train model from a file to perform predictions.
'''
self.model = joblib.load(
create_filename(filepath, '%s.pkl' % self.model_name))
self.scaler = joblib.load(
create_filename(filepath, '%s.pkl' % self.scaler_mlp))
def preprocess(self):
'''
Top of atmosphere is calculated and persisted into a file. Then a cloud
mask is created with the given algorithm.
'''
solar_zenith = self.get_sensor().parser.get_attribute(
rapideye.SOLAR_ZENITH)
data_acquisition_date = self.get_sensor().parser.get_attribute(
rapideye.ACQUISITION_DATE)
solar_azimuth = self.get_sensor().parser.get_attribute(
rapideye.SOLAR_AZIMUTH)
geotransform = self.get_raster().get_attribute(raster.GEOTRANSFORM)
data = self.get_raster().read_data_file_as_array()
sun_earth_distance = calculate_distance_sun_earth(
data_acquisition_date)
top_of_atmosphere_data = calculate_toa_rapideye(
calculate_rad_rapideye(data), sun_earth_distance, solar_zenith)
top_of_atmosphere_directory = create_filename(get_parent(self.path),
'TOA')
create_directory_path(top_of_atmosphere_directory)
output_file = create_filename(
top_of_atmosphere_directory,
get_basename(self.get_files()[2]) +
'_toa.tif') # TODO: change [2] in self.get_files()[2]
create_raster_from_reference(output_file,
top_of_atmosphere_data,
self.file_dictionary[_IMAGE],
data_type=NumericTypeCodeToGDALTypeCode(
numpy.float32))
LOGGER.debug('Top of atmosphere file was created.')
cloud_output_file = create_filename(
top_of_atmosphere_directory,
get_basename(self.get_files()[2]) + '_cloud.tif')
if self.algorithm == ANOMALY_DETECTION:
LOGGER.debug('Cloud mask by anomaly detection process.')
clouds = self.anomaly_detection_cloud_mask(top_of_atmosphere_data,
cloud_output_file,
solar_zenith,
solar_azimuth,
geotransform)
elif self.algorithm == TIME_SERIES:
LOGGER.debug('Cloud mask by reference with time series process.')
tile_id = self.get_sensor().get_attribute(TILE_ID)
clouds = self.masking_with_time_series(data, cloud_output_file,
solar_zenith, solar_azimuth,
geotransform, tile_id)
create_raster_from_reference(cloud_output_file,
clouds,
self.file_dictionary[_IMAGE],
data_type=NumericTypeCodeToGDALTypeCode(
numpy.float32))
LOGGER.info('Cloud mask was created.')
def train_model(X_train, X_test, y_train, y_test, output, model_name):
model = load_model(model_name)
persistence_directory = create_filename(output, model_name)
create_directory_path(persistence_directory)
model_instance = model.Model(persistence_directory)
model_instance.fit(X_train, y_train)
model_instance.save(persistence_directory)
predicted = model_instance.predict(X_test)
model_instance.create_report(
y_test, predicted, create_filename(persistence_directory,
'report.txt'))
def handle(self, **options):
'''
In this example command, the values that come from the user input are
added up and the result is printed in the screen.
'''
output = options['output'][0]
models = options['modelname']
model_directory = options['modeldir'][0]
pca_model = pca.Model(5)
pca_model.load(create_filename(model_directory, 'pca'))
for model_name in models:
persistence_directory = create_filename(model_directory,
model_name)
model = load_model(model_name)
model_instance = model.Model(persistence_directory)
model_instance.load(persistence_directory)
block_size = 500
for path in options['path']:
image_array = open_handle(path)
y_size = image_array.shape[1]
x_size = image_array.shape[2]
basename = get_basename(path)[:7]
warnings.filterwarnings('ignore')
final = numpy.zeros((x_size, y_size))
import time
start_time = time.time()
for i in range(0, y_size, block_size):
if i + block_size < y_size:
rows = block_size
else:
rows = y_size - i
for j in range(0, x_size, block_size):
if j + block_size < x_size:
cols = block_size
else:
cols = x_size - j
step = image_array[:, i:i + rows, j:j + cols]
step_ravel = step.reshape(10, -1)
prediction = model_instance.predict(
pca_model.transform(numpy.transpose(step_ravel)))
final[i:i + rows, j:j + cols] = prediction.reshape(
(rows, cols))
print("--- %s seconds ---" % (time.time() - start_time))
create_directory_path(output)
classification = create_filename(
output, '%s-%s.tif' % (basename, model_name))
create_raster_from_reference(classification,
final.reshape(x_size, y_size),
path,
data_type=gdal.GDT_Byte,
creating_options=['COMPRESS=LZW'])
def test_create_image(self):
import numpy
array = numpy.array(
[[
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
],
[
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
],
[
1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0,
1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1
],
[
1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0,
1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1
],
[
1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0,
1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1
],
[
1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0,
1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1
],
[
1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0,
1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1
],
[
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
],
[
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
],
[
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
]])
red = (1 - array) * 1000
green = (1 - array) * 1000
blue = array * 1000
create_raster(
create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'single.tif'),
array)
create_raster(
create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'multi.tif'),
numpy.array([red, green, blue]))
def raster_to_vector_mask(raster_object, output_path, no_data=[0]):
raster_array = raster_object.read_data_file_as_array()
for i in no_data:
raster_array[raster_array != i] = 0
raster_array[raster_array != 0] = 1
mask_file = create_filename(output_path, 'mask.tif')
create_raster_from_reference(mask_file, raster_array, raster_object.get_file())
mask_raster = raster.Data(mask_file)
ds = mask_raster._open_file()
rb = ds.GetRasterBand(1)
dst_layername = 'POLYGONIZED_STUFF'
drv = ogr.GetDriverByName(str('ESRI Shapefile'))
dst_ds = drv.CreateDataSource(output_path)
dst_layer = dst_ds.CreateLayer(dst_layername, srs = None )
#dst_layer.SetSpatialRef(raster.get_spatial_reference())
gdal.Polygonize(rb, None, dst_layer, -1, [])
def maybe_download_and_extract(target_directory, scene_url):
'''
This method will try to download the requested url, if
a file with the url name has already been downloaded it
will not proceed.
'''
filename = scene_url.split('/')[-1]
filepath = create_filename(target_directory, filename)
if not is_file(filepath):
def _progress(count, block_size, total_size):
sys.stdout.write('\rDownloading %s %.1f%%' %
(filename, float(count * block_size) /
float(total_size) * 100.0))
sys.stdout.flush()
filepath, _ = urllib.request.urlretrieve(scene_url, filepath,
_progress)
sys.stdout.write('')
sys.stdout.flush()
statinfo = os.stat(filepath)
logger.info('Successfully downloaded: %s %s bytes' %
(filename, statinfo.st_size))
else:
logger.info('%s was already downloaded at %s' %
(filename, target_directory))
def test_creat_image(self):
import numpy
filename = '/Users/agutierrez/Development/df/1448114/2015/2015-02-24/l3a/1448114_2015-02-24_RE3_3A_302417.tif'
width = get_width(filename)
height = get_height(filename)
plain = width * height
red = numpy.zeros(plain)
green = numpy.zeros(plain)
blue = numpy.zeros(plain)
ired = numpy.zeros(plain)
print 'before loop'
for i in range(plain):
if not i % 2:
red[i] = 1
if not i % 3:
green[i] = 1
if not i % 5:
blue[i] = 1
if not i % 7:
ired[i] = 1
print 'loop'
final = numpy.array([
numpy.reshape(red, (width, height)),
numpy.reshape(green, (width, height)),
numpy.reshape(blue, (width, height)),
numpy.reshape(ired, (width, height))
])
print final.shape
print 'hello'
create_raster_from_reference(
create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'sieve.tif'),
final, filename)
def split_shape_into_features(shape_name, destination_directory, column, name,
sep):
'''
This method will take a input shape and iterate over its features, creating
a new shape file with each one of them. It copies all the fields and the
same spatial reference from the original file. The created files are saved
in the destination directory using the number of the field given.
'''
driver = ogr.GetDriverByName(str('ESRI Shapefile'))
shape = driver.Open(shape_name, 0)
layer = shape.GetLayer()
layer_name = layer.GetName()
spatial_reference = layer.GetSpatialRef()
in_feature = layer.GetNextFeature()
shape_files = []
while in_feature:
encoding = 'utf-8'
in_feature_name = in_feature.GetField(column)
in_name = create_filename_from_string(
in_feature.GetField(name).decode(encoding))
final_path = destination_directory + str(in_feature.GetField(0))
create_directory_path(final_path)
output_name = create_filename(
final_path, '%s__%s.shp' % (in_feature_name, in_name))
shape_files.append(output_name)
if os.path.exists(output_name):
driver.DeleteDataSource(output_name)
datasource = driver.CreateDataSource(output_name)
outLayer = datasource.CreateLayer(layer_name,
spatial_reference,
geom_type=ogr.wkbPolygon)
inLayerDefn = layer.GetLayerDefn()
for i in range(0, inLayerDefn.GetFieldCount()):
fieldDefn = inLayerDefn.GetFieldDefn(i)
#LOGGER.debug(fieldDefn.GetName())
outLayer.CreateField(fieldDefn)
outLayerDefn = outLayer.GetLayerDefn()
geometry = in_feature.GetGeometryRef()
out_feature = ogr.Feature(outLayerDefn)
out_feature.SetGeometry(geometry)
for i in range(0, outLayerDefn.GetFieldCount()):
out_feature.SetField(
outLayerDefn.GetFieldDefn(i).GetNameRef(),
in_feature.GetField(i))
outLayer.CreateFeature(out_feature)
out_feature.Destroy()
in_feature.Destroy()
in_feature = layer.GetNextFeature()
shape.Destroy()
datasource.Destroy()
return shape_files
def get_convex_hull(shape_name, destination_directory):
'''
This method will read all objects from a shape file and create a new one
with the convex hull of all the geometry points of the first.
'''
driver = ogr.GetDriverByName(str('ESRI Shapefile'))
shape = driver.Open(shape_name, 0)
layer = shape.GetLayer()
layer_name = layer.GetName()
spatial_reference = layer.GetSpatialRef()
prefix = get_basename(shape_name)
output_name = create_filename(destination_directory,
'%s-hull.shp' % prefix)
geometries = ogr.Geometry(ogr.wkbGeometryCollection)
for feature in layer:
geometries.AddGeometry(feature.GetGeometryRef())
if os.path.exists(output_name):
driver.DeleteDataSource(output_name)
datasource = driver.CreateDataSource(output_name)
out_layer = datasource.CreateLayer(str('states_convexhull'),
spatial_reference,
geom_type=ogr.wkbPolygon)
out_layer.CreateField(ogr.FieldDefn(str('id'), ogr.OFTInteger))
featureDefn = out_layer.GetLayerDefn()
feature = ogr.Feature(featureDefn)
feature.SetGeometry(geometries.ConvexHull())
feature.SetField(str('id'), 1)
out_layer.CreateFeature(feature)
shape.Destroy()
datasource.Destroy()
def fit(self, X, y):
'''
This method creates the necessary files to train a c5 model with the
c5.0 executable. It creates only the names and data files.
'''
train_data = numpy.column_stack((X, y))
save_to_file(train_data,
create_filename(self.path, '%s.data' % self.model_name))
create_names_file(
numpy.unique(y), X.shape[1],
create_filename(self.path, '%s.names' % self.model_name))
launcher = LocalProcessLauncher()
shell_string = 'docker run --rm -v %s:/results madmex/c5_execution c5.0 -f /results/%s' % (
self.path, self.model_name)
LOGGER.debug('Docker command: %s', shell_string)
launcher.execute(shell_string)
def handle(self, **options):
output = options['output'][0]
#state = options['state'][0]
# for name in get_states_names():
# print name[0]
# for footprint in get_rapideye_footprints_from_state(name[0]):
# print footprint[0]
#
#
# import time
# time.sleep(2)
for image_path in options['path']:
print image_path
basename = '%s_ipcc.tif' % get_basename(image_path)
LOGGER.info(basename)
target = create_filename(output, basename)
#print target
start_time = time.time()
self.aggregate_by_block(image_path, target, INITIAL_IPCC_2015,
FINAL_IPCC_2015)
#self.method_by_block(image_path, target)
#self.mask_iterating_values(image_path, target, INITIAL_ARRAY, FINAL_ARRAY)
LOGGER.info("--- %s seconds ---" % (time.time() - start_time))
LOGGER.info('Dataset was written.')
def get_thumbnail_with_path(self, thumbnail_path):
'''
Creates a thumbnail for the scene in true color.
'''
from subprocess import call
thumnail_directory = create_filename(thumbnail_path, 'thumbnail')
create_directory_path(thumnail_directory)
filename = self.file_dictionary[_BROWSE]
thumbnail = create_filename(thumnail_directory,
'%s.jpg' % get_basename(filename))
resize_command = [
'/Library/Frameworks/GDAL.framework/Programs/gdal_translate',
filename, '-of', 'JPEG', thumbnail
]
call(resize_command)
def handle(self, *args, **options):
LOGGER.info('Hello World!')
directory = options['directory'][0]
for satellite, filename in FILES.iteritems():
filepath = create_filename(directory, filename)
populate_catalog_django(filepath, satellite)
print 'Done'
def _look_for_files(self):
'''
This method will be called by the constructor to look for files in the
given directory. In order for a bundle to be valid, a file for each
regular expression must be found.
'''
for key in self.file_dictionary.iterkeys():
for name in get_files_from_folder(self.path):
if re.match(key, name):
self.file_dictionary[key] = create_filename(
self.path, name)
def get_information_object(self):
row = self.get_sensor().get_attribute(olitirs.ROW)
path = self.get_sensor().get_attribute(olitirs.PATH)
basename_metadata = get_basename_of_file( self.file_dictionary[_BASE % (self.get_letter(), self.get_mission(), 'MTL.txt')])
information = Information(
metadata_path = create_filename(self.get_output_directory(),basename_metadata),
grid_id = unicode(path + row),
projection = self.get_raster().get_attribute(raster.PROJECTION),
cloud_percentage = self.get_sensor().get_attribute(olitirs.CLOUD_COVER),
geometry = self.get_raster().get_attribute(raster.FOOTPRINT),
elevation_angle = 0.0,
resolution = self.get_raster().get_attribute(raster.GEOTRANSFORM)[1]
)
return information
def predict(self, X):
'''
The c5 model creates a file with the tree extension and it
will be loaded by the predict executable.
'''
save_to_file_cases(
X, create_filename(self.path, '%s.cases' % self.model_name))
local = LocalProcessLauncher()
shell_string = 'docker run --rm -v %s:/results madmex/c5_execution predict -f /results/%s' % (
self.path, self.model_name)
LOGGER.debug('Docker command: %s', shell_string)
output = local.execute(shell_string)
by_line = output.split('\n')
y = []
for line in by_line:
array_predict = line.split()
if len(array_predict) == 4:
y.append(int(float(array_predict[2])))
return numpy.array(y)
def split(self, output_directory, column=0):
'''
This method will take a input shape and iterate over its features, creating
a new shape file with each one of them. It copies all the fields and the
same spatial reference from the original file. The created files are saved
in the destination directory using the number of the field given.
'''
layer = self.get_layer()
layer_name = layer.GetName()
spatial_reference = layer.GetSpatialRef()
in_feature = layer.GetNextFeature()
layer_definition = layer.GetLayerDefn()
field_definition = layer_definition.GetFieldDefn(0)
column_name = field_definition.GetName()
shape_files = []
create_directory_path(output_directory)
in_layer_definition = layer.GetLayerDefn()
while in_feature:
in_feature_name = in_feature.GetField(column_name)
output_name = create_filename(output_directory, '%s.shp' % in_feature_name)
shape_files.append(output_name)
if is_file(output_name):
self.driver.DeleteDataSource(output_name)
data_source = self.driver.CreateDataSource(output_name)
out_layer = data_source.CreateLayer(layer_name, spatial_reference, geom_type=ogr.wkbPolygon)
for i in range(0, in_layer_definition.GetFieldCount()):
fieldDefn = in_layer_definition.GetFieldDefn(i)
out_layer.CreateField(fieldDefn)
outLayerDefn = out_layer.GetLayerDefn()
geometry = in_feature.GetGeometryRef()
out_feature = ogr.Feature(outLayerDefn)
out_feature.SetGeometry(geometry)
for i in range(0, outLayerDefn.GetFieldCount()):
out_feature.SetField(outLayerDefn.GetFieldDefn(i).GetNameRef(), in_feature.GetField(i))
out_layer.CreateFeature(out_feature)
out_feature = None
in_feature = None
in_feature = layer.GetNextFeature()
self.close()
return [Data(filename) for filename in shape_files]
def create_shape_from_json(id, json, output_directory):
'''
Given a json string containing coordinates, this method creates a shape file.
'''
create_directory_path(output_directory)
filename = create_filename(output_directory, '%s.shp' % id)
shape = Data(filename)
if is_file(filename):
shape.driver.DeleteDataSource(filename)
data_source = shape.driver.CreateDataSource(filename)
spatial_reference = osr.SpatialReference()
spatial_reference.ImportFromEPSG(4326)
layer = data_source.CreateLayer(str('layer'), spatial_reference, geom_type=ogr.wkbPolygon)
layer.CreateField(ogr.FieldDefn(str('id'), ogr.OFTString))
feature = ogr.Feature(layer.GetLayerDefn())
feature.SetField(str('id'), str(id))
geometry = ogr.CreateGeometryFromJson(str(json))
feature.SetGeometry(geometry)
layer.CreateFeature(feature)
shape.close()
return shape
def handle(self, **options):
'''
In this example command, the values that come from the user input are
added up and the result is printed in the screen.
'''
output = options['output'][0]
zip = options['path'][0]
basename = get_basename(zip)
aux_name = create_filename(output, 'aux_%s' % basename)
real_name = create_filename(output, basename)
with ZipFile(zip, 'r') as unzipped:
unzipped.extractall(create_filename(output, 'aux_%s' % basename))
path = create_filename(aux_name, 'trend_tiles')
tifs = get_contents_from_folder(path)
create_directory_path(real_name)
for tif in tifs:
source = create_filename(path, tif)
target = create_filename(real_name, tif)
tile_map(source, target, 4000, 4000, 2, 2)
remove_directory(aux_name)
def handle(self, **options):
filepath = options['path'][0]
qa_file = get_files_from_folder_filter(filepath, r'.*qa.tif$')[0]
print qa_file
with rasterio.open(create_filename(filepath, qa_file)) as src:
qa_array = src.read()
profile = src.profile
print profile
profile.update(
dtype=rasterio.ubyte,
compress='lzw',
nodata=None)
my_values = map(lambda x: bin(x)[2:].zfill(16), numpy.unique(qa_array, return_counts=True)[0])
my_count = numpy.unique(qa_array, return_counts=True)[1]
for index in range(len(my_values)):
print '%s -> %s' % (my_values[index],my_count[index])
print qa_array.shape
cloud_mask = numpy.bitwise_and(qa_array, CLEAR) >> CLEAR_SHIFT
cloud_mask_file = create_filename(filepath, 'qa.tif')
print numpy.unique(cloud_mask, return_counts=True)
print cloud_mask_file
print cloud_mask.shape
with rasterio.open(cloud_mask_file, 'w', **profile) as dst:
dst.write(cloud_mask.astype(rasterio.ubyte))
files_of_interest = get_files_from_folder_filter(filepath, r'.*sr_band[0-9]+.tif$')
bands = []
print files_of_interest
for filename in files_of_interest:
print filename
complete_path = create_filename(filepath, filename)
with rasterio.open(complete_path) as src:
bands.append(src.read())
stack = numpy.array(bands)
print stack.shape
def handle(self, **options):
'''
In this example command, the values that come from the user input are
added up and the result is printed in the screen.
'''
shape_name = options['shape'][0]
destination = options['dest'][0]
paths = options['path']
years = []
for path in paths:
years.append(self.get_year_from_path(path))
if os.path.exists(shape_name):
LOGGER.info('The file %s was found.' % shape_name)
shape_files = split_shape_into_features(shape_name, destination,
str('id'), str('nombre'), '__')
process_launcher = LocalProcessLauncher()
import time
start_time = time.time()
cover_array = []
tth_array = []
cover_file = 'cover-stats.json'
tth_name = 'tth-stats.json'
json_directory = create_filename(destination, 'cover_stats')
json_file = create_filename(json_directory, cover_file.lower())
tth_file = create_filename(json_directory, tth_name.lower())
for shape_file in shape_files:
shape_name = get_basename(shape_file).split('__')
anp_id = shape_name[0]
anp_name = shape_name[1]
basename = '%s.tif' % anp_name
dataframe = DataFrame(index=years,
columns=[0, 1, 2, 3, 4, 5, 6, 7, 8])
create_directory_path(json_directory)
print shape_file
for path in paths:
#pixel_resolution, dataset = pixel_info(path)
year = self.get_year_from_path(path)
raster_dir = create_filename(
create_filename(destination, 'raster'), year)
create_directory_path(raster_dir)
raster_file = create_filename(raster_dir, basename)
shell_command = 'gdalwarp -ot Byte -co COMPRESS=LZW -cutline %s -crop_to_cutline %s %s' % (
shape_file, path, raster_file)
print shell_command
print process_launcher.execute(shell_command)
raster_array = open_handle(raster_file)
ds = gdal.Open(raster_file)
geotransform = ds.GetGeoTransform()
x_resolution = geotransform[1]
y_resolution = geotransform[5]
pixel_area = abs(x_resolution * y_resolution)
unique_values = numpy.unique(raster_array, return_counts=True)
indexes = unique_values[0]
counts = unique_values[1]
for i in range(len(indexes)):
key = indexes[i]
dataframe.set_value(year, key,
area(int(counts[i]), pixel_area))
dataframe = dataframe.sort_index()
columns = list(dataframe.columns.values)
index = list(dataframe.index.values)
print dataframe
cover_array.append(
self.dataframe_to_json(dataframe, anp_id, anp_name))
tth_dataframe = DataFrame(columns=columns)
for i in range(len(index) - 1):
label = '%s-%s' % (index[i], index[i + 1])
tth_column = calculate_thh(dataframe.ix[i],
dataframe.ix[i + 1],
int(index[i + 1]) - int(index[i]))
for j in range(len(tth_column)):
tth_dataframe.set_value(label, j, tth_column[j])
tth_dataframe = tth_dataframe.sort_index()
print tth_dataframe
tth_array.append(
self.dataframe_to_json(tth_dataframe, anp_id, anp_name))
self.json_to_file(json_file, cover_array)
self.json_to_file(tth_file, tth_array)
print("--- %s seconds ---" % (time.time() - start_time))
def handle(self, **options):
mapgrid = '1449619'
acq = get_pair_quality(mapgrid)
for image in acq:
print image.pk_id
print image.pk_id
id = options["id"][0]
image_path = options["image"][0]
reference_path = options["reference"][0]
output = options["output"][0]
print image_path
print reference_path
image_bundle = _get_bundle_from_path(image_path)
reference_bundle = _get_bundle_from_path(reference_path)
#extents = harmonize_images([image_bundle.get_raster(), reference_bundle.get_raster()])
#print extents
#print extents['x_offset']
#print extents['y_offset']
shape = image_bundle.get_raster().get_attribute(raster.DATA_SHAPE)
invariant_array = numpy.full((shape[0], shape[1]),
INV_MASK_VALUE,
dtype=np.int)
in1 = reference_bundle.get_raster_file()
in2 = image_bundle.get_raster_file()
in_invar = create_filename(output, 'invariantPixelMask.tif')
result = create_filename(output, 'crosscorrelation_next.tif')
to_polar = create_filename(output, 'crosscorrelation_polar.tif')
create_raster_from_reference(in_invar, invariant_array,
image_bundle.get_raster_file(),
gdal.GDT_Byte)
local = LocalProcessLauncher()
volume = '%s:%s' % (output, output)
shell_array = [
'docker', 'run', '--rm', '-v', volume, 'madmex/antares',
'correlation', '-in1', in1, '-in2', in2, '-in_invar', in_invar,
'-val_invar',
'%s' % INV_MASK_VALUE, '-out', result, '-window_size',
'%s' % WINDOW_SIZE, '-max_gap',
'%s' % MAX_GAP
]
shell_string = ' '.join(shell_array)
print shell_string
if not is_file(result):
log = local.execute(shell_string)
crosscorrelation = raster.Data(result, 'GTiff')
print crosscorrelation.get_attribute(raster.PROJECTION)
print crosscorrelation.get_attribute(raster.GEOTRANSFORM)
#tile_map(result, result)
correlation_array = crosscorrelation.read_data_file_as_array()
band_0 = correlation_array[0, :]
band_1 = correlation_array[1, :]
phi_band = phi(band_0, band_1)
rho_band = rho(band_0, band_1)
correlation_array[0, :] = phi_band
correlation_array[1, :] = rho_band
#create_raster_from_reference(to_polar, correlation_array, result)
crosscorrelation_polar = raster.Data(to_polar, 'GTiff')
extents = harmonize_images(
[crosscorrelation_polar,
reference_bundle.get_raster()])
x_offset = extents['x_offset'][1]
y_offset = extents['y_offset'][1]
x_tile_size = extents['x_range']
y_tile_size = extents['y_range']
aux_name = create_filename(output, 'auxiliar.tif')
tile_map(reference_bundle.get_raster_file(), aux_name, x_tile_size,
y_tile_size, x_offset, y_offset)
aux_array = raster.Data(aux_name, 'GTiff').read_data_file_as_array()
crosscorrelation_polar_array = crosscorrelation_polar.read_data_file_as_array(
)
stats = calculate_statistics_qa(crosscorrelation_polar_array,
aux_array, STAT_CLASSES, STAT_MIN,
STAT_MAX, THRESHOLD_COD, THRESHOLD_LOG)
desision = calculate_decision(stats['band_1']['histogram'],
stats['band_1']['histogram_bins'])
print stats
quality = QualityAssessment(
decision=desision,
max=adapt_numpy_float(stats['band_1']['maximum']),
min=adapt_numpy_float(stats['band_1']['minimum']),
median=adapt_numpy_float(stats['band_1']['median']),
mean=adapt_numpy_float(stats['band_1']['mean']),
standard_deviation=adapt_numpy_float(stats['band_1']['std']),
product_id=1,
reference_id=2)
persist_quality(quality)
print desision
def handle(self, **options):
'''
In this example command, the values that come from the user input are
added up and the result is printed in the screen.
'''
random_int = 1
training_template = options['training'][0]
target_template = options['target'][0]
index_template = options['index'][0]
output = options['output'][0]
models = options['model']
features = training_template % random_int
training = target_template % random_int
index = index_template % random_int
features_array = open_handle(features)
training_array = open_handle(training)
index_array = open_handle(index)
labels = numpy.unique(index_array)
features_flatten = numpy.ravel(features_array).reshape(10, 200 * 200)
training_flatten = numpy.ravel(training_array).reshape(200 * 200)
array_aux = []
for j in range(features_array.shape[0]):
means = scipy.ndimage.measurements.mean(features_array[j, :, :],
index_array, labels)
print means.shape
array_aux.append(means)
for j in range(features_array.shape[0]):
std = scipy.ndimage.measurements.standard_deviation(
features_array[j, :, :], index_array, labels)
print std.shape
array_aux.append(std)
print len(array_aux)
features_final = numpy.concatenate([array_aux], axis=0)
print features_final.shape
label_object = []
for id in labels:
values, counts = numpy.unique(training_array[index_array == id],
return_counts=True)
label_object.append(values[numpy.argmax(counts)])
features_total = features_final
training_total = label_object
for i in range(1, 53):
features = training_template % (i + 1)
training = target_template % (i + 1)
index = index_template % (i + 1)
features_array = open_handle(features)
training_array = open_handle(training)
index_array = open_handle(index)
labels = numpy.unique(index_array)
array_aux = []
for j in range(features_array.shape[0]):
array_aux.append(
scipy.ndimage.measurements.mean(features_array[j, :, :],
index_array, labels))
for j in range(features_array.shape[0]):
array_aux.append(
scipy.ndimage.measurements.standard_deviation(
features_array[j, :, :], index_array, labels))
features_final = numpy.concatenate([array_aux], axis=0)
print features_final.shape
label_object = []
for id in labels:
values, counts = numpy.unique(
training_array[index_array == id], return_counts=True)
#print '********* ', values[numpy.argmax(counts)], counts[numpy.argmax(counts)]
label_object.append(values[numpy.argmax(counts)])
print features_total.shape
print features_final.shape
features_total = numpy.concatenate(
(features_total, features_final), axis=1)
training_total = numpy.concatenate((training_total, label_object),
axis=1)
print 'label object', len(label_object)
#print scipy.ndimage.measurements.mean(training_array, index_array, labels)
features_flatten = numpy.concatenate(
(features_flatten, numpy.ravel(features_array).reshape(
10, 200 * 200)),
axis=1)
training_flatten = numpy.concatenate(
(training_flatten, numpy.ravel(training_array).reshape(
200 * 200)),
axis=0)
# Remove the elements that map to None
#mask = training_flatten!=0
#features_flatten = features_flatten[:,mask]
#training_flatten = training_flatten[mask]
mask = training_total != 0
features_flatten = features_total[:, mask]
training_flatten = training_total[mask]
print features_flatten.shape
print training_flatten.shape
X_train, X_test, y_train, y_test = train_test_split(
numpy.transpose(features_flatten),
training_flatten,
train_size=0.8,
test_size=0.2)
print X_train[0]
print y_train[0]
unsupervised = pca.Model(5)
unsupervised.fit(X_train)
'''
X_train = unsupervised.transform(X_train)
X_test = unsupervised.transform(X_test)
'''
pca_path = create_filename(output, 'pca')
create_directory_path(pca_path)
unsupervised.save(pca_path)
import time
for model_name in models:
start_time = time.time()
train_model(X_train, X_test, y_train, y_test, output, model_name)
print "--- %s seconds training %s model---" % (
(time.time() - start_time), model_name)
def handle(self, **options):
'''
In this example command, the values that come from the user input are
added up and the result is printed in the screen.
'''
output = options['output'][0]
for path in options['path']:
bundle = _get_bundle_from_path(path)
basename = get_basename(bundle.get_raster_file())
bundle.get_NDVI()
ndvi_file = create_filename(output, 'ndvi.tif')
red_edge_ndvi_file = create_filename(output, 'red_edge_ndvi.tif')
gndvi_file = create_filename(output, 'gndvi.tif')
ndre_file = create_filename(output, 'ndre.tif')
sovel_file = create_filename(output, 'sovel2.tif')
all_file = create_filename(output,
'%s_all_features.tif' % basename)
print all_file
image_array = bundle.get_raster().read_data_file_as_array()
ndvi_array = bundle.get_NDVI()
ndvi_array[ndvi_array <= -1] = -1
ndvi_array[ndvi_array >= 1] = 1
red_edge_ndvi_array = bundle.get_red_edge_NDVI()
red_edge_ndvi_array[red_edge_ndvi_array <= -1] = -1
red_edge_ndvi_array[red_edge_ndvi_array >= 1] = 1
gndvi_array = bundle.get_gndvi()
gndvi_array[gndvi_array <= -1] = -1
gndvi_array[gndvi_array >= 1] = 1
ndre_array = bundle.get_ndre()
ndre_array[ndre_array <= -1] = -1
ndre_array[ndre_array >= 1] = 1
sobel_filter_array = bundle.get_sobel_filter(sigma=2)
#sobel_filter_array[sobel_filter_array<=-1] = -1
#sobel_filter_array[sobel_filter_array>=1] = 1
#create_raster_from_reference(ndvi_file, ndvi_array, bundle.get_raster_file())
#create_raster_from_reference(red_edge_ndvi_file, red_edge_ndvi_array, bundle.get_raster_file())
#create_raster_from_reference(gndvi_file, gndvi_array, bundle.get_raster_file())
#create_raster_from_reference(ndre_file, ndre_array, bundle.get_raster_file())
create_raster_from_reference(sovel_file, sobel_filter_array,
bundle.get_raster_file())
all_features = numpy.array([
image_array[0], image_array[1], image_array[2], image_array[3],
image_array[4], ndvi_array, red_edge_ndvi_array, gndvi_array,
ndre_array, sobel_filter_array
])
print image_array[0].shape
print image_array[1].shape
print image_array[2].shape
print image_array[3].shape
print image_array[4].shape
print ndvi_array.shape
print red_edge_ndvi_array.shape
print gndvi_array.shape
print ndre_array.shape
print sobel_filter_array.shape
print all_features.shape
create_raster_from_reference(all_file,
all_features,
bundle.get_raster_file(),
creating_options=['BIGTIFF=YES'])
def handle(self, **options):
'''
In this example command, the values that come from the user input are
added up and the result is printed in the screen.
'''
output = options['output'][0]
models = options['modelname']
model_directory = options['modeldir'][0]
region = options['region'][0]
start_time = time.time()
for path in options['path']:
print path
scene_bundle = rapideye.Bundle(path)
directory = getattr(SETTINGS, 'TEMPORARY')
directory_helper = create_filename(directory, 'helper')
create_directory_path(directory_helper)
categories_file = create_filename(directory, 'categories.json')
categories_dictionaty = {
0: "AGRICULTURA DE RIEGO",
1: "AGRICULTURA DE TEMPORAL",
2: "AGUA",
3: "AREAS QUEMADAS",
4: "ASENTAMIENTOS HUMANOS",
5: "BOSQUE CULTIVADO",
6: "BOSQUE DE AYARIN",
7: "BOSQUE DE ENCINO",
8: "BOSQUE DE ENCINO-PINO",
9: "BOSQUE DE GALERIA",
10: "BOSQUE DE MEZQUITE",
11: "BOSQUE DE OYAMEL",
12: "BOSQUE DE PINO",
13: "BOSQUE DE PINO-ENCINO",
14: "BOSQUE INDUCIDO",
15: "BOSQUE MESOFILO DE MONTANA",
16: "DESPROVISTO DE VEGETACION",
17: "INDEFINIDO",
18: "MANGLAR",
19: "MATORRAL SUBTROPICAL",
20: "MEZQUITAL",
21: "NUBES",
22: "PASTIZAL CULTIVADO",
23: "PASTIZAL HALOFILO",
24: "PASTIZAL INDUCIDO",
25: "PASTIZAL NATURAL",
26: "PRADERA DE ALTA MONTANA",
27: "SABANOIDE",
28: "SELVA ALTA PERENNIFOLIA",
29: "SELVA ALTA SUBPERENNIFOLIA",
30: "SELVA BAJA CADUCIFOLIA",
31: "SELVA BAJA ESPINOSA CADUCIFOLIA",
32: "SELVA BAJA SUBCADUCIFOLIA",
33: "SELVA DE GALERIA",
34: "SELVA MEDIANA CADUCIFOLIA",
35: "SELVA MEDIANA SUBCADUCIFOLIA",
36: "SELVA MEDIANA SUBPERENNIFOLIA",
37: "SIN VEGETACION APARENTE",
38: "SOMBRAS",
39: "TULAR",
40: "VEGETACION DE DUNAS COSTERAS",
41: "VEGETACION HALOFILA HIDROFILA",
42: "ZONA URBANA"
}
basename = get_basename(scene_bundle.get_raster_file())
all_file = create_filename(directory_helper,
'%s_all_features.tif' % basename)
if not is_file(all_file):
scene_bundle.get_feature_array(all_file)
filename = get_basename(all_file)
if not is_file(
create_filename(directory_helper, '%s.shp' % filename)):
shell_string = 'docker run --rm -v %s:/data madmex/segment gdal-segment %s.tif -out helper/%s.shp -algo SLIC -region %s' % (
directory, filename, filename, region)
launcher = LocalProcessLauncher()
LOGGER.debug('Docker command: %s', shell_string)
launcher.execute(shell_string)
data = read_data_table(
create_filename(directory_helper, '%s.shp' % filename))
results = {}
for model_name in models:
persistence_directory = create_filename(
model_directory, model_name)
print model_name
model = load_model(model_name)
model_instance = model.Model(persistence_directory)
model_instance.load(persistence_directory)
prediction = model_instance.predict(data)
results[model_name] = prediction
print results
create_directory_path(output)
write_results(
create_filename(directory_helper, '%s.shp' % filename),
create_filename(output,
'%s_classification.shp' % filename[0:32]),
results, categories_dictionaty)
LOGGER.info("--- %s seconds ---" % (time.time() - start_time))
def handle(self, **options):
'''
This process will call the change detection process from a set of two
individual images. It will perform the harmonization and the multivariate
alteration detection on the images. It will then perform a maximum
correlation factor on them and work with the resulting bands.
'''
image_a = options['ima'][0]
image_b = options['imb'][0]
output_image = options['output'][0]
LOGGER.info('Image %s will be compared against image %s. Output will be available' \
' at %s.', image_a, image_b, output_image)
gdal_format = "GTiff"
image_a_data_class = raster.Data(image_a, gdal_format)
image_b_data_class = raster.Data(image_b, gdal_format)
# TODO : remove references to class harmonized
harmonized_class = harmonized.Data(image_a_data_class, image_b_data_class)
#band1 = image_a_data_class.GetRasterBand(1)
#band1 = band1.ReadAsArray(0, 0,image_a_data_class.RasterXSize,image_a_data_class.RasterYSize).astype(float)
#print(band1)
if harmonized_class:
#data_shape_harmonized = harmonized_class.get_attribute(harmonized.DATA_SHAPE)
#width, height, bands = data_shape_harmonized
#geotransform_harmonized = harmonized_class.get_attribute(raster.GEOTRANSFORM)
#projection_harmonized = harmonized_class.get_attribute(harmonized.PROJECTION)
image_a_data_array, image_b_data_array = harmonized_class.harmonized_arrays(image_a_data_class, image_b_data_class)
imad_class = imad.Transformation([image_a_data_array, image_b_data_array])
imad_class.execute()
mad_result = imad_class.output
LOGGER.debug('mad_result.shape: %s', mad_result.shape)
create_directory_path(getattr(SETTINGS, 'TEST_FOLDER'))
mad_output_file = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'mad.tif')
create_raster_from_reference(mad_output_file, mad_result,image_a)
maf_class = maf.Transformation(imad_class.output)
maf_class.execute()
maf_result = maf_class.output
pdf_file = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'maf_pdf.png')
thresholds = calc_threshold_grid(maf_result, pdf_file)
class_result = recode_classes_grid(maf_result, thresholds)
LOGGER.debug('maf_result.shape: %s', maf_result.shape)
LOGGER.debug('class_result.shape: %s', class_result.shape)
maf_outputfile = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'maf.tif')
class_outputfile = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'class.tif')
create_raster_from_reference(maf_outputfile, maf_result, image_a)
create_raster_from_reference(class_outputfile, class_result, image_a)
print 'Output written in: %s' % mad_output_file
print 'Shape is ', imad_class.output.shape
def handle(self, **options):
'''
This is the code that does the ingestion.
indexes --path /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/Landsat_2000_2008/L5_021_047_2000_022_sr/ --shape /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/AE_LT_new.shp --output
'''
path = options['path'][0]
for mask_file in os.listdir(path):
if mask_file.endswith('_cfmask.tif'):
print mask_file
basename = mask_file.replace('_cfmask.tif', '')
print basename
cloud_file_name = mask_file
LOGGER.info('Calculating indexes for Landsat scenes.')
band_name = basename + '_sr_band%s.tif'
final_path = options['output'][0]
create_directory_path(final_path)
cloud_file = create_filename(path, cloud_file_name)
cloud_array = open_handle(cloud_file)
cloud_mask = (cloud_array == 4)
LOGGER.debug('Recognize sensor depending on file name.')
if 'L8' in path:
LOGGER.debug('Landsat 8 scene was detected.')
blue_file = create_filename(path, band_name % BLUE_L8)
green_file = create_filename(path, band_name % GREEN_L8)
red_file = create_filename(path, band_name % RED_L8)
nir_file = create_filename(path, band_name % NIR_L8)
swir_file = create_filename(path, band_name % SWIR_L8)
else:
LOGGER.debug('Landsat 4, 5 or scene was detected.')
blue_file = create_filename(path, band_name % BLUE)
green_file = create_filename(path, band_name % GREEN)
red_file = create_filename(path, band_name % RED)
nir_file = create_filename(path, band_name % NIR)
swir_file = create_filename(path, band_name % SWIR)
LOGGER.debug('Loading bands of interest.')
green_array = open_handle(green_file)
red_array = open_handle(red_file)
nir_array = open_handle(nir_file)
swir_array = open_handle(swir_file)
LOGGER.debug('Calculating indexes.')
ndvi_array = calculate_index(nir_array, red_array)
mndwi_array = calculate_index(green_array, swir_array)
ndwig_array = calculate_index(nir_array, swir_array)
ndwim_array = calculate_index(green_array, nir_array)
LOGGER.debug('Setting cloud mask values to -999.')
ndvi_array[cloud_mask] = -999
mndwi_array[cloud_mask] = -999
ndwig_array[cloud_mask] = -999
ndwim_array[cloud_mask] = -999
LOGGER.debug('Creating files for indexes.')
ndvi_final_file = create_filename(final_path, basename + '_ndvi.tif')
mndwi_final_file = create_filename(final_path, basename + '_mndwi.tif')
ndwig_final_file = create_filename(final_path, basename + '_ndwig.tif')
ndwim_final_file = create_filename(final_path, basename + '_ndwim.tif')
ndvi_clipped_file = create_filename(final_path,
basename + '_ndvi_clipped.tif')
mndwi_clipped_file = create_filename(final_path,
basename + '_mndwi_clipped.tif')
ndwig_clipped_file = create_filename(final_path,
basename + '_ndwig_clipped.tif')
ndwim_clipped_file = create_filename(final_path,
basename + '_ndwim_clipped.tif')
ndvi_file = create_filename(final_path, basename + '_ndvi_pre.tif')
mndwi_file = create_filename(final_path, basename + '_mndwi_pre.tif')
ndwig_file = create_filename(final_path, basename + '_ndwig_pre.tif')
ndwim_file = create_filename(final_path, basename + '_ndwim_pre.tif')
files = [ndvi_file, mndwi_file, ndwig_file, ndwim_file]
clipped_files = [
ndvi_clipped_file, mndwi_clipped_file, ndwig_clipped_file,
ndwim_clipped_file
]
final_files = [
ndvi_final_file, mndwi_final_file, ndwig_final_file,
ndwim_final_file
]
LOGGER.debug('Writing information to files.')
create_raster_from_reference(ndvi_file, ndvi_array, green_file)
create_raster_from_reference(mndwi_file, mndwi_array, green_file)
create_raster_from_reference(ndwig_file, ndwig_array, green_file)
create_raster_from_reference(ndwim_file, ndwim_array, green_file)
LOGGER.debug('Deleting arrays to release memory.')
del ndvi_array
del mndwi_array
del ndwig_array
del ndwim_array
del cloud_array
LOGGER.debug('Reference rgb file creation.')
rgb_file = create_filename(final_path, basename + '_rgb.tif')
merge_command = [
'/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt',
'-separate', '-o', rgb_file, red_file, green_file, blue_file
]
call(merge_command)
shape = options['shape'][0]
LOGGER.debug('Cookie cut files using the given shape.')
for i in range(4):
clip_command = [
'/Library/Frameworks/GDAL.framework/Programs/gdalwarp',
'-crop_to_cutline', '-cutline', shape, files[i],
clipped_files[i]
]
call(clip_command)
for i in range(4):
aux_array = open_handle(clipped_files[i])
aux_array[(aux_array == 0)] = -9999
create_raster_from_reference(final_files[i], aux_array,
ndvi_clipped_file)
if not options['debug']:
LOGGER.info('Remove auxiliary files.')
os.remove(ndvi_clipped_file)
os.remove(mndwi_clipped_file)
os.remove(ndwig_clipped_file)
os.remove(ndwim_clipped_file)
os.remove(ndvi_file)
os.remove(mndwi_file)
os.remove(ndwig_file)
os.remove(ndwim_file)
print 'Done'
