def raster_to_vector_mask(raster_object, output_path, no_data=[0]):
raster_array = raster_object.read_data_file_as_array()
for i in no_data:
raster_array[raster_array != i] = 0
raster_array[raster_array != 0] = 1
mask_file = create_filename(output_path, 'mask.tif')
create_raster_from_reference(mask_file, raster_array, raster_object.get_file())
mask_raster = raster.Data(mask_file)
ds = mask_raster._open_file()
rb = ds.GetRasterBand(1)
dst_layername = 'POLYGONIZED_STUFF'
drv = ogr.GetDriverByName(str('ESRI Shapefile'))
dst_ds = drv.CreateDataSource(output_path)
dst_layer = dst_ds.CreateLayer(dst_layername, srs = None )
#dst_layer.SetSpatialRef(raster.get_spatial_reference())
gdal.Polygonize(rb, None, dst_layer, -1, [])
def test_creat_image(self):
import numpy
filename = '/Users/agutierrez/Development/df/1448114/2015/2015-02-24/l3a/1448114_2015-02-24_RE3_3A_302417.tif'
width = get_width(filename)
height = get_height(filename)
plain = width * height
red = numpy.zeros(plain)
green = numpy.zeros(plain)
blue = numpy.zeros(plain)
ired = numpy.zeros(plain)
print 'before loop'
for i in range(plain):
if not i % 2:
red[i] = 1
if not i % 3:
green[i] = 1
if not i % 5:
blue[i] = 1
if not i % 7:
ired[i] = 1
print 'loop'
final = numpy.array([
numpy.reshape(red, (width, height)),
numpy.reshape(green, (width, height)),
numpy.reshape(blue, (width, height)),
numpy.reshape(ired, (width, height))
])
print final.shape
print 'hello'
create_raster_from_reference(
create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'sieve.tif'),
final, filename)
def test_creat_image(self):
import numpy
filename = "/Users/agutierrez/Development/df/1448114/2015/2015-02-24/l3a/1448114_2015-02-24_RE3_3A_302417.tif"
width = get_width(filename)
height = get_height(filename)
plain = width * height
red = numpy.zeros(plain)
green = numpy.zeros(plain)
blue = numpy.zeros(plain)
ired = numpy.zeros(plain)
print "before loop"
for i in range(plain):
if not i % 2:
red[i] = 1
if not i % 3:
green[i] = 1
if not i % 5:
blue[i] = 1
if not i % 7:
ired[i] = 1
print "loop"
final = numpy.array(
[
numpy.reshape(red, (width, height)),
numpy.reshape(green, (width, height)),
numpy.reshape(blue, (width, height)),
numpy.reshape(ired, (width, height)),
]
)
print final.shape
print "hello"
create_raster_from_reference(create_file_name(getattr(SETTINGS, "TEST_FOLDER"), "sieve.tif"), final, filename)
def get_feature_array(self, outputfile):
'''
This method creates an array of features from the bands
of this raster.
'''
image_array = self.get_raster().read_data_file_as_array()
ndvi_array = self.get_NDVI()
ndvi_array[ndvi_array <= -1] = -1
ndvi_array[ndvi_array >= 1] = 1
red_edge_ndvi_array = self.get_red_edge_NDVI()
red_edge_ndvi_array[red_edge_ndvi_array <= -1] = -1
red_edge_ndvi_array[red_edge_ndvi_array >= 1] = 1
gndvi_array = self.get_gndvi()
gndvi_array[gndvi_array <= -1] = -1
gndvi_array[gndvi_array >= 1] = 1
ndre_array = self.get_ndre()
ndre_array[ndre_array <= -1] = -1
ndre_array[ndre_array >= 1] = 1
sobel_filter_array = self.get_sobel_filter(sigma=2)
all_features = numpy.array([
image_array[0], image_array[1], image_array[2], image_array[3],
image_array[4], ndvi_array, red_edge_ndvi_array, gndvi_array,
ndre_array, sobel_filter_array
])
create_raster_from_reference(outputfile,
all_features,
self.get_raster_file(),
creating_options=['BIGTIFF=YES'])
return raster.Data(outputfile)
def handle(self, **options):
paths = options['path']
output = options['output'][0]
print output
arrays = {}
import time
for path in paths:
start_time = time.time()
arrays[path] = open_handle(path)
print 'Opened %s' % path
print("--- %s seconds ---" % (time.time() - start_time))
size = arrays[paths[0]].shape
start_time = time.time()
result = numpy.full(size, NO_DATA)
mask = numpy.equal(arrays[paths[0]],arrays[paths[1]])
for p in range(2, len(paths)):
print 'Image %s ' % p
mask = numpy.logical_and(mask, numpy.equal(arrays[paths[0]],arrays[paths[p]]))
result[mask] = numpy.array(arrays[paths[0]])[mask]
for path in paths:
del arrays[path]
print("--- %s seconds ---" % (time.time() - start_time))
start_time = time.time()
create_raster_from_reference(output, result, paths[0], data_type=gdal.GDT_Byte)
print("--- %s seconds ---" % (time.time() - start_time))
def preprocess(self):
'''
Top of atmosphere is calculated and persisted into a file. Then a cloud
mask is created with the given algorithm.
'''
solar_zenith = self.get_sensor().parser.get_attribute(
rapideye.SOLAR_ZENITH)
data_acquisition_date = self.get_sensor().parser.get_attribute(
rapideye.ACQUISITION_DATE)
solar_azimuth = self.get_sensor().parser.get_attribute(
rapideye.SOLAR_AZIMUTH)
geotransform = self.get_raster().get_attribute(raster.GEOTRANSFORM)
data = self.get_raster().read_data_file_as_array()
sun_earth_distance = calculate_distance_sun_earth(
data_acquisition_date)
top_of_atmosphere_data = calculate_toa_rapideye(
calculate_rad_rapideye(data), sun_earth_distance, solar_zenith)
top_of_atmosphere_directory = create_filename(get_parent(self.path),
'TOA')
create_directory_path(top_of_atmosphere_directory)
output_file = create_filename(
top_of_atmosphere_directory,
get_basename(self.get_files()[2]) +
'_toa.tif') # TODO: change [2] in self.get_files()[2]
create_raster_from_reference(output_file,
top_of_atmosphere_data,
self.file_dictionary[_IMAGE],
data_type=NumericTypeCodeToGDALTypeCode(
numpy.float32))
LOGGER.debug('Top of atmosphere file was created.')
cloud_output_file = create_filename(
top_of_atmosphere_directory,
get_basename(self.get_files()[2]) + '_cloud.tif')
if self.algorithm == ANOMALY_DETECTION:
LOGGER.debug('Cloud mask by anomaly detection process.')
clouds = self.anomaly_detection_cloud_mask(top_of_atmosphere_data,
cloud_output_file,
solar_zenith,
solar_azimuth,
geotransform)
elif self.algorithm == TIME_SERIES:
LOGGER.debug('Cloud mask by reference with time series process.')
tile_id = self.get_sensor().get_attribute(TILE_ID)
clouds = self.masking_with_time_series(data, cloud_output_file,
solar_zenith, solar_azimuth,
geotransform, tile_id)
create_raster_from_reference(cloud_output_file,
clouds,
self.file_dictionary[_IMAGE],
data_type=NumericTypeCodeToGDALTypeCode(
numpy.float32))
LOGGER.info('Cloud mask was created.')
def method_five(self, path, output):
'''
Using numpy utilities, this method mask the desired value.
'''
data_array = open_handle(path)
data_array[data_array!=MASK] = 0
data_array[data_array==MASK] = 1
create_raster_from_reference(output, data_array, path, gdal.GDT_Byte)
del data_array
def method_five(self, path, output):
'''
Using numpy utilities, this method mask the desired value.
'''
data_array = open_handle(path)
data_array[data_array != MASK] = 0
data_array[data_array == MASK] = 1
create_raster_from_reference(output, data_array, path, gdal.GDT_Byte)
del data_array
def mask_iterating_values(self, path, output):
'''
Iterating over the values in the initial and final arrays this replaces the
desired values one by one.
'''
data_array = open_handle(path)
my_dictionary = dictionary_from_list(INITIAL_ARRAY, FINAL_ARRAY)
to_vector_array = replace_in_array(data_array, my_dictionary)
create_raster_from_reference(output, to_vector_array, path, gdal.GDT_Byte)
def mask_iterating_values(self, path, output, ini_arr, fin_arr):
'''
Iterating over the values in the initial and final arrays this replaces the
desired values one by one.
'''
data_array = open_handle(path)
my_dictionary = dictionary_from_list(ini_arr, fin_arr)
to_vector_array = replace_in_array(data_array, my_dictionary)
create_raster_from_reference(output, to_vector_array, path,
gdal.GDT_Byte)
def handle(self, **options):
'''
In this example command, the values that come from the user input are
added up and the result is printed in the screen.
'''
output = options['output'][0]
models = options['modelname']
model_directory = options['modeldir'][0]
pca_model = pca.Model(5)
pca_model.load(create_filename(model_directory, 'pca'))
for model_name in models:
persistence_directory = create_filename(model_directory,
model_name)
model = load_model(model_name)
model_instance = model.Model(persistence_directory)
model_instance.load(persistence_directory)
block_size = 500
for path in options['path']:
image_array = open_handle(path)
y_size = image_array.shape[1]
x_size = image_array.shape[2]
basename = get_basename(path)[:7]
warnings.filterwarnings('ignore')
final = numpy.zeros((x_size, y_size))
import time
start_time = time.time()
for i in range(0, y_size, block_size):
if i + block_size < y_size:
rows = block_size
else:
rows = y_size - i
for j in range(0, x_size, block_size):
if j + block_size < x_size:
cols = block_size
else:
cols = x_size - j
step = image_array[:, i:i + rows, j:j + cols]
step_ravel = step.reshape(10, -1)
prediction = model_instance.predict(
pca_model.transform(numpy.transpose(step_ravel)))
final[i:i + rows, j:j + cols] = prediction.reshape(
(rows, cols))
print("--- %s seconds ---" % (time.time() - start_time))
create_directory_path(output)
classification = create_filename(
output, '%s-%s.tif' % (basename, model_name))
create_raster_from_reference(classification,
final.reshape(x_size, y_size),
path,
data_type=gdal.GDT_Byte,
creating_options=['COMPRESS=LZW'])
def preprocess(self):
'''
Top of atmosphere is calculated and persisted into a file. Then a cloud
mask is created with the given algorithm.
'''
solar_zenith = self.get_sensor().parser.get_attribute(rapideye.SOLAR_ZENITH)
data_acquisition_date = self.get_sensor().parser.get_attribute(rapideye.ACQUISITION_DATE)
solar_azimuth = self.get_sensor().parser.get_attribute(rapideye.SOLAR_AZIMUTH)
geotransform = self.get_raster().get_attribute(raster.GEOTRANSFORM)
data = self.get_raster().read_data_file_as_array()
sun_earth_distance = calculate_distance_sun_earth(data_acquisition_date)
top_of_atmosphere_data = calculate_toa_rapideye(calculate_rad_rapideye(data), sun_earth_distance, solar_zenith)
top_of_atmosphere_directory = create_file_name(get_parent(self.path), 'TOA')
create_directory_path(top_of_atmosphere_directory)
output_file = create_file_name(top_of_atmosphere_directory, get_base_name(self.get_files()[2]) + '_toa.tif') # TODO: change [2] in self.get_files()[2]
create_raster_from_reference(output_file,
top_of_atmosphere_data,
self.file_dictionary[_IMAGE],
data_type=NumericTypeCodeToGDALTypeCode(numpy.float32)
)
LOGGER.debug('Top of atmosphere file was created.')
cloud_output_file = create_file_name(top_of_atmosphere_directory, get_base_name(self.get_files()[2]) + '_cloud.tif')
if self.algorithm == ANOMALY_DETECTION:
LOGGER.debug('Cloud mask by anomaly detection process.')
clouds = self.anomaly_detection_cloud_mask(top_of_atmosphere_data, cloud_output_file, solar_zenith, solar_azimuth, geotransform)
elif self.algorithm == TIME_SERIES:
LOGGER.debug('Cloud mask by reference with time series process.')
tile_id = self.get_sensor().get_attribute(TILE_ID)
clouds = self.masking_with_time_series(data, cloud_output_file, solar_zenith, solar_azimuth, geotransform, tile_id)
create_raster_from_reference(cloud_output_file,
clouds,
self.file_dictionary[_IMAGE],
data_type=NumericTypeCodeToGDALTypeCode(numpy.float32)
)
LOGGER.info('Cloud mask was created.')
def handle(self, **options):
paths = options['path']
output = options['output'][0]
print output
arrays = {}
image1 = open_handle(paths[0])
#import time
#for path in paths:
# start_time = time.time()
# arrays[path] = open_handle(path)
# print 'Opened %s' % path
# print("--- %s seconds ---" % (time.time() - start_time))
size = image1.shape
#start_time = time.time()
for p in range(2, len(paths)):
#print 'Image %s ' % p
image2 = open_handle(paths[p])
mask = numpy.logical_and(
numpy.equal(image1, open_handle(paths[1])),
numpy.equal(image1, image2))
image1[~mask] = NO_DATA
#for path in paths:
#del arrays[path]
#print("--- %s seconds ---" % (time.time() - start_time))
#start_time = time.time()
create_raster_from_reference(output,
image1y,
paths[0],
data_type=gdal.GDT_Byte)
def handle(self, **options):
mapgrid = '1449619'
acq = get_pair_quality(mapgrid)
for image in acq:
print image.pk_id
print image.pk_id
id = options["id"][0]
image_path = options["image"][0]
reference_path = options["reference"][0]
output = options["output"][0]
print image_path
print reference_path
image_bundle = _get_bundle_from_path(image_path)
reference_bundle = _get_bundle_from_path(reference_path)
#extents = harmonize_images([image_bundle.get_raster(), reference_bundle.get_raster()])
#print extents
#print extents['x_offset']
#print extents['y_offset']
shape = image_bundle.get_raster().get_attribute(raster.DATA_SHAPE)
invariant_array = numpy.full((shape[0], shape[1]),
INV_MASK_VALUE,
dtype=np.int)
in1 = reference_bundle.get_raster_file()
in2 = image_bundle.get_raster_file()
in_invar = create_filename(output, 'invariantPixelMask.tif')
result = create_filename(output, 'crosscorrelation_next.tif')
to_polar = create_filename(output, 'crosscorrelation_polar.tif')
create_raster_from_reference(in_invar, invariant_array,
image_bundle.get_raster_file(),
gdal.GDT_Byte)
local = LocalProcessLauncher()
volume = '%s:%s' % (output, output)
shell_array = [
'docker', 'run', '--rm', '-v', volume, 'madmex/antares',
'correlation', '-in1', in1, '-in2', in2, '-in_invar', in_invar,
'-val_invar',
'%s' % INV_MASK_VALUE, '-out', result, '-window_size',
'%s' % WINDOW_SIZE, '-max_gap',
'%s' % MAX_GAP
]
shell_string = ' '.join(shell_array)
print shell_string
if not is_file(result):
log = local.execute(shell_string)
crosscorrelation = raster.Data(result, 'GTiff')
print crosscorrelation.get_attribute(raster.PROJECTION)
print crosscorrelation.get_attribute(raster.GEOTRANSFORM)
#tile_map(result, result)
correlation_array = crosscorrelation.read_data_file_as_array()
band_0 = correlation_array[0, :]
band_1 = correlation_array[1, :]
phi_band = phi(band_0, band_1)
rho_band = rho(band_0, band_1)
correlation_array[0, :] = phi_band
correlation_array[1, :] = rho_band
#create_raster_from_reference(to_polar, correlation_array, result)
crosscorrelation_polar = raster.Data(to_polar, 'GTiff')
extents = harmonize_images(
[crosscorrelation_polar,
reference_bundle.get_raster()])
x_offset = extents['x_offset'][1]
y_offset = extents['y_offset'][1]
x_tile_size = extents['x_range']
y_tile_size = extents['y_range']
aux_name = create_filename(output, 'auxiliar.tif')
tile_map(reference_bundle.get_raster_file(), aux_name, x_tile_size,
y_tile_size, x_offset, y_offset)
aux_array = raster.Data(aux_name, 'GTiff').read_data_file_as_array()
crosscorrelation_polar_array = crosscorrelation_polar.read_data_file_as_array(
)
stats = calculate_statistics_qa(crosscorrelation_polar_array,
aux_array, STAT_CLASSES, STAT_MIN,
STAT_MAX, THRESHOLD_COD, THRESHOLD_LOG)
desision = calculate_decision(stats['band_1']['histogram'],
stats['band_1']['histogram_bins'])
print stats
quality = QualityAssessment(
decision=desision,
max=adapt_numpy_float(stats['band_1']['maximum']),
min=adapt_numpy_float(stats['band_1']['minimum']),
median=adapt_numpy_float(stats['band_1']['median']),
mean=adapt_numpy_float(stats['band_1']['mean']),
standard_deviation=adapt_numpy_float(stats['band_1']['std']),
product_id=1,
reference_id=2)
persist_quality(quality)
print desision
def handle(self, **options):
'''
In this example command, the values that come from the user input are
added up and the result is printed in the screen.
'''
sum_of_numbers = 0
#path_a = options['a'][0]
#path_b = options['b'][0]
tiles = ["086W_20N","086W_21N","087W_17N","087W_18N","087W_19N","087W_20N","087W_21N","088W_16N","088W_17N","088W_18N","088W_19N","088W_20N","088W_21N","089W_15N","089W_16N","089W_17N","089W_18N","089W_19N","089W_20N","089W_21N","090W_14N","090W_15N","090W_16N","090W_17N","090W_18N","090W_19N","090W_20N","090W_21N","091W_14N","091W_15N","091W_16N","091W_17N","091W_18N","091W_19N","092W_14N","092W_15N","092W_16N","092W_17N","092W_18N","093W_15N","093W_16N","093W_17N","093W_18N","094W_16N","094W_17N","094W_18N","095W_15N","095W_16N","095W_17N","095W_18N","095W_19N","096W_15N","096W_16N","096W_17N","096W_18N","096W_19N","096W_20N","097W_15N","097W_16N","097W_17N","097W_18N","097W_19N","097W_20N","097W_21N","097W_22N","097W_23N","097W_24N","097W_25N","097W_26N","097W_27N","098W_16N","098W_17N","098W_18N","098W_19N","098W_20N","098W_21N","098W_22N","098W_23N","098W_24N","098W_25N","098W_26N","098W_27N","098W_28N","099W_16N","099W_17N","099W_18N","099W_19N","099W_20N","099W_21N","099W_22N","099W_23N","099W_24N","099W_25N","099W_26N","099W_27N","099W_28N","099W_29N","099W_30N","100W_16N","100W_17N","100W_18N","100W_19N","100W_20N","100W_21N","100W_22N","100W_23N","100W_24N","100W_25N","100W_26N","100W_27N","100W_28N","100W_29N","100W_30N","101W_17N","101W_18N","101W_19N","101W_20N","101W_21N","101W_22N","101W_23N","101W_24N","101W_25N","101W_26N","101W_27N","101W_28N","101W_29N","101W_30N","102W_17N","102W_18N","102W_19N","102W_20N","102W_21N","102W_22N","102W_23N","102W_24N","102W_25N","102W_26N","102W_27N","102W_28N","102W_29N","102W_30N","103W_18N","103W_19N","103W_20N","103W_21N","103W_22N","103W_23N","103W_24N","103W_25N","103W_26N","103W_27N","103W_28N","103W_29N","103W_30N","103W_31N","104W_18N","104W_19N","104W_20N","104W_21N","104W_22N","104W_23N","104W_24N","104W_25N","104W_26N","104W_27N","104W_28N","104W_29N","104W_30N","104W_31N","105W_19N","105W_20N","105W_21N","105W_22N","105W_23N","105W_24N","105W_25N","105W_26N","105W_27N","105W_28N","105W_29N","105W_30N","105W_31N","105W_32N","106W_21N","106W_22N","106W_23N","106W_24N","106W_25N","106W_26N","106W_27N","106W_28N","106W_29N","106W_30N","106W_31N","106W_32N","107W_23N","107W_24N","107W_25N","107W_26N","107W_27N","107W_28N","107W_29N","107W_30N","107W_31N","107W_32N","108W_24N","108W_25N","108W_26N","108W_27N","108W_28N","108W_29N","108W_30N","108W_31N","108W_32N","109W_22N","109W_23N","109W_24N","109W_25N","109W_26N","109W_27N","109W_28N","109W_29N","109W_30N","109W_31N","109W_32N","110W_22N","110W_23N","110W_24N","110W_25N","110W_27N","110W_28N","110W_29N","110W_30N","110W_31N","110W_32N","111W_24N","111W_25N","111W_26N","111W_27N","111W_28N","111W_29N","111W_30N","111W_31N","111W_32N","112W_24N","112W_25N","112W_26N","112W_27N","112W_28N","112W_29N","112W_30N","112W_31N","112W_32N","113W_26N","113W_27N","113W_28N","113W_29N","113W_30N","113W_31N","113W_32N","113W_33N","114W_26N","114W_27N","114W_28N","114W_29N","114W_30N","114W_31N","114W_32N","114W_33N","115W_27N","115W_28N","115W_29N","115W_30N","115W_31N","115W_32N","115W_33N","116W_30N","116W_31N","116W_32N","116W_33N","117W_32N","117W_33N"]
print len(tiles)
path_1985 = '/Users/agutierrez/Dropbox/Multivariado/classification/classification_1985.tif'
path_2009 = '/Users/agutierrez/Dropbox/Multivariado/classification/classification_2009.tif'
path_change = '/Users/agutierrez/Dropbox/Multivariado/change.tif'
path_change_1985 = '/Users/agutierrez/Dropbox/Multivariado/classification/change_1985.tif'
path_change_2009 = '/Users/agutierrez/Dropbox/Multivariado/classification/change_2009.tif'
gdal_format = 'GTiff'
image_1985 = raster.Data(path_1985, gdal_format)
image_2009 = raster.Data(path_2009, gdal_format)
image_change = raster.Data(path_change, gdal_format)
array_1985 = image_1985.read_data_file_as_array()
array_2009 = image_2009.read_data_file_as_array()
array_change = image_change.read_data_file_as_array()
print array_1985.shape
print array_2009.shape
print numpy.unique(array_1985, return_counts=True)[0]
print numpy.unique(array_1985, return_counts=True)[1] * 30 * 30 * 0.0001
print numpy.unique(array_1985[array_change > .5], return_counts=True)
print numpy.unique(array_2009[array_change > .5], return_counts=True)
array_1985_masked = array_1985[array_change > .5]
array_2009_masked = array_2009[array_change > .5]
counts ={'1.0->2.0':0,
'1.0->3.0':0,
'1.0->4.0':0,
'1.0->5.0':0,
'2.0->1.0':0,
'2.0->3.0':0,
'2.0->4.0':0,
'2.0->5.0':0,
'3.0->1.0':0,
'3.0->2.0':0,
'3.0->4.0':0,
'3.0->5.0':0,
'4.0->1.0':0,
'4.0->2.0':0,
'4.0->3.0':0,
'4.0->5.0':0,
'5.0->1.0':0,
'5.0->2.0':0,
'5.0->3.0':0,
'5.0->4.0':0}
for i in range(len(array_1985_masked)):
if not array_1985_masked[i] == array_2009_masked[i]:
counts['%s->%s' % (array_1985_masked[i], array_2009_masked[i])] = counts['%s->%s' % (array_1985_masked[i], array_2009_masked[i])] + 30 * 30 * 0.0001
import json
print json.dumps(counts, indent=1)
array_1985[array_change < .5] = 0
array_2009[array_change < .5] = 0
create_raster_from_reference(path_change_1985, array_1985, path_1985, data_type=gdal.GDT_Byte)
create_raster_from_reference(path_change_2009, array_2009, path_1985, data_type=gdal.GDT_Byte)
'''
init = 1985
gdal_format = 'GTiff'
for tile in tiles:
for i in range(30):
path_a = '/LUSTRE/MADMEX/tasks/2016_tasks/matt_hansen_forests/tiles/%s/%s_%s.tif' % (tile, tile, init+i)
path_b = '/LUSTRE/MADMEX/tasks/2016_tasks/matt_hansen_forests/tiles/%s/%s_%s.tif' % (tile, tile, init+i+1)
path_c = '/LUSTRE/MADMEX/tasks/2016_tasks/matt_hansen_forests/tiles_example/%s/%s_%s_%s.tif' % (tile, tile, init+i,init+i+1)
directory = '/LUSTRE/MADMEX/tasks/2016_tasks/matt_hansen_forests/tiles_example/%s/' % tile
print path_a
print path_b
print path_c
#remove_file(path_c)
create_directory_path(directory)
if not check_if_file_exists(path_c):
image_a = raster.Data(path_a, gdal_format)
image_b = raster.Data(path_b, gdal_format)
array_a = image_a.read_data_file_as_array()
array_b = image_b.read_data_file_as_array()
#mask = ((array_a < 30) | (70 < array_a)) & ((array_b < 30) | (70 < array_b))
#array_a[array_a <= 30] = 0
#array_a[30 < array_a] = 1
#array_b[array_b <= 30] = 0
#array_b[30 < array_b] = 1
#diff = array_b - array_a
diff = numpy.zeros(numpy.shape(array_a))
upper = 30
lower = 10
diff[(array_b < lower) & (array_a > upper)] = 1
diff[(array_b > upper) & (array_a < lower)] = 2
#diff[mask] = -9999
create_raster_from_reference(path_c, diff, path_a, data_type=gdal.GDT_Byte)
else:
print 'File exists.'
print 'Done %s-%s' % (init+i,init+i)
'''
'''
total = {}
for state in ['Campeche','Chiapas','Oaxaca','Quintana_Roo','Yucatan']:
total_state = {}
for where in ['anps', 'corr-anp', 'est-corr-anp']:
raster_path = '/LUSTRE/MADMEX/staging/2017_tasks/corredor_biologico/analisis_por_estado-vegetacion/shapes_area_interes_por_estado/%s/%s_%s.tif' % (state, state, where)
print raster_path
image = raster.Data(raster_path, gdal_format)
array = image.read_data_file_as_array()
array[array > 0] = 1
counts = numpy.unique(array, return_counts=True)
print counts
total_state[where] = counts[1][1]
total[state] = total_state
print total
init = 1985
for state in ['Campeche','Chiapas','Oaxaca','Quintana_Roo','Yucatan']:
statistics_by_state = []
for where in ['anps', 'corr-anp', 'est-corr-anp']:
for i in range(30):
mask_path = '/LUSTRE/MADMEX/staging/2017_tasks/corredor_biologico/analisis_por_estado-vegetacion/shapes_area_interes_por_estado/%s/%s_%s.tif' % (state, state, where)
mask_image = raster.Data(mask_path, gdal_format)
mask_array = mask_image.read_data_file_as_array()
path_c = '/LUSTRE/MADMEX/staging/2017_tasks/corredor_biologico/analisis_por_estado-vegetacion/corte_por_area_interes_en_estado/diff-michael-repro/Mexico_TCC_%s_%s_%s_%s.tif' % (state, init+i,init+i+1,where)
print path_c
image_c = raster.Data(path_c, gdal_format)
array_c = image_c.read_data_file_as_array()
#array_c = array_c[array_c > -9999]
counts = numpy.unique(array_c, return_counts=True)
print counts
#size_non_zero = len(array_c)
#final = numpy.zeros(size_non_zero, dtype=numpy.int)
#final[(-40 < array_c) & (array_c < 40)] = 0
#final[40 < array_c] = 1
#final[array_c < -40] = 2
x_resolution = image_c.get_geotransform()[1]
y_resolution = image_c.get_geotransform()[5]
area = x_resolution * y_resolution
stats = {}
stats['period'] = '%s-%s' % (init+i,init+i+1)
stats['type'] = where
stats['resolution'] = area
if len(counts[1]) > 2:
stats['negative'] = counts[1][0]
stats['no-change'] = counts[1][1]
stats['positive'] = counts[1][2]
else:
stats['negative'] = 0
stats['no-change'] = counts[1][0]
stats['positive'] = 0
#stats['net'] = (counts[1][1] - counts[1][2])
stats['total'] = total[state][where]
statistics_by_state.append(stats)
print statistics_by_state
stats_path = '/LUSTRE/MADMEX/staging/2017_tasks/corredor_biologico/analisis_por_estado-vegetacion/shapes_area_interes_por_estado/stats/estadisticas-michael-repro-%s.json' % state
import json
with open(stats_path, 'w') as outfile:
json.dump(statistics_by_state, outfile)
'''
def handle(self, **options):
'''
In this example command, the values that come from the user input are
added up and the result is printed in the screen.
'''
output = options['output'][0]
for path in options['path']:
bundle = _get_bundle_from_path(path)
basename = get_basename(bundle.get_raster_file())
bundle.get_NDVI()
ndvi_file = create_filename(output, 'ndvi.tif')
red_edge_ndvi_file = create_filename(output, 'red_edge_ndvi.tif')
gndvi_file = create_filename(output, 'gndvi.tif')
ndre_file = create_filename(output, 'ndre.tif')
sovel_file = create_filename(output, 'sovel2.tif')
all_file = create_filename(output,
'%s_all_features.tif' % basename)
print all_file
image_array = bundle.get_raster().read_data_file_as_array()
ndvi_array = bundle.get_NDVI()
ndvi_array[ndvi_array <= -1] = -1
ndvi_array[ndvi_array >= 1] = 1
red_edge_ndvi_array = bundle.get_red_edge_NDVI()
red_edge_ndvi_array[red_edge_ndvi_array <= -1] = -1
red_edge_ndvi_array[red_edge_ndvi_array >= 1] = 1
gndvi_array = bundle.get_gndvi()
gndvi_array[gndvi_array <= -1] = -1
gndvi_array[gndvi_array >= 1] = 1
ndre_array = bundle.get_ndre()
ndre_array[ndre_array <= -1] = -1
ndre_array[ndre_array >= 1] = 1
sobel_filter_array = bundle.get_sobel_filter(sigma=2)
#sobel_filter_array[sobel_filter_array<=-1] = -1
#sobel_filter_array[sobel_filter_array>=1] = 1
#create_raster_from_reference(ndvi_file, ndvi_array, bundle.get_raster_file())
#create_raster_from_reference(red_edge_ndvi_file, red_edge_ndvi_array, bundle.get_raster_file())
#create_raster_from_reference(gndvi_file, gndvi_array, bundle.get_raster_file())
#create_raster_from_reference(ndre_file, ndre_array, bundle.get_raster_file())
create_raster_from_reference(sovel_file, sobel_filter_array,
bundle.get_raster_file())
all_features = numpy.array([
image_array[0], image_array[1], image_array[2], image_array[3],
image_array[4], ndvi_array, red_edge_ndvi_array, gndvi_array,
ndre_array, sobel_filter_array
])
print image_array[0].shape
print image_array[1].shape
print image_array[2].shape
print image_array[3].shape
print image_array[4].shape
print ndvi_array.shape
print red_edge_ndvi_array.shape
print gndvi_array.shape
print ndre_array.shape
print sobel_filter_array.shape
print all_features.shape
create_raster_from_reference(all_file,
all_features,
bundle.get_raster_file(),
creating_options=['BIGTIFF=YES'])
def handle(self, **options):
'''
This process will call the change detection process from a set of two
individual images. It will perform the harmonization and the multivariate
alteration detection on the images. It will then perform a maximum
correlation factor on them and work with the resulting bands.
'''
image_a = options['ima'][0]
image_b = options['imb'][0]
output_image = options['output'][0]
LOGGER.info('Image %s will be compared against image %s. Output will be available' \
' at %s.', image_a, image_b, output_image)
gdal_format = "GTiff"
image_a_data_class = raster.Data(image_a, gdal_format)
image_b_data_class = raster.Data(image_b, gdal_format)
# TODO : remove references to class harmonized
harmonized_class = harmonized.Data(image_a_data_class, image_b_data_class)
#band1 = image_a_data_class.GetRasterBand(1)
#band1 = band1.ReadAsArray(0, 0,image_a_data_class.RasterXSize,image_a_data_class.RasterYSize).astype(float)
#print(band1)
if harmonized_class:
#data_shape_harmonized = harmonized_class.get_attribute(harmonized.DATA_SHAPE)
#width, height, bands = data_shape_harmonized
#geotransform_harmonized = harmonized_class.get_attribute(raster.GEOTRANSFORM)
#projection_harmonized = harmonized_class.get_attribute(harmonized.PROJECTION)
image_a_data_array, image_b_data_array = harmonized_class.harmonized_arrays(image_a_data_class, image_b_data_class)
imad_class = imad.Transformation([image_a_data_array, image_b_data_array])
imad_class.execute()
mad_result = imad_class.output
LOGGER.debug('mad_result.shape: %s', mad_result.shape)
create_directory_path(getattr(SETTINGS, 'TEST_FOLDER'))
mad_output_file = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'mad.tif')
create_raster_from_reference(mad_output_file, mad_result,image_a)
maf_class = maf.Transformation(imad_class.output)
maf_class.execute()
maf_result = maf_class.output
pdf_file = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'maf_pdf.png')
thresholds = calc_threshold_grid(maf_result, pdf_file)
class_result = recode_classes_grid(maf_result, thresholds)
LOGGER.debug('maf_result.shape: %s', maf_result.shape)
LOGGER.debug('class_result.shape: %s', class_result.shape)
maf_outputfile = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'maf.tif')
class_outputfile = create_filename(getattr(SETTINGS, 'TEST_FOLDER'), 'class.tif')
create_raster_from_reference(maf_outputfile, maf_result, image_a)
create_raster_from_reference(class_outputfile, class_result, image_a)
print 'Output written in: %s' % mad_output_file
print 'Shape is ', imad_class.output.shape
def handle(self, **options):
'''
This is the code that does the ingestion.
indexes --path /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/Landsat_2000_2008/L5_021_047_2000_022_sr/ --shape /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/AE_LT_new.shp --output
'''
path = options['path'][0]
for mask_file in os.listdir(path):
if mask_file.endswith('_cfmask.tif'):
print mask_file
basename = mask_file.replace('_cfmask.tif', '')
print basename
cloud_file_name = mask_file
LOGGER.info('Calculating indexes for Landsat scenes.')
band_name = basename + '_sr_band%s.tif'
final_path = options['output'][0]
create_directory_path(final_path)
cloud_file = create_filename(path, cloud_file_name)
cloud_array = open_handle(cloud_file)
cloud_mask = (cloud_array == 4)
LOGGER.debug('Recognize sensor depending on file name.')
if 'L8' in path:
LOGGER.debug('Landsat 8 scene was detected.')
blue_file = create_filename(path, band_name % BLUE_L8)
green_file = create_filename(path, band_name % GREEN_L8)
red_file = create_filename(path, band_name % RED_L8)
nir_file = create_filename(path, band_name % NIR_L8)
swir_file = create_filename(path, band_name % SWIR_L8)
else:
LOGGER.debug('Landsat 4, 5 or scene was detected.')
blue_file = create_filename(path, band_name % BLUE)
green_file = create_filename(path, band_name % GREEN)
red_file = create_filename(path, band_name % RED)
nir_file = create_filename(path, band_name % NIR)
swir_file = create_filename(path, band_name % SWIR)
LOGGER.debug('Loading bands of interest.')
green_array = open_handle(green_file)
red_array = open_handle(red_file)
nir_array = open_handle(nir_file)
swir_array = open_handle(swir_file)
LOGGER.debug('Calculating indexes.')
ndvi_array = calculate_index(nir_array, red_array)
mndwi_array = calculate_index(green_array, swir_array)
ndwig_array = calculate_index(nir_array, swir_array)
ndwim_array = calculate_index(green_array, nir_array)
LOGGER.debug('Setting cloud mask values to -999.')
ndvi_array[cloud_mask] = -999
mndwi_array[cloud_mask] = -999
ndwig_array[cloud_mask] = -999
ndwim_array[cloud_mask] = -999
LOGGER.debug('Creating files for indexes.')
ndvi_final_file = create_filename(final_path, basename + '_ndvi.tif')
mndwi_final_file = create_filename(final_path, basename + '_mndwi.tif')
ndwig_final_file = create_filename(final_path, basename + '_ndwig.tif')
ndwim_final_file = create_filename(final_path, basename + '_ndwim.tif')
ndvi_clipped_file = create_filename(final_path,
basename + '_ndvi_clipped.tif')
mndwi_clipped_file = create_filename(final_path,
basename + '_mndwi_clipped.tif')
ndwig_clipped_file = create_filename(final_path,
basename + '_ndwig_clipped.tif')
ndwim_clipped_file = create_filename(final_path,
basename + '_ndwim_clipped.tif')
ndvi_file = create_filename(final_path, basename + '_ndvi_pre.tif')
mndwi_file = create_filename(final_path, basename + '_mndwi_pre.tif')
ndwig_file = create_filename(final_path, basename + '_ndwig_pre.tif')
ndwim_file = create_filename(final_path, basename + '_ndwim_pre.tif')
files = [ndvi_file, mndwi_file, ndwig_file, ndwim_file]
clipped_files = [
ndvi_clipped_file, mndwi_clipped_file, ndwig_clipped_file,
ndwim_clipped_file
]
final_files = [
ndvi_final_file, mndwi_final_file, ndwig_final_file,
ndwim_final_file
]
LOGGER.debug('Writing information to files.')
create_raster_from_reference(ndvi_file, ndvi_array, green_file)
create_raster_from_reference(mndwi_file, mndwi_array, green_file)
create_raster_from_reference(ndwig_file, ndwig_array, green_file)
create_raster_from_reference(ndwim_file, ndwim_array, green_file)
LOGGER.debug('Deleting arrays to release memory.')
del ndvi_array
del mndwi_array
del ndwig_array
del ndwim_array
del cloud_array
LOGGER.debug('Reference rgb file creation.')
rgb_file = create_filename(final_path, basename + '_rgb.tif')
merge_command = [
'/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt',
'-separate', '-o', rgb_file, red_file, green_file, blue_file
]
call(merge_command)
shape = options['shape'][0]
LOGGER.debug('Cookie cut files using the given shape.')
for i in range(4):
clip_command = [
'/Library/Frameworks/GDAL.framework/Programs/gdalwarp',
'-crop_to_cutline', '-cutline', shape, files[i],
clipped_files[i]
]
call(clip_command)
for i in range(4):
aux_array = open_handle(clipped_files[i])
aux_array[(aux_array == 0)] = -9999
create_raster_from_reference(final_files[i], aux_array,
ndvi_clipped_file)
if not options['debug']:
LOGGER.info('Remove auxiliary files.')
os.remove(ndvi_clipped_file)
os.remove(mndwi_clipped_file)
os.remove(ndwig_clipped_file)
os.remove(ndwim_clipped_file)
os.remove(ndvi_file)
os.remove(mndwi_file)
os.remove(ndwig_file)
os.remove(ndwim_file)
print 'Done'
def handle(self, **options):
"""
This is the code that does the ingestion.
indexes --path /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/Landsat_2000_2008/L5_021_047_2000_022_sr/ --shape /LUSTRE/MADMEX/staging/2016_tasks/Humedales_for_ctroche/LT/AE_LT_new.shp --output
"""
path = options["path"][0]
for mask_file in os.listdir(path):
if mask_file.endswith("_cfmask.tif"):
print mask_file
basename = mask_file.replace("_cfmask.tif", "")
print basename
cloud_file_name = mask_file
LOGGER.info("Calculating indexes for Landsat scenes.")
band_name = basename + "_sr_band%s.tif"
final_path = options["output"][0]
create_directory_path(final_path)
cloud_file = create_file_name(path, cloud_file_name)
cloud_array = open_handle(cloud_file)
cloud_mask = cloud_array == 4
LOGGER.debug("Recognize sensor depending on file name.")
if "L8" in path:
LOGGER.debug("Landsat 8 scene was detected.")
blue_file = create_file_name(path, band_name % BLUE_L8)
green_file = create_file_name(path, band_name % GREEN_L8)
red_file = create_file_name(path, band_name % RED_L8)
nir_file = create_file_name(path, band_name % NIR_L8)
swir_file = create_file_name(path, band_name % SWIR_L8)
else:
LOGGER.debug("Landsat 4, 5 or scene was detected.")
blue_file = create_file_name(path, band_name % BLUE)
green_file = create_file_name(path, band_name % GREEN)
red_file = create_file_name(path, band_name % RED)
nir_file = create_file_name(path, band_name % NIR)
swir_file = create_file_name(path, band_name % SWIR)
LOGGER.debug("Loading bands of interest.")
green_array = open_handle(green_file)
red_array = open_handle(red_file)
nir_array = open_handle(nir_file)
swir_array = open_handle(swir_file)
LOGGER.debug("Calculating indexes.")
ndvi_array = calculate_index(nir_array, red_array)
mndwi_array = calculate_index(green_array, swir_array)
ndwig_array = calculate_index(nir_array, swir_array)
ndwim_array = calculate_index(green_array, nir_array)
LOGGER.debug("Setting cloud mask values to -999.")
ndvi_array[cloud_mask] = -999
mndwi_array[cloud_mask] = -999
ndwig_array[cloud_mask] = -999
ndwim_array[cloud_mask] = -999
LOGGER.debug("Creating files for indexes.")
ndvi_final_file = create_file_name(final_path, basename + "_ndvi.tif")
mndwi_final_file = create_file_name(final_path, basename + "_mndwi.tif")
ndwig_final_file = create_file_name(final_path, basename + "_ndwig.tif")
ndwim_final_file = create_file_name(final_path, basename + "_ndwim.tif")
ndvi_clipped_file = create_file_name(final_path, basename + "_ndvi_clipped.tif")
mndwi_clipped_file = create_file_name(final_path, basename + "_mndwi_clipped.tif")
ndwig_clipped_file = create_file_name(final_path, basename + "_ndwig_clipped.tif")
ndwim_clipped_file = create_file_name(final_path, basename + "_ndwim_clipped.tif")
ndvi_file = create_file_name(final_path, basename + "_ndvi_pre.tif")
mndwi_file = create_file_name(final_path, basename + "_mndwi_pre.tif")
ndwig_file = create_file_name(final_path, basename + "_ndwig_pre.tif")
ndwim_file = create_file_name(final_path, basename + "_ndwim_pre.tif")
files = [ndvi_file, mndwi_file, ndwig_file, ndwim_file]
clipped_files = [ndvi_clipped_file, mndwi_clipped_file, ndwig_clipped_file, ndwim_clipped_file]
final_files = [ndvi_final_file, mndwi_final_file, ndwig_final_file, ndwim_final_file]
LOGGER.debug("Writing information to files.")
create_raster_from_reference(ndvi_file, ndvi_array, green_file)
create_raster_from_reference(mndwi_file, mndwi_array, green_file)
create_raster_from_reference(ndwig_file, ndwig_array, green_file)
create_raster_from_reference(ndwim_file, ndwim_array, green_file)
LOGGER.debug("Deleting arrays to release memory.")
del ndvi_array
del mndwi_array
del ndwig_array
del ndwim_array
del cloud_array
LOGGER.debug("Reference rgb file creation.")
rgb_file = create_file_name(final_path, basename + "_rgb.tif")
merge_command = [
"/Library/Frameworks/GDAL.framework/Programs/gdalbuildvrt",
"-separate",
"-o",
rgb_file,
red_file,
green_file,
blue_file,
]
call(merge_command)
shape = options["shape"][0]
LOGGER.debug("Cookie cut files using the given shape.")
for i in range(4):
clip_command = [
"/Library/Frameworks/GDAL.framework/Programs/gdalwarp",
"-crop_to_cutline",
"-cutline",
shape,
files[i],
clipped_files[i],
]
call(clip_command)
for i in range(4):
aux_array = open_handle(clipped_files[i])
aux_array[(aux_array == 0)] = -9999
create_raster_from_reference(final_files[i], aux_array, ndvi_clipped_file)
if not options["debug"]:
LOGGER.info("Remove auxiliary files.")
os.remove(ndvi_clipped_file)
os.remove(mndwi_clipped_file)
os.remove(ndwig_clipped_file)
os.remove(ndwim_clipped_file)
os.remove(ndvi_file)
os.remove(mndwi_file)
os.remove(ndwig_file)
os.remove(ndwim_file)
print "Done"
clouds = self.masking_with_time_series(data, cloud_output_file, solar_zenith, solar_azimuth, geotransform, tile_id)
create_raster_from_reference(cloud_output_file,
clouds,
self.file_dictionary[_IMAGE],
data_type=NumericTypeCodeToGDALTypeCode(numpy.float32)
)
LOGGER.info('Cloud mask was created.')
if __name__ == '__main__':
#path = '/Users/amaury/Documents/rapideye/df/1447813/2012/2012-03-14/l3a'
#bundle = Bundle(path)
#print bundle.get_files()
#print bundle.can_identify()
image = "/Users/agutierrez/Documents/rapideye/df/1447813/2012/2012-03-14/l3a/2012-03-14T182106_RE2_3A-NAC_11040070_149070.tif"
gdal_format = "GTiff"
data_class = raster.Data(image, gdal_format)
array = data_class.read_data_file_as_array()
width, height, bands = data_class.get_attribute(raster.DATA_SHAPE)
feature_bands = numpy.zeros([2, width, height])
from madmex.processing.raster import calculate_ndvi
feature_bands[0, :, :] = calculate_ndvi(array[4, :, :], array[2, :, :])
feature_bands[1, :, :] = calculate_ndvi(array[3, :, :], array[2, :, :])
out1 = get_parent(image) + 'result_ndvi_1.tif'
out2 = get_parent(image) + 'result_ndvi_2.tif'
create_raster_from_reference(out1, feature_bands[0, :, :], image)
create_raster_from_reference(out2, feature_bands[1, :, :], image)
print 'Done'
