def handle(self, **options): paths = options['path'] output = options['output'][0] print output arrays = {} import time for path in paths: start_time = time.time() arrays[path] = open_handle(path) print 'Opened %s' % path print("--- %s seconds ---" % (time.time() - start_time)) size = arrays[paths[0]].shape start_time = time.time() result = numpy.full(size, NO_DATA) mask = numpy.equal(arrays[paths[0]],arrays[paths[1]]) for p in range(2, len(paths)): print 'Image %s ' % p mask = numpy.logical_and(mask, numpy.equal(arrays[paths[0]],arrays[paths[p]])) result[mask] = numpy.array(arrays[paths[0]])[mask] for path in paths: del arrays[path] print("--- %s seconds ---" % (time.time() - start_time)) start_time = time.time() create_raster_from_reference(output, result, paths[0], data_type=gdal.GDT_Byte) print("--- %s seconds ---" % (time.time() - start_time))
def method_five(self, path, output): ''' Using numpy utilities, this method mask the desired value. ''' data_array = open_handle(path) data_array[data_array != MASK] = 0 data_array[data_array == MASK] = 1 create_raster_from_reference(output, data_array, path, gdal.GDT_Byte) del data_array
def method_five(self, path, output): ''' Using numpy utilities, this method mask the desired value. ''' data_array = open_handle(path) data_array[data_array!=MASK] = 0 data_array[data_array==MASK] = 1 create_raster_from_reference(output, data_array, path, gdal.GDT_Byte) del data_array
def mask_iterating_values(self, path, output): ''' Iterating over the values in the initial and final arrays this replaces the desired values one by one. ''' data_array = open_handle(path) my_dictionary = dictionary_from_list(INITIAL_ARRAY, FINAL_ARRAY) to_vector_array = replace_in_array(data_array, my_dictionary) create_raster_from_reference(output, to_vector_array, path, gdal.GDT_Byte)
def mask_iterating_values(self, path, output, ini_arr, fin_arr): ''' Iterating over the values in the initial and final arrays this replaces the desired values one by one. ''' data_array = open_handle(path) my_dictionary = dictionary_from_list(ini_arr, fin_arr) to_vector_array = replace_in_array(data_array, my_dictionary) create_raster_from_reference(output, to_vector_array, path, gdal.GDT_Byte)
def tile_map(image_path, output, x_tile_size, y_tile_size): from subprocess import call data_array = open_handle(image_path) width = data_array.shape[1] height = data_array.shape[0] for i in range(0, width, x_tile_size): for j in range(0, height, y_tile_size): gdaltranString = ['/Library/Frameworks/GDAL.framework/Programs/gdal_translate', '-of', 'GTIFF', '-srcwin', str(i), str(j) , str(x_tile_size) , str(y_tile_size) , image_path, output + 'utm_' + str(i) + '_' + str(j) + '.tif'] print ' '.join(gdaltranString) call(gdaltranString)
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' output = options['output'][0] models = options['modelname'] model_directory = options['modeldir'][0] pca_model = pca.Model(5) pca_model.load(create_filename(model_directory, 'pca')) for model_name in models: persistence_directory = create_filename(model_directory, model_name) model = load_model(model_name) model_instance = model.Model(persistence_directory) model_instance.load(persistence_directory) block_size = 500 for path in options['path']: image_array = open_handle(path) y_size = image_array.shape[1] x_size = image_array.shape[2] basename = get_basename(path)[:7] warnings.filterwarnings('ignore') final = numpy.zeros((x_size, y_size)) import time start_time = time.time() for i in range(0, y_size, block_size): if i + block_size < y_size: rows = block_size else: rows = y_size - i for j in range(0, x_size, block_size): if j + block_size < x_size: cols = block_size else: cols = x_size - j step = image_array[:, i:i + rows, j:j + cols] step_ravel = step.reshape(10, -1) prediction = model_instance.predict( pca_model.transform(numpy.transpose(step_ravel))) final[i:i + rows, j:j + cols] = prediction.reshape( (rows, cols)) print("--- %s seconds ---" % (time.time() - start_time)) create_directory_path(output) classification = create_filename( output, '%s-%s.tif' % (basename, model_name)) create_raster_from_reference(classification, final.reshape(x_size, y_size), path, data_type=gdal.GDT_Byte, creating_options=['COMPRESS=LZW'])
def handle(self, **options): paths = options['path'] output = options['output'][0] print output arrays = {} image1 = open_handle(paths[0]) #import time #for path in paths: # start_time = time.time() # arrays[path] = open_handle(path) # print 'Opened %s' % path # print("--- %s seconds ---" % (time.time() - start_time)) size = image1.shape #start_time = time.time() for p in range(2, len(paths)): #print 'Image %s ' % p image2 = open_handle(paths[p]) mask = numpy.logical_and( numpy.equal(image1, open_handle(paths[1])), numpy.equal(image1, image2)) image1[~mask] = NO_DATA #for path in paths: #del arrays[path] #print("--- %s seconds ---" % (time.time() - start_time)) #start_time = time.time() create_raster_from_reference(output, image1y, paths[0], data_type=gdal.GDT_Byte)
def tile_map(image_path, output, x_tile_size, y_tile_size): from subprocess import call data_array = open_handle(image_path) width = data_array.shape[1] height = data_array.shape[0] for i in range(0, width, x_tile_size): for j in range(0, height, y_tile_size): gdaltranString = [ "/Library/Frameworks/GDAL.framework/Programs/gdal_translate", "-of", "GTIFF", "-srcwin", str(i), str(j), str(x_tile_size), str(y_tile_size), image_path, output + "utm_" + str(i) + "_" + str(j) + ".tif", ] print " ".join(gdaltranString) call(gdaltranString)
def print_unique(self, path): data_array = open_handle(path) unique, counts = numpy.unique(data_array, return_counts=True) print unique, counts
def handle(self, **options): import time start_time = time.time() path = options['path'][0] print path features_array = [] bundle = Bundle(path) raster_path = bundle.file_dictionary[_IMAGE] data_array = open_handle(raster_path) data_array = numpy.array(data_array) print data_array, len(data_array) print len(data_array[data_array != 0]) for i in range(data_array.shape[0]): print 'band: %s' % (i + 1) band = data_array[i, :, :].ravel() band = band[band != 0] features_array.append(numpy.nanpercentile(band, 10)) features_array.append(numpy.nanpercentile(band, 25)) features_array.append(numpy.nanpercentile(band, 50)) features_array.append(numpy.nanpercentile(band, 75)) features_array.append(numpy.nanpercentile(band, 90)) features_array.append(numpy.mean(band)) features_array.append(numpy.min(band) * 1.0) features_array.append(numpy.max(band) * 1.0) geotransform = get_geotransform(raster_path) features_array.append(geotransform[0]) features_array.append(geotransform[3]) print features_array features_array.append((bundle.get_aquisition_date() - datetime.datetime(1970, 1, 1)).total_seconds()) tile_id = bundle.get_sensor().get_attribute(TILE_ID) features = RapideyeFeatures(band_1_quant_10=features_array[0], band_1_quant_25=features_array[1], band_1_quant_50=features_array[2], band_1_quant_75=features_array[3], band_1_quant_90=features_array[4], band_1_mean=features_array[5], band_1_min=features_array[6], band_1_max=features_array[7], band_2_quant_10=features_array[8], band_2_quant_25=features_array[9], band_2_quant_50=features_array[10], band_2_quant_75=features_array[11], band_2_quant_90=features_array[12], band_2_mean=features_array[13], band_2_min=features_array[14], band_2_max=features_array[15], band_3_quant_10=features_array[16], band_3_quant_25=features_array[17], band_3_quant_50=features_array[18], band_3_quant_75=features_array[19], band_3_quant_90=features_array[20], band_3_mean=features_array[21], band_3_min=features_array[22], band_3_max=features_array[23], band_4_quant_10=features_array[24], band_4_quant_25=features_array[25], band_4_quant_50=features_array[26], band_4_quant_75=features_array[27], band_4_quant_90=features_array[28], band_4_mean=features_array[29], band_4_min=features_array[30], band_4_max=features_array[31], band_5_quant_10=features_array[32], band_5_quant_25=features_array[33], band_5_quant_50=features_array[34], band_5_quant_75=features_array[35], band_5_quant_90=features_array[36], band_5_mean=features_array[37], band_5_min=features_array[38], band_5_max=features_array[39], top=features_array[40], left=features_array[41], time=features_array[42], footprint=tile_id, path=raster_path) klass = sessionmaker(bind=ENGINE, autoflush=False) session = klass() session.add(features) session.commit() print tile_id print features_array print len(features_array) print("--- %s seconds ---" % (time.time() - start_time))
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' shape_name = options['shape'][0] destination = options['dest'][0] paths = options['path'] years = [] for path in paths: years.append(self.get_year_from_path(path)) if os.path.exists(shape_name): LOGGER.info('The file %s was found.' % shape_name) shape_files = split_shape_into_features(shape_name, destination, str('id'), str('nombre'), '__') process_launcher = LocalProcessLauncher() import time start_time = time.time() cover_array = [] tth_array = [] cover_file = 'cover-stats.json' tth_name = 'tth-stats.json' json_directory = create_filename(destination, 'cover_stats') json_file = create_filename(json_directory, cover_file.lower()) tth_file = create_filename(json_directory, tth_name.lower()) for shape_file in shape_files: shape_name = get_basename(shape_file).split('__') anp_id = shape_name[0] anp_name = shape_name[1] basename = '%s.tif' % anp_name dataframe = DataFrame(index=years, columns=[0, 1, 2, 3, 4, 5, 6, 7, 8]) create_directory_path(json_directory) print shape_file for path in paths: #pixel_resolution, dataset = pixel_info(path) year = self.get_year_from_path(path) raster_dir = create_filename( create_filename(destination, 'raster'), year) create_directory_path(raster_dir) raster_file = create_filename(raster_dir, basename) shell_command = 'gdalwarp -ot Byte -co COMPRESS=LZW -cutline %s -crop_to_cutline %s %s' % ( shape_file, path, raster_file) print shell_command print process_launcher.execute(shell_command) raster_array = open_handle(raster_file) ds = gdal.Open(raster_file) geotransform = ds.GetGeoTransform() x_resolution = geotransform[1] y_resolution = geotransform[5] pixel_area = abs(x_resolution * y_resolution) unique_values = numpy.unique(raster_array, return_counts=True) indexes = unique_values[0] counts = unique_values[1] for i in range(len(indexes)): key = indexes[i] dataframe.set_value(year, key, area(int(counts[i]), pixel_area)) dataframe = dataframe.sort_index() columns = list(dataframe.columns.values) index = list(dataframe.index.values) print dataframe cover_array.append( self.dataframe_to_json(dataframe, anp_id, anp_name)) tth_dataframe = DataFrame(columns=columns) for i in range(len(index) - 1): label = '%s-%s' % (index[i], index[i + 1]) tth_column = calculate_thh(dataframe.ix[i], dataframe.ix[i + 1], int(index[i + 1]) - int(index[i])) for j in range(len(tth_column)): tth_dataframe.set_value(label, j, tth_column[j]) tth_dataframe = tth_dataframe.sort_index() print tth_dataframe tth_array.append( self.dataframe_to_json(tth_dataframe, anp_id, anp_name)) self.json_to_file(json_file, cover_array) self.json_to_file(tth_file, tth_array) print("--- %s seconds ---" % (time.time() - start_time))
def handle(self, **options): ''' In this example command, the values that come from the user input are added up and the result is printed in the screen. ''' random_int = 1 training_template = options['training'][0] target_template = options['target'][0] index_template = options['index'][0] output = options['output'][0] models = options['model'] features = training_template % random_int training = target_template % random_int index = index_template % random_int features_array = open_handle(features) training_array = open_handle(training) index_array = open_handle(index) labels = numpy.unique(index_array) features_flatten = numpy.ravel(features_array).reshape(10, 200 * 200) training_flatten = numpy.ravel(training_array).reshape(200 * 200) array_aux = [] for j in range(features_array.shape[0]): means = scipy.ndimage.measurements.mean(features_array[j, :, :], index_array, labels) print means.shape array_aux.append(means) for j in range(features_array.shape[0]): std = scipy.ndimage.measurements.standard_deviation( features_array[j, :, :], index_array, labels) print std.shape array_aux.append(std) print len(array_aux) features_final = numpy.concatenate([array_aux], axis=0) print features_final.shape label_object = [] for id in labels: values, counts = numpy.unique(training_array[index_array == id], return_counts=True) label_object.append(values[numpy.argmax(counts)]) features_total = features_final training_total = label_object for i in range(1, 53): features = training_template % (i + 1) training = target_template % (i + 1) index = index_template % (i + 1) features_array = open_handle(features) training_array = open_handle(training) index_array = open_handle(index) labels = numpy.unique(index_array) array_aux = [] for j in range(features_array.shape[0]): array_aux.append( scipy.ndimage.measurements.mean(features_array[j, :, :], index_array, labels)) for j in range(features_array.shape[0]): array_aux.append( scipy.ndimage.measurements.standard_deviation( features_array[j, :, :], index_array, labels)) features_final = numpy.concatenate([array_aux], axis=0) print features_final.shape label_object = [] for id in labels: values, counts = numpy.unique( training_array[index_array == id], return_counts=True) #print '********* ', values[numpy.argmax(counts)], counts[numpy.argmax(counts)] label_object.append(values[numpy.argmax(counts)]) print features_total.shape print features_final.shape features_total = numpy.concatenate( (features_total, features_final), axis=1) training_total = numpy.concatenate((training_total, label_object), axis=1) print 'label object', len(label_object) #print scipy.ndimage.measurements.mean(training_array, index_array, labels) features_flatten = numpy.concatenate( (features_flatten, numpy.ravel(features_array).reshape( 10, 200 * 200)), axis=1) training_flatten = numpy.concatenate( (training_flatten, numpy.ravel(training_array).reshape( 200 * 200)), axis=0) # Remove the elements that map to None #mask = training_flatten!=0 #features_flatten = features_flatten[:,mask] #training_flatten = training_flatten[mask] mask = training_total != 0 features_flatten = features_total[:, mask] training_flatten = training_total[mask] print features_flatten.shape print training_flatten.shape X_train, X_test, y_train, y_test = train_test_split( numpy.transpose(features_flatten), training_flatten, train_size=0.8, test_size=0.2) print X_train[0] print y_train[0] unsupervised = pca.Model(5) unsupervised.fit(X_train) ''' X_train = unsupervised.transform(X_train) X_test = unsupervised.transform(X_test) ''' pca_path = create_filename(output, 'pca') create_directory_path(pca_path) unsupervised.save(pca_path) import time for model_name in models: start_time = time.time() train_model(X_train, X_test, y_train, y_test, output, model_name) print "--- %s seconds training %s model---" % ( (time.time() - start_time), model_name)
def handle(self, **options): import time start_time = time.time() path = options['path'][0] print path features_array = [] bundle = Bundle(path) raster_path = bundle.file_dictionary[_IMAGE] data_array = open_handle(raster_path) data_array = numpy.array(data_array) print data_array, len(data_array) print len(data_array[data_array!=0]) for i in range(data_array.shape[0]): print 'band: %s' % (i + 1) band = data_array[i,:,:].ravel() band = band[band!=0] features_array.append(numpy.nanpercentile(band,10)) features_array.append(numpy.nanpercentile(band,25)) features_array.append(numpy.nanpercentile(band,50)) features_array.append(numpy.nanpercentile(band,75)) features_array.append(numpy.nanpercentile(band,90)) features_array.append(numpy.mean(band)) features_array.append(numpy.min(band) * 1.0) features_array.append(numpy.max(band) * 1.0) geotransform = get_geotransform(raster_path) features_array.append(geotransform[0]) features_array.append(geotransform[3]) print features_array features_array.append((bundle.get_aquisition_date() - datetime.datetime(1970, 1, 1)).total_seconds()) tile_id = bundle.get_sensor().get_attribute(TILE_ID) features = RapideyeFeatures(band_1_quant_10=features_array[0], band_1_quant_25=features_array[1], band_1_quant_50=features_array[2], band_1_quant_75=features_array[3], band_1_quant_90=features_array[4], band_1_mean=features_array[5], band_1_min=features_array[6], band_1_max=features_array[7], band_2_quant_10=features_array[8], band_2_quant_25=features_array[9], band_2_quant_50=features_array[10], band_2_quant_75=features_array[11], band_2_quant_90=features_array[12], band_2_mean=features_array[13], band_2_min=features_array[14], band_2_max=features_array[15], band_3_quant_10=features_array[16], band_3_quant_25=features_array[17], band_3_quant_50=features_array[18], band_3_quant_75=features_array[19], band_3_quant_90=features_array[20], band_3_mean=features_array[21], band_3_min=features_array[22], band_3_max=features_array[23], band_4_quant_10=features_array[24], band_4_quant_25=features_array[25], band_4_quant_50=features_array[26], band_4_quant_75=features_array[27], band_4_quant_90=features_array[28], band_4_mean=features_array[29], band_4_min=features_array[30], band_4_max=features_array[31], band_5_quant_10=features_array[32], band_5_quant_25=features_array[33], band_5_quant_50=features_array[34], band_5_quant_75=features_array[35], band_5_quant_90=features_array[36], band_5_mean=features_array[37], band_5_min=features_array[38], band_5_max=features_array[39], top=features_array[40], left=features_array[41], time=features_array[42], footprint=tile_id, path=raster_path) klass = sessionmaker(bind=ENGINE, autoflush=False) session = klass() session.add(features) session.commit() print tile_id print features_array print len(features_array) print("--- %s seconds ---" % (time.time() - start_time))