def annotate_ecoregions(ecoregion_uri, table_uri):
lookup_table = raster_utils.get_lookup_from_csv(table_uri, 'region')
print 'generating report'
esri_driver = ogr.GetDriverByName('ESRI Shapefile')
original_datasource = ogr.Open(ecoregion_uri)
updated_datasource_uri = os.path.join(os.path.dirname(ecoregion_uri), 'annotated_ecoregions_2.shp')
#If there is already an existing shapefile with the same name and path, delete it
#Copy the input shapefile into the designated output floder
if os.path.isfile(updated_datasource_uri):
os.remove(updated_datasource_uri)
datasource_copy = esri_driver.CopyDataSource(original_datasource, updated_datasource_uri)
layer = datasource_copy.GetLayer()
new_field_names = [('Magnitude', 'magnitude'), ('A(80)', 'A80'), ('A(90)', 'A90'), ('A(95)', 'A95')]
for table_field, field_name in new_field_names:
field_def = ogr.FieldDefn(field_name, ogr.OFTReal)
layer.CreateField(field_def)
for feature_id in xrange(layer.GetFeatureCount()):
feature = layer.GetFeature(feature_id)
feature_eco_name = raster_utils._smart_cast(feature.GetField('ECO_NAME'))
for table_field, field_name in new_field_names:
try:
value = lookup_table[feature_eco_name][table_field]
if feature_eco_name == 'Balsas Dry Forests':
print feature_eco_name, table_field, value
feature.SetField(field_name, float(value))
except KeyError as e:
if feature_eco_name == 'Balsas Dry Forests':
print e
feature.SetField(field_name, -1.0)
except TypeError:
if feature_eco_name == 'Balsas Dry Forests':
print e
feature.SetField(field_name, -1.0)
#Save back to datasource
layer.SetFeature(feature)
def average_layers():
base_table_uri = "C:/Users/rich/Desktop/all_grid_results_100km_clean_v2.csv"
base_table_file = open(base_table_uri, 'rU')
table_header = base_table_file.readline()
#need to mask the average layers to the biomass regions
giant_layer_uri = "C:/Users/rich/Desktop/average_layers_projected/giant_layer.tif"
af_uri = "C:/Users/rich/Desktop/af_biov2ct1.tif"
am_uri = "C:/Users/rich/Desktop/am_biov2ct1.tif"
as_uri = "C:/Users/rich/Desktop/as_biov2ct1.tif"
cell_size = raster_utils.get_cell_size_from_uri(am_uri)
#raster_utils.vectorize_datasets(
# [af_uri, am_uri, as_uri], lambda x,y,z: x+y+z, giant_layer_uri, gdal.GDT_Float32,
# -1, cell_size, 'union', vectorize_op=False)
table_uri = base_table_uri
table_file = open(table_uri, 'rU')
table_header = table_file.readline().rstrip()
lookup_table = raster_utils.get_lookup_from_csv(table_uri, 'ID100km')
out_table_uri = "C:/Users/rich/Desktop/all_grid_results_100km_human_elevation.csv"
out_table_file = codecs.open(out_table_uri, 'w', 'utf-8')
average_raster_list = [
("C:/Users/rich/Desktop/average_layers_projected/lighted_area_luminosity.tif", 'Lighted area density'),
("C:/Users/rich/Desktop/average_layers_projected/fi_average.tif", 'Fire densities'),
("C:/Users/rich/Desktop/average_layers_projected/glbctd1t0503m.tif", 'FAO_Cattle'),
("C:/Users/rich/Desktop/average_layers_projected/glbgtd1t0503m.tif", 'FAO_Goat'),
("C:/Users/rich/Desktop/average_layers_projected/glbpgd1t0503m.tif", 'FAO_Pig'),
("C:/Users/rich/Desktop/average_layers_projected/glbshd1t0503m.tif", 'FAO_Sheep'),
("C:/Users/rich/Desktop/average_layers_projected/glds00ag.tif", 'Human population density AG'),
("C:/Users/rich/Desktop/average_layers_projected/glds00g.tif", 'Human population density G'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_11.tif', '"11: Urban, Dense settlement"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_12.tif', '"12: Dense settlements, Dense settlements"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_22.tif', '"22: Irrigated villages, Villages"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_23.tif', '"23: Cropped & pastoral villages, Villages"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_24.tif', '"24: Pastoral villages, Villages"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_25.tif', '"25: Rainfed villages, Villages"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_26.tif', '"26: Rainfed mosaic villages, Villages"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_31.tif', '"31: Residential irrigated cropland, Croplands"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_32.tif', '"32: Residential rainfed mosaic, Croplands"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_33.tif', '"33: Populated irrigated cropland, Croplands"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_34.tif', '"34: Populated rainfed cropland, Croplands"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_35.tif', '"35: Remote croplands, Croplands"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_41.tif', '"41: Residential rangelands, Rangelands"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_42.tif', '"42: Populated rangelands, Rangelands"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_43.tif', '"43: Remote rangelands, Rangelands"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_51.tif', '"51: Populated forests, Forested"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_52.tif', '"52: Remote forests, Forested"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_61.tif', '"61: Wild forests, Wildlands"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_62.tif', '"62: Sparse trees, Wildlands"'),
('C:/Users/rich/Desktop/average_layers_projected/anthrome_63.tif', '"63: Barren, Wildlands"'),
("C:/Users/rich/Desktop/average_layers_projected/5km_global_pantropic_dem.tif", '"Average Elevation"'),
]
clipped_raster_list = []
for average_raster_uri, header in average_raster_list:
print 'clipping ' + average_raster_uri
clipped_raster_uri = os.path.join(os.path.dirname(average_raster_uri), 'temp', os.path.basename(average_raster_uri))
cell_size = raster_utils.get_cell_size_from_uri(average_raster_uri)
raster_utils.vectorize_datasets(
[average_raster_uri, giant_layer_uri], lambda x,y: x, clipped_raster_uri, gdal.GDT_Float32,
-1, cell_size, 'intersection', vectorize_op=False)
clipped_raster_list.append((clipped_raster_uri, header))
dataset_list = [gdal.Open(uri) for uri, label in clipped_raster_list]
band_list = [ds.GetRasterBand(1) for ds in dataset_list]
nodata_list = [band.GetNoDataValue() for band in band_list]
extended_table_headers = ','.join([header for _, header in average_raster_list])
def write_to_file(value):
try:
out_table_file.write(value)
except UnicodeDecodeError as e:
out_table_file.write(value.decode('latin-1'))
write_to_file(table_header + ',' + extended_table_headers + '\n')
#print table_header + ',' + extended_table_headers
for line in table_file:
split_line = line.rstrip().split(',')
grid_id = split_line[2]
#for grid_id in lookup_table:
try:
split_grid_id = grid_id.split('-')
grid_row_index, grid_col_index = map(int, split_grid_id)
except ValueError as e:
month_to_number = {
'Jan': 1,
'Feb': 2,
'Mar': 3,
'Apr': 4,
'May': 5,
'Jun': 6,
'Jul': 7,
'Aug': 8,
'Sep': 9,
'Oct': 10,
'Nov': 11,
'Dec': 12,
}
grid_row_index, grid_col_index = month_to_number[split_grid_id[0]], int(split_grid_id[1])
print 'processing grid id ' + grid_id
ds = dataset_list[0]
base_srs = osr.SpatialReference(ds.GetProjection())
lat_lng_srs = base_srs.CloneGeogCS()
coord_transform = osr.CoordinateTransformation(
base_srs, lat_lng_srs)
gt = ds.GetGeoTransform()
grid_resolution = 100 #100km
row_coord = grid_row_index * grid_resolution * 1000 + GLOBAL_UPPER_LEFT_ROW
col_coord = grid_col_index * grid_resolution * 1000 + GLOBAL_UPPER_LEFT_COL
lng_coord, lat_coord, _ = coord_transform.TransformPoint(
col_coord, row_coord)
write_to_file(','.join(split_line[0:2]) + ',%d-%d,' % (grid_row_index, grid_col_index) + ','.join(split_line[3:11]) +',%f,%f,' % (lat_coord, lng_coord)+','.join(split_line[13:]))
for (_, header), band, ds, nodata in zip(clipped_raster_list, band_list, dataset_list, nodata_list):
gt = ds.GetGeoTransform()
n_rows = ds.RasterYSize
n_cols = ds.RasterXSize
xoff = int(grid_col_index * (grid_resolution * 1000.0) / (gt[1]))
yoff = int(grid_row_index * (grid_resolution * 1000.0) / (-gt[5]))
win_xsize = int((grid_resolution * 1000.0) / (gt[1]))
win_ysize = int((grid_resolution * 1000.0) / (gt[1]))
if xoff + win_xsize > n_cols:
win_xsize = n_cols - xoff
if yoff + win_ysize > n_rows:
win_ysize = n_rows - yoff
block = band.ReadAsArray(
xoff=xoff, yoff=yoff, win_xsize=win_xsize, win_ysize=win_ysize)
block_average = numpy.average(block[block != nodata])
write_to_file(',%f' % block_average)
write_to_file('\n')
def _make_magnitude_maps(base_uri, table_uri):
grid_resolution = 100
base_ds = gdal.Open(base_uri)
projection = base_ds.GetProjection()
driver = gdal.GetDriverByName('GTiff')
gt = base_ds.GetGeoTransform()
#gt = (
# GLOBAL_UPPER_LEFT_COL, gt[1], gt[2],
# GLOBAL_UPPER_LEFT_ROW, gt[4], gt[5]
#)
n_rows = base_ds.RasterYSize
n_cols = base_ds.RasterXSize
lookup_table = raster_utils.get_lookup_from_csv(table_uri, 'ID100km')
for map_type in ['Magnitude', 'A80', 'A90', 'A95', 'Average Elevation']:
output_dir, base_filename = os.path.split(base_uri)
basename = os.path.basename(base_filename)
output_uri = os.path.join(output_dir, map_type + '.tif')
n_rows_grid = int(-gt[5] * n_rows / (grid_resolution * 1000.0))
n_cols_grid = int(gt[1] * n_cols / (grid_resolution * 1000.0))
new_geotransform = (
gt[0], grid_resolution * 1000.0, gt[2],
gt[3], gt[4], -grid_resolution * 1000.0)
n_rows_grid = -1
n_cols_grid = -1
for grid_id in lookup_table:
try:
split_grid_id = grid_id.split('-')
grid_row_index, grid_col_index = map(int, split_grid_id)
except ValueError as e:
month_to_number = {
'Jan': 1,
'Feb': 2,
'Mar': 3,
'Apr': 4,
'May': 5,
'Jun': 6,
'Jul': 7,
'Aug': 8,
'Sep': 9,
'Oct': 10,
'Nov': 11,
'Dec': 12,
}
try:
grid_row_index, grid_col_index = month_to_number[split_grid_id[0]], int(split_grid_id[1])
except KeyError as e:
continue
n_rows_grid = max(n_rows_grid, grid_row_index)
n_cols_grid = max(n_cols_grid, grid_col_index)
n_rows_grid += 1
n_cols_grid += 1
output_ds = driver.Create(
output_uri.encode('utf-8'), n_cols_grid, n_rows_grid, 1, gdal.GDT_Float32)
output_ds.SetProjection(projection)
output_ds.SetGeoTransform(new_geotransform)
output_band = output_ds.GetRasterBand(1)
output_nodata = -9999
output_band.SetNoDataValue(output_nodata)
output_band.Fill(output_nodata)
last_time = time.time()
for grid_id in lookup_table:
current_time = time.time()
if current_time - last_time > 5.0:
print "%s working..." % (map_type,)
last_time = current_time
try:
split_grid_id = grid_id.split('-')
grid_row_index, grid_col_index = map(int, split_grid_id)
except ValueError as e:
month_to_number = {
'Jan': 1,
'Feb': 2,
'Mar': 3,
'Apr': 4,
'May': 5,
'Jun': 6,
'Jul': 7,
'Aug': 8,
'Sep': 9,
'Oct': 10,
'Nov': 11,
'Dec': 12,
}
try:
grid_row_index, grid_col_index = month_to_number[split_grid_id[0]], int(split_grid_id[1])
except KeyError as e:
continue
#grid_row_index, grid_col_index = map(int, grid_id.split('-'))
try:
output_band.WriteArray(
numpy.array([[float(lookup_table[grid_id][map_type])]]),
xoff=grid_col_index, yoff=n_rows_grid - grid_row_index - 1)
except ValueError:
pass
def run(self):
biomass_ds = gdal.Open(GLOBAL_BIOMASS_URI, gdal.GA_ReadOnly)
n_rows, n_cols = raster_utils.get_row_col_from_uri(GLOBAL_BIOMASS_URI)
base_srs = osr.SpatialReference(biomass_ds.GetProjection())
lat_lng_srs = base_srs.CloneGeogCS()
coord_transform = osr.CoordinateTransformation(
base_srs, lat_lng_srs)
geo_trans = biomass_ds.GetGeoTransform()
biomass_band = biomass_ds.GetRasterBand(1)
biomass_nodata = biomass_band.GetNoDataValue()
forest_table = raster_utils.get_lookup_from_csv(
self.forest_only_table_uri, 'gridID')
forest_headers = list(forest_table.values()[0].keys())
nonexistant_files = []
for uri in ALIGNED_LAYERS_TO_AVERAGE:
if not os.path.isfile(uri):
nonexistant_files.append(uri)
if len(nonexistant_files) > 0:
raise Exception(
"The following files don't exist: %s" %
(str(nonexistant_files)))
average_dataset_list = [
gdal.Open(uri) for uri in ALIGNED_LAYERS_TO_AVERAGE]
average_band_list = [ds.GetRasterBand(1) for ds in average_dataset_list]
average_nodata_list = [
band.GetNoDataValue() for band in average_band_list]
max_dataset_list = [gdal.Open(uri) for uri in ALIGNED_LAYERS_TO_MAX]
max_band_list = [ds.GetRasterBand(1) for ds in max_dataset_list]
max_nodata_list = [band.GetNoDataValue() for band in max_band_list]
for global_grid_resolution, grid_output_filename in \
zip(GRID_RESOLUTION_LIST, self.grid_output_file_list):
try:
grid_output_file = open(grid_output_filename, 'w')
grid_output_file.write('grid id,lat_coord,lng_coord')
for filename in (
ALIGNED_LAYERS_TO_AVERAGE + ALIGNED_LAYERS_TO_MAX):
grid_output_file.write(
',%s' % os.path.splitext(
os.path.basename(filename))[0][len('aligned_'):])
for header in forest_headers:
grid_output_file.write(',%s' % header)
grid_output_file.write('\n')
n_grid_rows = int(
(-geo_trans[5] * n_rows) / (global_grid_resolution * 1000))
n_grid_cols = int(
(geo_trans[1] * n_cols) / (global_grid_resolution * 1000))
grid_row_stepsize = int(n_rows / float(n_grid_rows))
grid_col_stepsize = int(n_cols / float(n_grid_cols))
for grid_row in xrange(n_grid_rows):
for grid_col in xrange(n_grid_cols):
#first check to make sure there is biomass at all!
global_row = grid_row * grid_row_stepsize
global_col = grid_col * grid_col_stepsize
global_col_size = min(
grid_col_stepsize, n_cols - global_col)
global_row_size = min(
grid_row_stepsize, n_rows - global_row)
array = biomass_band.ReadAsArray(
global_col, global_row, global_col_size,
global_row_size)
if numpy.count_nonzero(array != biomass_nodata) == 0:
continue
grid_id = '%d-%d' % (grid_row, grid_col)
grid_row_center = (
-(grid_row + 0.5) * (global_grid_resolution*1000) +
geo_trans[3])
grid_col_center = (
(grid_col + 0.5) * (global_grid_resolution*1000) +
geo_trans[0])
grid_lng_coord, grid_lat_coord, _ = (
coord_transform.TransformPoint(
grid_col_center, grid_row_center))
grid_output_file.write(
'%s,%s,%s' % (grid_id, grid_lat_coord,
grid_lng_coord))
#take the average values
for band, nodata, layer_uri in zip(
average_band_list, average_nodata_list,
ALIGNED_LAYERS_TO_AVERAGE +
ALIGNED_LAYERS_TO_MAX):
nodata = band.GetNoDataValue()
array = band.ReadAsArray(
global_col, global_row, global_col_size,
global_row_size)
layer_name = os.path.splitext(
os.path.basename(layer_uri)) \
[0][len('aligned_'):]
pure_average_layers = [
'global_elevation', 'global_water_capacity',
'fi_average', 'lighted_area_luminosity',
'glbctd1t0503m', 'glbgtd1t0503m',
'glbpgd1t0503m', 'glbshd1t0503m', 'glds00ag',
'glds00g']
if layer_name not in pure_average_layers:
array[array == nodata] = 0.0
valid_values = array[array != nodata]
if valid_values.size != 0:
value = numpy.average(valid_values)
else:
value = -9999.
grid_output_file.write(',%f' % value)
#take the mode values
for band, nodata in zip(max_band_list, max_nodata_list):
nodata = band.GetNoDataValue()
array = band.ReadAsArray(
global_col, global_row, global_col_size,
global_row_size)
#get the most common value
valid_values = array[array != nodata]
if valid_values.size != 0:
value = scipy.stats.mode(valid_values)[0][0]
grid_output_file.write(',%f' % value)
else:
grid_output_file.write(',-9999')
#add the forest_only values
for header in forest_headers:
try:
value = forest_table[grid_id][header]
if type(value) == unicode:
grid_output_file.write(
',%s' % forest_table[grid_id][header].\
encode('latin-1', 'replace'))
else:
grid_output_file.write(
',%s' % forest_table[grid_id][header])
except KeyError:
grid_output_file.write(',-9999')
grid_output_file.write('\n')
grid_output_file.close()
except IndexError as exception:
grid_output_file.close()
os.remove(grid_output_filename)
raise exception