def create_overall_suitability(): global p p.overall_suitability_dir = p.cur_dir hb.create_directories(p.overall_suitability_dir) p.overall_suitability_paths = [] if p.run_this and p.run_this_zone: # NOTE, here the methods assume ONLY crop will be changing insofar as the physical suitability is defined wrt crops; 0 is 1 becasue already got rid of 0 in unique values physical_suitability_array = hb.as_array( p.current_physical_suitability_path) for i in p.classes_with_change: suitability_path = hb.ruri( os.path.join(p.overall_suitability_dir, 'overall_suitability_' + str(i) + '.tif')) p.overall_suitability_paths.append(suitability_path) combined_adjacency_effect_path = os.path.join( p.convolution_inputs_dir, 'combined_adjacency_effect_' + str(i) + '.tif') adjacency_effect_array = hb.as_array( combined_adjacency_effect_path) adjacency_effect_array = seals_utils.normalize_array( adjacency_effect_array ) # Didn't put this in HB because didn't want to redo the 0.4.0 release. conversion_eligibility_raster_path = os.path.join( p.create_conversion_eligibility_dir, str(i) + '_conversion_eligibility.tif') conversion_eligibility_array = hb.as_array( conversion_eligibility_raster_path) try: physical_suitability_importance = float( p.physical_suitability_importance) except: physical_suitability_importance = 0.5 L.warning( 'Could not interpret physical suitability importance. Using default of 0.5' ) physical_suitability_array = seals_utils.normalize_array( physical_suitability_array) overall_suitability_array = ( adjacency_effect_array + (physical_suitability_importance * physical_suitability_array)) * conversion_eligibility_array overall_suitability_array = np.where( np.isnan(overall_suitability_array), 0, overall_suitability_array) overall_suitability_array = np.where(overall_suitability_array < 0, 0, overall_suitability_array) hb.save_array_as_geotiff(overall_suitability_array, suitability_path, p.match_float_path, compress=True)
def force_geotiff_to_match_projection_ndv_and_datatype(input_path, match_path, output_path, output_datatype=None, output_ndv=None): """Rather than actually projecting, just change the metadata so it matches exactly. This only will be useful if there was a data error and something got a projection defined when the underlying data wasnt actually transofmred into that shape. NOTE that the output will keep the same geotransform as input, and only the projection, no data and datatype will change. """ if not output_datatype: output_datatype = hb.get_datatype_from_uri(match_path) if not output_ndv: output_ndv = hb.get_ndv_from_path(match_path) match_wkt = hb.get_dataset_projection_wkt_uri(match_path) input_geotransform = hb.get_geotransform_uri(input_path) # Load the array, but use numpy to convert it to the new datatype input_array = hb.as_array(input_path).astype(hb.gdal_number_to_numpy_type[output_datatype]) if not output_ndv: output_ndv = -9999 hb.save_array_as_geotiff(input_array, output_path, data_type=output_datatype, ndv=output_ndv, geotransform_override=input_geotransform, projection_override=match_wkt)
def create_conversion_eligibility(): global p p.conversion_eligibility_dir = p.cur_dir if p.run_this and p.run_this_zone: # Prevent illogical conversion eg new ag onto existing ag, or new ag onto urban conversion_eligibility_params = hb.file_to_python_object( p.conversion_eligibility_path, declare_type='DD', output_key_data_type=str, output_value_data_type=int) simplified_lulc_array = hb.as_array(p.lulc_simplified_path) for i in p.classes_with_change: conversion_eligibility_raster_path = os.path.join( p.conversion_eligibility_dir, str(i) + '_conversion_eligibility.tif') conversion_eligibility_array = np.zeros( simplified_lulc_array.shape).astype(np.float64) for j in p.classes_with_effect: conversion_eligibility_array = np.where( simplified_lulc_array == j, conversion_eligibility_params[str(j)][str(i)], conversion_eligibility_array) hb.save_array_as_geotiff(conversion_eligibility_array, conversion_eligibility_raster_path, p.match_int_path, compress=True)
def force_global_angular_data_to_equal_area_earth_grid(input_path, output_path): output_datatype = hb.get_datatype_from_uri(input_path) output_ndv = hb.get_ndv_from_path(input_path) match_wkt = hb.get_dataset_projection_wkt_uri(input_path) match_wkt = hb.get_wkt_from_epsg_code(6933) input_geotransform = hb.get_geotransform_uri(input_path) output_geotransform = list(hb.common_geotransforms['wec_30s']) output_geotransform[1] = input_geotransform[1] * hb.size_of_one_arcdegree_at_equator_in_meters output_geotransform[5] = input_geotransform[5] * hb.size_of_one_arcdegree_at_equator_in_meters # Load the array, but use numpy to convert it to the new datatype input_array = hb.as_array(input_path).astype(hb.gdal_number_to_numpy_type[output_datatype]) if not output_ndv: output_ndv = -9999 hb.save_array_as_geotiff(input_array, output_path, data_type=output_datatype, ndv=output_ndv, geotransform_override=output_geotransform, projection_override=match_wkt)
def create_calories_per_ha(p): uris = hb.get_list_of_file_uris_recursively( 'earthstat/crop_production', filter_extensions='.tif', filter_strings='YieldPerHectare') nutritional_content_uri = 'crop_nutritional_contents.csv' nutritional_content_odict = hb.file_to_python_object( nutritional_content_uri, declare_type='DD') total_calories_per_ha_masked_uri = os.path.join( p.cur_dir, 'total_calories_per_ha_masked.tif') total_calories_per_ha_masked_array = np.zeros(hb.as_array( uris[0]).shape).astype(np.float64) for uri in uris: earthstat_name = hb.explode_uri(uri)['file_root'].split('_')[0] kcal_per_ton = ( float(nutritional_content_odict[earthstat_name]["Kcal/Kg"]) * 1000.0) yield_per_ha = hb.as_array(uri).astype(np.float64) output_array = yield_per_ha * kcal_per_ton mask_uri = uri.replace('YieldPerHectare', 'HarvestedAreaFraction') mask_array = hb.as_array(mask_uri) mask_array = np.where(mask_array < 0.01, 0, 1) output_array *= mask_array print(output_array.dtype, earthstat_name) output_uri = os.path.join( p.cur_dir, earthstat_name + '_calories_per_ha_masked.tif') hb.save_array_as_geotiff(output_array, output_uri, uri, data_type=7) total_calories_per_ha_masked_array += output_array hb.save_array_as_geotiff(total_calories_per_ha_masked_array, total_calories_per_ha_masked_uri, uris[0], data_type=7)
def convert_simplified_to_original_classes(): global p lulc_class_types_odict = hb.file_to_python_object(p.lulc_class_types_path, declare_type='DD') p.simple_classes_to_projected_original_classes = OrderedDict() for original_class, csv_odict in lulc_class_types_odict.items(): if csv_odict['output_class_id'] != '': p.simple_classes_to_projected_original_classes[int( csv_odict['lulc_class_type'])] = int( csv_odict['output_class_id']) if p.run_this and p.run_this_zone: lulc_original_classes_array = hb.as_array( p.base_year_current_zone_lulc_path) for c, path in enumerate(p.change_array_paths): change_array = hb.as_array(path) change_array_ndv = hb.get_nodata_from_uri(path) lulc_projected_original_classes_array = np.where( (change_array > 0) & (change_array != change_array_ndv), p.simple_classes_to_projected_original_classes[ p.classes_projected_to_change[c]], lulc_original_classes_array) p.lulc_projected_original_classes_path = os.path.join( p.cur_dir, 'lulc_projected_original_classes.tif') hb.save_array_as_geotiff(lulc_projected_original_classes_array, p.lulc_projected_original_classes_path, p.match_int_path) p.layers_to_stitch.append(p.lulc_projected_original_classes_path) # ALSO NOTE that we only return this once, because separate batched tasks are appending to it return ( 'layers_to_stitch', 'append_to_list', p.layers_to_stitch ) # WARNING the only intended use of returns in a tasks is if its a return resource to be synced among parallel tasks.
# ssp5_pasture_clipped_path = os.path.join(run_dir, r"MAGPIE-ssp585\2050\pastr ^ area_fraction ^ managed pasture_clipped.tif") # hb.clip_dataset_uri(base_ag_path, aoi_path, base_ag_clipped_path) # hb.clip_dataset_uri(base_urban_path, aoi_path, base_urban_clipped_path) # hb.clip_dataset_uri(base_pasture_path, aoi_path, base_pasture_clipped_path) # hb.clip_dataset_uri(ssp1_ag_path, aoi_path, ssp1_ag_clipped_path) # hb.clip_dataset_uri(ssp3_ag_path, aoi_path, ssp3_ag_clipped_path) # hb.clip_dataset_uri(ssp5_ag_path, aoi_path, ssp5_ag_clipped_path) # hb.clip_dataset_uri(ssp1_urban_path, aoi_path, ssp1_urban_clipped_path) # hb.clip_dataset_uri(ssp3_urban_path, aoi_path, ssp3_urban_clipped_path) # hb.clip_dataset_uri(ssp5_urban_path, aoi_path, ssp5_urban_clipped_path) # hb.clip_dataset_uri(ssp1_pasture_path, aoi_path, ssp1_pasture_clipped_path) # hb.clip_dataset_uri(ssp3_pasture_path, aoi_path, ssp3_pasture_clipped_path) # hb.clip_dataset_uri(ssp5_pasture_path, aoi_path, ssp5_pasture_clipped_path) base_ag_array = hb.as_array(base_ag_path) base_urban_array = hb.as_array(base_urban_path) base_pasture_array = hb.as_array(base_pasture_path) ssp1_ag_array = hb.as_array(ssp1_ag_path) ssp3_ag_array = hb.as_array(ssp3_ag_path) ssp5_ag_array = hb.as_array(ssp5_ag_path) ssp1_urban_array = hb.as_array(ssp1_urban_path) ssp3_urban_array = hb.as_array(ssp3_urban_path) ssp5_urban_array = hb.as_array(ssp5_urban_path) ssp1_pasture_array = hb.as_array(ssp1_pasture_path) ssp3_pasture_array = hb.as_array(ssp3_pasture_path) ssp5_pasture_array = hb.as_array(ssp5_pasture_path) ssp1_ag_change_array = ssp1_ag_array - base_ag_array ssp3_ag_change_array = ssp3_ag_array - base_ag_array ssp5_ag_change_array = ssp5_ag_array - base_ag_array
def execute(kw, ui): if not kw: kw = create_default_kw(ui) kw['output_folder'] = kw['workspace_dir'] ui.update_run_log('Calculating crop-specific production') baseline_lulc_uri = ui.root_app.project_key_raster if not kw.get('model_base_data_dir'): kw['model_base_data_dir'] = os.path.join(ui.root_app.base_data_folder, 'models', 'nutritional_adequacy') bounding_box = hb.get_datasource_bounding_box(ui.root_app.project_aoi) ui.update_run_log('Loading input maps. Bounding box set to: ' + str(bounding_box)) # TODO Unimplemented switch here. run_full_nutritional_model = False # OTW just cal calories if run_full_nutritional_model: try: os.remove( os.path.join(kw['output_folder'], 'crop_proportion_baseline_500m.tif')) os.remove( os.path.join(kw['output_folder'], 'crop_proportion_baseline_1km.tif')) os.remove( os.path.join(kw['output_folder'], 'crop_proportion_500m.tif')) os.remove( os.path.join(kw['output_folder'], 'crop_proportion_1km.tif')) except: 'no' match_uri = kw['lulc_uri'] # Load the LULC array as the baseline lulc_baseline = hb.as_array(baseline_lulc_uri) no_data_value = hb.get_nodata_from_uri(baseline_lulc_uri) # Extract the nan_mask for later lulc_nan_mask = np.where(lulc_baseline == no_data_value, 1, 0).astype(np.int8) # Calculate the proportion of the grid-cell that is in cultivation as a function of the LULC. # For MODIS, this means 12 and 14 are 1.0 and 0.5 respectively. crop_proportion_baseline = np.where(lulc_baseline == 12, 1.0, 0.0) # TODO START HERE, i missed a nan mask and now the results have near infinite value. create a robust solution # crop_proportion_baseline[lulc_nan_mask] = np.nan crop_proportion_baseline = np.where(lulc_baseline == 14, .5, crop_proportion_baseline) # BUG If the files are not in the normal folder and onl linked to, it fails to find them. try: os.mkdir(os.path.join(kw['output_folder'], 'YieldTonsPerCell')) except: 'Dir already exists.' clipped_dir = os.path.join(kw['output_folder'], 'crop_production_and_harvested_area') try: os.mkdir(clipped_dir) except: 'Dir already exists.' try: os.mkdir(os.path.join(kw['output_folder'], 'nutrient_production')) except: 'Dir already exists.' crop_proportion_baseline_500m_uri = os.path.join( kw['output_folder'], 'YieldTonsPerCell', 'crop_proportion_baseline_500m.tif') utilities.save_array_as_geotiff(crop_proportion_baseline, crop_proportion_baseline_500m_uri, kw['lulc_uri']) crop_proportion_baseline_1km_uri = os.path.join( kw['output_folder'], 'YieldTonsPerCell', 'crop_proportion_baseline_1km.tif') population_bounding_box = utilities.get_bounding_box( kw['population_uri']) cell_size = utilities.get_cell_size_from_uri(kw['population_uri']) hb.resize_and_resample_dataset_uri(crop_proportion_baseline_500m_uri, population_bounding_box, cell_size, crop_proportion_baseline_1km_uri, 'bilinear') if kw['lulc_uri'] != baseline_lulc_uri: lulc_scenario = utilities.as_array(kw['lulc_uri']) crop_proportion = np.where(lulc_scenario == 12, 1.0, 0.0) crop_proportion = np.where(lulc_scenario == 14, .5, crop_proportion) crop_proportion_500m_uri = os.path.join( kw['output_folder'], 'YieldTonsPerCell', 'crop_proportion_500m.tif') utilities.save_array_as_geotiff(crop_proportion, crop_proportion_500m_uri, kw['lulc_uri']) crop_proportion_1km_uri = os.path.join(kw['output_folder'], 'YieldTonsPerCell', 'crop_proportion_1km.tif') # original_dataset_uri, bounding_box, out_pixel_size, output_uri, resample_method hb.resize_and_resample_dataset_uri(crop_proportion_500m_uri, population_bounding_box, cell_size, crop_proportion_1km_uri, 'bilinear') crop_proportion_baseline_1km = utilities.as_array( crop_proportion_baseline_1km_uri) crop_proportion_1km = utilities.as_array(crop_proportion_1km_uri) change_ratio = np.where( crop_proportion_baseline_1km > 0, crop_proportion_1km / crop_proportion_baseline_1km, 1.0) change_ratio_mean = np.mean(change_ratio) else: change_ratio_mean = 1.0 ui.update_run_log('Loading input maps') crop_maps_folder = kw['crop_maps_folder'] nutritional_content_odict = utilities.file_to_python_object( kw['nutritional_content_table_uri'], declare_type='2d_odict' ) # outputs as OrderedDict([('almond', OrderedDict([('fraction_refuse', '0.6'), ('Protein', '212.2'), ('Lipid', '494.2'), etc nutritional_requirements_odict = utilities.file_to_python_object( kw['nutritional_requirements_table_uri'], declare_type='2d_indexed_odict') population = utilities.as_array(kw['population_uri']) demographic_groups_list = kw['demographic_groups_list'] demographics_folder = kw['demographics_folder'] ui.update_run_log('Calculating crop-specific production') lulc_array = utilities.as_array(kw['lulc_uri']) lulc_wkt = hb.get_dataset_projection_wkt_uri(kw['lulc_uri']) harvested_area_ha_filenames = [] harvested_area_fraction_filenames = [] yield_tons_per_ha_filenames = [] yield_tons_per_cell_filenames = [] ha_array = 0 yield_per_ha_array = 0 # Calculate ha per cell cell_size = hb.get_cell_size_from_uri(kw['lulc_uri']) ha_per_cell = np.ones(lulc_array.shape) * (cell_size**2 / 10000) ha_per_cell_uri = os.path.join(kw['output_folder'], 'ha_per_cell.tif') utilities.save_array_as_geotiff(ha_per_cell, ha_per_cell_uri, kw['lulc_uri']) force_recalculation = False for folder_name in os.listdir(crop_maps_folder): current_folder = os.path.join(crop_maps_folder, folder_name) if os.path.isdir(current_folder): print('current_folder', current_folder) current_crop_name = folder_name.split('_', 1)[0] input_harvested_area_fraction_uri = os.path.join( current_folder, current_crop_name + '_HarvestedAreaFraction.tif') clipped_harvested_area_fraction_uri = os.path.join( clipped_dir, current_crop_name + '_HarvestedAreaFraction.tif') input_yield_tons_per_ha_uri = os.path.join( current_folder, current_crop_name + '_YieldPerHectare.tif') clipped_yield_tons_per_ha_uri = os.path.join( clipped_dir, current_crop_name + '_YieldPerHectare.tif') yield_tons_per_cell_uri = os.path.join( clipped_dir, current_crop_name + '_YieldTonsPerCell.tif') if not os.path.exists( clipped_harvested_area_fraction_uri ) or not os.path.exists( clipped_yield_tons_per_ha_uri) or not os.path.exists( yield_tons_per_cell_uri) or force_recalculation: # hb.clip_dataset_uri(input_harvested_area_fraction_uri, kw['aoi_uri'], clipped_harvested_area_fraction_uri) utilities.clip_by_shape_with_buffered_intermediate_uri( input_harvested_area_fraction_uri, kw['aoi_uri'], clipped_harvested_area_fraction_uri, match_uri, resampling_method='bilinear') harvested_area_fraction_array = utilities.as_array( clipped_harvested_area_fraction_uri) # hb.clip_dataset_uri(input_yield_tons_per_ha_uri, kw['aoi_uri'], clipped_yield_tons_per_ha_uri) utilities.clip_by_shape_with_buffered_intermediate_uri( input_yield_tons_per_ha_uri, kw['aoi_uri'], clipped_yield_tons_per_ha_uri, match_uri, resampling_method='bilinear') yield_tons_per_ha_array = utilities.as_array( clipped_yield_tons_per_ha_uri) nan1 = utilities.get_nodata_from_uri( input_harvested_area_fraction_uri) nan2 = utilities.get_nodata_from_uri( input_yield_tons_per_ha_uri) nan_mask = np.where((yield_tons_per_ha_array == nan1) & ( harvested_area_fraction_array == nan2)) yield_tons_per_cell_array = yield_tons_per_ha_array * harvested_area_fraction_array * ha_per_cell yield_tons_per_cell_array[nan_mask] == nan1 # NOTE forcing ndv to zero for calcualtions utilities.save_array_as_geotiff(yield_tons_per_cell_array, yield_tons_per_cell_uri, kw['lulc_uri'], data_type_override=7, no_data_value_override=0) harvested_area_fraction_filenames.append( clipped_harvested_area_fraction_uri) yield_tons_per_ha_filenames.append( clipped_yield_tons_per_ha_uri) yield_tons_per_cell_filenames.append(yield_tons_per_cell_uri) ui.update_run_log('Creating yield (tons) map for ' + folder_name) match_5min_uri = os.path.join(kw['output_folder'], 'crop_production_and_harvested_area', 'maize_HarvestedAreaFraction.tif') # match_5min_uri = os.path.join(ui.root_app.base_data_folder, 'models/crop_production/global_dataset/observed_yield/rice_yield_map.tif') match_array = utilities.as_array(match_5min_uri) # TODO Figure out if nans all right nan3 = utilities.get_nodata_from_uri( clipped_harvested_area_fraction_uri) array = utilities.as_array(clipped_harvested_area_fraction_uri) nan_mask = np.where(array == nan3, True, False).astype(np.bool) # TODO Why default to run always? # Fat = np.zeros(match_array.shape).astype(np.float64) #[Fatnan_mask] = nan3 Energy = np.zeros(match_array.shape).astype(np.float64) Energy[nan_mask] = nan3 Protein = np.zeros(match_array.shape).astype(np.float64) Protein[nan_mask] = nan3 VitA = np.zeros(match_array.shape).astype(np.float64) VitA[nan_mask] = nan3 VitC = np.zeros(match_array.shape).astype(np.float64) VitC[nan_mask] = nan3 VitE = np.zeros(match_array.shape).astype(np.float64) VitE[nan_mask] = nan3 Thiamin = np.zeros(match_array.shape).astype(np.float64) Thiamin[nan_mask] = nan3 Riboflavin = np.zeros(match_array.shape).astype(np.float64) Riboflavin[nan_mask] = nan3 Niacin = np.zeros(match_array.shape).astype(np.float64) Niacin[nan_mask] = nan3 VitB6 = np.zeros(match_array.shape).astype(np.float64) VitB6[nan_mask] = nan3 Folate = np.zeros(match_array.shape).astype(np.float64) Folate[nan_mask] = nan3 VitB12 = np.zeros(match_array.shape).astype(np.float64) VitB12[nan_mask] = nan3 Ca = np.zeros(match_array.shape).astype(np.float64) Ca[nan_mask] = nan3 Ph = np.zeros(match_array.shape).astype(np.float64) Ph[nan_mask] = nan3 Mg = np.zeros(match_array.shape).astype(np.float64) Mg[nan_mask] = nan3 K = np.zeros(match_array.shape).astype(np.float64) K[nan_mask] = nan3 Na = np.zeros(match_array.shape).astype(np.float64) Na[nan_mask] = nan3 Fe = np.zeros(match_array.shape).astype(np.float64) Fe[nan_mask] = nan3 Zn = np.zeros(match_array.shape).astype(np.float64) Zn[nan_mask] = nan3 Cu = np.zeros(match_array.shape).astype(np.float64) Cu[nan_mask] = nan3 for i in range(len(yield_tons_per_cell_filenames)): current_crop_name = os.path.splitext( os.path.split( harvested_area_fraction_filenames[i])[1])[0].split('_', 1)[0] ui.update_run_log('Calculating nutritional contribution of ' + current_crop_name) if current_crop_name in nutritional_content_odict.keys(): print('adding Nutritional content of ' + current_crop_name) # Fat += utilities.as_array(yield_tons_per_cell_filenames[i]) * 1000.0 * float(nutritional_content_odict[current_crop_name]['Fat']) Energy += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Energy']) Protein += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name] ['Protein']) VitA += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['VitA']) VitC += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['VitC']) VitE += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['VitE']) Thiamin += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name] ['Thiamin']) Riboflavin += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name] ['Riboflavin']) Niacin += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Niacin']) VitB6 += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['VitB6']) Folate += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Folate']) VitB12 += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['VitB12']) Ca += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Ca']) Ph += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Ph']) Mg += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Energy']) K += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Mg']) Na += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['K']) Fe += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Fe']) Zn += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Energy']) Cu += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Zn']) # TODO make this happen earlier in calcs # Fat *= change_ratio_mean Energy *= change_ratio_mean Protein *= change_ratio_mean VitA *= change_ratio_mean VitC *= change_ratio_mean VitE *= change_ratio_mean Thiamin *= change_ratio_mean Riboflavin *= change_ratio_mean Niacin *= change_ratio_mean VitB6 *= change_ratio_mean Folate *= change_ratio_mean VitB12 *= change_ratio_mean Ca *= change_ratio_mean Ph *= change_ratio_mean Mg *= change_ratio_mean K *= change_ratio_mean Na *= change_ratio_mean Fe *= change_ratio_mean Zn *= change_ratio_mean Cu *= change_ratio_mean # Fat *= change_ratio_mean Energy[nan_mask] = 0 Protein[nan_mask] = 0 VitA[nan_mask] = 0 VitC[nan_mask] = 0 VitE[nan_mask] = 0 Thiamin[nan_mask] = 0 Riboflavin[nan_mask] = 0 Niacin[nan_mask] = 0 VitB6[nan_mask] = 0 Folate[nan_mask] = 0 VitB12[nan_mask] = 0 Ca[nan_mask] = 0 Ph[nan_mask] = 0 Mg[nan_mask] = 0 K[nan_mask] = 0 Na[nan_mask] = 0 Fe[nan_mask] = 0 Zn[nan_mask] = 0 Cu[nan_mask] = 0 match_1km_uri = os.path.join(kw['output_folder'], 'YieldTonsPerCell', 'crop_proportion_baseline_1km.tif') utilities.save_array_as_geotiff( Energy, os.path.join(kw['output_folder'], 'nutrient_production', 'Energy_per_cell_5min.tif'), match_5min_uri, data_type_override=7, no_data_value_override=nan3) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Energy_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Energy_per_cell_1km.tif'), match_1km_uri) # utilities.save_array_as_geotiff(Fat, os.path.join(kw['output_folder'], 'nutrient_production', 'Fat_per_cell_5min.tif'), match_5min_uri) # utilities.resample_preserve_sum(os.path.join(kw['output_folder'], 'nutrient_production', 'Fat_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Fat_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Protein, os.path.join(kw['output_folder'], 'nutrient_production', 'Protein_per_cell_5min.tif'), match_5min_uri, no_data_value_override=nan3) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Protein_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Protein_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( VitA, os.path.join(kw['output_folder'], 'nutrient_production', 'VitA_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'VitA_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'VitA_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( VitC, os.path.join(kw['output_folder'], 'nutrient_production', 'VitC_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'VitC_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'VitC_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( VitE, os.path.join(kw['output_folder'], 'nutrient_production', 'VitE_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'VitE_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'VitE_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Thiamin, os.path.join(kw['output_folder'], 'nutrient_production', 'Thiamin_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Thiamin_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Thiamin_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Riboflavin, os.path.join(kw['output_folder'], 'nutrient_production', 'Riboflavin_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Riboflavin_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Riboflavin_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Niacin, os.path.join(kw['output_folder'], 'nutrient_production', 'Niacin_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Niacin_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Niacin_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( VitB6, os.path.join(kw['output_folder'], 'nutrient_production', 'VitB6_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'VitB6_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'VitB6_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Folate, os.path.join(kw['output_folder'], 'nutrient_production', 'Folate_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Folate_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Folate_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( VitB12, os.path.join(kw['output_folder'], 'nutrient_production', 'VitB12_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'VitB12_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'VitB12_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Ca, os.path.join(kw['output_folder'], 'nutrient_production', 'Ca_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Ca_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Ca_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Ph, os.path.join(kw['output_folder'], 'nutrient_production', 'Ph_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Ph_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Ph_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Mg, os.path.join(kw['output_folder'], 'nutrient_production', 'Mg_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Mg_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Mg_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( K, os.path.join(kw['output_folder'], 'nutrient_production', 'K_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'K_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'K_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Na, os.path.join(kw['output_folder'], 'nutrient_production', 'Na_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Na_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Na_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Fe, os.path.join(kw['output_folder'], 'nutrient_production', 'Fe_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Fe_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Fe_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Zn, os.path.join(kw['output_folder'], 'nutrient_production', 'Zn_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Zn_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Zn_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Cu, os.path.join(kw['output_folder'], 'nutrient_production', 'Cu_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Cu_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Cu_per_cell_1km.tif'), match_1km_uri) # calculate demand calculate_demand = True if calculate_demand: # TODO FOR HONDURAS need to switch to a non-population-map derived resolution. Incorporate decision making unit map to set the desired resolution ui.update_run_log('Calculating total nutrient demand') overall_nutrient_sum = 0 overall_nutrient_requirement_sum = 0 overall_ratio_array = np.zeros(population.shape) overall_ratio = 0 population_zero_normalized = np.where(population < 0, 0, population) for nutrient in nutritional_requirements_odict: nutrient_uri = os.path.join(kw['output_folder'], 'nutrient_production', nutrient + '_per_cell_1km.tif') nutrient_array = utilities.as_array(nutrient_uri) nutrient_requirement_array = population_zero_normalized * float( nutritional_requirements_odict[nutrient] ['recommended_daily_allowance']) * 365.0 # nutrient_requirement_array[nan_mask] = np.nan # nutrient_requirement_array[nutrient_requirement_array<=0] = np.nan nutrient_provision_ratio = np.where( (nutrient_array / nutrient_requirement_array > 0) & (nutrient_array / nutrient_requirement_array < 999999999999999999999999999999), nutrient_array / nutrient_requirement_array, 0) overall_ratio_array += nutrient_provision_ratio nutrient_sum = np.nansum(nutrient_array) overall_nutrient_sum += nutrient_sum nutrient_requirement_sum = np.nansum( nutrient_requirement_array) overall_nutrient_requirement_sum += nutrient_requirement_sum output_string = 'Full landscape produced ' + str( nutrient_sum ) + ' of ' + nutrient + ' compared to a national requirement of ' + str( nutrient_requirement_sum ) + ', yielding nutritional adequacy ratio of ' + str( nutrient_sum / nutrient_requirement_sum) + '.' ui.update_run_log(output_string) overall_ratio += nutrient_provision_ratio utilities.save_array_as_geotiff( nutrient_provision_ratio, nutrient_uri.replace('_per_cell_1km.tif', '_adequacy_ratio.tif'), nutrient_uri) overall_ratio_array *= 1.0 / 19.0 overall_ratio = (1.0 / 19.0) * (overall_nutrient_sum / overall_nutrient_requirement_sum) output_string = 'Overall nutrion adequacy ratio: ' + str( overall_ratio) + '.' ui.update_run_log(output_string) overall_ratio_uri = os.path.join(kw['output_folder'], 'overall_adequacy_ratio.tif') utilities.save_array_as_geotiff(overall_ratio_array, overall_ratio_uri, nutrient_uri) run_calories_only_model = True if run_calories_only_model: calc_caloric_production_from_lulc_uri(kw['lulc_uri'], ui=ui, **kw) return
r"C:\OneDrive\Projects\ipbes\output\SSP5xRCP8.5_MAGPIE_global_2050_c3per_kcal_production_per_cell.tif", r"C:\OneDrive\Projects\ipbes\output\SSP5xRCP8.5_MAGPIE_global_2050_c3per_percent_change.tif", r"C:\OneDrive\Projects\ipbes\output\SSP5xRCP8.5_MAGPIE_global_2050_c4ann_kcal_production_per_cell.tif", r"C:\OneDrive\Projects\ipbes\output\SSP5xRCP8.5_MAGPIE_global_2050_c4ann_percent_change.tif", r"C:\OneDrive\Projects\ipbes\output\SSP5xRCP8.5_MAGPIE_global_2050_c4per_kcal_production_per_cell.tif", r"C:\OneDrive\Projects\ipbes\output\SSP5xRCP8.5_MAGPIE_global_2050_c4per_percent_change.tif", r"C:\OneDrive\Projects\ipbes\output\SSP1xRCP2.6_IMAGE_global_2050_c3ann_kcal_production_per_cell.tif", r"C:\OneDrive\Projects\ipbes\output\SSP1xRCP2.6_IMAGE_global_2050_c3ann_percent_change.tif", r"C:\OneDrive\Projects\ipbes\output\SSP1xRCP2.6_IMAGE_global_2050_c3nfx_kcal_production_per_cell.tif", r"C:\OneDrive\Projects\ipbes\output\SSP1xRCP2.6_IMAGE_global_2050_c3nfx_percent_change.tif", r"C:\OneDrive\Projects\ipbes\output\SSP1xRCP2.6_IMAGE_global_2050_c3per_kcal_production_per_cell.tif", r"C:\OneDrive\Projects\ipbes\output\SSP1xRCP2.6_IMAGE_global_2050_c3per_percent_change.tif", r"C:\OneDrive\Projects\ipbes\output\SSP1xRCP2.6_IMAGE_global_2050_c4ann_kcal_production_per_cell.tif", r"C:\OneDrive\Projects\ipbes\output\SSP1xRCP2.6_IMAGE_global_2050_c4ann_percent_change.tif", r"C:\OneDrive\Projects\ipbes\output\SSP1xRCP2.6_IMAGE_global_2050_c4per_kcal_production_per_cell.tif", r"C:\OneDrive\Projects\ipbes\output\SSP1xRCP2.6_IMAGE_global_2050_c4per_percent_change.tif", ] for path in paths: file_root = hb.explode_path(path)['file_root'] scenario_name = file_root.split('_')[0].replace('x', ' ').upper() data_name = file_root.split('_', 3)[3:][0].replace('_', ' ') print('scenario_name', scenario_name) print('data_name', data_name) a = hb.as_array(path) output_uri = path.replace('.tif', '.png') if 'percent_change' in path: ge.show_array(a, output_uri=output_uri, use_basemap=True, resolution='i', title=scenario_name, cbar_label=data_name, vmin=0, vmax=0.2, move_ticks_in=True) else: ge.show_array(a, output_uri=output_uri, use_basemap=True, resolution='i', title=scenario_name, cbar_label=data_name, vmin=0, move_ticks_in=True)
def aggregate_crops_by_type(**kw): """CMIP6 and the land-use harmonization project have centered on 5 crop types: c3 annual, c3 perennial, c4 annual, c4 perennial, nitrogen fixer Aggregate the 15 crops to those four categories by modifying the baseline_regression_data.""" vars_names_to_aggregate = [ 'production_value_per_ha', 'calories_per_ha', 'proportion_cultivated', 'PotassiumApplication_Rate', 'PhosphorusApplication_Rate', 'NitrogenApplication_Rate', ] crop_membership = OrderedDict() crop_membership['c3_annual'] = [ 'aniseetc', 'artichoke', 'asparagus', 'bambara', 'barley', 'buckwheat', 'cabbage', 'canaryseed', 'carob', 'carrot', 'cassava', 'cauliflower', 'cerealnes', 'chestnut', 'cinnamon', 'cucumberetc', 'currant', 'date', 'eggplant', 'fonio', 'garlic', 'ginger', 'mixedgrain', 'hazelnut', 'hempseed', 'hop', 'kapokseed', 'linseed', 'mango', 'mate', 'mustard', 'nutmeg', 'okra', 'onion', 'greenonion', 'peppermint', 'potato', 'pumpkinetc', 'pyrethrum', 'ramie', 'rapeseed', 'rice', 'safflower', 'sisal', 'sorghumfor', 'sourcherry', 'spinach', 'sugarbeet', 'sunflower', 'taro', 'tobacco', 'tomato', 'triticale', 'tung', 'vanilla', 'vetch', 'walnut', 'watermelon', 'wheat', 'yam', 'yautia', ] crop_membership['c3_perennial'] = [ 'almond', 'apple', 'apricot', 'areca', 'avocado', 'banana', 'blueberry', 'brazil', 'cashewapple', 'cashew', 'cherry', 'chicory', 'chilleetc', 'citrusnes', 'clove', 'cocoa', 'coconut', 'coffee', 'cotton', 'cranberry', 'fig', 'flax', 'grapefruitetc', 'grape', 'jute', 'karite', 'kiwi', 'kolanut', 'lemonlime', 'lettuce', 'abaca', 'melonetc', 'melonseed', 'oats', 'oilpalm', 'oilseedfor', 'olive', 'orange', 'papaya', 'peachetc', 'pear', 'pepper', 'persimmon', 'pineapple', 'pistachio', 'plantain', 'plum', 'poppy', 'quince', 'quinoa', 'rasberry', 'rubber', 'rye', 'stonefruitnes', 'strawberry', 'stringbean', 'sweetpotato', 'tangetc', 'tea', ] crop_membership['c4_annual'] = [ 'maize', 'millet', 'sorghum', ] crop_membership['c4_perennial'] = [ 'greencorn', 'sugarcane', ] crop_membership['nitrogen_fixer'] = [ 'bean', 'greenbean', 'soybean', 'chickpea', 'clover', 'cowpea', 'groundnut', 'lupin', 'pea', 'greenpea', 'pigeonpea', 'lentil', 'legumenes', 'broadbean', 'castor', ] match_path = kw['5min_floats_match_path'] match_array = hb.as_array(match_path) # Iterate through crop_types if kw['runtime_conditionals']['aggregate_crops_by_type']: df = pd.DataFrame(index=range(1, 100), columns=crop_membership.keys()) for crop_type, crops in crop_membership.items(): L.info('Aggregating ' + str(crop_type) + ' ' + str(crops)) crop_type_calories_output_path = os.path.join( dirs['aggregate_crops_by_type'], crop_type + '_calories.tif') crop_type_calories_array = np.zeros(match_array.shape) current_crop_calories_array = None for crop in crops: crop_calories_path = os.path.join( hb.BASE_DATA_DIR, 'crops/crop_calories', crop + '_calories_per_ha_masked.tif') current_crop_calories_array = hb.as_array(crop_calories_path) current_crop_calories_array[np.isnan( current_crop_calories_array)] = 0.0 current_crop_calories_array[ current_crop_calories_array > 1e+14] = 0.0 current_crop_calories_array[ current_crop_calories_array < 0] = 0.0 current_crop_climate_bins_path = os.path.join( hb.BASE_DATA_DIR, r'crops\invest\extended_climate_bin_maps\extendedclimatebins' + crop + '.tif') current_crop_climate_bins = hb.as_array( current_crop_climate_bins_path) for i in range(1, 101): sum_ = np.sum( np.where(current_crop_climate_bins == i, current_crop_calories_array, 0)) # print(np.sum(current_crop_climate_bins)) crop_type_calories_array += current_crop_calories_array # print('crop_calories_path', crop_calories_path, np.sum(current_crop_calories_array), np.sum(crop_type_calories_array)) # # print(crop_type, np.sum(crop_type_calories_array)) hb.save_array_as_geotiff(crop_type_calories_array, crop_type_calories_output_path, match_path) return kw
r'c:/onedrive/projects\ipbes\intermediate\maps_for_each_rcp_ssp_pair\sspcur_rcp85_c4_annual.tif', r'c:/onedrive/projects\ipbes\intermediate\maps_for_each_rcp_ssp_pair\sspcur_rcp85_c4_annual_2015_extent.tif', r'c:/onedrive/projects\ipbes\intermediate\maps_for_each_rcp_ssp_pair\sspcur_rcp85_c4_annual_2050_extent.tif', r'c:/onedrive/projects\ipbes\intermediate\maps_for_each_rcp_ssp_pair\sspcur_rcp85_c4_perennial.tif', r'c:/onedrive/projects\ipbes\intermediate\maps_for_each_rcp_ssp_pair\sspcur_rcp85_c4_perennial_2015_extent.tif', r'c:/onedrive/projects\ipbes\intermediate\maps_for_each_rcp_ssp_pair\sspcur_rcp85_c4_perennial_2050_extent.tif', r'c:/onedrive/projects\ipbes\intermediate\maps_for_each_rcp_ssp_pair\sspcur_rcp85_nitrogen_fixer.tif', r'c:/onedrive/projects\ipbes\intermediate\maps_for_each_rcp_ssp_pair\sspcur_rcp85_nitrogen_fixer_2015_extent.tif', r'c:/onedrive/projects\ipbes\intermediate\maps_for_each_rcp_ssp_pair\sspcur_rcp85_nitrogen_fixer_2050_extent.tif', ] # for file_path in hb.list_filtered_paths_recursively(maps_dir, include_extensions='.tif'): # print('file_path', file_path) for i in luc_and_cc_paths: a = hb.as_array(i) output_uri = os.path.join(kw['output_dir'], os.path.split(i)[1].replace('.tif', '.png')) ge.show_array(a, output_uri=output_uri, use_basemap=True, resolution='i', cbar_label='Change in kcal per ha') for i in luc_no_cc_paths: a = hb.as_array(i) output_uri = os.path.join(kw['output_dir'], os.path.split(i)[1].replace('.tif', '.png')) ge.show_array(a, output_uri=output_uri, use_basemap=True,
import os, sys sys.path.extend(['C:/OneDrive/Projects']) import numpy as np import hazelbean as hb global_random_floats_15m_32bit_path = os.path.join(hb.TEST_DATA_DIR, 'global_random_floats_15m_32bit.tif') two_poly_eckert_iv_aoi_path = os.path.join(hb.TEST_DATA_DIR, 'two_poly_eckert_iv_aoi.shp') two_poly_wgs84_aoi_path = os.path.join(hb.TEST_DATA_DIR, 'two_poly_wgs84_aoi.shp') a = hb.as_array(global_random_floats_15m_32bit_path) # Old clip method for reference # hb.clip_dataset_uri(global_random_floats_15m_32bit_path, two_poly_wgs84_aoi_path, hb.temp('.tif', 'clip1', False, 'tests')) base_raster_path_list = [global_random_floats_15m_32bit_path] target_raster_path_list = [hb.temp('.tif', 'clip1', False, 'tests')] resample_method_list = ['bilinear'] target_pixel_size = hb.get_raster_info(global_random_floats_15m_32bit_path)['pixel_size'] bounding_box_mode = 'intersection' base_vector_path_list = [two_poly_wgs84_aoi_path] raster_align_index = 0 hb.align_and_resize_raster_stack( base_raster_path_list, target_raster_path_list, resample_method_list,
def caloric_production_change(**kw): if kw['runtime_conditionals']['caloric_production_change']: base_year = 2015 for scenario in kw['scenario_names']: for year in kw['years']: if year != base_year: for c, crop_type in enumerate(kw['crop_types_short']): base_dir = os.path.join(dirs['resample_lulc'], scenario, str(base_year)) base_year_path = hb.list_filtered_paths_recursively( base_dir, include_strings=crop_type, include_extensions='.tif')[0] base_year_array = hb.as_array(base_year_path) base_year_array[np.isnan(base_year_array)] = 0.0 base_year_array[base_year_array > 1e+14] = 0.0 base_year_array[base_year_array < 0] = 0.0 input_dir = os.path.join(dirs['resample_lulc'], scenario, str(year)) input_path = hb.list_filtered_paths_recursively( input_dir, include_strings=crop_type, include_extensions='.tif')[0] input_array = hb.as_array(input_path) input_array[np.isnan(input_array)] = 0.0 input_array[input_array > 1e+14] = 0.0 input_array[input_array < 0] = 0.0 calories_per_ha_array = hb.as_array( os.path.join(dirs['aggregate_crops_by_type'], kw['crop_types'][c] + '_calories.tif')) calories_per_ha_array[np.isnan( calories_per_ha_array)] = 0.0 calories_per_ha_array[ calories_per_ha_array > 1e+14] = 0.0 calories_per_ha_array[calories_per_ha_array < 0] = 0.0 ha_per_cell_array = hb.as_array( os.path.join(hb.BASE_DATA_DIR, 'misc', 'ha_per_cell_5m.tif')) extent_difference_array = base_year_array - input_array baseline_calorie_provision = calories_per_ha_array * ha_per_cell_array * base_year_array calorie_provision_per_cell = calories_per_ha_array * ha_per_cell_array * input_array caloric_change_per_cell = calories_per_ha_array * ha_per_cell_array * extent_difference_array # calorie_provision_percent_change = (calorie_provision_per_cell / baseline_calorie_provision) * 100.0 - 100.0 calorie_provision_percent_change = np.divide( calorie_provision_per_cell, baseline_calorie_provision, out=np.zeros_like(calorie_provision_per_cell), where=baseline_calorie_provision != 0) calorie_provision_percent_change = np.multiply( calorie_provision_percent_change, 100.0, out=np.zeros_like(calorie_provision_per_cell), where=baseline_calorie_provision != 0) calorie_provision_percent_change = np.subtract( calorie_provision_percent_change, 100.0, out=np.zeros_like(calorie_provision_per_cell), where=baseline_calorie_provision != 0) hb.create_dirs( os.path.join(dirs['caloric_production_change'], scenario, str(year))) extent_difference_path = os.path.join( dirs['caloric_production_change'], scenario, str(year), crop_type + '_extent_difference.tif') hb.save_array_as_geotiff( extent_difference_array, extent_difference_path, kw['5min_floats_match_path'], no_data_value_override=-9999.0) caloric_change_per_cell_path = os.path.join( dirs['caloric_production_change'], scenario, str(year), crop_type + '_caloric_change_per_cell.tif') hb.save_array_as_geotiff( caloric_change_per_cell, caloric_change_per_cell_path, kw['5min_floats_match_path'], no_data_value_override=-9999.0) caloric_production_per_cell_path = os.path.join( dirs['caloric_production_change'], scenario, str(year), crop_type + '_calories_per_cell.tif') hb.save_array_as_geotiff( calorie_provision_per_cell, caloric_production_per_cell_path, kw['5min_floats_match_path'], no_data_value_override=-9999.0) calorie_provision_percent_change_path = os.path.join( dirs['caloric_production_change'], scenario, str(year), crop_type + '_calories_percent_change.tif') hb.save_array_as_geotiff( calorie_provision_percent_change, calorie_provision_percent_change_path, kw['5min_floats_match_path'], no_data_value_override=-9999.0, data_type_override=6) produce_final = True if produce_final: overlay_shp_uri = os.path.join( hb.BASE_DATA_DIR, 'misc', 'countries') scenario_string = scenario.split( '-')[1][0:4].upper() + 'xRCP' + scenario.split( '-')[1][4] + '.' + scenario.split( '-')[1][5] + '_' + scenario.split( '-')[0] + '_global_' + str(year) kw['output_dir'] = kw['output_dir'].replace( '\\', '/') output_path = os.path.join( kw['output_dir'], scenario_string + '_' + crop_type + '_kcal_production_per_cell.tif') shutil.copy(caloric_production_per_cell_path, output_path) ge.show_raster_uri( output_path, output_uri=output_path.replace('.tif', '.png'), title=hb.explode_uri(output_path) ['file_root'].replace('_', ' ').title(), cbar_label= 'Kcal production per grid-cell given 2050 land-use', cbar_percentiles=[2, 50, 98], overlay_shp_uri=overlay_shp_uri, use_basemap=True, bounding_box='clip_poles' ) #cbar_percentiles=[1, 50, 99], output_path = os.path.join( kw['output_dir'], scenario_string + '_' + crop_type + '_percent_change.tif') shutil.copy(calorie_provision_percent_change_path, output_path) ge.show_raster_uri( output_path, output_uri=output_path.replace('.tif', '.png'), title=hb.explode_uri(output_path) ['file_root'].replace('_', ' ').title(), cbar_label= 'Percent change in kcal production from land-use change', vmin=-50, vmid=0, vmax=50, overlay_shp_uri=overlay_shp_uri, use_basemap=True, bounding_box='clip_poles' ) #cbar_percentiles=[1, 50, 99], return kw
def create_lulc(): global p L.info('Creating class-types lulc.') p.name_from_iterator_replacements = hb.file_root(p.area_of_interest_path) p.base_year_current_zone_lulc_path = os.path.join( p.cur_dir, 'base_year_' + p.name_from_iterator_replacements + '.tif') # Create match paths of both data types p.match_int_path = p.base_year_current_zone_lulc_path p.lulc_simplified_path = os.path.join(p.cur_dir, 'lulc_simplified.tif') # p.lulc_simplified_path = p.base_year_current_zone_lulc_path p.valid_mask_path = os.path.join(p.cur_dir, 'valid_mask_high_res.tif') p.proportion_valid_fine_per_coarse_cell_path = os.path.join( p.cur_dir, 'proportion_valid_fine_per_coarse_cell.tif') if p.run_this: hb.clip_while_aligning_to_coarser( p.base_year_lulc_path, p.base_year_current_zone_lulc_path, p.area_of_interest_path, p.current_change_map_paths[0], resample_method='nearest', output_data_type=1, nodata_target=255, all_touched=True, verbose=True, ensure_fits=True, gtiff_creation_options=hb.DEFAULT_GTIFF_CREATION_OPTIONS) # Set NDV masking based on AOI of current zone. hb.create_valid_mask_from_vector_path( p.area_of_interest_path, p.base_year_current_zone_lulc_path, p.valid_mask_path) p.valid_mask = hb.as_array(p.valid_mask_path) hb.set_ndv_by_mask_path(p.base_year_current_zone_lulc_path, p.valid_mask_path) p.proportion_valid_fine_per_coarse_cell = hazelbean.pyramids.calc_proportion_of_coarse_res_with_valid_fine_res( p.current_change_map_paths[0], p.valid_mask_path) hb.save_array_as_geotiff(p.proportion_valid_fine_per_coarse_cell, p.proportion_valid_fine_per_coarse_cell_path, p.current_change_map_paths[0]) lulc_ds = gdal.Open(p.base_year_current_zone_lulc_path) lulc_band = lulc_ds.GetRasterBand(1) lulc_array = lulc_band.ReadAsArray().astype(np.int) p.scaled_proportion_to_allocate_paths = [] for path in p.current_change_map_paths: unscaled = hb.as_array(path).astype(np.float64) scaled_proportion_to_allocate = p.proportion_valid_fine_per_coarse_cell * unscaled scaled_proportion_to_allocate_path = os.path.join( p.cur_dir, os.path.split(hb.suri(path, 'scaled'))[1]) hb.save_array_as_geotiff(scaled_proportion_to_allocate, scaled_proportion_to_allocate_path, path, ndv=-9999.0, data_type=7) p.scaled_proportion_to_allocate_paths.append( scaled_proportion_to_allocate_path) if os.path.exists(p.lulc_class_types_path): # load the simplified class correspondnce as a nested dictionary. lulc_class_types_odict = hb.file_to_python_object( p.lulc_class_types_path, declare_type='DD') # For cythonization reasons, I need to ensure this comes in as ints lulc_class_types_ints_dict = dict() p.lulc_unsimplified_classes_list = [] for row_name in lulc_class_types_odict.keys(): lulc_class_types_ints_dict[int(row_name)] = int( lulc_class_types_odict[row_name]['lulc_class_type']) p.lulc_unsimplified_classes_list.append(int(row_name)) p.max_unsimplified_lulc_classes = max( p.lulc_unsimplified_classes_list) p.new_unsimplified_lulc_addition_value = 10**( len(str(p.max_unsimplified_lulc_classes)) + 1 ) / 10 # DOCUMENTATION, new classes are defined here as adding 1 order # # 1 is agriculture, 2 is mixed ag/natural, 3 is natural, 4 is urban, 0 is no data lulc_simplified_array = hb.reclassify_int_array_by_dict_to_ints( lulc_array, lulc_class_types_ints_dict) no_data_value_override = hb.get_nodata_from_uri( p.base_year_current_zone_lulc_path) hb.save_array_as_geotiff(lulc_simplified_array, p.lulc_simplified_path, p.base_year_current_zone_lulc_path, data_type=1, set_inf_to_no_data_value=False, ndv=no_data_value_override, compress=True) else: L.warn( 'No lulc_class_types_path specified. Assuming you want to run every class uniquely.' ) # If we don't run this zone, we know we will need to use the unmodified lulc when stitching everything back together if p.run_this_zone is False: p.layers_to_stitch.append(p.base_year_current_zone_lulc_path) else: p.lulc_simplified_path = p.base_year_current_zone_lulc_path
def calc_standardized_ecological_uncertainty(): global p if not os.path.exists(p.output_dir): hb.create_directories(p.output_dir) p.standardized_ecological_uncertainty_analysis_res_path = os.path.join( p.cur_dir, 'standardized_ecological_uncertainty_analysis_res.tif') hb.raster_calculator_hb( [(p.percent_of_overall_forest_cover_within_threshold_path, 1)], standardized_ecological_uncertainty, p.standardized_ecological_uncertainty_analysis_res_path, 7, -9999.0, gtiff_creation_options=hb.DEFAULT_GTIFF_CREATION_OPTIONS) p.standardized_ecological_uncertainty_unmasked_path = os.path.join( p.cur_dir, 'standardized_ecological_uncertainty_unmasked.tif') hb.resample_to_match( p.standardized_ecological_uncertainty_analysis_res_path, p.clipped_lulc_path, p.standardized_ecological_uncertainty_unmasked_path) def mask_op(x, y): return np.where(x != 255, x * y, 255) p.standardized_ecological_uncertainty_unnormalized_path = os.path.join( p.cur_dir, 'standardized_ecological_uncertainty_unnormalized.tif') hb.raster_calculator_hb( [(p.standardized_ecological_uncertainty_unmasked_path, 1), (p.is_restorable_path, 1)], mask_op, p.standardized_ecological_uncertainty_unnormalized_path, 7, -9999.0, gtiff_creation_options=hb.DEFAULT_GTIFF_CREATION_OPTIONS) x = hb.as_array(p.standardized_ecological_uncertainty_unnormalized_path) min = np.min(x) max = np.max(x) desired_max = 100.0 scalar = desired_max / max o = np.where(x != -9999.0, x * scalar, -9999.0) x = None p.standardized_ecological_uncertainty_path = os.path.join( p.cur_dir, 'standardized_ecological_uncertainty.tif') hb.save_array_as_geotiff( o, p.standardized_ecological_uncertainty_path, p.standardized_ecological_uncertainty_unnormalized_path, data_type=7, ndv=-9999.0) r = hb.as_array(p.is_restorable_path) keys_where = np.where( r == 1 ) # Not the NDV cause we're calculating deciles of the actually restorable land size = len(keys_where[0]) output = np.ones(o.shape) * 255 stride = int(size / 10.0) sorted_keys_1dim = o[keys_where].argsort(axis=None) sorted_keys = (keys_where[0][sorted_keys_1dim], keys_where[1][sorted_keys_1dim]) for i in range(10): L.info('Calculating percentile ' + str((i + 1) * 10)) output[sorted_keys[0][i * stride:(i + 1) * stride], sorted_keys[1][i * stride:(i + 1) * stride]] = i + 1 output = output.reshape(o.shape) p.restoration_success_deciles_pre_final_mask_path = os.path.join( p.cur_dir, 'restoration_success_deciles_pre_final_mask.tif') hb.save_array_as_geotiff( output, p.restoration_success_deciles_pre_final_mask_path, p.standardized_ecological_uncertainty_unnormalized_path, data_type=1, ndv=255) ## NOTE FOR NEXT RELEASE: The following section was written to be memory safe and fast via raster_calculator_hb. However, ## This would make each tile in the calculation independent of others, which would incorrectly identify max value for as the LOCAL value not global. ## This resulted in tiling artifacts. And then again, the percentile calculaiton was messed up too. Thus, here, I reverted for time sake ## to non-memory safe numpy arrays. # def normalize_op(x): # min = np.min(x) # max = np.max(x) # desired_max = 100.0 # scalar = desired_max / max # return np.where(x != -9999.0, x * scalar, -9999.0) # # p.standardized_ecological_uncertainty_path = os.path.join(p.cur_dir, 'standardized_ecological_uncertainty.tif') # hb.raster_calculator_hb([(p.standardized_ecological_uncertainty_unnormalized_path, 1)], normalize_op, p.standardized_ecological_uncertainty_path, 7, -9999.0) # # def make_deciles(x, y): # # keys_where = np.where(y == 1) # Not the NDV cause we're calculating deciles of the actually restorable land # # keys_where = np.where(x != -9999.0) # size = len(keys_where[0]) # output = np.ones(x.shape) * -9999.0 # stride = int(size / 10.0) # # sorted_keys_1dim = x[keys_where].argsort(axis=None) # sorted_keys = (keys_where[0][sorted_keys_1dim], keys_where[1][sorted_keys_1dim]) # for i in range(10): # L.info('Calculating percentile ' + str((i + 1) * 10)) # # output[sorted_keys[0][i * stride: (i + 1) * stride], sorted_keys[1][i * stride: (i + 1) * stride]] = i + 1 # output = output.reshape(x.shape) # # return output # p.restoration_success_deciles_pre_final_mask_path = os.path.join(p.cur_dir, 'restoration_success_deciles_pre_final_mask.tif') # hb.raster_calculator_hb([(p.standardized_ecological_uncertainty_path, 1), # (p.is_restorable_path, 1), # ], make_deciles, p.restoration_success_deciles_pre_final_mask_path, 1, 255) p.restoration_success_deciles_path = os.path.join( p.output_dir, 'restoration_success_deciles.tif') hb.set_ndv_by_mask_path(p.restoration_success_deciles_pre_final_mask_path, p.valid_mask_input_res_path, p.restoration_success_deciles_path) def cast_int(x): return np.byte(x) p.standardized_ecological_uncertainty_ints_pre_final_mask_path = os.path.join( p.cur_dir, 'standardized_ecological_uncertainty_ints_pre_final_mask.tif') hb.raster_calculator_hb( [(p.standardized_ecological_uncertainty_path, 1)], cast_int, p.standardized_ecological_uncertainty_ints_pre_final_mask_path, 1, 255) p.standardized_ecological_uncertainty_ints_path = os.path.join( p.output_dir, 'standardized_ecological_uncertainty_percent.tif') hb.set_ndv_by_mask_path( p.standardized_ecological_uncertainty_ints_pre_final_mask_path, p.valid_mask_input_res_path, p.standardized_ecological_uncertainty_ints_path)
def zonal_statistics_rasterized(zone_ids_raster_path, values_raster_path, zones_ndv=None, values_ndv=None, zone_ids_data_type=None, values_data_type=None, use_iterblocks=True, unique_zone_ids=None, verbose=True, max_possible_zone_value=None): """ Calculate zonal statistics using a pre-generated raster ID array. NOTE that by construction, this type of zonal statistics cannot handle overlapping polygons (each polygon is just represented by its id int value in the raster). """ if use_iterblocks: if verbose: L.info('Starting to run zonal_statistics_rasterized using iterblocks.') if unique_zone_ids is None: if verbose: L.info('Load zone_ids_raster and compute unique values in it. Could be slow (and could be pregenerated for speed if desired).') zone_ids = hb.as_array(zone_ids_raster_path) unique_zone_ids = np.unique(zone_ids).astype(np.int64) zone_ids = None # Get dimensions of rasters for callback reporting' zone_ds = gdal.OpenEx(zone_ids_raster_path) n_cols = zone_ds.RasterYSize n_rows = zone_ds.RasterXSize n_pixels = n_cols * n_rows # Create new arrays to hold results. # NOTE THAT this creates an array as long as the MAX VALUE in unique_zone_ids, which means there could be many zero values. This # is intended as it increases computation speed to not have to do an additional lookup. aggregated_sums = np.zeros(max(unique_zone_ids) + 1, dtype=np.float64) aggregated_counts = np.zeros(max(unique_zone_ids) + 1, dtype=np.int64) last_time = time.time() pixels_processed = 0 # Iterate through block_offsets zone_ids_raster_path_band = (zone_ids_raster_path, 1) for block_offset in hb.iterblocks(zone_ids_raster_path_band, offset_only=True): # L.info('block_offset ' + str(block_offset)) # NOTE CHANGE FROM PGP on buf_ vs win_ block_offset_new_gdal_api = { 'xoff': block_offset['xoff'], 'yoff': block_offset['yoff'], 'buf_ysize': block_offset['win_ysize'], 'buf_xsize': block_offset['win_xsize'], } zones_ds = gdal.OpenEx(zone_ids_raster_path) values_ds = gdal.OpenEx(values_raster_path) # No idea why, but using **block_offset_new_gdal_api failed, so I unpack it manually here. zones_array = zones_ds.ReadAsArray(block_offset_new_gdal_api['xoff'], block_offset_new_gdal_api['yoff'], block_offset_new_gdal_api['buf_xsize'], block_offset_new_gdal_api['buf_ysize']).astype(np.int64) values_array = values_ds.ReadAsArray(block_offset_new_gdal_api['xoff'], block_offset_new_gdal_api['yoff'], block_offset_new_gdal_api['buf_xsize'], block_offset_new_gdal_api['buf_ysize']).astype(np.float64) unique_zone_ids_np = np.asarray(unique_zone_ids, dtype=np.int64) unique_ids, sums, counts = hb.zonal_stats_cythonized_iterblocks_from_arrays(zones_array, values_array, unique_zone_ids_np, zones_ndv, values_ndv) aggregated_sums += sums aggregated_counts += counts pixels_processed += block_offset_new_gdal_api['buf_xsize'] * block_offset_new_gdal_api['buf_ysize'] last_time = hb.invoke_timed_callback( last_time, lambda: L.info('%.2f%% complete', float(pixels_processed) / n_pixels * 100.0), 3) sums = aggregated_sums counts = aggregated_counts else: L.info('Running zonal_statistics_rasterized without using iterblocks. This allows smarter type detection but can be slower and hit memory errors.') if zones_ndv is None: zones_ndv = np.int64(-9999) # INT if values_ndv is None: values_ndv = np.float64(-9999.0) if unique_zone_ids is None: if max_possible_zone_value is None: max_possible_zone_value = 100000 unique_zone_ids = np.arange(0, max_possible_zone_value, dtype=np.int64) # unique_zone_ids = np.concatenate(np.asarray([zones_ndv], dtype=np.int64), np.arange(0, max_possible_zone_value, dtype=np.int64)) hb.assert_gdal_paths_in_same_projection(zone_ids_raster_path, values_raster_path) unique_ids, sums, counts = hb.zonal_stats_cythonized_iterblocks(zone_ids_raster_path, values_raster_path, unique_zone_ids, zones_ndv, values_ndv) zone_ids_raster_path = None values_raster_path = None zones_array = None values_array = None return unique_ids, sums, counts
def process_coarse_change_maps(): global p L.info('process_coarse_change_maps.') # Change maps are in this directory and must be of the format [CLASS_ID_INT]_[someting, but anything else].tif if not os.path.isdir(p.coarse_change_maps_dir): p.coarse_change_maps_dir = os.path.split(p.coarse_change_maps_dir)[0] if not os.path.isdir(p.coarse_change_maps_dir): raise NameError('Unable to parse coarse_change_maps_dir.') tifs_in_dir = hb.list_filtered_paths_nonrecursively( p.coarse_change_maps_dir, include_extensions='.tif') p.change_map_paths = [] for path in tifs_in_dir: try: rendered_int = int(hb.file_root(path).split('_')[0]) except: rendered_int = None if isinstance(rendered_int, int): p.change_map_paths.append(path) p.change_map_raster_infos = [ hb.get_raster_info(i) for i in p.change_map_paths ] # Test that all the change maps are the same properties. if len(set([i['geotransform'] for i in p.change_map_raster_infos])) != 1: for j in [i['geotransform'] for i in p.change_map_raster_infos]: L.critical('geotransform: ' + str(j)) # raise NameError('The maps in coarse change maps dir are not all the same shape, projection, etc, or they have been improperly named/formatted.') # p.current_change_in_crop_extent_path = os.path.join(p.cur_dir, 'change_in_crop_extent.tif') p.current_change_map_paths = [] p.float_ndv = None p.int_ndv = 255 L.info('change_map_paths: ' + str(p.change_map_paths)) p.zone_transition_sums = OrderedDict() p.classes_projected_to_change = [] for path in p.change_map_paths: changing_class_id = int(os.path.split(path)[1].split('_')[0]) p.classes_projected_to_change.append(changing_class_id) if not p.float_ndv: p.float_ndv = hb.get_nodata_from_uri(path) if p.float_ndv is None: p.float_ndv = -9999.0 new_path = os.path.join(p.cur_dir, os.path.split(path)[1]) p.current_change_map_paths.append(new_path) if p.run_this: # NOTE NONSTANDARD placement of run_this hb.clip_raster_by_vector( str(path), str(new_path), str(p.area_of_interest_path), resample_method='nearest', all_touched=True, verbose=True, ensure_fits=True, gtiff_creation_options=hb.DEFAULT_GTIFF_CREATION_OPTIONS) # To make the model not run in zones with zero change, we collect these sums and prevent runing if all of them are zero current_coarse_array = hb.as_array(new_path) current_sum = np.sum( current_coarse_array[current_coarse_array != p.float_ndv]) p.zone_transition_sums[changing_class_id] = current_sum p.run_this_zone = True if np.sum([float(i) for i in p.zone_transition_sums.values()]) <= 0: p.run_this_zone = False L.info('current_change_map_paths' + str(p.current_change_map_paths))
def create_allocation_from_change_map(): global p p.projected_lulc_simplified_path = hb.ruri( os.path.join(p.cur_dir, 'projected_lulc_simplified.tif')) # AGROSERVE shortcut note: assumed that it happens in SEQUENCE first cropland then pasture. if p.run_this and p.run_this_zone: lulc_array = hb.as_array(p.lulc_simplified_path) new_lulc_array = np.copy(lulc_array) p.change_array_paths = [] for change_map_index, change_map_path in enumerate( p.scaled_proportion_to_allocate_paths): change_to_allocate_array = hb.as_array(change_map_path) # Often it is the case that the number of cells that will be allocated is greater than the amount of high-res cells actually available for conversion. This happens only if the # conversion_elligibility.csv rules out cells (it will not happen if only adjacency and physical suitability is done, as there will be SOME places allbethem terrible. num_cells_skipped = np.zeros(change_to_allocate_array.shape) class_to_allocate = int( os.path.split(change_map_path)[1].split('_')[0]) current_overall_suitability_path = p.overall_suitability_paths[ change_map_index] overall_suitability_array = hb.as_array( current_overall_suitability_path) # Test that map resolutions are workable multiples of each other aspect_ratio_test_result = int( round(overall_suitability_array.shape[0] / change_to_allocate_array.shape[0])) == int( round(overall_suitability_array.shape[1] / change_to_allocate_array.shape[1])) if not aspect_ratio_test_result: warnings.warn('aspect_ratio_test_value FAILED.') aspect_ratio = int( round(overall_suitability_array.shape[0] / change_to_allocate_array.shape[0])) L.info('Beginning allocation using allocation ratio of ' + str(aspect_ratio)) L.info('Sizes involved: overall_suitability_array, ' + str(overall_suitability_array.shape) + ' change_to_allocate_array, ' + str(change_to_allocate_array.shape)) ha_per_source_cell = 300**2 / 100**2 change_array = np.zeros(lulc_array.shape) combined_rank_array = np.zeros(lulc_array.shape).astype(np.int64) # TODOO Note that i ignored smaller-than-chunk shards. for change_map_region_row in range( change_to_allocate_array.shape[0]): L.info('Starting horizontal row ' + str(change_map_region_row)) for change_map_region_col in range( change_to_allocate_array.shape[1]): if not change_to_allocate_array[change_map_region_row, change_map_region_col] > 0: num_cells_to_allocate = 0 else: num_cells_to_allocate = int( round( change_to_allocate_array[change_map_region_row, change_map_region_col] / ha_per_source_cell)) if num_cells_to_allocate > 0: source_map_starting_row = change_map_region_row * aspect_ratio source_map_starting_col = change_map_region_col * aspect_ratio combined_adjacency_effect_chunk = overall_suitability_array[ source_map_starting_row:source_map_starting_row + aspect_ratio, source_map_starting_col:source_map_starting_col + aspect_ratio] ranked_chunk, sorted_keys = hb.get_rank_array_and_keys( combined_adjacency_effect_chunk, ndv=0) if num_cells_to_allocate > len(sorted_keys[0]): previous_num_cells_to_allocate = num_cells_to_allocate num_skipped = num_cells_to_allocate - len( sorted_keys[0]) num_cells_to_allocate = len(sorted_keys[0]) L.warning( 'Allocation algorithm requested to allocate more cells than were available for transition given the suitability constraints. Num requested: ' + str(previous_num_cells_to_allocate) + ', Num allocated: ' + str(len(sorted_keys[0])) + ', Num skipped ' + str(num_skipped)) num_cells_skipped[ change_map_region_row, change_map_region_col] = num_skipped sorted_keys_array = np.array(sorted_keys) # Create a tuple (ready for use as a numpy key) of the top allocation_amoutn keys keys_to_change = ( sorted_keys_array[0][0:num_cells_to_allocate], sorted_keys_array[1][0:num_cells_to_allocate]) change_chunk = np.zeros(ranked_chunk.shape) change_chunk[keys_to_change] = 1 ## TODOO this was useful but there was a 29x29 vs 30x30 error. Renable after fix. # Just for visualization purposes, who what all the ranked zones look like together when mosaiced. combined_rank_array[ source_map_starting_row:source_map_starting_row + aspect_ratio, source_map_starting_col:source_map_starting_col + aspect_ratio] = ranked_chunk # TODOO BUG, there's a slight shift to the right that comes in here. change_array[ source_map_starting_row:source_map_starting_row + aspect_ratio, source_map_starting_col:source_map_starting_col + aspect_ratio] = change_chunk L.info('Processing outputted results.') p.new_classes_int_list = [13] p.final_lulc_addition_value = 13 new_lulc_array = np.where( (change_array == 1), p.final_lulc_addition_value, new_lulc_array) # NOTE, pasture will be 8 thus, crops 9 change_array_path = os.path.join( p.cur_dir, str(class_to_allocate) + '_change_array.tif') p.change_array_paths.append(change_array_path) hb.save_array_as_geotiff(change_array, change_array_path, p.match_int_path, compress=True) p.num_cells_skipped_path = hb.ruri( os.path.join(p.cur_dir, str(class_to_allocate) + '_num_cells_skipped.tif')) hb.save_array_as_geotiff(num_cells_skipped, p.num_cells_skipped_path, change_map_path, compress=True) p.combined_rank_array_path = hb.ruri( os.path.join( p.cur_dir, str(class_to_allocate) + '_combined_rank_array.tif')) hb.save_array_as_geotiff(combined_rank_array, p.combined_rank_array_path, p.match_int_path, compress=True, data_type=7) hb.save_array_as_geotiff(new_lulc_array, p.projected_lulc_simplified_path, p.match_int_path, compress=True)
def create_physical_suitability(): global p L.info('Creating physical suitability layer from base data.') # physical suitability calculations, though for speed it's included as a base datum. dem_unaligned_path = hb.temp( '.tif', folder=p.workspace_dir, remove_at_exit=True) #hb.temp('.tif', remove_at_exit=True) stats_to_calculate = ['TRI'] hb.clip_hydrosheds_dem_from_aoi(dem_unaligned_path, p.area_of_interest_path, p.match_float_path) hb.calculate_topographic_stats_from_dem( dem_unaligned_path, p.physical_suitability_dir, stats_to_calculate=stats_to_calculate, output_suffix='unaligned') dem_path = os.path.join(p.physical_suitability_dir, 'dem.tif') hb.align_dataset_to_match(dem_unaligned_path, p.match_float_path, dem_path, aoi_uri=p.area_of_interest_path) for stat in stats_to_calculate: stat_unaligned_path = os.path.join(p.physical_suitability_dir, stat + '_unaligned.tif') hb.delete_path_at_exit(stat_unaligned_path) stat_path = os.path.join(p.physical_suitability_dir, stat + '.tif') hb.align_dataset_to_match(stat_unaligned_path, p.match_float_path, stat_path, resample_method='bilinear', align_to_match=True, aoi_uri=p.area_of_interest_path) soc_path = os.path.join(p.physical_suitability_dir, 'soc.tif') hb.align_dataset_to_match(p.base_data_soc_path, p.match_int_path, soc_path, aoi_uri=p.area_of_interest_path, output_data_type=7) tri_path = os.path.join(p.physical_suitability_dir, 'tri.tif') hb.align_dataset_to_match(p.base_data_tri_path, p.match_int_path, tri_path, aoi_uri=p.area_of_interest_path, output_data_type=7) # TODOO Create cythonized array_sum_product() p.physical_suitability_path = os.path.join(p.physical_suitability_dir, 'physical_suitability.tif') soc_array = hb.as_array(soc_path) tri_array = hb.as_array(tri_path) physical_suitability_array = np.log(soc_array) - np.log(tri_array) # p.global_physical_suitability_path = os.path.join(p.model_base_data_dir, 'physical_suitability_compressed.tif') p.clipped_physical_suitability_path = os.path.join( p.cur_dir, 'physical_suitability.tif') if p.run_this and p.run_this_zone: # hb.clip_raster_by_vector(p.global_physical_suitability_path, p.physical_suitability_path, p.coarse_res_aoi_path, all_touched=True) hb.clip_while_aligning_to_coarser( p.physical_suitability_path, p.clipped_physical_suitability_path, p.area_of_interest_path, p.current_change_map_paths[0], resample_method='nearest', all_touched=True, verbose=True, ensure_fits=True, gtiff_creation_options=hb.DEFAULT_GTIFF_CREATION_OPTIONS) p.current_physical_suitability_path = p.clipped_physical_suitability_path # NOTE awkward naming # hb.clip_dataset_uri(p.global_physical_suitability_path, p.coarse_res_aoi_path, p.physical_suitability_path, False, all_touched=False) physical_suitability_array = hb.as_array( p.current_physical_suitability_path) p.match_float_path = p.current_physical_suitability_path np.seterr(divide='ignore', invalid='ignore') physical_suitability_array = np.where( physical_suitability_array > -1000, physical_suitability_array, 0) physical_suitability_array = np.where( physical_suitability_array < 100000000, physical_suitability_array, 0) hb.save_array_as_geotiff(physical_suitability_array, p.current_physical_suitability_path, p.match_float_path, compress=True)
def create_convolution_inputs(): global p p.convolution_inputs_dir = p.cur_dir if p.run_this and p.run_this_zone: lulc_array = hb.as_array(p.lulc_simplified_path) ndv = hb.get_nodata_from_uri(p.lulc_simplified_path) # Get which values exist in simplified_lulc unique_values = list(hb.enumerate_array_as_odict(lulc_array).keys()) unique_values = [int(i) for i in unique_values] try: p.classes_to_ignore = [ int(i) for i in p.classes_to_ignore.split(' ') ] except: p.classes_to_ignore = [] # TODOO Better approach than ignoring classes would be to encode ALL such information into the different CSVs. This would allow more grandular control over how, e.g. water DOES have attraction effect but does not necessarily expand. ignore_values = [ndv] + p.classes_to_ignore p.simplified_lulc_classes = [ i for i in unique_values if i not in ignore_values ] # HACK p.classes_to_ignore = [0] p.classes_with_effect = [ i for i in p.simplified_lulc_classes if i not in p.classes_to_ignore ] L.info('Creating binaries for classes ' + str(p.classes_with_effect)) try: p.max_simplified_lulc_classes = max(p.simplified_lulc_classes) except: p.max_simplified_lulc_classes = 20 p.new_simplified_lulc_addition_value = 10**( len(str(p.max_simplified_lulc_classes)) + 1 ) / 10 # DOCUMENTATION, new classes are defined here as adding 1 order of magnitude larger value (2 becomes 12 if the max is 5. 2 becomes 102 if the max is 15. p.classes_with_change = [ int(os.path.split(i)[1].split('_')[0]) for i in p.current_change_map_paths ] binary_paths = [] for unique_value in p.classes_with_effect: # binary_array = np.zeros(lulc_array.shape) binary_array = np.where(lulc_array == unique_value, 1, 0).astype(np.uint8) binary_path = os.path.join( p.convolution_inputs_dir, 'class_' + str(unique_value) + '_binary.tif') binary_paths.append(binary_path) hb.save_array_as_geotiff(binary_array, binary_path, p.lulc_simplified_path, compress=True) convolution_params = hb.file_to_python_object( p.class_proximity_parameters_path, declare_type='DD', output_key_data_type=str, output_value_data_type=float) convolution_paths = [] for i, v in enumerate(p.classes_with_effect): L.info('Calculating convolution for class ' + str(v)) binary_array = hb.as_array(binary_paths[i]) convolution_metric = seals_utils.distance_from_blurred_threshold( binary_array, convolution_params[str(v)]['clustering'], 0.5, convolution_params[str(v)]['decay']) convolution_path = os.path.join( p.convolution_inputs_dir, 'class_' + str(p.classes_with_effect[i]) + '_convolution.tif') convolution_paths.append(convolution_path) hb.save_array_as_geotiff(convolution_metric, convolution_path, p.match_float_path, compress=True) pairwise_params = hb.file_to_python_object( p.pairwise_class_relationships_path, declare_type='DD', output_key_data_type=str, output_value_data_type=float) for i in p.classes_with_effect: i_convolution_path = os.path.join( p.convolution_inputs_dir, 'class_' + str(i) + '_convolution.tif') i_convolution_array = hb.as_array(i_convolution_path) for j in p.classes_with_change: L.info('Processing effect of ' + str(i) + ' on ' + str(j)) adjacency_effect_path = os.path.join( p.convolution_inputs_dir, 'adjacency_effect_of_' + str(i) + '_on_' + str(j) + '.tif') adjacency_effect_array = i_convolution_array * pairwise_params[ str(i)][str(j)] hb.save_array_as_geotiff(adjacency_effect_array, adjacency_effect_path, p.match_float_path, compress=True) for i in p.classes_with_change: L.info('Combining adjacency effects for class ' + str(i)) combined_adjacency_effect_array = np.ones(lulc_array.shape) combined_adjacency_effect_path = os.path.join( p.convolution_inputs_dir, 'combined_adjacency_effect_' + str(i) + '.tif') for j in p.classes_with_effect: current_uri = os.path.join( p.convolution_inputs_dir, 'adjacency_effect_of_' + str(j) + '_on_' + str(i) + '.tif') # NOTICE SWITCHED I and J current_effect = hb.as_array(current_uri) combined_adjacency_effect_array *= current_effect + 1.0 # Center on 1 so that 0.0 has no effect hb.save_array_as_geotiff(combined_adjacency_effect_array, combined_adjacency_effect_path, p.match_float_path, compress=True)