def convert_simplified_to_original_classes(): global p lulc_class_types_odict = hb.file_to_python_object(p.lulc_class_types_path, declare_type='DD') p.simple_classes_to_projected_original_classes = OrderedDict() for original_class, csv_odict in lulc_class_types_odict.items(): if csv_odict['output_class_id'] != '': p.simple_classes_to_projected_original_classes[int( csv_odict['lulc_class_type'])] = int( csv_odict['output_class_id']) if p.run_this and p.run_this_zone: lulc_original_classes_array = hb.as_array( p.base_year_current_zone_lulc_path) for c, path in enumerate(p.change_array_paths): change_array = hb.as_array(path) change_array_ndv = hb.get_nodata_from_uri(path) lulc_projected_original_classes_array = np.where( (change_array > 0) & (change_array != change_array_ndv), p.simple_classes_to_projected_original_classes[ p.classes_projected_to_change[c]], lulc_original_classes_array) p.lulc_projected_original_classes_path = os.path.join( p.cur_dir, 'lulc_projected_original_classes.tif') hb.save_array_as_geotiff(lulc_projected_original_classes_array, p.lulc_projected_original_classes_path, p.match_int_path) p.layers_to_stitch.append(p.lulc_projected_original_classes_path) # ALSO NOTE that we only return this once, because separate batched tasks are appending to it return ( 'layers_to_stitch', 'append_to_list', p.layers_to_stitch ) # WARNING the only intended use of returns in a tasks is if its a return resource to be synced among parallel tasks.
def raster_calculator_flex( input_, op, output_path, **kwargs ): #, datatype=None, ndv=None, gtiff_creation_options=None, compress=False # If input is a string, put it into a list if isinstance(input_, str): input_ = [input_] elif isinstance(input_, hb.ArrayFrame): input_ = input_.path final_input = [''] * len(input_) for c, i in enumerate(input_): if isinstance(i, hb.ArrayFrame): final_input[c] = i.path else: final_input[c] = i input_ = final_input # Determine size of inputs if isinstance(input_, str) or isinstance(input_, hb.ArrayFrame): input_size = 1 elif isinstance(input_, list): input_size = len(input_) else: raise NameError( 'input_ given to raster_calculator_flex() not understood. Give a path or list of paths.' ) # Check that files exist. for i in input_: if not os.path.exists(i): raise FileNotFoundError( str(input_) + ' not found by raster_calculator_flex()') # Verify datatypes datatype = kwargs.get('datatype', None) if not datatype: datatypes = [hb.get_datatype_from_uri(i) for i in input_] if len(set(datatypes)) > 1: L.info( 'Rasters given to raster_calculator_flex() were not all of the same type. Defaulting to using first input datatype.' ) datatype = datatypes[0] # Check NDVs. ndv = kwargs.get('ndv', None) if not ndv: ndvs = [hb.get_nodata_from_uri(i) for i in input_] if len(set(ndvs)) > 1: L.info( 'NDVs used in rasters given to raster_calculator_flex() were not all the same. Defaulting to using first value.' ) ndv = ndvs[0] gtiff_creation_options = kwargs.get('gtiff_creation_options', None) if not gtiff_creation_options: gtiff_creation_options = ['TILED=YES', 'BIGTIFF=IF_SAFER'] #, 'COMPRESS=lzw'] compress = kwargs.get('compress', None) if compress: gtiff_creation_options.append('COMPRESS=lzw') # Build tuples to match the required format of raster_calculator. if input_size == 1: if isinstance(input_[0], str): input_tuples_list = [(input_[0], 1)] else: input_tuples_list = [(input_[0].path, 1)] else: if isinstance(input_[0], str): input_tuples_list = [(i, 1) for i in input_] else: input_tuples_list = [(i.path, 1) for i in input_] # Check that the op matches the number of rasters. if len(inspect.signature(op).parameters) != input_size: raise NameError( 'op given to raster_calculator_flex() did not have the same number of parameters as the number of rasters given.' ) hb.raster_calculator(input_tuples_list, op, output_path, datatype, ndv, gtiff_creation_options=gtiff_creation_options) output_af = hb.ArrayFrame(output_path) return output_af
def execute(kw, ui): if not kw: kw = create_default_kw(ui) kw['output_folder'] = kw['workspace_dir'] ui.update_run_log('Calculating crop-specific production') baseline_lulc_uri = ui.root_app.project_key_raster if not kw.get('model_base_data_dir'): kw['model_base_data_dir'] = os.path.join(ui.root_app.base_data_folder, 'models', 'nutritional_adequacy') bounding_box = hb.get_datasource_bounding_box(ui.root_app.project_aoi) ui.update_run_log('Loading input maps. Bounding box set to: ' + str(bounding_box)) # TODO Unimplemented switch here. run_full_nutritional_model = False # OTW just cal calories if run_full_nutritional_model: try: os.remove( os.path.join(kw['output_folder'], 'crop_proportion_baseline_500m.tif')) os.remove( os.path.join(kw['output_folder'], 'crop_proportion_baseline_1km.tif')) os.remove( os.path.join(kw['output_folder'], 'crop_proportion_500m.tif')) os.remove( os.path.join(kw['output_folder'], 'crop_proportion_1km.tif')) except: 'no' match_uri = kw['lulc_uri'] # Load the LULC array as the baseline lulc_baseline = hb.as_array(baseline_lulc_uri) no_data_value = hb.get_nodata_from_uri(baseline_lulc_uri) # Extract the nan_mask for later lulc_nan_mask = np.where(lulc_baseline == no_data_value, 1, 0).astype(np.int8) # Calculate the proportion of the grid-cell that is in cultivation as a function of the LULC. # For MODIS, this means 12 and 14 are 1.0 and 0.5 respectively. crop_proportion_baseline = np.where(lulc_baseline == 12, 1.0, 0.0) # TODO START HERE, i missed a nan mask and now the results have near infinite value. create a robust solution # crop_proportion_baseline[lulc_nan_mask] = np.nan crop_proportion_baseline = np.where(lulc_baseline == 14, .5, crop_proportion_baseline) # BUG If the files are not in the normal folder and onl linked to, it fails to find them. try: os.mkdir(os.path.join(kw['output_folder'], 'YieldTonsPerCell')) except: 'Dir already exists.' clipped_dir = os.path.join(kw['output_folder'], 'crop_production_and_harvested_area') try: os.mkdir(clipped_dir) except: 'Dir already exists.' try: os.mkdir(os.path.join(kw['output_folder'], 'nutrient_production')) except: 'Dir already exists.' crop_proportion_baseline_500m_uri = os.path.join( kw['output_folder'], 'YieldTonsPerCell', 'crop_proportion_baseline_500m.tif') utilities.save_array_as_geotiff(crop_proportion_baseline, crop_proportion_baseline_500m_uri, kw['lulc_uri']) crop_proportion_baseline_1km_uri = os.path.join( kw['output_folder'], 'YieldTonsPerCell', 'crop_proportion_baseline_1km.tif') population_bounding_box = utilities.get_bounding_box( kw['population_uri']) cell_size = utilities.get_cell_size_from_uri(kw['population_uri']) hb.resize_and_resample_dataset_uri(crop_proportion_baseline_500m_uri, population_bounding_box, cell_size, crop_proportion_baseline_1km_uri, 'bilinear') if kw['lulc_uri'] != baseline_lulc_uri: lulc_scenario = utilities.as_array(kw['lulc_uri']) crop_proportion = np.where(lulc_scenario == 12, 1.0, 0.0) crop_proportion = np.where(lulc_scenario == 14, .5, crop_proportion) crop_proportion_500m_uri = os.path.join( kw['output_folder'], 'YieldTonsPerCell', 'crop_proportion_500m.tif') utilities.save_array_as_geotiff(crop_proportion, crop_proportion_500m_uri, kw['lulc_uri']) crop_proportion_1km_uri = os.path.join(kw['output_folder'], 'YieldTonsPerCell', 'crop_proportion_1km.tif') # original_dataset_uri, bounding_box, out_pixel_size, output_uri, resample_method hb.resize_and_resample_dataset_uri(crop_proportion_500m_uri, population_bounding_box, cell_size, crop_proportion_1km_uri, 'bilinear') crop_proportion_baseline_1km = utilities.as_array( crop_proportion_baseline_1km_uri) crop_proportion_1km = utilities.as_array(crop_proportion_1km_uri) change_ratio = np.where( crop_proportion_baseline_1km > 0, crop_proportion_1km / crop_proportion_baseline_1km, 1.0) change_ratio_mean = np.mean(change_ratio) else: change_ratio_mean = 1.0 ui.update_run_log('Loading input maps') crop_maps_folder = kw['crop_maps_folder'] nutritional_content_odict = utilities.file_to_python_object( kw['nutritional_content_table_uri'], declare_type='2d_odict' ) # outputs as OrderedDict([('almond', OrderedDict([('fraction_refuse', '0.6'), ('Protein', '212.2'), ('Lipid', '494.2'), etc nutritional_requirements_odict = utilities.file_to_python_object( kw['nutritional_requirements_table_uri'], declare_type='2d_indexed_odict') population = utilities.as_array(kw['population_uri']) demographic_groups_list = kw['demographic_groups_list'] demographics_folder = kw['demographics_folder'] ui.update_run_log('Calculating crop-specific production') lulc_array = utilities.as_array(kw['lulc_uri']) lulc_wkt = hb.get_dataset_projection_wkt_uri(kw['lulc_uri']) harvested_area_ha_filenames = [] harvested_area_fraction_filenames = [] yield_tons_per_ha_filenames = [] yield_tons_per_cell_filenames = [] ha_array = 0 yield_per_ha_array = 0 # Calculate ha per cell cell_size = hb.get_cell_size_from_uri(kw['lulc_uri']) ha_per_cell = np.ones(lulc_array.shape) * (cell_size**2 / 10000) ha_per_cell_uri = os.path.join(kw['output_folder'], 'ha_per_cell.tif') utilities.save_array_as_geotiff(ha_per_cell, ha_per_cell_uri, kw['lulc_uri']) force_recalculation = False for folder_name in os.listdir(crop_maps_folder): current_folder = os.path.join(crop_maps_folder, folder_name) if os.path.isdir(current_folder): print('current_folder', current_folder) current_crop_name = folder_name.split('_', 1)[0] input_harvested_area_fraction_uri = os.path.join( current_folder, current_crop_name + '_HarvestedAreaFraction.tif') clipped_harvested_area_fraction_uri = os.path.join( clipped_dir, current_crop_name + '_HarvestedAreaFraction.tif') input_yield_tons_per_ha_uri = os.path.join( current_folder, current_crop_name + '_YieldPerHectare.tif') clipped_yield_tons_per_ha_uri = os.path.join( clipped_dir, current_crop_name + '_YieldPerHectare.tif') yield_tons_per_cell_uri = os.path.join( clipped_dir, current_crop_name + '_YieldTonsPerCell.tif') if not os.path.exists( clipped_harvested_area_fraction_uri ) or not os.path.exists( clipped_yield_tons_per_ha_uri) or not os.path.exists( yield_tons_per_cell_uri) or force_recalculation: # hb.clip_dataset_uri(input_harvested_area_fraction_uri, kw['aoi_uri'], clipped_harvested_area_fraction_uri) utilities.clip_by_shape_with_buffered_intermediate_uri( input_harvested_area_fraction_uri, kw['aoi_uri'], clipped_harvested_area_fraction_uri, match_uri, resampling_method='bilinear') harvested_area_fraction_array = utilities.as_array( clipped_harvested_area_fraction_uri) # hb.clip_dataset_uri(input_yield_tons_per_ha_uri, kw['aoi_uri'], clipped_yield_tons_per_ha_uri) utilities.clip_by_shape_with_buffered_intermediate_uri( input_yield_tons_per_ha_uri, kw['aoi_uri'], clipped_yield_tons_per_ha_uri, match_uri, resampling_method='bilinear') yield_tons_per_ha_array = utilities.as_array( clipped_yield_tons_per_ha_uri) nan1 = utilities.get_nodata_from_uri( input_harvested_area_fraction_uri) nan2 = utilities.get_nodata_from_uri( input_yield_tons_per_ha_uri) nan_mask = np.where((yield_tons_per_ha_array == nan1) & ( harvested_area_fraction_array == nan2)) yield_tons_per_cell_array = yield_tons_per_ha_array * harvested_area_fraction_array * ha_per_cell yield_tons_per_cell_array[nan_mask] == nan1 # NOTE forcing ndv to zero for calcualtions utilities.save_array_as_geotiff(yield_tons_per_cell_array, yield_tons_per_cell_uri, kw['lulc_uri'], data_type_override=7, no_data_value_override=0) harvested_area_fraction_filenames.append( clipped_harvested_area_fraction_uri) yield_tons_per_ha_filenames.append( clipped_yield_tons_per_ha_uri) yield_tons_per_cell_filenames.append(yield_tons_per_cell_uri) ui.update_run_log('Creating yield (tons) map for ' + folder_name) match_5min_uri = os.path.join(kw['output_folder'], 'crop_production_and_harvested_area', 'maize_HarvestedAreaFraction.tif') # match_5min_uri = os.path.join(ui.root_app.base_data_folder, 'models/crop_production/global_dataset/observed_yield/rice_yield_map.tif') match_array = utilities.as_array(match_5min_uri) # TODO Figure out if nans all right nan3 = utilities.get_nodata_from_uri( clipped_harvested_area_fraction_uri) array = utilities.as_array(clipped_harvested_area_fraction_uri) nan_mask = np.where(array == nan3, True, False).astype(np.bool) # TODO Why default to run always? # Fat = np.zeros(match_array.shape).astype(np.float64) #[Fatnan_mask] = nan3 Energy = np.zeros(match_array.shape).astype(np.float64) Energy[nan_mask] = nan3 Protein = np.zeros(match_array.shape).astype(np.float64) Protein[nan_mask] = nan3 VitA = np.zeros(match_array.shape).astype(np.float64) VitA[nan_mask] = nan3 VitC = np.zeros(match_array.shape).astype(np.float64) VitC[nan_mask] = nan3 VitE = np.zeros(match_array.shape).astype(np.float64) VitE[nan_mask] = nan3 Thiamin = np.zeros(match_array.shape).astype(np.float64) Thiamin[nan_mask] = nan3 Riboflavin = np.zeros(match_array.shape).astype(np.float64) Riboflavin[nan_mask] = nan3 Niacin = np.zeros(match_array.shape).astype(np.float64) Niacin[nan_mask] = nan3 VitB6 = np.zeros(match_array.shape).astype(np.float64) VitB6[nan_mask] = nan3 Folate = np.zeros(match_array.shape).astype(np.float64) Folate[nan_mask] = nan3 VitB12 = np.zeros(match_array.shape).astype(np.float64) VitB12[nan_mask] = nan3 Ca = np.zeros(match_array.shape).astype(np.float64) Ca[nan_mask] = nan3 Ph = np.zeros(match_array.shape).astype(np.float64) Ph[nan_mask] = nan3 Mg = np.zeros(match_array.shape).astype(np.float64) Mg[nan_mask] = nan3 K = np.zeros(match_array.shape).astype(np.float64) K[nan_mask] = nan3 Na = np.zeros(match_array.shape).astype(np.float64) Na[nan_mask] = nan3 Fe = np.zeros(match_array.shape).astype(np.float64) Fe[nan_mask] = nan3 Zn = np.zeros(match_array.shape).astype(np.float64) Zn[nan_mask] = nan3 Cu = np.zeros(match_array.shape).astype(np.float64) Cu[nan_mask] = nan3 for i in range(len(yield_tons_per_cell_filenames)): current_crop_name = os.path.splitext( os.path.split( harvested_area_fraction_filenames[i])[1])[0].split('_', 1)[0] ui.update_run_log('Calculating nutritional contribution of ' + current_crop_name) if current_crop_name in nutritional_content_odict.keys(): print('adding Nutritional content of ' + current_crop_name) # Fat += utilities.as_array(yield_tons_per_cell_filenames[i]) * 1000.0 * float(nutritional_content_odict[current_crop_name]['Fat']) Energy += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Energy']) Protein += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name] ['Protein']) VitA += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['VitA']) VitC += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['VitC']) VitE += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['VitE']) Thiamin += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name] ['Thiamin']) Riboflavin += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name] ['Riboflavin']) Niacin += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Niacin']) VitB6 += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['VitB6']) Folate += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Folate']) VitB12 += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['VitB12']) Ca += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Ca']) Ph += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Ph']) Mg += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Energy']) K += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Mg']) Na += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['K']) Fe += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Fe']) Zn += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Energy']) Cu += utilities.as_array( yield_tons_per_cell_filenames[i]) * 1000.0 * float( nutritional_content_odict[current_crop_name]['Zn']) # TODO make this happen earlier in calcs # Fat *= change_ratio_mean Energy *= change_ratio_mean Protein *= change_ratio_mean VitA *= change_ratio_mean VitC *= change_ratio_mean VitE *= change_ratio_mean Thiamin *= change_ratio_mean Riboflavin *= change_ratio_mean Niacin *= change_ratio_mean VitB6 *= change_ratio_mean Folate *= change_ratio_mean VitB12 *= change_ratio_mean Ca *= change_ratio_mean Ph *= change_ratio_mean Mg *= change_ratio_mean K *= change_ratio_mean Na *= change_ratio_mean Fe *= change_ratio_mean Zn *= change_ratio_mean Cu *= change_ratio_mean # Fat *= change_ratio_mean Energy[nan_mask] = 0 Protein[nan_mask] = 0 VitA[nan_mask] = 0 VitC[nan_mask] = 0 VitE[nan_mask] = 0 Thiamin[nan_mask] = 0 Riboflavin[nan_mask] = 0 Niacin[nan_mask] = 0 VitB6[nan_mask] = 0 Folate[nan_mask] = 0 VitB12[nan_mask] = 0 Ca[nan_mask] = 0 Ph[nan_mask] = 0 Mg[nan_mask] = 0 K[nan_mask] = 0 Na[nan_mask] = 0 Fe[nan_mask] = 0 Zn[nan_mask] = 0 Cu[nan_mask] = 0 match_1km_uri = os.path.join(kw['output_folder'], 'YieldTonsPerCell', 'crop_proportion_baseline_1km.tif') utilities.save_array_as_geotiff( Energy, os.path.join(kw['output_folder'], 'nutrient_production', 'Energy_per_cell_5min.tif'), match_5min_uri, data_type_override=7, no_data_value_override=nan3) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Energy_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Energy_per_cell_1km.tif'), match_1km_uri) # utilities.save_array_as_geotiff(Fat, os.path.join(kw['output_folder'], 'nutrient_production', 'Fat_per_cell_5min.tif'), match_5min_uri) # utilities.resample_preserve_sum(os.path.join(kw['output_folder'], 'nutrient_production', 'Fat_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Fat_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Protein, os.path.join(kw['output_folder'], 'nutrient_production', 'Protein_per_cell_5min.tif'), match_5min_uri, no_data_value_override=nan3) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Protein_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Protein_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( VitA, os.path.join(kw['output_folder'], 'nutrient_production', 'VitA_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'VitA_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'VitA_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( VitC, os.path.join(kw['output_folder'], 'nutrient_production', 'VitC_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'VitC_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'VitC_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( VitE, os.path.join(kw['output_folder'], 'nutrient_production', 'VitE_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'VitE_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'VitE_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Thiamin, os.path.join(kw['output_folder'], 'nutrient_production', 'Thiamin_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Thiamin_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Thiamin_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Riboflavin, os.path.join(kw['output_folder'], 'nutrient_production', 'Riboflavin_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Riboflavin_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Riboflavin_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Niacin, os.path.join(kw['output_folder'], 'nutrient_production', 'Niacin_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Niacin_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Niacin_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( VitB6, os.path.join(kw['output_folder'], 'nutrient_production', 'VitB6_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'VitB6_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'VitB6_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Folate, os.path.join(kw['output_folder'], 'nutrient_production', 'Folate_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Folate_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Folate_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( VitB12, os.path.join(kw['output_folder'], 'nutrient_production', 'VitB12_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'VitB12_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'VitB12_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Ca, os.path.join(kw['output_folder'], 'nutrient_production', 'Ca_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Ca_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Ca_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Ph, os.path.join(kw['output_folder'], 'nutrient_production', 'Ph_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Ph_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Ph_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Mg, os.path.join(kw['output_folder'], 'nutrient_production', 'Mg_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Mg_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Mg_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( K, os.path.join(kw['output_folder'], 'nutrient_production', 'K_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'K_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'K_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Na, os.path.join(kw['output_folder'], 'nutrient_production', 'Na_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Na_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Na_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Fe, os.path.join(kw['output_folder'], 'nutrient_production', 'Fe_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Fe_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Fe_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Zn, os.path.join(kw['output_folder'], 'nutrient_production', 'Zn_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Zn_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Zn_per_cell_1km.tif'), match_1km_uri) utilities.save_array_as_geotiff( Cu, os.path.join(kw['output_folder'], 'nutrient_production', 'Cu_per_cell_5min.tif'), match_5min_uri) utilities.resample_preserve_sum( os.path.join(kw['output_folder'], 'nutrient_production', 'Cu_per_cell_5min.tif'), os.path.join(kw['output_folder'], 'nutrient_production', 'Cu_per_cell_1km.tif'), match_1km_uri) # calculate demand calculate_demand = True if calculate_demand: # TODO FOR HONDURAS need to switch to a non-population-map derived resolution. Incorporate decision making unit map to set the desired resolution ui.update_run_log('Calculating total nutrient demand') overall_nutrient_sum = 0 overall_nutrient_requirement_sum = 0 overall_ratio_array = np.zeros(population.shape) overall_ratio = 0 population_zero_normalized = np.where(population < 0, 0, population) for nutrient in nutritional_requirements_odict: nutrient_uri = os.path.join(kw['output_folder'], 'nutrient_production', nutrient + '_per_cell_1km.tif') nutrient_array = utilities.as_array(nutrient_uri) nutrient_requirement_array = population_zero_normalized * float( nutritional_requirements_odict[nutrient] ['recommended_daily_allowance']) * 365.0 # nutrient_requirement_array[nan_mask] = np.nan # nutrient_requirement_array[nutrient_requirement_array<=0] = np.nan nutrient_provision_ratio = np.where( (nutrient_array / nutrient_requirement_array > 0) & (nutrient_array / nutrient_requirement_array < 999999999999999999999999999999), nutrient_array / nutrient_requirement_array, 0) overall_ratio_array += nutrient_provision_ratio nutrient_sum = np.nansum(nutrient_array) overall_nutrient_sum += nutrient_sum nutrient_requirement_sum = np.nansum( nutrient_requirement_array) overall_nutrient_requirement_sum += nutrient_requirement_sum output_string = 'Full landscape produced ' + str( nutrient_sum ) + ' of ' + nutrient + ' compared to a national requirement of ' + str( nutrient_requirement_sum ) + ', yielding nutritional adequacy ratio of ' + str( nutrient_sum / nutrient_requirement_sum) + '.' ui.update_run_log(output_string) overall_ratio += nutrient_provision_ratio utilities.save_array_as_geotiff( nutrient_provision_ratio, nutrient_uri.replace('_per_cell_1km.tif', '_adequacy_ratio.tif'), nutrient_uri) overall_ratio_array *= 1.0 / 19.0 overall_ratio = (1.0 / 19.0) * (overall_nutrient_sum / overall_nutrient_requirement_sum) output_string = 'Overall nutrion adequacy ratio: ' + str( overall_ratio) + '.' ui.update_run_log(output_string) overall_ratio_uri = os.path.join(kw['output_folder'], 'overall_adequacy_ratio.tif') utilities.save_array_as_geotiff(overall_ratio_array, overall_ratio_uri, nutrient_uri) run_calories_only_model = True if run_calories_only_model: calc_caloric_production_from_lulc_uri(kw['lulc_uri'], ui=ui, **kw) return
def process_coarse_change_maps(): global p L.info('process_coarse_change_maps.') # Change maps are in this directory and must be of the format [CLASS_ID_INT]_[someting, but anything else].tif if not os.path.isdir(p.coarse_change_maps_dir): p.coarse_change_maps_dir = os.path.split(p.coarse_change_maps_dir)[0] if not os.path.isdir(p.coarse_change_maps_dir): raise NameError('Unable to parse coarse_change_maps_dir.') tifs_in_dir = hb.list_filtered_paths_nonrecursively( p.coarse_change_maps_dir, include_extensions='.tif') p.change_map_paths = [] for path in tifs_in_dir: try: rendered_int = int(hb.file_root(path).split('_')[0]) except: rendered_int = None if isinstance(rendered_int, int): p.change_map_paths.append(path) p.change_map_raster_infos = [ hb.get_raster_info(i) for i in p.change_map_paths ] # Test that all the change maps are the same properties. if len(set([i['geotransform'] for i in p.change_map_raster_infos])) != 1: for j in [i['geotransform'] for i in p.change_map_raster_infos]: L.critical('geotransform: ' + str(j)) # raise NameError('The maps in coarse change maps dir are not all the same shape, projection, etc, or they have been improperly named/formatted.') # p.current_change_in_crop_extent_path = os.path.join(p.cur_dir, 'change_in_crop_extent.tif') p.current_change_map_paths = [] p.float_ndv = None p.int_ndv = 255 L.info('change_map_paths: ' + str(p.change_map_paths)) p.zone_transition_sums = OrderedDict() p.classes_projected_to_change = [] for path in p.change_map_paths: changing_class_id = int(os.path.split(path)[1].split('_')[0]) p.classes_projected_to_change.append(changing_class_id) if not p.float_ndv: p.float_ndv = hb.get_nodata_from_uri(path) if p.float_ndv is None: p.float_ndv = -9999.0 new_path = os.path.join(p.cur_dir, os.path.split(path)[1]) p.current_change_map_paths.append(new_path) if p.run_this: # NOTE NONSTANDARD placement of run_this hb.clip_raster_by_vector( str(path), str(new_path), str(p.area_of_interest_path), resample_method='nearest', all_touched=True, verbose=True, ensure_fits=True, gtiff_creation_options=hb.DEFAULT_GTIFF_CREATION_OPTIONS) # To make the model not run in zones with zero change, we collect these sums and prevent runing if all of them are zero current_coarse_array = hb.as_array(new_path) current_sum = np.sum( current_coarse_array[current_coarse_array != p.float_ndv]) p.zone_transition_sums[changing_class_id] = current_sum p.run_this_zone = True if np.sum([float(i) for i in p.zone_transition_sums.values()]) <= 0: p.run_this_zone = False L.info('current_change_map_paths' + str(p.current_change_map_paths))
def create_convolution_inputs(): global p p.convolution_inputs_dir = p.cur_dir if p.run_this and p.run_this_zone: lulc_array = hb.as_array(p.lulc_simplified_path) ndv = hb.get_nodata_from_uri(p.lulc_simplified_path) # Get which values exist in simplified_lulc unique_values = list(hb.enumerate_array_as_odict(lulc_array).keys()) unique_values = [int(i) for i in unique_values] try: p.classes_to_ignore = [ int(i) for i in p.classes_to_ignore.split(' ') ] except: p.classes_to_ignore = [] # TODOO Better approach than ignoring classes would be to encode ALL such information into the different CSVs. This would allow more grandular control over how, e.g. water DOES have attraction effect but does not necessarily expand. ignore_values = [ndv] + p.classes_to_ignore p.simplified_lulc_classes = [ i for i in unique_values if i not in ignore_values ] # HACK p.classes_to_ignore = [0] p.classes_with_effect = [ i for i in p.simplified_lulc_classes if i not in p.classes_to_ignore ] L.info('Creating binaries for classes ' + str(p.classes_with_effect)) try: p.max_simplified_lulc_classes = max(p.simplified_lulc_classes) except: p.max_simplified_lulc_classes = 20 p.new_simplified_lulc_addition_value = 10**( len(str(p.max_simplified_lulc_classes)) + 1 ) / 10 # DOCUMENTATION, new classes are defined here as adding 1 order of magnitude larger value (2 becomes 12 if the max is 5. 2 becomes 102 if the max is 15. p.classes_with_change = [ int(os.path.split(i)[1].split('_')[0]) for i in p.current_change_map_paths ] binary_paths = [] for unique_value in p.classes_with_effect: # binary_array = np.zeros(lulc_array.shape) binary_array = np.where(lulc_array == unique_value, 1, 0).astype(np.uint8) binary_path = os.path.join( p.convolution_inputs_dir, 'class_' + str(unique_value) + '_binary.tif') binary_paths.append(binary_path) hb.save_array_as_geotiff(binary_array, binary_path, p.lulc_simplified_path, compress=True) convolution_params = hb.file_to_python_object( p.class_proximity_parameters_path, declare_type='DD', output_key_data_type=str, output_value_data_type=float) convolution_paths = [] for i, v in enumerate(p.classes_with_effect): L.info('Calculating convolution for class ' + str(v)) binary_array = hb.as_array(binary_paths[i]) convolution_metric = seals_utils.distance_from_blurred_threshold( binary_array, convolution_params[str(v)]['clustering'], 0.5, convolution_params[str(v)]['decay']) convolution_path = os.path.join( p.convolution_inputs_dir, 'class_' + str(p.classes_with_effect[i]) + '_convolution.tif') convolution_paths.append(convolution_path) hb.save_array_as_geotiff(convolution_metric, convolution_path, p.match_float_path, compress=True) pairwise_params = hb.file_to_python_object( p.pairwise_class_relationships_path, declare_type='DD', output_key_data_type=str, output_value_data_type=float) for i in p.classes_with_effect: i_convolution_path = os.path.join( p.convolution_inputs_dir, 'class_' + str(i) + '_convolution.tif') i_convolution_array = hb.as_array(i_convolution_path) for j in p.classes_with_change: L.info('Processing effect of ' + str(i) + ' on ' + str(j)) adjacency_effect_path = os.path.join( p.convolution_inputs_dir, 'adjacency_effect_of_' + str(i) + '_on_' + str(j) + '.tif') adjacency_effect_array = i_convolution_array * pairwise_params[ str(i)][str(j)] hb.save_array_as_geotiff(adjacency_effect_array, adjacency_effect_path, p.match_float_path, compress=True) for i in p.classes_with_change: L.info('Combining adjacency effects for class ' + str(i)) combined_adjacency_effect_array = np.ones(lulc_array.shape) combined_adjacency_effect_path = os.path.join( p.convolution_inputs_dir, 'combined_adjacency_effect_' + str(i) + '.tif') for j in p.classes_with_effect: current_uri = os.path.join( p.convolution_inputs_dir, 'adjacency_effect_of_' + str(j) + '_on_' + str(i) + '.tif') # NOTICE SWITCHED I and J current_effect = hb.as_array(current_uri) combined_adjacency_effect_array *= current_effect + 1.0 # Center on 1 so that 0.0 has no effect hb.save_array_as_geotiff(combined_adjacency_effect_array, combined_adjacency_effect_path, p.match_float_path, compress=True)
def create_lulc(): global p L.info('Creating class-types lulc.') p.name_from_iterator_replacements = hb.file_root(p.area_of_interest_path) p.base_year_current_zone_lulc_path = os.path.join( p.cur_dir, 'base_year_' + p.name_from_iterator_replacements + '.tif') # Create match paths of both data types p.match_int_path = p.base_year_current_zone_lulc_path p.lulc_simplified_path = os.path.join(p.cur_dir, 'lulc_simplified.tif') # p.lulc_simplified_path = p.base_year_current_zone_lulc_path p.valid_mask_path = os.path.join(p.cur_dir, 'valid_mask_high_res.tif') p.proportion_valid_fine_per_coarse_cell_path = os.path.join( p.cur_dir, 'proportion_valid_fine_per_coarse_cell.tif') if p.run_this: hb.clip_while_aligning_to_coarser( p.base_year_lulc_path, p.base_year_current_zone_lulc_path, p.area_of_interest_path, p.current_change_map_paths[0], resample_method='nearest', output_data_type=1, nodata_target=255, all_touched=True, verbose=True, ensure_fits=True, gtiff_creation_options=hb.DEFAULT_GTIFF_CREATION_OPTIONS) # Set NDV masking based on AOI of current zone. hb.create_valid_mask_from_vector_path( p.area_of_interest_path, p.base_year_current_zone_lulc_path, p.valid_mask_path) p.valid_mask = hb.as_array(p.valid_mask_path) hb.set_ndv_by_mask_path(p.base_year_current_zone_lulc_path, p.valid_mask_path) p.proportion_valid_fine_per_coarse_cell = hazelbean.pyramids.calc_proportion_of_coarse_res_with_valid_fine_res( p.current_change_map_paths[0], p.valid_mask_path) hb.save_array_as_geotiff(p.proportion_valid_fine_per_coarse_cell, p.proportion_valid_fine_per_coarse_cell_path, p.current_change_map_paths[0]) lulc_ds = gdal.Open(p.base_year_current_zone_lulc_path) lulc_band = lulc_ds.GetRasterBand(1) lulc_array = lulc_band.ReadAsArray().astype(np.int) p.scaled_proportion_to_allocate_paths = [] for path in p.current_change_map_paths: unscaled = hb.as_array(path).astype(np.float64) scaled_proportion_to_allocate = p.proportion_valid_fine_per_coarse_cell * unscaled scaled_proportion_to_allocate_path = os.path.join( p.cur_dir, os.path.split(hb.suri(path, 'scaled'))[1]) hb.save_array_as_geotiff(scaled_proportion_to_allocate, scaled_proportion_to_allocate_path, path, ndv=-9999.0, data_type=7) p.scaled_proportion_to_allocate_paths.append( scaled_proportion_to_allocate_path) if os.path.exists(p.lulc_class_types_path): # load the simplified class correspondnce as a nested dictionary. lulc_class_types_odict = hb.file_to_python_object( p.lulc_class_types_path, declare_type='DD') # For cythonization reasons, I need to ensure this comes in as ints lulc_class_types_ints_dict = dict() p.lulc_unsimplified_classes_list = [] for row_name in lulc_class_types_odict.keys(): lulc_class_types_ints_dict[int(row_name)] = int( lulc_class_types_odict[row_name]['lulc_class_type']) p.lulc_unsimplified_classes_list.append(int(row_name)) p.max_unsimplified_lulc_classes = max( p.lulc_unsimplified_classes_list) p.new_unsimplified_lulc_addition_value = 10**( len(str(p.max_unsimplified_lulc_classes)) + 1 ) / 10 # DOCUMENTATION, new classes are defined here as adding 1 order # # 1 is agriculture, 2 is mixed ag/natural, 3 is natural, 4 is urban, 0 is no data lulc_simplified_array = hb.reclassify_int_array_by_dict_to_ints( lulc_array, lulc_class_types_ints_dict) no_data_value_override = hb.get_nodata_from_uri( p.base_year_current_zone_lulc_path) hb.save_array_as_geotiff(lulc_simplified_array, p.lulc_simplified_path, p.base_year_current_zone_lulc_path, data_type=1, set_inf_to_no_data_value=False, ndv=no_data_value_override, compress=True) else: L.warn( 'No lulc_class_types_path specified. Assuming you want to run every class uniquely.' ) # If we don't run this zone, we know we will need to use the unmodified lulc when stitching everything back together if p.run_this_zone is False: p.layers_to_stitch.append(p.base_year_current_zone_lulc_path) else: p.lulc_simplified_path = p.base_year_current_zone_lulc_path