def organize_geo(self): domain_collection_list = {} for domain_name_step in self.domain_name_list: file_path_collections = self.define_domain_collection( domain_name_step) if not os.path.exists(file_path_collections): domain_info = self.get_domain_info(domain_name_step) domain_drainage_area = self.get_domain_drainage_area( domain_name_step) domain_collection = {**domain_drainage_area, **domain_info} folder_name_collections, file_name_collections = os.path.split( file_path_collections) make_folder(folder_name_collections) write_obj(file_path_collections, domain_collection) else: domain_collection = read_obj(file_path_collections) domain_collection_list[domain_name_step] = domain_collection return domain_collection_list
def dump_scenarios(self, scenarios_collections): logging.info(' ----> Dump scenarios [' + str(self.time_run) + '] ... ') scenarios_file_path = self.file_path_scenarios_data_collections scenarios_collection_tmp = None for group_data_key in self.structure_data_group.keys(): logging.info(' -----> Alert Area ' + group_data_key + ' ... ') file_path = scenarios_file_path[group_data_key] if isinstance(file_path, list): file_path = file_path[0] if self.flag_dest_updating: if os.path.exists(file_path): os.remove(file_path) if not os.path.exists(file_path): file_data = scenarios_collections[group_data_key] if file_data is not None: folder_name, file_name = os.path.split(file_path) make_folder(folder_name) write_file_csv(file_path, file_data) logging.info(' -----> Alert Area ' + group_data_key + ' ... DONE') else: logging.info(' -----> Alert Area ' + group_data_key + ' ... SKIPPED. Datasets are undefined.') else: logging.info(' -----> Alert Area ' + group_data_key + ' ... SKIPPED. Datasets previously saved.') logging.info(' ----> Dump scenarios [' + str(self.time_run) + '] ... DONE')
def set_logging_file(logger_file=logger_file_default, logger_name=logger_name_default, logger_handle=logger_handle_default, logger_formatter=logger_formatter_default, logger_history=False, logger_history_maxfiles=12, logger_extra_tags=None): # Set to flush progressbar output in logging stream handle # progressbar.streams.wrap_stderr() if logger_extra_tags is not None: for extra_key, extra_value in logger_extra_tags.items(): logger_file = logger_file.replace(extra_key, ':') string_count = logger_file.count(':') extra_value = [extra_value] * string_count logger_file = logger_file.format(*extra_value) logger_folder_name, logger_file_name = os.path.split(logger_file) make_folder(logger_folder_name) # Save old logger file (to check run in the past) if logger_history: store_logging_file(logger_file, logger_file_max=logger_history_maxfiles) # Remove old logging file if os.path.exists(logger_file): os.remove(logger_file) # Open logger logging.getLogger(logger_name) logging.root.setLevel(logging.DEBUG) # Set logger handle type if logger_handle == 'file': # Set logger handler obj logger_handle_1 = logging.FileHandler(logger_file, 'w') logger_handle_2 = logging.StreamHandler() # Set logger level logger_handle_1.setLevel(logging.DEBUG) logger_handle_2.setLevel(logging.DEBUG) # Set logger formatter logger_handle_1.setFormatter(logging.Formatter(logger_formatter)) logger_handle_2.setFormatter(logging.Formatter(logger_formatter)) # Add handle to logger logging.getLogger('').addHandler(logger_handle_1) logging.getLogger('').addHandler(logger_handle_2) elif logger_handle == 'stream': # Set logger handler obj logging.StreamHandler() # Set logger level logger_handle.setLevel(logging.DEBUG) # Set logger formatter logger_handle.setFormatter(logging.Formatter(logger_formatter)) # Add handle to logger logging.getLogger('').addHandler(logger_handle) else: # Set logger handler obj logging.NullHandler() # Add handle to logger logging.getLogger('').addHandler(logger_handle)
def organize_forcing(self, var_name='soil_moisture'): logging.info(' ----> Organize soil moisture forcing ... ') time_range = self.time_range file_data_geo = self.file_data_geo file_data_basin = self.file_data_basin file_path_src_list = self.file_path_src_list file_path_ancillary_list = self.file_path_ancillary_list for (group_key_basin, group_basin), (group_key_geo, group_geo), group_file, group_ancillary in zip( file_data_basin.items(), file_data_geo.items(), file_path_src_list.values(), file_path_ancillary_list.values()): logging.info(' -----> Alert Area ' + group_key_basin + ' ... ') basin_list = list(group_basin.keys()) geo_mask_ref = group_geo geo_x_ref = group_geo['west_east'] geo_y_ref = group_geo['south_north'] basin_collections = {} file_ancillary_collections = {} if basin_list: for basin_name in basin_list: logging.info(' ------> BasinName ' + basin_name + ' ... ') file_basin_geo = group_basin[basin_name] file_basin_list = group_file[basin_name] file_ancillary_list = group_ancillary[basin_name] for time_step, file_basin_step, file_ancillary_step in zip(time_range, file_basin_list, file_ancillary_list): logging.info(' -------> TimeStep: ' + str(time_step) + ' ... ') if self.flag_ancillary_updating: if os.path.exists(file_ancillary_step): os.remove(file_ancillary_step) if not os.path.exists(file_ancillary_step): if file_basin_step.endswith(self.file_extension_zip): file_basin_out = change_extension(file_basin_step, self.file_extension_unzip) else: file_basin_out = file_basin_step if os.path.exists(file_basin_step): unzip_filename(file_basin_step, file_basin_out) data_vtot = read_file_binary( file_basin_out, data_geo=file_basin_geo[self.var_name_terrain].values) data_vmax = convert_cn2s( file_basin_geo[self.var_name_curve_number].values, file_basin_geo[self.var_name_terrain].values) data_sm = data_vtot / data_vmax data_sm[file_basin_geo[self.var_name_channels_network].values == 1] = -1 da_sm_base = create_darray_2d(data_sm, file_basin_geo[self.var_name_x], file_basin_geo[self.var_name_y], coord_name_x='west_east', coord_name_y='south_north', dim_name_x='west_east', dim_name_y='south_north') da_sm_interp = da_sm_base.interp(south_north=geo_y_ref, west_east=geo_x_ref, method='nearest') if time_step not in list(basin_collections.keys()): basin_collections[time_step] = [da_sm_interp] file_ancillary_collections[time_step] = [file_ancillary_step] else: data_tmp = basin_collections[time_step] data_tmp.append(da_sm_interp) basin_collections[time_step] = data_tmp file_tmp = file_ancillary_collections[time_step] file_tmp.append(file_ancillary_step) file_tmp = list(set(file_tmp)) file_ancillary_collections[time_step] = file_tmp logging.info(' -------> TimeStep: ' + str(time_step) + ' ... DONE') else: logging.info(' -------> TimeStep: ' + str(time_step) + ' ... FAILED') logging.warning(' ==> File: ' + file_basin_step + ' does not exist') else: logging.info(' -------> TimeStep: ' + str(time_step) + ' ... PREVIOUSLY DONE') logging.info(' ------> BasinName ' + basin_name + ' ... DONE') logging.info(' -----> Alert Area ' + group_key_basin + ' ... DONE') else: logging.info(' -----> Alert Area ' + group_key_basin + ' ... SKIPPED') logging.warning(' ==> Datasets are not defined') logging.info(' -----> Compose grid datasets from basins to alert area domain ... ') for (time_step, data_list), file_path_ancillary in zip( basin_collections.items(), file_ancillary_collections.values()): logging.info(' ------> TimeStep: ' + str(time_step) + ' ... ') if isinstance(file_path_ancillary, list) and file_path_ancillary.__len__() == 1: file_path_ancillary = file_path_ancillary[0] else: logging.error(' ===> Soil moisture ancillary file are not correctly defined.') raise IOError('Ancillary file is not unique') if self.flag_ancillary_updating: if os.path.exists(file_path_ancillary): os.remove(file_path_ancillary) if not os.path.exists(file_path_ancillary): logging.info(' -------> Merge grid datasets ... ') array_merge = np.zeros([geo_mask_ref.values.shape[0] * geo_mask_ref.values.shape[1]]) array_merge[:] = np.nan for data_step in data_list: array_values = data_step.values.ravel() idx_finite = np.isfinite(array_values) array_merge[idx_finite] = array_values[idx_finite] grid_merge = np.reshape(array_merge, [geo_mask_ref.values.shape[0], geo_mask_ref.values.shape[1]]) idx_choice = np.where(grid_merge == -1) grid_merge[idx_choice[0], idx_choice[1]] = np.nan idx_filter = np.where((geo_mask_ref.values == 1) & (np.isnan(grid_merge))) grid_merge[idx_filter[0], idx_filter[1]] = np.nanmean(grid_merge) grid_merge[(geo_mask_ref.values == 0)] = np.nan grid_merge[idx_choice[0], idx_choice[1]] = np.nan logging.info(' -------> Merge grid datasets ... DONE') logging.info(' -------> Save grid datasets ... ') dset_merge = create_dset( grid_merge, geo_mask_ref.values, geo_x_ref.values, geo_y_ref.values, var_data_time=time_step, var_data_name=var_name, var_geo_name='mask', var_data_attrs=None, var_geo_attrs=None, coord_name_x='longitude', coord_name_y='latitude', coord_name_time='time', dim_name_x='west_east', dim_name_y='south_north', dim_name_time='time', dims_order_2d=None, dims_order_3d=None) folder_name_ancillary, file_name_ancillary = os.path.split(file_path_ancillary) make_folder(folder_name_ancillary) if file_path_ancillary.endswith('.nc'): write_dset( file_path_ancillary, dset_merge, dset_mode='w', dset_engine='h5netcdf', dset_compression=0, dset_format='NETCDF4', dim_key_time='time', no_data=-9999.0) logging.info(' ------> Save grid datasets ... DONE. [NETCDF]') elif file_path_ancillary.endswith('.tiff'): save_file_tiff(file_path_ancillary, np.flipud(dset_merge[var_name].values), geo_x_ref.values, np.flipud(geo_y_ref.values), file_metadata=self.file_metadata, file_epsg_code=self.file_epsg_code) logging.info(' ------> Save grid datasets ... DONE. [GEOTIFF]') else: logging.info(' ------> Save grid datasets ... FAILED') logging.error(' ===> Filename format is not allowed') raise NotImplementedError('Format is not implemented yet') self.file_path_processed.append(file_path_ancillary) logging.info(' -------> Save grid datasets ... DONE') logging.info(' ------> TimeStep: ' + str(time_step) + ' ... DONE') else: logging.info(' ------> TimeStep: ' + str(time_step) + ' ... PREVIOUSLY DONE') logging.info(' -----> Compose grid datasets from basins to alert area domain ... DONE') logging.info(' ----> Organize soil moisture forcing ... DONE')
def organize_discharge(self): time = self.time_run geo_data_collection = self.geo_data_collection logging.info(' --> Organize discharge datasets [' + str(time) + '] ... ') file_path_discharge = self.file_path_discharge file_time_discharge = self.file_time_discharge section_collection = {} for domain_name_step in self.domain_name_list: logging.info(' ---> Domain ' + domain_name_step + ' ... ') file_path_discharge = self.file_path_discharge[domain_name_step] file_path_ancillary = self.file_path_ancillary[domain_name_step] if self.flag_cleaning_dynamic_ancillary: if os.path.exists(file_path_ancillary): os.remove(file_path_ancillary) if not os.path.exists(file_path_ancillary): domain_discharge_index = geo_data_collection[domain_name_step][ self.domain_discharge_index_tag] domain_grid_rows = geo_data_collection[domain_name_step][ self.domain_grid_x_tag].shape[0] domain_grid_cols = geo_data_collection[domain_name_step][ self.domain_grid_y_tag].shape[1] domain_section_db = geo_data_collection[domain_name_step][ self.domain_sections_db_tag] section_workspace = {} for section_key, section_data in domain_section_db.items(): section_description = section_data['description'] section_name = section_data['name'] section_idx = section_data['idx'] section_discharge_default = section_data[ 'discharge_default'] section_id = self.format_group.format( section_data['group']['id']) logging.info(' ----> Section ' + section_description + ' ... ') section_file_path_list = file_path_discharge[section_id] if section_file_path_list: section_dframe = pd.DataFrame( index=file_time_discharge) for section_file_path_step in section_file_path_list: section_folder_name_step, section_file_name_step = os.path.split( section_file_path_step) section_file_ts_start, section_file_ts_end, \ section_file_mask, section_file_ens = parse_file_parts(section_file_name_step) section_file_tag = create_file_tag( section_file_ts_start, section_file_ts_end, section_file_ens) section_ts = read_file_hydro( section_name, section_file_path_step) section_dframe[section_file_tag] = section_ts section_workspace[section_description] = section_dframe logging.info(' ----> Section ' + section_description + ' ... DONE') else: logging.info(' ----> Section ' + section_description + ' ... SKIPPED. Datasets are empty') section_workspace[section_description] = None folder_name_ancillary, file_name_ancillary = os.path.split( file_path_ancillary) make_folder(folder_name_ancillary) flag_save_obj = True for section_key, section_data in section_workspace.items(): if section_data is None: flag_save_obj = False break if flag_save_obj: write_obj(file_path_ancillary, section_workspace) logging.info(' ---> Domain ' + domain_name_step + ' ... DONE') else: logging.info( ' ---> Domain ' + domain_name_step + ' ... SKIPPED. All or some datasets are empty') else: section_workspace = read_obj(file_path_ancillary) logging.info(' ---> Domain ' + domain_name_step + ' ... SKIPPED. Data previously computed') section_collection[domain_name_step] = section_workspace logging.info(' --> Organize discharge datasets [' + str(time) + '] ... DONE') return section_collection
def save_analysis(self, group_analysis_sm, group_analysis_rain_map, group_analysis_rain_point, group_soilslip): logging.info(' ----> Save analysis [' + str(self.time_step) + '] ... ') time_step = self.time_step geo_data_alert_area = self.geo_data_alert_area group_data_alert_area = self.structure_data_group for (group_data_key, group_data_items), geo_data_dframe in zip( group_data_alert_area.items(), geo_data_alert_area.values()): logging.info(' -----> Alert Area ' + group_data_key + ' ... ') file_path_dest = collect_file_list( time_step, self.folder_name_dest_indicators_raw, self.file_name_dest_indicators_raw, self.alg_template_tags, alert_area_name=group_data_key)[0] if not os.path.exists(file_path_dest): group_soilslip_select = group_soilslip[group_data_key] if group_analysis_sm[group_data_key] is not None: group_analysis_sm_select = group_analysis_sm[ group_data_key][self.template_struct_obj] else: group_analysis_sm_select = None if group_analysis_rain_map[group_data_key] is not None: group_analysis_rain_map_select = group_analysis_rain_map[ group_data_key][self.template_struct_obj] else: group_analysis_rain_map_select = None if group_analysis_rain_point[group_data_key] is not None: group_analysis_rain_point_select = group_analysis_rain_point[ group_data_key] else: group_analysis_rain_point_select = None if group_analysis_sm_select is not None: if time_step in list(group_soilslip_select.index): soilslip_select = group_soilslip_select.loc[ time_step.strftime('%Y-%m-%d 00:00:00')] else: soilslip_select = None else: soilslip_select = None if (group_analysis_sm_select is not None) and ( (group_analysis_rain_map_select is not None) and (group_analysis_rain_point_select is not None)): analysis_sm = self.unpack_analysis( group_analysis_sm_select) analysis_rain_map = self.unpack_analysis( group_analysis_rain_map_select) analysis_rain_point = self.unpack_analysis( group_analysis_rain_point_select) analysis_data = { **analysis_sm, **analysis_rain_map, **analysis_rain_point } else: analysis_data = None if (group_analysis_sm_select is None) and ( (group_analysis_rain_map_select is not None) and (group_analysis_rain_point_select is not None)): logging.warning( ' ===> SoilMoisture datasets is undefined') elif (group_analysis_rain_map_select is None) and (group_analysis_sm_select is not None): logging.warning(' ===> Rain map datasets is undefined') elif (group_analysis_rain_point_select is None) and (group_analysis_sm_select is not None): logging.warning( ' ===> Rain point datasets is undefined') else: logging.warning( ' ===> Rain and SoilMoisture datasets are undefined' ) if soilslip_select is not None: analysis_event = self.unpack_analysis( soilslip_select, ['event_n', 'event_threshold', 'event_index']) else: analysis_event = self.analysis_event_undefined logging.warning( ' ===> SoilSlip datasets is null. No events reported') if (analysis_data is not None) and (analysis_event is not None): analysis_obj = { self.flag_indicators_time: time_step, self.flag_indicators_data: analysis_data, self.flag_indicators_event: analysis_event } folder_name_dest, file_name_dest = os.path.split( file_path_dest) make_folder(folder_name_dest) write_obj(file_path_dest, analysis_obj) logging.info(' -----> Alert Area ' + group_data_key + ' ... DONE') else: logging.info(' -----> Alert Area ' + group_data_key + ' ... SKIPPED. Some datasets are undefined') else: logging.info(' -----> Alert Area ' + group_data_key + ' ... SKIPPED. Analysis file created previously') logging.info(' ----> Save analysis [' + str(self.time_step) + '] ... DONE')
def organize_data(self): # Starting info logging.info(' ----> Organize soil slips point information ... ') geo_point_db = self.dset_geo_point time_point_expected = self.dset_time_point file_path_dst = self.file_path_dst if self.flag_geo_updating: if os.path.exists(file_path_dst): os.remove(file_path_dst) if not os.path.exists(file_path_dst): soil_slip_collections = {} for group_key, group_data in self.structure_group_data.items(): group_selection = group_data[self.structure_group_tag_name] group_threshold = group_data[self.structure_group_tag_threshold] group_index = group_data[self.structure_group_tag_index] geo_point_selection = geo_point_db.loc[geo_point_db[self.column_db_tag_alert_area] == group_selection] # geo_point_selection = geo_point_selection.reset_index() # geo_point_selection = geo_point_selection.set_index(self.column_db_tag_time) time_point_selection = pd.DatetimeIndex(geo_point_selection[ self.column_db_tag_time].values).unique().sort_values() soil_slip_n = [] soil_slip_features = [] soil_slip_threshold = [] soil_slip_index = [] for time_point_step in time_point_selection: time_str_step = time_point_step.strftime('%Y-%m-%d') geo_point_step = geo_point_selection.loc[ geo_point_selection[self.column_db_tag_time] == time_str_step] geo_point_threshold = find_category(geo_point_step.shape[0], group_threshold) geo_point_index = find_value(geo_point_threshold, group_index) soil_slip_n.append(geo_point_step.shape[0]) soil_slip_features.append(geo_point_step) soil_slip_threshold.append(geo_point_threshold) soil_slip_index.append(geo_point_index) data_soilslip = {'event_n': soil_slip_n, 'event_threshold': soil_slip_threshold, 'event_index': soil_slip_index, 'event_features': soil_slip_features} dframe_soilslip = pd.DataFrame(data_soilslip, index=time_point_selection) soil_slip_collections[group_key] = {} soil_slip_collections[group_key] = dframe_soilslip # Write soil slips collections to disk folder_name_dst, file_name_dst = os.path.split(file_path_dst) make_folder(folder_name_dst) write_obj(file_path_dst, soil_slip_collections) # Ending info logging.info(' ----> Organize soil slips point information ... DONE') else: # Read soil slips collections from disk soil_slip_collections = read_obj(file_path_dst) logging.info(' ----> Organize soil slips point information ... LOADED. Datasets was previously computed.') return soil_slip_collections
def main(): # ------------------------------------------------------------------------------------- # Get algorithm settings alg_settings, alg_time = get_args() # Set algorithm settings data_settings = read_file_json(alg_settings) # Set algorithm logging make_folder(data_settings['data']['log']['folder_name']) set_logging( logger_file=os.path.join(data_settings['data']['log']['folder_name'], data_settings['data']['log']['file_name'])) # Set algorithm library dependencies set_deps(data_settings['algorithm']['dependencies'], env_extra=['PROJ_LIB', 'GDAL_DATA']) # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Info algorithm logging.info( ' ============================================================================ ' ) logging.info(' ==> ' + alg_name + ' (Version: ' + alg_version + ' Release_Date: ' + alg_release + ')') logging.info(' ==> START ... ') logging.info(' ') # Time algorithm information start_time = time.time() # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Organize time run time_run, time_range = set_time( time_run_args=alg_time, time_run_file=data_settings['time']['time_now'], time_run_file_start=data_settings['time']['time_start'], time_run_file_end=data_settings['time']['time_end'], time_format=time_format, time_period=data_settings['time']['time_period'], time_frequency=data_settings['time']['time_frequency'], time_rounding=data_settings['time']['time_rounding']) # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Geographical datasets driver_data_geo_grid = DriverGeoGrid( src_dict=data_settings['data']['static']['source'], dst_dict=data_settings['data']['static']['destination'], group_data=data_settings['algorithm']['ancillary']['group'], alg_template_tags=data_settings['algorithm']['template'], flag_geo_updating=data_settings['algorithm']['flags'] ['updating_static_data']) geo_data_collection = driver_data_geo_grid.organize_data() # Weather stations point datasets driver_data_geo_point_weather_stations = DriverGeoPoint_WeatherStations( src_dict=data_settings['data']['static']['source'], dst_dict=data_settings['data']['static']['destination'], group_data=data_settings['algorithm']['ancillary']['group'], alg_template_tags=data_settings['algorithm']['template'], flag_geo_updating=data_settings['algorithm']['flags'] ['updating_static_data']) geo_point_collection_weather_stations = driver_data_geo_point_weather_stations.organize_data( ) # Soil-slips point datasets driver_data_geo_point_soil_slips = DriverGeoPoint_SoilSlips( src_dict=data_settings['data']['static']['source'], dst_dict=data_settings['data']['static']['destination'], group_data=data_settings['algorithm']['ancillary']['group'], flag_geo_updating=data_settings['algorithm']['flags'] ['updating_static_data']) geo_point_collection_soil_slips = driver_data_geo_point_soil_slips.organize_data( ) # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Activate analyzer mode if activate_algorithm_step( ['organizer', 'analyzer'], data_settings['algorithm']['flags']['running_mode']): # Iterate over time(s) for time_step in time_range: # Rain datasets driver_data_forcing_rain = DriverForcingRain( time_step, src_dict=data_settings['data']['dynamic']['source'], ancillary_dict=data_settings['data']['dynamic']['ancillary'], dst_dict=data_settings['data']['dynamic']['destination'], tmp_dict=data_settings['data']['tmp'], time_data=data_settings['data']['dynamic']['time'], geo_data_region=geo_data_collection['geo_region'], geo_data_weather_station=geo_point_collection_weather_stations, group_data=data_settings['algorithm']['ancillary']['group'], alg_template_tags=data_settings['algorithm']['template'], flag_ancillary_updating=data_settings['algorithm']['flags'] ['updating_dynamic_ancillary_rain']) if activate_algorithm_step( ['organizer'], data_settings['algorithm']['flags']['running_mode']): driver_data_forcing_rain.organize_forcing() # Soil moisture datasets driver_data_forcing_sm = DriverForcingSM( time_step, src_dict=data_settings['data']['dynamic']['source'], ancillary_dict=data_settings['data']['dynamic']['ancillary'], dst_dict=data_settings['data']['dynamic']['destination'], time_data=data_settings['data']['dynamic']['time'], basin_data=geo_data_collection['geo_basin'], geo_data=geo_data_collection['geo_alert_area'], group_data=data_settings['algorithm']['ancillary']['group'], alg_template_tags=data_settings['algorithm']['template'], flag_ancillary_updating=data_settings['algorithm']['flags'] ['updating_dynamic_ancillary_sm']) if activate_algorithm_step( ['organizer'], data_settings['algorithm']['flags']['running_mode']): driver_data_forcing_sm.organize_forcing() # Analysis datasets to define indicators driver_analysis_indicators = DriverAnalysisIndicators( time_step, file_list_rain_map=driver_data_forcing_rain. file_path_processed_map, file_list_rain_point=driver_data_forcing_rain. file_path_processed_point, file_list_sm=driver_data_forcing_sm.file_path_processed, ancillary_dict=data_settings['data']['dynamic']['ancillary'], dst_dict=data_settings['data']['dynamic']['destination'], time_data=data_settings['data']['dynamic']['time'], geo_data_region=geo_data_collection['geo_region'], geo_data_alert_area=geo_data_collection['geo_alert_area'], geo_data_weather_station=geo_point_collection_weather_stations, index_data_alert_area=geo_data_collection['index_alert_area'], group_data=data_settings['algorithm']['ancillary']['group'], alg_template_tags=data_settings['algorithm']['template'], flag_dest_updating=data_settings['algorithm']['flags'] ['updating_dynamic_indicators']) if activate_algorithm_step( ['analyzer'], data_settings['algorithm']['flags']['running_mode']): analysis_data_rain_point = driver_analysis_indicators.organize_analysis_rain_point( ) analysis_data_rain_map = driver_analysis_indicators.organize_analysis_rain_map( ) analysis_data_sm = driver_analysis_indicators.organize_analysis_sm( ) driver_analysis_indicators.save_analysis( analysis_data_sm, analysis_data_rain_map, analysis_data_rain_point, geo_point_collection_soil_slips) # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Activate publisher mode if activate_algorithm_step( ['publisher'], data_settings['algorithm']['flags']['running_mode']): # ------------------------------------------------------------------------------------- # Analysis datasets to define scenarios driver_analysis_scenarios = DriverAnalysisScenarios( time_run, time_range, ancillary_dict=data_settings['data']['dynamic']['ancillary'], dst_dict=data_settings['data']['dynamic']['destination'], geo_data_region=driver_data_geo_grid.dset_geo_region, geo_data_alert_area=driver_data_geo_grid.dset_geo_alert_area, group_data=data_settings['algorithm']['ancillary']['group'], plot_data=data_settings['algorithm']['ancillary']['plot'], alg_template_tags=data_settings['algorithm']['template'], flag_dest_updating=data_settings['algorithm']['flags'] ['updating_dynamic_scenarios']) scenarios_data = driver_analysis_scenarios.collect_scenarios() driver_analysis_scenarios.dump_scenarios(scenarios_data) driver_analysis_scenarios.plot_scenarios(scenarios_data) # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Info algorithm time_elapsed = round(time.time() - start_time, 1) logging.info(' ') logging.info(' ==> ' + alg_name + ' (Version: ' + alg_version + ' Release_Date: ' + alg_release + ')') logging.info(' ==> TIME ELAPSED: ' + str(time_elapsed) + ' seconds') logging.info(' ==> ... END') logging.info(' ==> Bye, Bye') logging.info( ' ============================================================================ ' )
def organize_dynamic_data(self): time_str = self.time_str time_period = self.time_period geo_da_dst = self.geo_da_dst src_dict = self.src_dict var_name_obj = self.var_name_obj file_path_obj_src = self.file_path_obj_src file_path_obj_anc = self.file_path_obj_anc flag_cleaning_ancillary = self.flag_cleaning_dynamic_ancillary log_stream.info(' ---> Organize dynamic datasets [' + time_str + '] ... ') # Check if ancillary file already exists file_check_list = [] for file_path_tmp in file_path_obj_anc: if os.path.exists(file_path_tmp): if flag_cleaning_ancillary: os.remove(file_path_tmp) file_check_list.append(False) else: file_check_list.append(True) else: file_check_list.append(False) file_check = all(file_check_list) # If statement on ancillary availability if not file_check: dset_collection = {} for var_name in var_name_obj: log_stream.info(' ----> Variable "' + var_name + '" ... ') var_compute, var_tag, var_scale_factor, var_shift, file_compression, \ file_geo_reference, file_type, file_coords, file_freq, compute_quality, var_decimal_digits \ = self.extract_var_fields(src_dict[var_name]) var_file_path_src = file_path_obj_src[var_name] if var_compute: var_geo_data = None for var_time, var_file_path_in in zip( time_period, var_file_path_src): log_stream.info( ' -----> Time "' + var_time.strftime(time_format_algorithm) + '" ... ') if os.path.exists(var_file_path_in): #copy to tmp var_file_path, var_file_name = os.path.split( var_file_path_in) var_file_path_in_tempcopy = self.domain + '_' + var_file_name copyfile( var_file_path_in, os.path.join(var_file_path, var_file_path_in_tempcopy)) var_file_path_in = os.path.join( var_file_path, var_file_path_in_tempcopy) if file_compression: var_file_path_out = self.define_file_name_unzip( var_file_path_in) unzip_filename(var_file_path_in, var_file_path_out) else: var_file_path_out = deepcopy(var_file_path_in) if file_type == 'binary': if var_geo_data is None: log_stream.info( ' ------> Select geo reference for binary datasets ... ' ) var_geo_name = search_geo_reference( var_file_path_out, self.static_data_src, tag_geo_reference=file_geo_reference) log_stream.info( ' -------> Geo reference name: ' + var_geo_name) var_geo_data, var_geo_x, var_geo_y, var_geo_attrs = \ self.set_geo_attributes(self.static_data_src[var_geo_name]) log_stream.info( ' ------> Select geo reference for binary datasets ... DONE' ) var_da_src = read_data_binary( var_file_path_out, var_geo_x, var_geo_y, var_geo_attrs, var_scale_factor=var_scale_factor, var_time=var_time, var_name=var_name, coord_name_geo_x=self.coord_name_geo_x, coord_name_geo_y=self.coord_name_geo_y, coord_name_time=self.coord_name_time, dim_name_geo_x=self.dim_name_geo_x, dim_name_geo_y=self.dim_name_geo_y, dim_name_time=self.dim_name_time, dims_order=self.dims_order_3d) elif file_type == 'netcdf': if var_geo_data is None: log_stream.info( ' ------> Select geo reference for netcdf datasets ... ' ) var_geo_data, var_geo_x, var_geo_y, var_geo_attrs = \ self.set_geo_attributes(self.static_data_src[file_geo_reference]) log_stream.info( ' ------> Select geo reference for netcdf datasets ... DONE' ) var_da_src = read_data_nc( var_file_path_out, var_geo_x, var_geo_y, var_geo_attrs, var_coords=file_coords, var_scale_factor=var_scale_factor, var_name=var_tag, var_time=var_time, coord_name_geo_x=self.coord_name_geo_x, coord_name_geo_y=self.coord_name_geo_y, coord_name_time=self.coord_name_time, dim_name_geo_x=self.dim_name_geo_x, dim_name_geo_y=self.dim_name_geo_y, dim_name_time=self.dim_name_time, dims_order=self.dims_order_3d) elif file_type == 'tiff' or file_type == 'asc': var_da_src = read_data_tiff( var_file_path_out, var_scale_factor=var_scale_factor, var_name=var_tag, var_time=var_time, coord_name_geo_x=self.coord_name_geo_x, coord_name_geo_y=self.coord_name_geo_y, coord_name_time=self.coord_name_time, dim_name_geo_x=self.dim_name_geo_x, dim_name_geo_y=self.dim_name_geo_y, dim_name_time=self.dim_name_time, dims_order=self.dims_order_3d, decimal_round_data=2, decimal_round_geo=7) elif file_type == 'mat': var_da_src = read_data_mat( var_file_path_out, var_scale_factor=var_scale_factor, var_name=var_tag, var_time=var_time, coord_name_geo_x=self.coord_name_geo_x, coord_name_geo_y=self.coord_name_geo_y, coord_name_time=self.coord_name_time, dim_name_geo_x=self.dim_name_geo_x, dim_name_geo_y=self.dim_name_geo_y, dim_name_time=self.dim_name_time, dims_order=self.dims_order_3d, decimal_round_data=2, decimal_round_geo=7, src_dict=src_dict[var_name]) else: log_stream.info( ' -----> Time "' + var_time.strftime(time_format_algorithm) + '" ... FAILED') log_stream.error(' ===> File type "' + file_type + '"is not allowed.') raise NotImplementedError( 'Case not implemented yet') # Delete (if needed the uncompressed file(s) if var_file_path_in != var_file_path_out: if os.path.exists(var_file_path_out): os.remove(var_file_path_out) # Delete temporary file os.remove(var_file_path_in) # Apply scale factor and shift to values if var_shift is not None: var_da_src.values = var_da_src.values + var_shift if var_scale_factor is not None: var_da_src.values = var_da_src.values / var_scale_factor #if var_shift is not None: # var_da_src=var_da_src + var_shift #if var_scale_factor is not None: # var_da_src=var_da_src / var_scale_factor # Organize destination dataset if var_da_src is not None: # Active (if needed) interpolation method to the variable source data-array active_interp = active_var_interp( var_da_src.attrs, geo_da_dst.attrs) # Apply the interpolation method to the variable source data-array if active_interp: var_da_dst = apply_var_interp( var_da_src, geo_da_dst, var_name=var_name, dim_name_geo_x=self.dim_name_geo_x, dim_name_geo_y=self.dim_name_geo_y, coord_name_geo_x=self.coord_name_geo_x, coord_name_geo_y=self.coord_name_geo_y, interp_method=self.interp_method) else: if var_tag != var_name: var_da_dst = deepcopy(var_da_src) var_da_dst.name = var_name else: var_da_dst = deepcopy(var_da_src) # Mask the variable destination data-array var_nodata = None if 'nodata_value' in list( var_da_dst.attrs.keys()): var_nodata = var_da_dst.attrs[ 'nodata_value'] geo_nodata = None if 'nodata_value' in list( geo_da_dst.attrs.keys()): geo_nodata = geo_da_dst.attrs[ 'nodata_value'] if (geo_nodata is not None) and (var_nodata is not None): var_da_masked = var_da_dst.where( (geo_da_dst.values[:, :, np.newaxis] != geo_nodata) & (var_da_dst != var_nodata)) else: var_da_masked = deepcopy(var_da_dst) #Sanity check to remove nans var_da_masked.values = \ np.where(np.isnan(var_da_masked.values), var_nodata, var_da_masked.values) #Round var_da_masked.values = np.round( var_da_masked.values, var_decimal_digits) # plt.figure(1) # plt.imshow(var_da_dst.values[:, :, 0]) # plt.colorbar() # plt.figure(2) # plt.imshow(var_da_src.values[:, :, 0]) # plt.colorbar() # plt.figure(3) # plt.imshow(var_da_masked.values[:, :, 0]) # plt.colorbar() # plt.show() # plt.figure(4) # plt.imshow(geo_da_dst.values) # plt.colorbar() # plt.show() # Organize data in a common datasets var_dset_masked = create_dset( var_data_time=var_time, var_data_name=var_name, var_data_values=var_da_masked, var_data_attrs=None, var_geo_1d=False, file_attributes=geo_da_dst.attrs, var_geo_name='terrain', var_geo_values=geo_da_dst.values, var_geo_x=geo_da_dst['longitude'].values, var_geo_y=geo_da_dst['latitude'].values, var_geo_attrs=None) # Organize data in merged datasets if var_time not in list( dset_collection.keys()): dset_collection[var_time] = var_dset_masked else: var_dset_tmp = deepcopy( dset_collection[var_time]) var_dset_tmp = var_dset_tmp.merge( var_dset_masked, join='right') dset_collection[var_time] = var_dset_tmp #Compute SQA if needed if compute_quality: log_stream.info(' ----> Variable "' + var_name + '" ... computing quality ') SQA = compute_SQA(var_da_masked.values, geo_da_dst.values, self.SQA_ground_and_snow) SQA_dset = create_dset( var_data_time=var_time, var_data_name='SQA', var_data_values=SQA, var_data_attrs=None, var_geo_1d=False, file_attributes=geo_da_dst.attrs, var_geo_name='terrain', var_geo_values=geo_da_dst.values, var_geo_x=geo_da_dst['longitude']. values, var_geo_y=geo_da_dst['latitude']. values, var_geo_attrs=None) var_dset_tmp = deepcopy( dset_collection[var_time]) var_dset_tmp = var_dset_tmp.merge( SQA_dset, join='right') dset_collection[var_time] = var_dset_tmp log_stream.info( ' -----> Time "' + var_time.strftime(time_format_algorithm) + '" ... DONE') else: log_stream.info( ' -----> Time "' + var_time.strftime(time_format_algorithm) + '" ... Datasets is not defined') else: var_da_src = None log_stream.info( ' -----> Time "' + var_time.strftime(time_format_algorithm) + '" ... Datasets is not defined') log_stream.info(' ----> Variable "' + var_name + '" ... DONE') else: log_stream.info( ' ----> Variable "' + var_name + '" ... SKIPPED. Compute flag not activated.') # Save ancillary datasets for file_path_anc, (dset_time, dset_anc) in zip(file_path_obj_anc, dset_collection.items()): folder_name_anc, file_name_anc = os.path.split(file_path_anc) if not os.path.exists(folder_name_anc): make_folder(folder_name_anc) write_obj(file_path_anc, dset_anc) log_stream.info(' ---> Organize dynamic datasets [' + time_str + '] ... DONE') else: log_stream.info( ' ---> Organize dynamic datasets [' + time_str + '] ... SKIPPED. All datasets are previously computed')
def dump_dynamic_data(self): time_str = self.time_str time_period = self.time_period dst_dict = self.dst_dict file_path_obj_anc = self.file_path_obj_anc file_path_obj_dst = self.file_path_obj_dst flag_cleaning_dynamic = self.flag_cleaning_dynamic_data log_stream.info(' ---> Dump dynamic datasets [' + time_str + '] ... ') for time_step, file_path_anc, file_path_dst in zip( time_period, file_path_obj_anc, file_path_obj_dst): log_stream.info(' -----> Time "' + time_step.strftime(time_format_algorithm) + '" ... ') file_path_zip = self.define_file_name_zip(file_path_dst) if flag_cleaning_dynamic: if os.path.exists(file_path_dst): os.remove(file_path_dst) if os.path.exists(file_path_zip): os.remove(file_path_zip) if os.path.exists(file_path_anc): dset_obj = read_obj(file_path_anc) folder_name_dst, file_name_dst = os.path.split(file_path_dst) if not os.path.exists(folder_name_dst): make_folder(folder_name_dst) log_stream.info(' ------> Save filename "' + file_name_dst + '" ... ') if not (os.path.exists(file_path_dst) or os.path.exists(file_path_zip)): # Squeeze time dimensions (if equal == 1) --> continuum expects 2d variables in forcing variables if self.dim_name_time in list(dset_obj.dims): time_array = dset_obj[self.dim_name_time].values if time_array.shape[0] == 1: dset_obj = dset_obj.squeeze(self.dim_name_time) dset_obj = dset_obj.drop(self.dim_name_time) write_dset(file_path_dst, dset_obj, dset_engine=self.nc_type_engine, dset_format=self.nc_type_file, dset_compression=self.nc_compression_level, fill_data=-9999.0, dset_type='float32') log_stream.info(' ------> Save filename "' + file_name_dst + '" ... DONE') log_stream.info(' ------> Zip filename "' + file_name_dst + '" ... ') if dst_dict[self.file_compression_tag]: zip_filename(file_path_dst, file_path_zip) if os.path.exists(file_path_zip) and (file_path_zip != file_name_dst): os.remove(file_path_dst) log_stream.info(' ------> Zip filename "' + file_name_dst + '" ... DONE') else: log_stream.info(' ------> Zip filename "' + file_name_dst + '" ... SKIPPED. Zip not activated') else: log_stream.info(' ------> Save filename "' + file_name_dst + '" ... SKIPPED. Filename previously saved') log_stream.info(' -----> Time "' + time_step.strftime(time_format_algorithm) + '" ... DONE') else: log_stream.info(' -----> Time "' + time_step.strftime(time_format_algorithm) + '" ... SKIPPED. Datasets not available') log_stream.info(' ---> Dump dynamic datasets [' + time_str + '] ... DONE')
def __init__(self, src_dict, dst_dict=None, group_data=None, flag_point_data_src='weather_stations_data', flag_grid_data='geo_data', flag_point_data_dst='weather_stations_data', alg_template_tags=None, flag_geo_updating=True, search_radius_km=10): self.flag_point_data_src = flag_point_data_src self.flag_point_data_dst = flag_point_data_dst self.flag_grid_data = flag_grid_data self.file_name_tag = 'file_name' self.folder_name_tag = 'folder_name' self.point_registry_tag = 'registry' self.point_alert_area_tree_tag = 'alert_area_tree' self.grid_vector_tag = 'alert_area_vector' self.point_code_tag = 'code' self.point_name_tag = 'name' self.point_longitude_tag = 'longitude' self.point_latitude_tag = 'latitude' self.point_alert_area_tag = 'alert_area' self.group_data = group_data self.alg_template_tags = alg_template_tags self.flag_geo_updating = flag_geo_updating self.search_radius_km = search_radius_km self.file_name_point_registry_src = src_dict[self.flag_point_data_src][ self.point_registry_tag][self.file_name_tag] self.folder_name_point_registry_src = src_dict[ self.flag_point_data_src][self.point_registry_tag][ self.folder_name_tag] self.file_path_point_registry_src = os.path.join( self.folder_name_point_registry_src, self.file_name_point_registry_src) self.file_name_point_registry_dst = dst_dict[self.flag_point_data_dst][ self.point_registry_tag][self.file_name_tag] self.folder_name_point_registry_dst = dst_dict[ self.flag_point_data_dst][self.point_registry_tag][ self.folder_name_tag] self.file_path_point_registry_dst = os.path.join( self.folder_name_point_registry_dst, self.file_name_point_registry_dst) self.file_name_point_alert_area_tree_dst = dst_dict[ self.flag_point_data_dst][self.point_alert_area_tree_tag][ self.file_name_tag] self.folder_name_point_alert_area_tree_dst = dst_dict[ self.flag_point_data_dst][self.point_alert_area_tree_tag][ self.folder_name_tag] self.file_path_point_alert_area_tree_dst = os.path.join( self.folder_name_point_alert_area_tree_dst, self.file_name_point_alert_area_tree_dst) self.file_name_grid = dst_dict[self.flag_grid_data][ self.grid_vector_tag][self.file_name_tag] self.folder_name_grid = dst_dict[self.flag_grid_data][ self.grid_vector_tag][self.folder_name_tag] self.file_path_grid = self.collect_file_obj(self.folder_name_grid, self.file_name_grid) self.search_radius_degree = km_2_degree(self.search_radius_km) if self.flag_geo_updating: if os.path.exists(self.file_path_point_registry_dst): os.remove(self.file_path_point_registry_dst) logging.info(' -----> Define geo points registry ... ') if not os.path.exists(self.file_path_point_registry_dst): df_geo_point = self.dset_geo_point = self.read_geo_point() self.df_geo_point = self.join_geo_point2grid(df_geo_point) make_folder(self.folder_name_point_registry_dst) self.df_geo_point.to_csv(self.file_path_point_registry_dst) logging.info(' -----> Define geo points registry ... DONE') else: self.df_geo_point = pd.read_csv(self.file_path_point_registry_dst) logging.info( ' -----> Define geo points registry ... LOADED. Datasets was previously computed.' ) self.tag_sep = ':'
def organize_data(self): # Starting info logging.info(' ----> Organize weather stations point information ... ') df_point = self.df_geo_point max_distance = self.search_radius_degree inf_distance = float("inf") file_path_point = self.file_path_point_alert_area_tree_dst flag_geo_updating = self.flag_geo_updating if flag_geo_updating: if os.path.exists(file_path_point): os.remove(file_path_point) if not os.path.exists(file_path_point): code_points = df_point[self.point_code_tag].values name_points = df_point[self.point_name_tag].values lats_points = df_point[self.point_latitude_tag].values lons_points = df_point[self.point_longitude_tag].values aa_points = df_point[self.point_alert_area_tag].values coord_points = np.dstack( [lats_points.ravel(), lons_points.ravel()])[0] coord_tree = cKDTree(coord_points) weather_stations_collections = {} for code_point, aa_point, coord_point in zip( code_points, aa_points, coord_points): distances, indices = coord_tree.query( coord_point, len(coord_points), p=2, distance_upper_bound=max_distance) code_points_neighbors = [] name_points_neighbors = [] coord_points_neighbors = [] lats_points_neighbors = [] lons_points_neighbors = [] aa_points_neighbors = [] for index, distance in zip(indices, distances): if distance == inf_distance: break coord_points_neighbors.append(coord_points[index]) code_points_neighbors.append(code_points[index]) name_points_neighbors.append(name_points[index]) lons_points_neighbors.append(lons_points[index]) lats_points_neighbors.append(lats_points[index]) aa_points_neighbors.append(aa_points[index]) coord_dict = { self.point_code_tag: code_points_neighbors, self.point_name_tag: name_points_neighbors, self.point_latitude_tag: lats_points_neighbors, self.point_longitude_tag: lons_points_neighbors, self.point_alert_area_tag: aa_points_neighbors } coord_dframe = pd.DataFrame(data=coord_dict) if aa_point not in list(weather_stations_collections.keys()): weather_stations_collections[aa_point] = {} weather_stations_collections[aa_point][ code_point] = coord_dframe folder_name, file_name = os.path.split(file_path_point) make_folder(folder_name) write_obj(file_path_point, weather_stations_collections) # Ending info logging.info( ' ----> Organize weather stations point information ... DONE') else: # Ending info weather_stations_collections = read_obj(file_path_point) logging.info( ' ----> Organize weather stations point information ... LOADED. ' 'Datasets was previously computed.') return weather_stations_collections
def plot_scenarios(self, scenarios_collections): logging.info(' ----> Plot scenarios [' + str(self.time_run) + '] ... ') season_list = self.list_season event_n_min = self.event_n_min event_n_max = self.event_n_max scenarios_sm2event_file_path = self.file_path_scenarios_graph_sm2event_collections scenarios_rain2event_file_path = self.file_path_scenarios_graph_rain2event_collections scenarios_rain2sm_file_path = self.file_path_scenarios_graph_rain2sm_collections for group_data_key, group_data_alert_value in self.structure_data_group.items(): logging.info(' -----> Alert Area ' + group_data_key + ' ... ') file_path_sm2event = scenarios_sm2event_file_path[group_data_key] file_path_rain2event = scenarios_rain2event_file_path[group_data_key] file_path_rain2sm = scenarios_rain2sm_file_path[group_data_key] file_data = scenarios_collections[group_data_key] rain_period_list = group_data_alert_value['rain_datasets']['search_period'] template_rain_point = [] for rain_period_step in rain_period_list: template_rain_step = self.template_rain_point_accumulated.format(rain_period_step) template_rain_point.append(template_rain_step) template_sm_point = self.template_sm_point_avg if file_data is not None: logging.info(' ------> Plot rain against sm ... ') for season_step, file_path_rain2sm_step in zip(season_list, file_path_rain2sm): logging.info(' ------> Season ' + season_step + ' ... ') file_data_step = filter_scenarios_dataframe( file_data, tag_column_sm=template_sm_point, tag_column_rain=template_rain_point, filter_rain=True, filter_sm=True, filter_event=True, filter_season=self.filter_season, tag_column_event='event_n', value_min_event=event_n_min, value_max_event=event_n_max, season_lut=self.lut_season, season_name=season_step) folder_name_rain2sm, file_name_rain2sm = os.path.split(file_path_rain2sm_step) make_folder(folder_name_rain2sm) plot_scenarios_rain2sm(file_data_step, file_path_rain2sm_step, var_x=self.template_sm_point_avg, var_y=self.template_rain_point_accumulated, var_z='event_index', event_n_min=event_n_min, event_n_max=event_n_max, event_label=self.event_label, season_label=season_step, figure_dpi=60, extra_args={'rain_type': rain_period_list, 'soil_moisture_type': 'average'}) logging.info(' ------> Season ' + season_step + ' ... DONE') logging.info(' ------> Plot rain against sm ... DONE') ''' #plot_scenarios_rain2sm(file_data, file_path_rain2sm, # var_x='soil_moisture_maximum', var_y=self.template_rain_accumulated, # var_z='event_index', # event_n_min=self.event_n_min, event_label=self.event_label, # figure_dpi=60, # extra_args={'rain_type': rain_period_list, # 'soil_moisture_type': 'max'}) logging.info(' ------> Plot sm against events ... ') folder_name_sm2event, file_name_sm2event = os.path.split(file_path_sm2event) make_folder(folder_name_sm2event) plot_scenarios_sm2event(file_data, file_path_sm2event, event_n_min=self.event_n_min, event_label=self.event_label, figure_dpi=120) logging.info(' ------> Plot sm against events ... DONE') logging.info(' ------> Plot rain against events ... ') folder_name_rain2event, file_name_rain2event = os.path.split(file_path_rain2event) make_folder(folder_name_rain2event) plot_scenarios_rain2event(file_data, file_path_rain2event) logging.info(' ------> Plot rain against events ... DONE') ''' logging.info(' -----> Alert Area ' + group_data_key + ' ... DONE') else: logging.info(' -----> Alert Area ' + group_data_key + ' ... SKIPPED. Datasets are undefined.') logging.info(' ----> Plot scenarios [' + str(self.time_run) + '] ... DONE')
def organize_forcing(self, var_name='rain', var_min=0, var_max=None): logging.info(' ----> Organize rain forcing ... ') geoy_out_1d = self.file_data_geo['south_north'].values geox_out_1d = self.file_data_geo['west_east'].values mask_out_2d = self.file_data_geo.values point_weather_section = self.geo_data_weather_station geox_out_2d, geoy_out_2d = np.meshgrid(geox_out_1d, geoy_out_1d) for datetime_step, file_path_src, file_path_ancillary_map, file_path_ancillary_point in zip( self.time_range, self.file_path_src_list, self.file_path_ancillary_map_list, self.file_path_ancillary_point_list): logging.info(' -----> TimeStep: ' + str(datetime_step) + ' ... ') if self.flag_ancillary_updating: if os.path.exists(file_path_ancillary_map): os.remove(file_path_ancillary_map) if os.path.exists(file_path_ancillary_point): os.remove(file_path_ancillary_point) if not (os.path.exists(file_path_ancillary_map or os.path.exists(file_path_ancillary_point))): if os.path.exists(file_path_src): file_dframe = read_file_csv(file_path_src, datetime_step, file_header=self.columns_src, file_sep=self.file_columns_sep, scale_factor_longitude=self.file_scale_factor_longitude, scale_factor_latitude=self.file_scale_factor_latitude, scale_factor_data=self.file_scale_factor_rain) # Filter data using variable limits (if defined) if var_min is not None: file_dframe = file_dframe[(file_dframe['data'] >= var_min)] if var_max is not None: file_dframe = file_dframe[(file_dframe['data'] <= var_max)] if file_dframe is not None: file_time_src = file_dframe.index.unique() else: file_time_src = None else: file_dframe = None file_time_src = None logging.warning(' ===> File datasets of rain weather stations is not available.') if (file_time_src is not None) and (file_time_src.__len__() > 1): logging.warning(' ===> Time step selected are greater than 1. Errors could arise in the script') if file_dframe is not None: logging.info(' ------> Interpolate points to map datasets ... ') map_out_2d = interpolate_rain_dataframe(file_dframe, mask_out_2d, geox_out_2d, geoy_out_2d, folder_tmp=self.folder_tmp) logging.info(' ------> Interpolate points to map datasets ... DONE') logging.info(' ------> Save map datasets ... ') folder_name_map, file_name_map = os.path.split(file_path_ancillary_map) make_folder(folder_name_map) if file_path_ancillary_map.endswith('.nc'): dset_out = create_dset( map_out_2d, mask_out_2d, geox_out_2d, geoy_out_2d, var_data_time=datetime_step, var_data_name=var_name, var_geo_name='mask', var_data_attrs=None, var_geo_attrs=None, coord_name_x='longitude', coord_name_y='latitude', coord_name_time='time', dim_name_x='west_east', dim_name_y='south_north', dim_name_time='time', dims_order_2d=None, dims_order_3d=None) write_dset( file_path_ancillary_map, dset_out, dset_mode='w', dset_engine='h5netcdf', dset_compression=0, dset_format='NETCDF4', dim_key_time='time', no_data=-9999.0) logging.info(' ------> Save map datasets ... DONE. [NETCDF]') elif file_path_ancillary_map.endswith('.tiff'): ### error in saving ERROR 1: Only OGC WKT Projections supported for writing to GeoTIFF. EPSG:4326 not supported. save_file_tiff(file_path_ancillary_map, map_out_2d, geox_out_2d, geoy_out_2d, file_metadata=self.file_metadata, file_epsg_code=self.file_epsg_code) logging.info(' ------> Save map datasets ... DONE. [GEOTIFF]') else: logging.info(' ------> Save map datasets ... FAILED') logging.error(' ===> Filename format is not allowed') raise NotImplementedError('Format is not implemented yet') self.file_path_processed_map.append(file_path_ancillary_map) logging.info(' ------> Save points datasets ... ') folder_name_point, file_name_point = os.path.split(file_path_ancillary_point) make_folder(folder_name_point) if file_path_ancillary_point.endswith('.csv'): write_file_csv(file_path_ancillary_point, file_dframe) logging.info(' ------> Save points datasets... DONE') else: logging.info(' ------> Save points datasets ... FAILED') logging.error(' ===> Filename format is not allowed') raise NotImplementedError('Format is not implemented yet') self.file_path_processed_point.append(file_path_ancillary_point) logging.info(' -----> TimeStep: ' + str(datetime_step) + ' ... DONE') else: logging.info(' -----> TimeStep: ' + str(datetime_step) + ' ... FAILED') logging.warning(' ===> File datasets of rain weather stations is not available.') else: logging.info(' -----> TimeStep: ' + str(datetime_step) + ' ... PREVIOUSLY DONE') logging.info(' ----> Organize rain forcing ... DONE')
def main(): # ------------------------------------------------------------------------------------- # Get algorithm settings alg_settings, alg_time = get_args() # Set algorithm settings data_settings = read_file_json(alg_settings) # Set algorithm logging make_folder(data_settings['data']['log']['folder_name']) set_logging( logger_file=os.path.join(data_settings['data']['log']['folder_name'], data_settings['data']['log']['file_name'])) # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Info algorithm logging.info( ' ============================================================================ ' ) logging.info(' ==> ' + alg_name + ' (Version: ' + alg_version + ' Release_Date: ' + alg_release + ')') logging.info(' ==> START ... ') logging.info(' ') # Time algorithm information start_time = time.time() # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Organize time run time_now, time_run, time_range = set_time( time_run_args=alg_time, time_run_file=data_settings['time']['time_now'], time_format=time_format, time_period=data_settings['time']['time_period'], time_frequency=data_settings['time']['time_frequency'], time_rounding=data_settings['time']['time_rounding']) # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Geographical datasets driver_data_geo = DriverGeo( src_dict=data_settings['data']['static']['source'], dst_dict=data_settings['data']['static']['destination'], alg_ancillary=data_settings['algorithm']['ancillary'], alg_template_tags=data_settings['algorithm']['template'], flag_cleaning_geo=data_settings['algorithm']['flags'] ['cleaning_geo_data']) geo_data_collection = driver_data_geo.organize_geo() # ------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------- # Iterate over time range for time_step in time_range: # ------------------------------------------------------------------------------------- # Discharge datasets driver_data_source_discharge = DriverDischarge( time_now=time_now, time_run=time_step, geo_data_collection=geo_data_collection, src_dict=data_settings['data']['dynamic']['source'], ancillary_dict=data_settings['data']['dynamic']['ancillary'], alg_ancillary=data_settings['algorithm']['ancillary'], alg_template_tags=data_settings['algorithm']['template'], flag_cleaning_dynamic_ancillary=data_settings['algorithm']['flags'] ['cleaning_dynamic_data_ancillary']) discharge_data_collection = driver_data_source_discharge.organize_discharge( ) # Scenario datasets driver_data_destination_scenario = DriverScenario( time_now=time_now, time_run=time_step, discharge_data_collection=discharge_data_collection, geo_data_collection=geo_data_collection, src_dict=data_settings['data']['static']['source'], dst_dict=data_settings['data']['dynamic']['destination'], alg_ancillary=data_settings['algorithm']['ancillary'], alg_template_tags=data_settings['algorithm']['template'], flag_cleaning_scenario=data_settings['algorithm']['flags'] ['cleaning_scenario_data']) scenario_info_collection = driver_data_destination_scenario.organize_scenario_datasets( ) scenario_map_collection = driver_data_destination_scenario.compute_scenario_map( scenario_info_collection) driver_data_destination_scenario.dump_scenario_map( scenario_map_collection, scenario_info_collection) # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Info algorithm time_elapsed = round(time.time() - start_time, 1) logging.info(' ') logging.info(' ==> ' + alg_name + ' (Version: ' + alg_version + ' Release_Date: ' + alg_release + ')') logging.info(' ==> TIME ELAPSED: ' + str(time_elapsed) + ' seconds') logging.info(' ==> ... END') logging.info(' ==> Bye, Bye') logging.info( ' ============================================================================ ' )