def organize_geo(self):
domain_collection_list = {}
for domain_name_step in self.domain_name_list:
file_path_collections = self.define_domain_collection(
domain_name_step)
if not os.path.exists(file_path_collections):
domain_info = self.get_domain_info(domain_name_step)
domain_drainage_area = self.get_domain_drainage_area(
domain_name_step)
domain_collection = {**domain_drainage_area, **domain_info}
folder_name_collections, file_name_collections = os.path.split(
file_path_collections)
make_folder(folder_name_collections)
write_obj(file_path_collections, domain_collection)
else:
domain_collection = read_obj(file_path_collections)
domain_collection_list[domain_name_step] = domain_collection
return domain_collection_list
def dump_scenarios(self, scenarios_collections):
logging.info(' ----> Dump scenarios [' + str(self.time_run) + '] ... ')
scenarios_file_path = self.file_path_scenarios_data_collections
scenarios_collection_tmp = None
for group_data_key in self.structure_data_group.keys():
logging.info(' -----> Alert Area ' + group_data_key + ' ... ')
file_path = scenarios_file_path[group_data_key]
if isinstance(file_path, list):
file_path = file_path[0]
if self.flag_dest_updating:
if os.path.exists(file_path):
os.remove(file_path)
if not os.path.exists(file_path):
file_data = scenarios_collections[group_data_key]
if file_data is not None:
folder_name, file_name = os.path.split(file_path)
make_folder(folder_name)
write_file_csv(file_path, file_data)
logging.info(' -----> Alert Area ' + group_data_key + ' ... DONE')
else:
logging.info(' -----> Alert Area ' + group_data_key + ' ... SKIPPED. Datasets are undefined.')
else:
logging.info(' -----> Alert Area ' + group_data_key + ' ... SKIPPED. Datasets previously saved.')
logging.info(' ----> Dump scenarios [' + str(self.time_run) + '] ... DONE')
def set_logging_file(logger_file=logger_file_default,
logger_name=logger_name_default,
logger_handle=logger_handle_default,
logger_formatter=logger_formatter_default,
logger_history=False,
logger_history_maxfiles=12,
logger_extra_tags=None):
# Set to flush progressbar output in logging stream handle
# progressbar.streams.wrap_stderr()
if logger_extra_tags is not None:
for extra_key, extra_value in logger_extra_tags.items():
logger_file = logger_file.replace(extra_key, ':')
string_count = logger_file.count(':')
extra_value = [extra_value] * string_count
logger_file = logger_file.format(*extra_value)
logger_folder_name, logger_file_name = os.path.split(logger_file)
make_folder(logger_folder_name)
# Save old logger file (to check run in the past)
if logger_history:
store_logging_file(logger_file,
logger_file_max=logger_history_maxfiles)
# Remove old logging file
if os.path.exists(logger_file):
os.remove(logger_file)
# Open logger
logging.getLogger(logger_name)
logging.root.setLevel(logging.DEBUG)
# Set logger handle type
if logger_handle == 'file':
# Set logger handler obj
logger_handle_1 = logging.FileHandler(logger_file, 'w')
logger_handle_2 = logging.StreamHandler()
# Set logger level
logger_handle_1.setLevel(logging.DEBUG)
logger_handle_2.setLevel(logging.DEBUG)
# Set logger formatter
logger_handle_1.setFormatter(logging.Formatter(logger_formatter))
logger_handle_2.setFormatter(logging.Formatter(logger_formatter))
# Add handle to logger
logging.getLogger('').addHandler(logger_handle_1)
logging.getLogger('').addHandler(logger_handle_2)
elif logger_handle == 'stream':
# Set logger handler obj
logging.StreamHandler()
# Set logger level
logger_handle.setLevel(logging.DEBUG)
# Set logger formatter
logger_handle.setFormatter(logging.Formatter(logger_formatter))
# Add handle to logger
logging.getLogger('').addHandler(logger_handle)
else:
# Set logger handler obj
logging.NullHandler()
# Add handle to logger
logging.getLogger('').addHandler(logger_handle)
def organize_forcing(self, var_name='soil_moisture'):
logging.info(' ----> Organize soil moisture forcing ... ')
time_range = self.time_range
file_data_geo = self.file_data_geo
file_data_basin = self.file_data_basin
file_path_src_list = self.file_path_src_list
file_path_ancillary_list = self.file_path_ancillary_list
for (group_key_basin, group_basin), (group_key_geo, group_geo), group_file, group_ancillary in zip(
file_data_basin.items(), file_data_geo.items(),
file_path_src_list.values(), file_path_ancillary_list.values()):
logging.info(' -----> Alert Area ' + group_key_basin + ' ... ')
basin_list = list(group_basin.keys())
geo_mask_ref = group_geo
geo_x_ref = group_geo['west_east']
geo_y_ref = group_geo['south_north']
basin_collections = {}
file_ancillary_collections = {}
if basin_list:
for basin_name in basin_list:
logging.info(' ------> BasinName ' + basin_name + ' ... ')
file_basin_geo = group_basin[basin_name]
file_basin_list = group_file[basin_name]
file_ancillary_list = group_ancillary[basin_name]
for time_step, file_basin_step, file_ancillary_step in zip(time_range,
file_basin_list, file_ancillary_list):
logging.info(' -------> TimeStep: ' + str(time_step) + ' ... ')
if self.flag_ancillary_updating:
if os.path.exists(file_ancillary_step):
os.remove(file_ancillary_step)
if not os.path.exists(file_ancillary_step):
if file_basin_step.endswith(self.file_extension_zip):
file_basin_out = change_extension(file_basin_step, self.file_extension_unzip)
else:
file_basin_out = file_basin_step
if os.path.exists(file_basin_step):
unzip_filename(file_basin_step, file_basin_out)
data_vtot = read_file_binary(
file_basin_out,
data_geo=file_basin_geo[self.var_name_terrain].values)
data_vmax = convert_cn2s(
file_basin_geo[self.var_name_curve_number].values,
file_basin_geo[self.var_name_terrain].values)
data_sm = data_vtot / data_vmax
data_sm[file_basin_geo[self.var_name_channels_network].values == 1] = -1
da_sm_base = create_darray_2d(data_sm,
file_basin_geo[self.var_name_x], file_basin_geo[self.var_name_y],
coord_name_x='west_east', coord_name_y='south_north',
dim_name_x='west_east', dim_name_y='south_north')
da_sm_interp = da_sm_base.interp(south_north=geo_y_ref, west_east=geo_x_ref, method='nearest')
if time_step not in list(basin_collections.keys()):
basin_collections[time_step] = [da_sm_interp]
file_ancillary_collections[time_step] = [file_ancillary_step]
else:
data_tmp = basin_collections[time_step]
data_tmp.append(da_sm_interp)
basin_collections[time_step] = data_tmp
file_tmp = file_ancillary_collections[time_step]
file_tmp.append(file_ancillary_step)
file_tmp = list(set(file_tmp))
file_ancillary_collections[time_step] = file_tmp
logging.info(' -------> TimeStep: ' + str(time_step) + ' ... DONE')
else:
logging.info(' -------> TimeStep: ' + str(time_step) + ' ... FAILED')
logging.warning(' ==> File: ' + file_basin_step + ' does not exist')
else:
logging.info(' -------> TimeStep: ' + str(time_step) + ' ... PREVIOUSLY DONE')
logging.info(' ------> BasinName ' + basin_name + ' ... DONE')
logging.info(' -----> Alert Area ' + group_key_basin + ' ... DONE')
else:
logging.info(' -----> Alert Area ' + group_key_basin + ' ... SKIPPED')
logging.warning(' ==> Datasets are not defined')
logging.info(' -----> Compose grid datasets from basins to alert area domain ... ')
for (time_step, data_list), file_path_ancillary in zip(
basin_collections.items(), file_ancillary_collections.values()):
logging.info(' ------> TimeStep: ' + str(time_step) + ' ... ')
if isinstance(file_path_ancillary, list) and file_path_ancillary.__len__() == 1:
file_path_ancillary = file_path_ancillary[0]
else:
logging.error(' ===> Soil moisture ancillary file are not correctly defined.')
raise IOError('Ancillary file is not unique')
if self.flag_ancillary_updating:
if os.path.exists(file_path_ancillary):
os.remove(file_path_ancillary)
if not os.path.exists(file_path_ancillary):
logging.info(' -------> Merge grid datasets ... ')
array_merge = np.zeros([geo_mask_ref.values.shape[0] * geo_mask_ref.values.shape[1]])
array_merge[:] = np.nan
for data_step in data_list:
array_values = data_step.values.ravel()
idx_finite = np.isfinite(array_values)
array_merge[idx_finite] = array_values[idx_finite]
grid_merge = np.reshape(array_merge, [geo_mask_ref.values.shape[0], geo_mask_ref.values.shape[1]])
idx_choice = np.where(grid_merge == -1)
grid_merge[idx_choice[0], idx_choice[1]] = np.nan
idx_filter = np.where((geo_mask_ref.values == 1) & (np.isnan(grid_merge)))
grid_merge[idx_filter[0], idx_filter[1]] = np.nanmean(grid_merge)
grid_merge[(geo_mask_ref.values == 0)] = np.nan
grid_merge[idx_choice[0], idx_choice[1]] = np.nan
logging.info(' -------> Merge grid datasets ... DONE')
logging.info(' -------> Save grid datasets ... ')
dset_merge = create_dset(
grid_merge,
geo_mask_ref.values, geo_x_ref.values, geo_y_ref.values,
var_data_time=time_step,
var_data_name=var_name,
var_geo_name='mask', var_data_attrs=None, var_geo_attrs=None,
coord_name_x='longitude', coord_name_y='latitude', coord_name_time='time',
dim_name_x='west_east', dim_name_y='south_north', dim_name_time='time',
dims_order_2d=None, dims_order_3d=None)
folder_name_ancillary, file_name_ancillary = os.path.split(file_path_ancillary)
make_folder(folder_name_ancillary)
if file_path_ancillary.endswith('.nc'):
write_dset(
file_path_ancillary,
dset_merge, dset_mode='w', dset_engine='h5netcdf', dset_compression=0, dset_format='NETCDF4',
dim_key_time='time', no_data=-9999.0)
logging.info(' ------> Save grid datasets ... DONE. [NETCDF]')
elif file_path_ancillary.endswith('.tiff'):
save_file_tiff(file_path_ancillary,
np.flipud(dset_merge[var_name].values),
geo_x_ref.values, np.flipud(geo_y_ref.values),
file_metadata=self.file_metadata, file_epsg_code=self.file_epsg_code)
logging.info(' ------> Save grid datasets ... DONE. [GEOTIFF]')
else:
logging.info(' ------> Save grid datasets ... FAILED')
logging.error(' ===> Filename format is not allowed')
raise NotImplementedError('Format is not implemented yet')
self.file_path_processed.append(file_path_ancillary)
logging.info(' -------> Save grid datasets ... DONE')
logging.info(' ------> TimeStep: ' + str(time_step) + ' ... DONE')
else:
logging.info(' ------> TimeStep: ' + str(time_step) + ' ... PREVIOUSLY DONE')
logging.info(' -----> Compose grid datasets from basins to alert area domain ... DONE')
logging.info(' ----> Organize soil moisture forcing ... DONE')
def organize_discharge(self):
time = self.time_run
geo_data_collection = self.geo_data_collection
logging.info(' --> Organize discharge datasets [' + str(time) +
'] ... ')
file_path_discharge = self.file_path_discharge
file_time_discharge = self.file_time_discharge
section_collection = {}
for domain_name_step in self.domain_name_list:
logging.info(' ---> Domain ' + domain_name_step + ' ... ')
file_path_discharge = self.file_path_discharge[domain_name_step]
file_path_ancillary = self.file_path_ancillary[domain_name_step]
if self.flag_cleaning_dynamic_ancillary:
if os.path.exists(file_path_ancillary):
os.remove(file_path_ancillary)
if not os.path.exists(file_path_ancillary):
domain_discharge_index = geo_data_collection[domain_name_step][
self.domain_discharge_index_tag]
domain_grid_rows = geo_data_collection[domain_name_step][
self.domain_grid_x_tag].shape[0]
domain_grid_cols = geo_data_collection[domain_name_step][
self.domain_grid_y_tag].shape[1]
domain_section_db = geo_data_collection[domain_name_step][
self.domain_sections_db_tag]
section_workspace = {}
for section_key, section_data in domain_section_db.items():
section_description = section_data['description']
section_name = section_data['name']
section_idx = section_data['idx']
section_discharge_default = section_data[
'discharge_default']
section_id = self.format_group.format(
section_data['group']['id'])
logging.info(' ----> Section ' + section_description +
' ... ')
section_file_path_list = file_path_discharge[section_id]
if section_file_path_list:
section_dframe = pd.DataFrame(
index=file_time_discharge)
for section_file_path_step in section_file_path_list:
section_folder_name_step, section_file_name_step = os.path.split(
section_file_path_step)
section_file_ts_start, section_file_ts_end, \
section_file_mask, section_file_ens = parse_file_parts(section_file_name_step)
section_file_tag = create_file_tag(
section_file_ts_start, section_file_ts_end,
section_file_ens)
section_ts = read_file_hydro(
section_name, section_file_path_step)
section_dframe[section_file_tag] = section_ts
section_workspace[section_description] = section_dframe
logging.info(' ----> Section ' + section_description +
' ... DONE')
else:
logging.info(' ----> Section ' + section_description +
' ... SKIPPED. Datasets are empty')
section_workspace[section_description] = None
folder_name_ancillary, file_name_ancillary = os.path.split(
file_path_ancillary)
make_folder(folder_name_ancillary)
flag_save_obj = True
for section_key, section_data in section_workspace.items():
if section_data is None:
flag_save_obj = False
break
if flag_save_obj:
write_obj(file_path_ancillary, section_workspace)
logging.info(' ---> Domain ' + domain_name_step +
' ... DONE')
else:
logging.info(
' ---> Domain ' + domain_name_step +
' ... SKIPPED. All or some datasets are empty')
else:
section_workspace = read_obj(file_path_ancillary)
logging.info(' ---> Domain ' + domain_name_step +
' ... SKIPPED. Data previously computed')
section_collection[domain_name_step] = section_workspace
logging.info(' --> Organize discharge datasets [' + str(time) +
'] ... DONE')
return section_collection
def save_analysis(self, group_analysis_sm, group_analysis_rain_map,
group_analysis_rain_point, group_soilslip):
logging.info(' ----> Save analysis [' + str(self.time_step) + '] ... ')
time_step = self.time_step
geo_data_alert_area = self.geo_data_alert_area
group_data_alert_area = self.structure_data_group
for (group_data_key, group_data_items), geo_data_dframe in zip(
group_data_alert_area.items(), geo_data_alert_area.values()):
logging.info(' -----> Alert Area ' + group_data_key + ' ... ')
file_path_dest = collect_file_list(
time_step,
self.folder_name_dest_indicators_raw,
self.file_name_dest_indicators_raw,
self.alg_template_tags,
alert_area_name=group_data_key)[0]
if not os.path.exists(file_path_dest):
group_soilslip_select = group_soilslip[group_data_key]
if group_analysis_sm[group_data_key] is not None:
group_analysis_sm_select = group_analysis_sm[
group_data_key][self.template_struct_obj]
else:
group_analysis_sm_select = None
if group_analysis_rain_map[group_data_key] is not None:
group_analysis_rain_map_select = group_analysis_rain_map[
group_data_key][self.template_struct_obj]
else:
group_analysis_rain_map_select = None
if group_analysis_rain_point[group_data_key] is not None:
group_analysis_rain_point_select = group_analysis_rain_point[
group_data_key]
else:
group_analysis_rain_point_select = None
if group_analysis_sm_select is not None:
if time_step in list(group_soilslip_select.index):
soilslip_select = group_soilslip_select.loc[
time_step.strftime('%Y-%m-%d 00:00:00')]
else:
soilslip_select = None
else:
soilslip_select = None
if (group_analysis_sm_select is not None) and (
(group_analysis_rain_map_select is not None) and
(group_analysis_rain_point_select is not None)):
analysis_sm = self.unpack_analysis(
group_analysis_sm_select)
analysis_rain_map = self.unpack_analysis(
group_analysis_rain_map_select)
analysis_rain_point = self.unpack_analysis(
group_analysis_rain_point_select)
analysis_data = {
**analysis_sm,
**analysis_rain_map,
**analysis_rain_point
}
else:
analysis_data = None
if (group_analysis_sm_select is None) and (
(group_analysis_rain_map_select is not None) and
(group_analysis_rain_point_select is not None)):
logging.warning(
' ===> SoilMoisture datasets is undefined')
elif (group_analysis_rain_map_select is
None) and (group_analysis_sm_select is not None):
logging.warning(' ===> Rain map datasets is undefined')
elif (group_analysis_rain_point_select is
None) and (group_analysis_sm_select is not None):
logging.warning(
' ===> Rain point datasets is undefined')
else:
logging.warning(
' ===> Rain and SoilMoisture datasets are undefined'
)
if soilslip_select is not None:
analysis_event = self.unpack_analysis(
soilslip_select,
['event_n', 'event_threshold', 'event_index'])
else:
analysis_event = self.analysis_event_undefined
logging.warning(
' ===> SoilSlip datasets is null. No events reported')
if (analysis_data is not None) and (analysis_event
is not None):
analysis_obj = {
self.flag_indicators_time: time_step,
self.flag_indicators_data: analysis_data,
self.flag_indicators_event: analysis_event
}
folder_name_dest, file_name_dest = os.path.split(
file_path_dest)
make_folder(folder_name_dest)
write_obj(file_path_dest, analysis_obj)
logging.info(' -----> Alert Area ' + group_data_key +
' ... DONE')
else:
logging.info(' -----> Alert Area ' + group_data_key +
' ... SKIPPED. Some datasets are undefined')
else:
logging.info(' -----> Alert Area ' + group_data_key +
' ... SKIPPED. Analysis file created previously')
logging.info(' ----> Save analysis [' + str(self.time_step) +
'] ... DONE')
def organize_data(self):
# Starting info
logging.info(' ----> Organize soil slips point information ... ')
geo_point_db = self.dset_geo_point
time_point_expected = self.dset_time_point
file_path_dst = self.file_path_dst
if self.flag_geo_updating:
if os.path.exists(file_path_dst):
os.remove(file_path_dst)
if not os.path.exists(file_path_dst):
soil_slip_collections = {}
for group_key, group_data in self.structure_group_data.items():
group_selection = group_data[self.structure_group_tag_name]
group_threshold = group_data[self.structure_group_tag_threshold]
group_index = group_data[self.structure_group_tag_index]
geo_point_selection = geo_point_db.loc[geo_point_db[self.column_db_tag_alert_area] == group_selection]
# geo_point_selection = geo_point_selection.reset_index()
# geo_point_selection = geo_point_selection.set_index(self.column_db_tag_time)
time_point_selection = pd.DatetimeIndex(geo_point_selection[
self.column_db_tag_time].values).unique().sort_values()
soil_slip_n = []
soil_slip_features = []
soil_slip_threshold = []
soil_slip_index = []
for time_point_step in time_point_selection:
time_str_step = time_point_step.strftime('%Y-%m-%d')
geo_point_step = geo_point_selection.loc[
geo_point_selection[self.column_db_tag_time] == time_str_step]
geo_point_threshold = find_category(geo_point_step.shape[0], group_threshold)
geo_point_index = find_value(geo_point_threshold, group_index)
soil_slip_n.append(geo_point_step.shape[0])
soil_slip_features.append(geo_point_step)
soil_slip_threshold.append(geo_point_threshold)
soil_slip_index.append(geo_point_index)
data_soilslip = {'event_n': soil_slip_n, 'event_threshold': soil_slip_threshold,
'event_index': soil_slip_index, 'event_features': soil_slip_features}
dframe_soilslip = pd.DataFrame(data_soilslip, index=time_point_selection)
soil_slip_collections[group_key] = {}
soil_slip_collections[group_key] = dframe_soilslip
# Write soil slips collections to disk
folder_name_dst, file_name_dst = os.path.split(file_path_dst)
make_folder(folder_name_dst)
write_obj(file_path_dst, soil_slip_collections)
# Ending info
logging.info(' ----> Organize soil slips point information ... DONE')
else:
# Read soil slips collections from disk
soil_slip_collections = read_obj(file_path_dst)
logging.info(' ----> Organize soil slips point information ... LOADED. Datasets was previously computed.')
return soil_slip_collections
def main():
# -------------------------------------------------------------------------------------
# Get algorithm settings
alg_settings, alg_time = get_args()
# Set algorithm settings
data_settings = read_file_json(alg_settings)
# Set algorithm logging
make_folder(data_settings['data']['log']['folder_name'])
set_logging(
logger_file=os.path.join(data_settings['data']['log']['folder_name'],
data_settings['data']['log']['file_name']))
# Set algorithm library dependencies
set_deps(data_settings['algorithm']['dependencies'],
env_extra=['PROJ_LIB', 'GDAL_DATA'])
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Info algorithm
logging.info(
' ============================================================================ '
)
logging.info(' ==> ' + alg_name + ' (Version: ' + alg_version +
' Release_Date: ' + alg_release + ')')
logging.info(' ==> START ... ')
logging.info(' ')
# Time algorithm information
start_time = time.time()
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Organize time run
time_run, time_range = set_time(
time_run_args=alg_time,
time_run_file=data_settings['time']['time_now'],
time_run_file_start=data_settings['time']['time_start'],
time_run_file_end=data_settings['time']['time_end'],
time_format=time_format,
time_period=data_settings['time']['time_period'],
time_frequency=data_settings['time']['time_frequency'],
time_rounding=data_settings['time']['time_rounding'])
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Geographical datasets
driver_data_geo_grid = DriverGeoGrid(
src_dict=data_settings['data']['static']['source'],
dst_dict=data_settings['data']['static']['destination'],
group_data=data_settings['algorithm']['ancillary']['group'],
alg_template_tags=data_settings['algorithm']['template'],
flag_geo_updating=data_settings['algorithm']['flags']
['updating_static_data'])
geo_data_collection = driver_data_geo_grid.organize_data()
# Weather stations point datasets
driver_data_geo_point_weather_stations = DriverGeoPoint_WeatherStations(
src_dict=data_settings['data']['static']['source'],
dst_dict=data_settings['data']['static']['destination'],
group_data=data_settings['algorithm']['ancillary']['group'],
alg_template_tags=data_settings['algorithm']['template'],
flag_geo_updating=data_settings['algorithm']['flags']
['updating_static_data'])
geo_point_collection_weather_stations = driver_data_geo_point_weather_stations.organize_data(
)
# Soil-slips point datasets
driver_data_geo_point_soil_slips = DriverGeoPoint_SoilSlips(
src_dict=data_settings['data']['static']['source'],
dst_dict=data_settings['data']['static']['destination'],
group_data=data_settings['algorithm']['ancillary']['group'],
flag_geo_updating=data_settings['algorithm']['flags']
['updating_static_data'])
geo_point_collection_soil_slips = driver_data_geo_point_soil_slips.organize_data(
)
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Activate analyzer mode
if activate_algorithm_step(
['organizer', 'analyzer'],
data_settings['algorithm']['flags']['running_mode']):
# Iterate over time(s)
for time_step in time_range:
# Rain datasets
driver_data_forcing_rain = DriverForcingRain(
time_step,
src_dict=data_settings['data']['dynamic']['source'],
ancillary_dict=data_settings['data']['dynamic']['ancillary'],
dst_dict=data_settings['data']['dynamic']['destination'],
tmp_dict=data_settings['data']['tmp'],
time_data=data_settings['data']['dynamic']['time'],
geo_data_region=geo_data_collection['geo_region'],
geo_data_weather_station=geo_point_collection_weather_stations,
group_data=data_settings['algorithm']['ancillary']['group'],
alg_template_tags=data_settings['algorithm']['template'],
flag_ancillary_updating=data_settings['algorithm']['flags']
['updating_dynamic_ancillary_rain'])
if activate_algorithm_step(
['organizer'],
data_settings['algorithm']['flags']['running_mode']):
driver_data_forcing_rain.organize_forcing()
# Soil moisture datasets
driver_data_forcing_sm = DriverForcingSM(
time_step,
src_dict=data_settings['data']['dynamic']['source'],
ancillary_dict=data_settings['data']['dynamic']['ancillary'],
dst_dict=data_settings['data']['dynamic']['destination'],
time_data=data_settings['data']['dynamic']['time'],
basin_data=geo_data_collection['geo_basin'],
geo_data=geo_data_collection['geo_alert_area'],
group_data=data_settings['algorithm']['ancillary']['group'],
alg_template_tags=data_settings['algorithm']['template'],
flag_ancillary_updating=data_settings['algorithm']['flags']
['updating_dynamic_ancillary_sm'])
if activate_algorithm_step(
['organizer'],
data_settings['algorithm']['flags']['running_mode']):
driver_data_forcing_sm.organize_forcing()
# Analysis datasets to define indicators
driver_analysis_indicators = DriverAnalysisIndicators(
time_step,
file_list_rain_map=driver_data_forcing_rain.
file_path_processed_map,
file_list_rain_point=driver_data_forcing_rain.
file_path_processed_point,
file_list_sm=driver_data_forcing_sm.file_path_processed,
ancillary_dict=data_settings['data']['dynamic']['ancillary'],
dst_dict=data_settings['data']['dynamic']['destination'],
time_data=data_settings['data']['dynamic']['time'],
geo_data_region=geo_data_collection['geo_region'],
geo_data_alert_area=geo_data_collection['geo_alert_area'],
geo_data_weather_station=geo_point_collection_weather_stations,
index_data_alert_area=geo_data_collection['index_alert_area'],
group_data=data_settings['algorithm']['ancillary']['group'],
alg_template_tags=data_settings['algorithm']['template'],
flag_dest_updating=data_settings['algorithm']['flags']
['updating_dynamic_indicators'])
if activate_algorithm_step(
['analyzer'],
data_settings['algorithm']['flags']['running_mode']):
analysis_data_rain_point = driver_analysis_indicators.organize_analysis_rain_point(
)
analysis_data_rain_map = driver_analysis_indicators.organize_analysis_rain_map(
)
analysis_data_sm = driver_analysis_indicators.organize_analysis_sm(
)
driver_analysis_indicators.save_analysis(
analysis_data_sm, analysis_data_rain_map,
analysis_data_rain_point, geo_point_collection_soil_slips)
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Activate publisher mode
if activate_algorithm_step(
['publisher'], data_settings['algorithm']['flags']['running_mode']):
# -------------------------------------------------------------------------------------
# Analysis datasets to define scenarios
driver_analysis_scenarios = DriverAnalysisScenarios(
time_run,
time_range,
ancillary_dict=data_settings['data']['dynamic']['ancillary'],
dst_dict=data_settings['data']['dynamic']['destination'],
geo_data_region=driver_data_geo_grid.dset_geo_region,
geo_data_alert_area=driver_data_geo_grid.dset_geo_alert_area,
group_data=data_settings['algorithm']['ancillary']['group'],
plot_data=data_settings['algorithm']['ancillary']['plot'],
alg_template_tags=data_settings['algorithm']['template'],
flag_dest_updating=data_settings['algorithm']['flags']
['updating_dynamic_scenarios'])
scenarios_data = driver_analysis_scenarios.collect_scenarios()
driver_analysis_scenarios.dump_scenarios(scenarios_data)
driver_analysis_scenarios.plot_scenarios(scenarios_data)
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Info algorithm
time_elapsed = round(time.time() - start_time, 1)
logging.info(' ')
logging.info(' ==> ' + alg_name + ' (Version: ' + alg_version +
' Release_Date: ' + alg_release + ')')
logging.info(' ==> TIME ELAPSED: ' + str(time_elapsed) + ' seconds')
logging.info(' ==> ... END')
logging.info(' ==> Bye, Bye')
logging.info(
' ============================================================================ '
)
def organize_dynamic_data(self):
time_str = self.time_str
time_period = self.time_period
geo_da_dst = self.geo_da_dst
src_dict = self.src_dict
var_name_obj = self.var_name_obj
file_path_obj_src = self.file_path_obj_src
file_path_obj_anc = self.file_path_obj_anc
flag_cleaning_ancillary = self.flag_cleaning_dynamic_ancillary
log_stream.info(' ---> Organize dynamic datasets [' + time_str +
'] ... ')
# Check if ancillary file already exists
file_check_list = []
for file_path_tmp in file_path_obj_anc:
if os.path.exists(file_path_tmp):
if flag_cleaning_ancillary:
os.remove(file_path_tmp)
file_check_list.append(False)
else:
file_check_list.append(True)
else:
file_check_list.append(False)
file_check = all(file_check_list)
# If statement on ancillary availability
if not file_check:
dset_collection = {}
for var_name in var_name_obj:
log_stream.info(' ----> Variable "' + var_name + '" ... ')
var_compute, var_tag, var_scale_factor, var_shift, file_compression, \
file_geo_reference, file_type, file_coords, file_freq, compute_quality, var_decimal_digits \
= self.extract_var_fields(src_dict[var_name])
var_file_path_src = file_path_obj_src[var_name]
if var_compute:
var_geo_data = None
for var_time, var_file_path_in in zip(
time_period, var_file_path_src):
log_stream.info(
' -----> Time "' +
var_time.strftime(time_format_algorithm) +
'" ... ')
if os.path.exists(var_file_path_in):
#copy to tmp
var_file_path, var_file_name = os.path.split(
var_file_path_in)
var_file_path_in_tempcopy = self.domain + '_' + var_file_name
copyfile(
var_file_path_in,
os.path.join(var_file_path,
var_file_path_in_tempcopy))
var_file_path_in = os.path.join(
var_file_path, var_file_path_in_tempcopy)
if file_compression:
var_file_path_out = self.define_file_name_unzip(
var_file_path_in)
unzip_filename(var_file_path_in,
var_file_path_out)
else:
var_file_path_out = deepcopy(var_file_path_in)
if file_type == 'binary':
if var_geo_data is None:
log_stream.info(
' ------> Select geo reference for binary datasets ... '
)
var_geo_name = search_geo_reference(
var_file_path_out,
self.static_data_src,
tag_geo_reference=file_geo_reference)
log_stream.info(
' -------> Geo reference name: ' +
var_geo_name)
var_geo_data, var_geo_x, var_geo_y, var_geo_attrs = \
self.set_geo_attributes(self.static_data_src[var_geo_name])
log_stream.info(
' ------> Select geo reference for binary datasets ... DONE'
)
var_da_src = read_data_binary(
var_file_path_out,
var_geo_x,
var_geo_y,
var_geo_attrs,
var_scale_factor=var_scale_factor,
var_time=var_time,
var_name=var_name,
coord_name_geo_x=self.coord_name_geo_x,
coord_name_geo_y=self.coord_name_geo_y,
coord_name_time=self.coord_name_time,
dim_name_geo_x=self.dim_name_geo_x,
dim_name_geo_y=self.dim_name_geo_y,
dim_name_time=self.dim_name_time,
dims_order=self.dims_order_3d)
elif file_type == 'netcdf':
if var_geo_data is None:
log_stream.info(
' ------> Select geo reference for netcdf datasets ... '
)
var_geo_data, var_geo_x, var_geo_y, var_geo_attrs = \
self.set_geo_attributes(self.static_data_src[file_geo_reference])
log_stream.info(
' ------> Select geo reference for netcdf datasets ... DONE'
)
var_da_src = read_data_nc(
var_file_path_out,
var_geo_x,
var_geo_y,
var_geo_attrs,
var_coords=file_coords,
var_scale_factor=var_scale_factor,
var_name=var_tag,
var_time=var_time,
coord_name_geo_x=self.coord_name_geo_x,
coord_name_geo_y=self.coord_name_geo_y,
coord_name_time=self.coord_name_time,
dim_name_geo_x=self.dim_name_geo_x,
dim_name_geo_y=self.dim_name_geo_y,
dim_name_time=self.dim_name_time,
dims_order=self.dims_order_3d)
elif file_type == 'tiff' or file_type == 'asc':
var_da_src = read_data_tiff(
var_file_path_out,
var_scale_factor=var_scale_factor,
var_name=var_tag,
var_time=var_time,
coord_name_geo_x=self.coord_name_geo_x,
coord_name_geo_y=self.coord_name_geo_y,
coord_name_time=self.coord_name_time,
dim_name_geo_x=self.dim_name_geo_x,
dim_name_geo_y=self.dim_name_geo_y,
dim_name_time=self.dim_name_time,
dims_order=self.dims_order_3d,
decimal_round_data=2,
decimal_round_geo=7)
elif file_type == 'mat':
var_da_src = read_data_mat(
var_file_path_out,
var_scale_factor=var_scale_factor,
var_name=var_tag,
var_time=var_time,
coord_name_geo_x=self.coord_name_geo_x,
coord_name_geo_y=self.coord_name_geo_y,
coord_name_time=self.coord_name_time,
dim_name_geo_x=self.dim_name_geo_x,
dim_name_geo_y=self.dim_name_geo_y,
dim_name_time=self.dim_name_time,
dims_order=self.dims_order_3d,
decimal_round_data=2,
decimal_round_geo=7,
src_dict=src_dict[var_name])
else:
log_stream.info(
' -----> Time "' +
var_time.strftime(time_format_algorithm) +
'" ... FAILED')
log_stream.error(' ===> File type "' +
file_type +
'"is not allowed.')
raise NotImplementedError(
'Case not implemented yet')
# Delete (if needed the uncompressed file(s)
if var_file_path_in != var_file_path_out:
if os.path.exists(var_file_path_out):
os.remove(var_file_path_out)
# Delete temporary file
os.remove(var_file_path_in)
# Apply scale factor and shift to values
if var_shift is not None:
var_da_src.values = var_da_src.values + var_shift
if var_scale_factor is not None:
var_da_src.values = var_da_src.values / var_scale_factor
#if var_shift is not None:
# var_da_src=var_da_src + var_shift
#if var_scale_factor is not None:
# var_da_src=var_da_src / var_scale_factor
# Organize destination dataset
if var_da_src is not None:
# Active (if needed) interpolation method to the variable source data-array
active_interp = active_var_interp(
var_da_src.attrs, geo_da_dst.attrs)
# Apply the interpolation method to the variable source data-array
if active_interp:
var_da_dst = apply_var_interp(
var_da_src,
geo_da_dst,
var_name=var_name,
dim_name_geo_x=self.dim_name_geo_x,
dim_name_geo_y=self.dim_name_geo_y,
coord_name_geo_x=self.coord_name_geo_x,
coord_name_geo_y=self.coord_name_geo_y,
interp_method=self.interp_method)
else:
if var_tag != var_name:
var_da_dst = deepcopy(var_da_src)
var_da_dst.name = var_name
else:
var_da_dst = deepcopy(var_da_src)
# Mask the variable destination data-array
var_nodata = None
if 'nodata_value' in list(
var_da_dst.attrs.keys()):
var_nodata = var_da_dst.attrs[
'nodata_value']
geo_nodata = None
if 'nodata_value' in list(
geo_da_dst.attrs.keys()):
geo_nodata = geo_da_dst.attrs[
'nodata_value']
if (geo_nodata is not None) and (var_nodata
is not None):
var_da_masked = var_da_dst.where(
(geo_da_dst.values[:, :, np.newaxis] !=
geo_nodata)
& (var_da_dst != var_nodata))
else:
var_da_masked = deepcopy(var_da_dst)
#Sanity check to remove nans
var_da_masked.values = \
np.where(np.isnan(var_da_masked.values), var_nodata, var_da_masked.values)
#Round
var_da_masked.values = np.round(
var_da_masked.values, var_decimal_digits)
# plt.figure(1)
# plt.imshow(var_da_dst.values[:, :, 0])
# plt.colorbar()
# plt.figure(2)
# plt.imshow(var_da_src.values[:, :, 0])
# plt.colorbar()
# plt.figure(3)
# plt.imshow(var_da_masked.values[:, :, 0])
# plt.colorbar()
# plt.show()
# plt.figure(4)
# plt.imshow(geo_da_dst.values)
# plt.colorbar()
# plt.show()
# Organize data in a common datasets
var_dset_masked = create_dset(
var_data_time=var_time,
var_data_name=var_name,
var_data_values=var_da_masked,
var_data_attrs=None,
var_geo_1d=False,
file_attributes=geo_da_dst.attrs,
var_geo_name='terrain',
var_geo_values=geo_da_dst.values,
var_geo_x=geo_da_dst['longitude'].values,
var_geo_y=geo_da_dst['latitude'].values,
var_geo_attrs=None)
# Organize data in merged datasets
if var_time not in list(
dset_collection.keys()):
dset_collection[var_time] = var_dset_masked
else:
var_dset_tmp = deepcopy(
dset_collection[var_time])
var_dset_tmp = var_dset_tmp.merge(
var_dset_masked, join='right')
dset_collection[var_time] = var_dset_tmp
#Compute SQA if needed
if compute_quality:
log_stream.info(' ----> Variable "' +
var_name +
'" ... computing quality ')
SQA = compute_SQA(var_da_masked.values,
geo_da_dst.values,
self.SQA_ground_and_snow)
SQA_dset = create_dset(
var_data_time=var_time,
var_data_name='SQA',
var_data_values=SQA,
var_data_attrs=None,
var_geo_1d=False,
file_attributes=geo_da_dst.attrs,
var_geo_name='terrain',
var_geo_values=geo_da_dst.values,
var_geo_x=geo_da_dst['longitude'].
values,
var_geo_y=geo_da_dst['latitude'].
values,
var_geo_attrs=None)
var_dset_tmp = deepcopy(
dset_collection[var_time])
var_dset_tmp = var_dset_tmp.merge(
SQA_dset, join='right')
dset_collection[var_time] = var_dset_tmp
log_stream.info(
' -----> Time "' +
var_time.strftime(time_format_algorithm) +
'" ... DONE')
else:
log_stream.info(
' -----> Time "' +
var_time.strftime(time_format_algorithm) +
'" ... Datasets is not defined')
else:
var_da_src = None
log_stream.info(
' -----> Time "' +
var_time.strftime(time_format_algorithm) +
'" ... Datasets is not defined')
log_stream.info(' ----> Variable "' + var_name +
'" ... DONE')
else:
log_stream.info(
' ----> Variable "' + var_name +
'" ... SKIPPED. Compute flag not activated.')
# Save ancillary datasets
for file_path_anc, (dset_time,
dset_anc) in zip(file_path_obj_anc,
dset_collection.items()):
folder_name_anc, file_name_anc = os.path.split(file_path_anc)
if not os.path.exists(folder_name_anc):
make_folder(folder_name_anc)
write_obj(file_path_anc, dset_anc)
log_stream.info(' ---> Organize dynamic datasets [' + time_str +
'] ... DONE')
else:
log_stream.info(
' ---> Organize dynamic datasets [' + time_str +
'] ... SKIPPED. All datasets are previously computed')
def dump_dynamic_data(self):
time_str = self.time_str
time_period = self.time_period
dst_dict = self.dst_dict
file_path_obj_anc = self.file_path_obj_anc
file_path_obj_dst = self.file_path_obj_dst
flag_cleaning_dynamic = self.flag_cleaning_dynamic_data
log_stream.info(' ---> Dump dynamic datasets [' + time_str + '] ... ')
for time_step, file_path_anc, file_path_dst in zip(
time_period, file_path_obj_anc, file_path_obj_dst):
log_stream.info(' -----> Time "' +
time_step.strftime(time_format_algorithm) +
'" ... ')
file_path_zip = self.define_file_name_zip(file_path_dst)
if flag_cleaning_dynamic:
if os.path.exists(file_path_dst):
os.remove(file_path_dst)
if os.path.exists(file_path_zip):
os.remove(file_path_zip)
if os.path.exists(file_path_anc):
dset_obj = read_obj(file_path_anc)
folder_name_dst, file_name_dst = os.path.split(file_path_dst)
if not os.path.exists(folder_name_dst):
make_folder(folder_name_dst)
log_stream.info(' ------> Save filename "' + file_name_dst +
'" ... ')
if not (os.path.exists(file_path_dst)
or os.path.exists(file_path_zip)):
# Squeeze time dimensions (if equal == 1) --> continuum expects 2d variables in forcing variables
if self.dim_name_time in list(dset_obj.dims):
time_array = dset_obj[self.dim_name_time].values
if time_array.shape[0] == 1:
dset_obj = dset_obj.squeeze(self.dim_name_time)
dset_obj = dset_obj.drop(self.dim_name_time)
write_dset(file_path_dst,
dset_obj,
dset_engine=self.nc_type_engine,
dset_format=self.nc_type_file,
dset_compression=self.nc_compression_level,
fill_data=-9999.0,
dset_type='float32')
log_stream.info(' ------> Save filename "' +
file_name_dst + '" ... DONE')
log_stream.info(' ------> Zip filename "' + file_name_dst +
'" ... ')
if dst_dict[self.file_compression_tag]:
zip_filename(file_path_dst, file_path_zip)
if os.path.exists(file_path_zip) and (file_path_zip !=
file_name_dst):
os.remove(file_path_dst)
log_stream.info(' ------> Zip filename "' +
file_name_dst + '" ... DONE')
else:
log_stream.info(' ------> Zip filename "' +
file_name_dst +
'" ... SKIPPED. Zip not activated')
else:
log_stream.info(' ------> Save filename "' +
file_name_dst +
'" ... SKIPPED. Filename previously saved')
log_stream.info(' -----> Time "' +
time_step.strftime(time_format_algorithm) +
'" ... DONE')
else:
log_stream.info(' -----> Time "' +
time_step.strftime(time_format_algorithm) +
'" ... SKIPPED. Datasets not available')
log_stream.info(' ---> Dump dynamic datasets [' + time_str +
'] ... DONE')
def __init__(self,
src_dict,
dst_dict=None,
group_data=None,
flag_point_data_src='weather_stations_data',
flag_grid_data='geo_data',
flag_point_data_dst='weather_stations_data',
alg_template_tags=None,
flag_geo_updating=True,
search_radius_km=10):
self.flag_point_data_src = flag_point_data_src
self.flag_point_data_dst = flag_point_data_dst
self.flag_grid_data = flag_grid_data
self.file_name_tag = 'file_name'
self.folder_name_tag = 'folder_name'
self.point_registry_tag = 'registry'
self.point_alert_area_tree_tag = 'alert_area_tree'
self.grid_vector_tag = 'alert_area_vector'
self.point_code_tag = 'code'
self.point_name_tag = 'name'
self.point_longitude_tag = 'longitude'
self.point_latitude_tag = 'latitude'
self.point_alert_area_tag = 'alert_area'
self.group_data = group_data
self.alg_template_tags = alg_template_tags
self.flag_geo_updating = flag_geo_updating
self.search_radius_km = search_radius_km
self.file_name_point_registry_src = src_dict[self.flag_point_data_src][
self.point_registry_tag][self.file_name_tag]
self.folder_name_point_registry_src = src_dict[
self.flag_point_data_src][self.point_registry_tag][
self.folder_name_tag]
self.file_path_point_registry_src = os.path.join(
self.folder_name_point_registry_src,
self.file_name_point_registry_src)
self.file_name_point_registry_dst = dst_dict[self.flag_point_data_dst][
self.point_registry_tag][self.file_name_tag]
self.folder_name_point_registry_dst = dst_dict[
self.flag_point_data_dst][self.point_registry_tag][
self.folder_name_tag]
self.file_path_point_registry_dst = os.path.join(
self.folder_name_point_registry_dst,
self.file_name_point_registry_dst)
self.file_name_point_alert_area_tree_dst = dst_dict[
self.flag_point_data_dst][self.point_alert_area_tree_tag][
self.file_name_tag]
self.folder_name_point_alert_area_tree_dst = dst_dict[
self.flag_point_data_dst][self.point_alert_area_tree_tag][
self.folder_name_tag]
self.file_path_point_alert_area_tree_dst = os.path.join(
self.folder_name_point_alert_area_tree_dst,
self.file_name_point_alert_area_tree_dst)
self.file_name_grid = dst_dict[self.flag_grid_data][
self.grid_vector_tag][self.file_name_tag]
self.folder_name_grid = dst_dict[self.flag_grid_data][
self.grid_vector_tag][self.folder_name_tag]
self.file_path_grid = self.collect_file_obj(self.folder_name_grid,
self.file_name_grid)
self.search_radius_degree = km_2_degree(self.search_radius_km)
if self.flag_geo_updating:
if os.path.exists(self.file_path_point_registry_dst):
os.remove(self.file_path_point_registry_dst)
logging.info(' -----> Define geo points registry ... ')
if not os.path.exists(self.file_path_point_registry_dst):
df_geo_point = self.dset_geo_point = self.read_geo_point()
self.df_geo_point = self.join_geo_point2grid(df_geo_point)
make_folder(self.folder_name_point_registry_dst)
self.df_geo_point.to_csv(self.file_path_point_registry_dst)
logging.info(' -----> Define geo points registry ... DONE')
else:
self.df_geo_point = pd.read_csv(self.file_path_point_registry_dst)
logging.info(
' -----> Define geo points registry ... LOADED. Datasets was previously computed.'
)
self.tag_sep = ':'
def organize_data(self):
# Starting info
logging.info(' ----> Organize weather stations point information ... ')
df_point = self.df_geo_point
max_distance = self.search_radius_degree
inf_distance = float("inf")
file_path_point = self.file_path_point_alert_area_tree_dst
flag_geo_updating = self.flag_geo_updating
if flag_geo_updating:
if os.path.exists(file_path_point):
os.remove(file_path_point)
if not os.path.exists(file_path_point):
code_points = df_point[self.point_code_tag].values
name_points = df_point[self.point_name_tag].values
lats_points = df_point[self.point_latitude_tag].values
lons_points = df_point[self.point_longitude_tag].values
aa_points = df_point[self.point_alert_area_tag].values
coord_points = np.dstack(
[lats_points.ravel(), lons_points.ravel()])[0]
coord_tree = cKDTree(coord_points)
weather_stations_collections = {}
for code_point, aa_point, coord_point in zip(
code_points, aa_points, coord_points):
distances, indices = coord_tree.query(
coord_point,
len(coord_points),
p=2,
distance_upper_bound=max_distance)
code_points_neighbors = []
name_points_neighbors = []
coord_points_neighbors = []
lats_points_neighbors = []
lons_points_neighbors = []
aa_points_neighbors = []
for index, distance in zip(indices, distances):
if distance == inf_distance:
break
coord_points_neighbors.append(coord_points[index])
code_points_neighbors.append(code_points[index])
name_points_neighbors.append(name_points[index])
lons_points_neighbors.append(lons_points[index])
lats_points_neighbors.append(lats_points[index])
aa_points_neighbors.append(aa_points[index])
coord_dict = {
self.point_code_tag: code_points_neighbors,
self.point_name_tag: name_points_neighbors,
self.point_latitude_tag: lats_points_neighbors,
self.point_longitude_tag: lons_points_neighbors,
self.point_alert_area_tag: aa_points_neighbors
}
coord_dframe = pd.DataFrame(data=coord_dict)
if aa_point not in list(weather_stations_collections.keys()):
weather_stations_collections[aa_point] = {}
weather_stations_collections[aa_point][
code_point] = coord_dframe
folder_name, file_name = os.path.split(file_path_point)
make_folder(folder_name)
write_obj(file_path_point, weather_stations_collections)
# Ending info
logging.info(
' ----> Organize weather stations point information ... DONE')
else:
# Ending info
weather_stations_collections = read_obj(file_path_point)
logging.info(
' ----> Organize weather stations point information ... LOADED. '
'Datasets was previously computed.')
return weather_stations_collections
def plot_scenarios(self, scenarios_collections):
logging.info(' ----> Plot scenarios [' + str(self.time_run) + '] ... ')
season_list = self.list_season
event_n_min = self.event_n_min
event_n_max = self.event_n_max
scenarios_sm2event_file_path = self.file_path_scenarios_graph_sm2event_collections
scenarios_rain2event_file_path = self.file_path_scenarios_graph_rain2event_collections
scenarios_rain2sm_file_path = self.file_path_scenarios_graph_rain2sm_collections
for group_data_key, group_data_alert_value in self.structure_data_group.items():
logging.info(' -----> Alert Area ' + group_data_key + ' ... ')
file_path_sm2event = scenarios_sm2event_file_path[group_data_key]
file_path_rain2event = scenarios_rain2event_file_path[group_data_key]
file_path_rain2sm = scenarios_rain2sm_file_path[group_data_key]
file_data = scenarios_collections[group_data_key]
rain_period_list = group_data_alert_value['rain_datasets']['search_period']
template_rain_point = []
for rain_period_step in rain_period_list:
template_rain_step = self.template_rain_point_accumulated.format(rain_period_step)
template_rain_point.append(template_rain_step)
template_sm_point = self.template_sm_point_avg
if file_data is not None:
logging.info(' ------> Plot rain against sm ... ')
for season_step, file_path_rain2sm_step in zip(season_list, file_path_rain2sm):
logging.info(' ------> Season ' + season_step + ' ... ')
file_data_step = filter_scenarios_dataframe(
file_data,
tag_column_sm=template_sm_point,
tag_column_rain=template_rain_point,
filter_rain=True, filter_sm=True, filter_event=True,
filter_season=self.filter_season,
tag_column_event='event_n', value_min_event=event_n_min, value_max_event=event_n_max,
season_lut=self.lut_season, season_name=season_step)
folder_name_rain2sm, file_name_rain2sm = os.path.split(file_path_rain2sm_step)
make_folder(folder_name_rain2sm)
plot_scenarios_rain2sm(file_data_step, file_path_rain2sm_step,
var_x=self.template_sm_point_avg,
var_y=self.template_rain_point_accumulated,
var_z='event_index',
event_n_min=event_n_min, event_n_max=event_n_max,
event_label=self.event_label, season_label=season_step,
figure_dpi=60,
extra_args={'rain_type': rain_period_list,
'soil_moisture_type': 'average'})
logging.info(' ------> Season ' + season_step + ' ... DONE')
logging.info(' ------> Plot rain against sm ... DONE')
'''
#plot_scenarios_rain2sm(file_data, file_path_rain2sm,
# var_x='soil_moisture_maximum', var_y=self.template_rain_accumulated,
# var_z='event_index',
# event_n_min=self.event_n_min, event_label=self.event_label,
# figure_dpi=60,
# extra_args={'rain_type': rain_period_list,
# 'soil_moisture_type': 'max'})
logging.info(' ------> Plot sm against events ... ')
folder_name_sm2event, file_name_sm2event = os.path.split(file_path_sm2event)
make_folder(folder_name_sm2event)
plot_scenarios_sm2event(file_data, file_path_sm2event,
event_n_min=self.event_n_min, event_label=self.event_label,
figure_dpi=120)
logging.info(' ------> Plot sm against events ... DONE')
logging.info(' ------> Plot rain against events ... ')
folder_name_rain2event, file_name_rain2event = os.path.split(file_path_rain2event)
make_folder(folder_name_rain2event)
plot_scenarios_rain2event(file_data, file_path_rain2event)
logging.info(' ------> Plot rain against events ... DONE')
'''
logging.info(' -----> Alert Area ' + group_data_key + ' ... DONE')
else:
logging.info(' -----> Alert Area ' + group_data_key + ' ... SKIPPED. Datasets are undefined.')
logging.info(' ----> Plot scenarios [' + str(self.time_run) + '] ... DONE')
def organize_forcing(self, var_name='rain', var_min=0, var_max=None):
logging.info(' ----> Organize rain forcing ... ')
geoy_out_1d = self.file_data_geo['south_north'].values
geox_out_1d = self.file_data_geo['west_east'].values
mask_out_2d = self.file_data_geo.values
point_weather_section = self.geo_data_weather_station
geox_out_2d, geoy_out_2d = np.meshgrid(geox_out_1d, geoy_out_1d)
for datetime_step, file_path_src, file_path_ancillary_map, file_path_ancillary_point in zip(
self.time_range, self.file_path_src_list,
self.file_path_ancillary_map_list, self.file_path_ancillary_point_list):
logging.info(' -----> TimeStep: ' + str(datetime_step) + ' ... ')
if self.flag_ancillary_updating:
if os.path.exists(file_path_ancillary_map):
os.remove(file_path_ancillary_map)
if os.path.exists(file_path_ancillary_point):
os.remove(file_path_ancillary_point)
if not (os.path.exists(file_path_ancillary_map or os.path.exists(file_path_ancillary_point))):
if os.path.exists(file_path_src):
file_dframe = read_file_csv(file_path_src, datetime_step, file_header=self.columns_src,
file_sep=self.file_columns_sep,
scale_factor_longitude=self.file_scale_factor_longitude,
scale_factor_latitude=self.file_scale_factor_latitude,
scale_factor_data=self.file_scale_factor_rain)
# Filter data using variable limits (if defined)
if var_min is not None:
file_dframe = file_dframe[(file_dframe['data'] >= var_min)]
if var_max is not None:
file_dframe = file_dframe[(file_dframe['data'] <= var_max)]
if file_dframe is not None:
file_time_src = file_dframe.index.unique()
else:
file_time_src = None
else:
file_dframe = None
file_time_src = None
logging.warning(' ===> File datasets of rain weather stations is not available.')
if (file_time_src is not None) and (file_time_src.__len__() > 1):
logging.warning(' ===> Time step selected are greater than 1. Errors could arise in the script')
if file_dframe is not None:
logging.info(' ------> Interpolate points to map datasets ... ')
map_out_2d = interpolate_rain_dataframe(file_dframe,
mask_out_2d, geox_out_2d, geoy_out_2d,
folder_tmp=self.folder_tmp)
logging.info(' ------> Interpolate points to map datasets ... DONE')
logging.info(' ------> Save map datasets ... ')
folder_name_map, file_name_map = os.path.split(file_path_ancillary_map)
make_folder(folder_name_map)
if file_path_ancillary_map.endswith('.nc'):
dset_out = create_dset(
map_out_2d,
mask_out_2d, geox_out_2d, geoy_out_2d,
var_data_time=datetime_step,
var_data_name=var_name,
var_geo_name='mask', var_data_attrs=None, var_geo_attrs=None,
coord_name_x='longitude', coord_name_y='latitude', coord_name_time='time',
dim_name_x='west_east', dim_name_y='south_north', dim_name_time='time',
dims_order_2d=None, dims_order_3d=None)
write_dset(
file_path_ancillary_map,
dset_out, dset_mode='w', dset_engine='h5netcdf', dset_compression=0, dset_format='NETCDF4',
dim_key_time='time', no_data=-9999.0)
logging.info(' ------> Save map datasets ... DONE. [NETCDF]')
elif file_path_ancillary_map.endswith('.tiff'):
### error in saving ERROR 1: Only OGC WKT Projections supported for writing to GeoTIFF. EPSG:4326 not supported.
save_file_tiff(file_path_ancillary_map, map_out_2d, geox_out_2d, geoy_out_2d,
file_metadata=self.file_metadata, file_epsg_code=self.file_epsg_code)
logging.info(' ------> Save map datasets ... DONE. [GEOTIFF]')
else:
logging.info(' ------> Save map datasets ... FAILED')
logging.error(' ===> Filename format is not allowed')
raise NotImplementedError('Format is not implemented yet')
self.file_path_processed_map.append(file_path_ancillary_map)
logging.info(' ------> Save points datasets ... ')
folder_name_point, file_name_point = os.path.split(file_path_ancillary_point)
make_folder(folder_name_point)
if file_path_ancillary_point.endswith('.csv'):
write_file_csv(file_path_ancillary_point, file_dframe)
logging.info(' ------> Save points datasets... DONE')
else:
logging.info(' ------> Save points datasets ... FAILED')
logging.error(' ===> Filename format is not allowed')
raise NotImplementedError('Format is not implemented yet')
self.file_path_processed_point.append(file_path_ancillary_point)
logging.info(' -----> TimeStep: ' + str(datetime_step) + ' ... DONE')
else:
logging.info(' -----> TimeStep: ' + str(datetime_step) + ' ... FAILED')
logging.warning(' ===> File datasets of rain weather stations is not available.')
else:
logging.info(' -----> TimeStep: ' + str(datetime_step) + ' ... PREVIOUSLY DONE')
logging.info(' ----> Organize rain forcing ... DONE')
def main():
# -------------------------------------------------------------------------------------
# Get algorithm settings
alg_settings, alg_time = get_args()
# Set algorithm settings
data_settings = read_file_json(alg_settings)
# Set algorithm logging
make_folder(data_settings['data']['log']['folder_name'])
set_logging(
logger_file=os.path.join(data_settings['data']['log']['folder_name'],
data_settings['data']['log']['file_name']))
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Info algorithm
logging.info(
' ============================================================================ '
)
logging.info(' ==> ' + alg_name + ' (Version: ' + alg_version +
' Release_Date: ' + alg_release + ')')
logging.info(' ==> START ... ')
logging.info(' ')
# Time algorithm information
start_time = time.time()
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Organize time run
time_now, time_run, time_range = set_time(
time_run_args=alg_time,
time_run_file=data_settings['time']['time_now'],
time_format=time_format,
time_period=data_settings['time']['time_period'],
time_frequency=data_settings['time']['time_frequency'],
time_rounding=data_settings['time']['time_rounding'])
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Geographical datasets
driver_data_geo = DriverGeo(
src_dict=data_settings['data']['static']['source'],
dst_dict=data_settings['data']['static']['destination'],
alg_ancillary=data_settings['algorithm']['ancillary'],
alg_template_tags=data_settings['algorithm']['template'],
flag_cleaning_geo=data_settings['algorithm']['flags']
['cleaning_geo_data'])
geo_data_collection = driver_data_geo.organize_geo()
# -------------------------------------------------------------------------------------
# ----------------------------------------------------------------------------------
# Iterate over time range
for time_step in time_range:
# -------------------------------------------------------------------------------------
# Discharge datasets
driver_data_source_discharge = DriverDischarge(
time_now=time_now,
time_run=time_step,
geo_data_collection=geo_data_collection,
src_dict=data_settings['data']['dynamic']['source'],
ancillary_dict=data_settings['data']['dynamic']['ancillary'],
alg_ancillary=data_settings['algorithm']['ancillary'],
alg_template_tags=data_settings['algorithm']['template'],
flag_cleaning_dynamic_ancillary=data_settings['algorithm']['flags']
['cleaning_dynamic_data_ancillary'])
discharge_data_collection = driver_data_source_discharge.organize_discharge(
)
# Scenario datasets
driver_data_destination_scenario = DriverScenario(
time_now=time_now,
time_run=time_step,
discharge_data_collection=discharge_data_collection,
geo_data_collection=geo_data_collection,
src_dict=data_settings['data']['static']['source'],
dst_dict=data_settings['data']['dynamic']['destination'],
alg_ancillary=data_settings['algorithm']['ancillary'],
alg_template_tags=data_settings['algorithm']['template'],
flag_cleaning_scenario=data_settings['algorithm']['flags']
['cleaning_scenario_data'])
scenario_info_collection = driver_data_destination_scenario.organize_scenario_datasets(
)
scenario_map_collection = driver_data_destination_scenario.compute_scenario_map(
scenario_info_collection)
driver_data_destination_scenario.dump_scenario_map(
scenario_map_collection, scenario_info_collection)
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Info algorithm
time_elapsed = round(time.time() - start_time, 1)
logging.info(' ')
logging.info(' ==> ' + alg_name + ' (Version: ' + alg_version +
' Release_Date: ' + alg_release + ')')
logging.info(' ==> TIME ELAPSED: ' + str(time_elapsed) + ' seconds')
logging.info(' ==> ... END')
logging.info(' ==> Bye, Bye')
logging.info(
' ============================================================================ '
)
