def generate_hles_obs_variants():
period = Period(datetime(1980, 11, 1), datetime(2009, 2, 1))
# should be continuous??
months_of_interest = list(range(1, 13))
period.months_of_interest = months_of_interest
vname_to_level_erai = {
T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID),
U_WE: VerticalLevel(1, level_kinds.HYBRID),
V_SN: VerticalLevel(1, level_kinds.HYBRID),
}
vname_map = {}
vname_map.update(vname_map_CRCM5)
# set the paths to the data
label_to_base_folder = OrderedDict()
common_dir = Path(
"/home/data/big1/obs_data_for_HLES/interploated_to_the_same_grid/")
# label_to_base_folder["obs_anuspmaurer_erai"] = common_dir / "obs_anuspmaurer_erai"
label_to_base_folder["obs_daymet_erai"] = common_dir / "obs_daymet_erai"
# label_to_base_folder["obs_anuspmaurer_narr"] = common_dir / "obs_anuspmaurer_narr"
# label_to_base_folder["obs_daymet_narr"] = common_dir / "obs_daymet_narr"
# ---
for label, base_folder in label_to_base_folder.items():
__obs_case_monthly(period, vname_to_level_erai, vname_map, label,
base_folder)
def generate_hles_obs_variants():
period = Period(
datetime(1980, 11, 1), datetime(2009, 2, 1)
)
# should be continuous??
months_of_interest = list(range(1, 13))
period.months_of_interest = months_of_interest
vname_to_level_erai = {
T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID),
U_WE: VerticalLevel(1, level_kinds.HYBRID),
V_SN: VerticalLevel(1, level_kinds.HYBRID),
}
vname_map = {}
vname_map.update(vname_map_CRCM5)
# set the paths to the data
label_to_base_folder = OrderedDict()
common_dir = Path("/HOME/huziy/skynet3_rech1/obs_data_for_HLES/interploated_to_the_same_grid/")
label_to_base_folder["obs_anuspmaurer_erai"] = common_dir / "obs_anuspmaurer_erai"
label_to_base_folder["obs_daymet_erai"] = common_dir / "obs_daymet_erai"
label_to_base_folder["obs_anuspmaurer_narr"] = common_dir / "obs_anuspmaurer_narr"
label_to_base_folder["obs_daymet_narr"] = common_dir / "obs_daymet_narr"
# ---
for label, base_folder in label_to_base_folder.items():
__obs_case(period, vname_to_level_erai, vname_map, label, base_folder)
def main_crcm5_hl():
label = "CRCM5_HL"
period = Period(datetime(1980, 1, 1), datetime(2009, 12, 31))
pool = Pool(processes=12)
input_params = []
for month_start in period.range("months"):
month_end = month_start.add(months=1).subtract(seconds=1)
current_month_period = Period(month_start, month_end)
current_month_period.months_of_interest = [
month_start.month,
]
vname_to_level_erai = {
T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID),
U_WE: VerticalLevel(1, level_kinds.HYBRID),
V_SN: VerticalLevel(1, level_kinds.HYBRID),
}
vname_map = {}
vname_map.update(vname_map_CRCM5)
vname_map = {}
vname_map.update(vname_map_CRCM5)
vname_map.update({default_varname_mappings.SNOWFALL_RATE: "U3"})
label_to_config = OrderedDict([(label, {
DataManager.SP_BASE_FOLDER:
"/RECH2/huziy/coupling/GL_440x260_0.1deg_GL_with_Hostetler/Samples_selected",
DataManager.SP_DATASOURCE_TYPE:
data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT_VNAME_IN_FNAME,
DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING:
vname_map,
DataManager.SP_LEVEL_MAPPING:
vname_to_level_erai,
DataManager.SP_OFFSET_MAPPING:
vname_to_offset_CRCM5,
DataManager.SP_MULTIPLIER_MAPPING:
vname_to_multiplier_CRCM5,
"out_folder":
"lake_effect_analysis_{}_{}-{}_monthly".format(
label, period.start.year, period.end.year)
})])
kwargs = dict(
label_to_config=label_to_config,
period=current_month_period,
months_of_interest=current_month_period.months_of_interest,
nprocs_to_use=1)
print(current_month_period.months_of_interest)
input_params.append(kwargs)
# execute in parallel
pool.map(monthly_func, input_params)
def main_obs():
label = "Obs_monthly_icefix_test2_1proc_speedtest_3"
period = Period(datetime(1980, 1, 1), datetime(2010, 12, 31))
pool = Pool(processes=20)
input_params = []
for month_start in period.range("months"):
month_end = month_start.add(months=1).subtract(seconds=1)
current_month_period = Period(month_start, month_end)
current_month_period.months_of_interest = [
month_start.month,
]
vname_to_level_erai = {
T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID),
U_WE: VerticalLevel(1, level_kinds.HYBRID),
V_SN: VerticalLevel(1, level_kinds.HYBRID),
}
vname_map = {}
vname_map.update(vname_map_CRCM5)
label_to_config = OrderedDict([(label, {
DataManager.SP_BASE_FOLDER:
"/HOME/huziy/skynet3_rech1/obs_data_for_HLES/interploated_to_the_same_grid/GL_0.1_452x260_icefix",
DataManager.SP_DATASOURCE_TYPE:
data_source_types.
ALL_VARS_IN_A_FOLDER_IN_NETCDF_FILES_OPEN_EACH_FILE_SEPARATELY,
DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING:
vname_map,
DataManager.SP_LEVEL_MAPPING:
vname_to_level_erai,
DataManager.SP_OFFSET_MAPPING:
vname_to_offset_CRCM5,
DataManager.SP_MULTIPLIER_MAPPING:
vname_to_multiplier_CRCM5,
DataManager.SP_VARNAME_TO_FILENAME_PREFIX_MAPPING:
vname_to_fname_prefix_CRCM5,
"out_folder":
"lake_effect_analysis_daily_{}_{}-{}".format(
label, period.start.year, period.end.year)
})])
kwargs = dict(
label_to_config=label_to_config,
period=current_month_period,
months_of_interest=current_month_period.months_of_interest,
nprocs_to_use=1)
print(current_month_period.months_of_interest)
input_params.append(kwargs)
# execute in parallel
pool.map(monthly_func, input_params)
def __obs_case_monthly(period,
vname_to_level: dict,
vname_map: dict,
label: str,
base_dir: str,
nprocs: int = 4):
"""
find hles for each month of interest
:param period:
:param vname_to_level:
:param vname_map:
:param label:
:param base_dir:
"""
pool = multiprocessing.Pool(processes=nprocs)
input_params = []
for month_start in period.range("months"):
month_end = month_start.add(months=1).subtract(seconds=1)
current_month_period = Period(month_start, month_end)
current_month_period.months_of_interest = [
month_start.month,
]
label_to_config = OrderedDict([(label, {
DataManager.SP_BASE_FOLDER:
base_dir,
DataManager.SP_DATASOURCE_TYPE:
data_source_types.
ALL_VARS_IN_A_FOLDER_IN_NETCDF_FILES_OPEN_EACH_FILE_SEPARATELY,
DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING:
vname_map,
DataManager.SP_LEVEL_MAPPING:
vname_to_level,
DataManager.SP_OFFSET_MAPPING:
vname_to_offset_CRCM5,
DataManager.SP_MULTIPLIER_MAPPING:
vname_to_multiplier_CRCM5,
DataManager.SP_VARNAME_TO_FILENAME_PREFIX_MAPPING:
vname_to_fname_prefix_CRCM5,
"out_folder":
"lake_effect_analysis_{}_{}-{}".format(label, period.start.year,
period.end.year)
})])
kwargs = dict(
label_to_config=label_to_config,
period=current_month_period,
months_of_interest=current_month_period.months_of_interest,
nprocs_to_use=1)
print(current_month_period.months_of_interest)
input_params.append(kwargs)
# execute in parallel
pool.map(monthly_func, input_params)
def main_crcm5_nemo():
label = "CRCM5_NEMO"
period = Period(
datetime(1980, 1, 1), datetime(2015, 12, 31)
)
pool = Pool(processes=10)
input_params = []
for month_start in period.range("months"):
month_end = month_start.add(months=1).subtract(seconds=1)
current_month_period = Period(month_start, month_end)
current_month_period.months_of_interest = [month_start.month, ]
vname_to_level_erai = {
T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID),
U_WE: VerticalLevel(1, level_kinds.HYBRID),
V_SN: VerticalLevel(1, level_kinds.HYBRID),
}
vname_map = {}
vname_map.update(vname_map_CRCM5)
vname_map = {}
vname_map.update(vname_map_CRCM5)
vname_map.update({
default_varname_mappings.SNOWFALL_RATE: "SN"
})
label_to_config = OrderedDict([(
label, {
DataManager.SP_BASE_FOLDER: "/snow3/huziy/NEI/GL/erai0.75deg_driven/GL_with_NEMO_dtN_1h_and_30min/Samples",
DataManager.SP_DATASOURCE_TYPE: data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT,
DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING: vname_map,
DataManager.SP_LEVEL_MAPPING: vname_to_level_erai,
DataManager.SP_OFFSET_MAPPING: vname_to_offset_CRCM5,
DataManager.SP_MULTIPLIER_MAPPING: vname_to_multiplier_CRCM5,
DataManager.SP_VARNAME_TO_FILENAME_PREFIX_MAPPING: default_varname_mappings.vname_to_fname_prefix_CRCM5,
"out_folder": "lake_effect_analysis_{}_{}-{}_monthly".format(label, period.start.year, period.end.year)
}
)])
kwargs = dict(
label_to_config=label_to_config, period=current_month_period, months_of_interest=current_month_period.months_of_interest, nprocs_to_use=1
)
print(current_month_period.months_of_interest)
input_params.append(kwargs)
# execute in parallel
pool.map(monthly_func, input_params)
def main_crcm5_hl():
label = "CRCM5_HL"
period = Period(
datetime(1980, 1, 1), datetime(2009, 12, 31)
)
pool = Pool(processes=12)
input_params = []
for month_start in period.range("months"):
month_end = month_start.add(months=1).subtract(seconds=1)
current_month_period = Period(month_start, month_end)
current_month_period.months_of_interest = [month_start.month, ]
vname_to_level_erai = {
T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID),
U_WE: VerticalLevel(1, level_kinds.HYBRID),
V_SN: VerticalLevel(1, level_kinds.HYBRID),
}
vname_map = {}
vname_map.update(vname_map_CRCM5)
vname_map = {}
vname_map.update(vname_map_CRCM5)
vname_map.update({
default_varname_mappings.SNOWFALL_RATE: "U3"
})
label_to_config = OrderedDict([(
label, {
DataManager.SP_BASE_FOLDER: "/RECH2/huziy/coupling/GL_440x260_0.1deg_GL_with_Hostetler/Samples_selected",
DataManager.SP_DATASOURCE_TYPE: data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT_VNAME_IN_FNAME,
DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING: vname_map,
DataManager.SP_LEVEL_MAPPING: vname_to_level_erai,
DataManager.SP_OFFSET_MAPPING: vname_to_offset_CRCM5,
DataManager.SP_MULTIPLIER_MAPPING: vname_to_multiplier_CRCM5,
"out_folder": "lake_effect_analysis_{}_{}-{}_monthly".format(label, period.start.year, period.end.year)
}
)])
kwargs = dict(
label_to_config=label_to_config,
period=current_month_period,
months_of_interest=current_month_period.months_of_interest,
nprocs_to_use=1
)
print(current_month_period.months_of_interest)
input_params.append(kwargs)
# execute in parallel
pool.map(monthly_func, input_params)
def main_future(nprocs=20):
period = Period(
datetime(2079, 1, 1), datetime(2100, 12, 31)
)
label = "CRCM5_NEMO_fix_TT_PR_CanESM2_RCP85_{}-{}_monthly".format(period.start.year, period.end.year)
vname_to_level_erai = {
T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID),
U_WE: VerticalLevel(1, level_kinds.HYBRID),
V_SN: VerticalLevel(1, level_kinds.HYBRID),
}
base_folder = "/scratch/huziy/Output/GL_CC_CanESM2_RCP85/coupled-GL-future_CanESM2/Samples"
vname_map = {}
vname_map.update(vname_map_CRCM5)
# vname_map[default_varname_mappings.SNOWFALL_RATE] = "SN"
vname_map[default_varname_mappings.SNOWFALL_RATE] = "XXX"
pool = Pool(processes=nprocs)
input_params = []
for month_start in period.range("months"):
month_end = month_start.add(months=1).subtract(seconds=1)
current_month_period = Period(month_start, month_end)
current_month_period.months_of_interest = [month_start.month, ]
label_to_config = OrderedDict([(
label, {
# "base_folder": "/HOME/huziy/skynet3_rech1/CRCM5_outputs/cc_canesm2_rcp85_gl/coupled-GL-future_CanESM2/Samples",
DataManager.SP_BASE_FOLDER: base_folder,
DataManager.SP_DATASOURCE_TYPE: data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT,
DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING: vname_map,
DataManager.SP_LEVEL_MAPPING: vname_to_level_erai,
DataManager.SP_OFFSET_MAPPING: vname_to_offset_CRCM5,
DataManager.SP_MULTIPLIER_MAPPING: vname_to_multiplier_CRCM5,
DataManager.SP_VARNAME_TO_FILENAME_PREFIX_MAPPING: vname_to_fname_prefix_CRCM5,
"out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year)
}
)])
kwargs = dict(
label_to_config=label_to_config, period=current_month_period, months_of_interest=current_month_period.months_of_interest, nprocs_to_use=1
)
print(current_month_period.months_of_interest)
input_params.append(kwargs)
# execute in parallel
pool.map(monthly_func, input_params)
def main_future(nprocs=20):
period = Period(
datetime(2079, 1, 1), datetime(2100, 12, 31)
)
label = "CRCM5_HL_CanESM2_RCP85_{}-{}_monthly".format(period.start.year, period.end.year)
vname_to_level_erai = {
T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID),
U_WE: VerticalLevel(1, level_kinds.HYBRID),
V_SN: VerticalLevel(1, level_kinds.HYBRID),
}
base_folder = "/scratch/huziy/NEI/GL_samples_only/GL_CC_CanESM2_RCP85/HL-GL-current_CanESM2/Samples"
vname_map = {}
vname_map.update(vname_map_CRCM5)
vname_map[default_varname_mappings.SNOWFALL_RATE] = "SN"
pool = Pool(processes=nprocs)
input_params = []
for month_start in period.range("months"):
month_end = month_start.add(months=1).subtract(seconds=1)
current_month_period = Period(month_start, month_end)
current_month_period.months_of_interest = [month_start.month, ]
label_to_config = OrderedDict([(
label, {
# "base_folder": "/HOME/huziy/skynet3_rech1/CRCM5_outputs/cc_canesm2_rcp85_gl/coupled-GL-future_CanESM2/Samples",
DataManager.SP_BASE_FOLDER: base_folder,
DataManager.SP_DATASOURCE_TYPE: data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT,
DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING: vname_map,
DataManager.SP_LEVEL_MAPPING: vname_to_level_erai,
DataManager.SP_OFFSET_MAPPING: vname_to_offset_CRCM5,
DataManager.SP_MULTIPLIER_MAPPING: vname_to_multiplier_CRCM5,
DataManager.SP_VARNAME_TO_FILENAME_PREFIX_MAPPING: vname_to_fname_prefix_CRCM5,
"out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year)
}
)])
kwargs = dict(
label_to_config=label_to_config, period=current_month_period, months_of_interest=current_month_period.months_of_interest, nprocs_to_use=1
)
print(current_month_period.months_of_interest)
input_params.append(kwargs)
# execute in parallel
pool.map(monthly_func, input_params)
def main_obs():
label = "Obs_monthly_icefix_test2_1proc_speedtest_3"
period = Period(
datetime(1980, 1, 1), datetime(2010, 12, 31)
)
pool = Pool(processes=20)
input_params = []
for month_start in period.range("months"):
month_end = month_start.add(months=1).subtract(seconds=1)
current_month_period = Period(month_start, month_end)
current_month_period.months_of_interest = [month_start.month, ]
vname_to_level_erai = {
T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID),
U_WE: VerticalLevel(1, level_kinds.HYBRID),
V_SN: VerticalLevel(1, level_kinds.HYBRID),
}
vname_map = {}
vname_map.update(vname_map_CRCM5)
label_to_config = OrderedDict([(
label, {
DataManager.SP_BASE_FOLDER: "/HOME/huziy/skynet3_rech1/obs_data_for_HLES/interploated_to_the_same_grid/GL_0.1_452x260_icefix",
DataManager.SP_DATASOURCE_TYPE: data_source_types.ALL_VARS_IN_A_FOLDER_IN_NETCDF_FILES_OPEN_EACH_FILE_SEPARATELY,
DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING: vname_map,
DataManager.SP_LEVEL_MAPPING: vname_to_level_erai,
DataManager.SP_OFFSET_MAPPING: vname_to_offset_CRCM5,
DataManager.SP_MULTIPLIER_MAPPING: vname_to_multiplier_CRCM5,
DataManager.SP_VARNAME_TO_FILENAME_PREFIX_MAPPING: vname_to_fname_prefix_CRCM5,
"out_folder": "lake_effect_analysis_daily_{}_{}-{}".format(label, period.start.year, period.end.year)
}
)])
kwargs = dict(
label_to_config=label_to_config, period=current_month_period, months_of_interest=current_month_period.months_of_interest, nprocs_to_use=1
)
print(current_month_period.months_of_interest)
input_params.append(kwargs)
# execute in parallel
pool.map(monthly_func, input_params)
def enh_lakeffect_snfall_calculator_proc(args):
"""
For multiprocessing
:param args:
"""
data_manager, label, period, out_folder = args
if not isinstance(period, Period):
p = Period(start=period[0], end=period[1])
p.months_of_interest = period[2]
period = p
print("Start calculations for {} ... {}".format(period.start, period.end))
calculate_enh_lakeffect_snowfall_for_a_datasource(data_mngr=data_manager, label=label,
period=period,
out_folder=out_folder)
print("Finish calculations for {} ... {}".format(period.start, period.end))
def main():
label = "Obs"
period = Period(datetime(1980, 11, 1), datetime(1981, 2, 1))
# should be continuous??
months_of_interest = [11, 12, 1]
period.months_of_interest = months_of_interest
vname_to_level_erai = {
T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID),
U_WE: VerticalLevel(1, level_kinds.HYBRID),
V_SN: VerticalLevel(1, level_kinds.HYBRID),
}
vname_map = {}
vname_map.update(vname_map_CRCM5)
label_to_config = OrderedDict([(label, {
DataManager.SP_BASE_FOLDER:
"/HOME/huziy/skynet3_rech1/obs_data_for_HLES/interploated_to_the_same_grid/GL_0.1_452x260",
DataManager.SP_DATASOURCE_TYPE:
data_source_types.
ALL_VARS_IN_A_FOLDER_IN_NETCDF_FILES_OPEN_EACH_FILE_SEPARATELY,
DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING:
vname_map,
DataManager.SP_LEVEL_MAPPING:
vname_to_level_erai,
DataManager.SP_OFFSET_MAPPING:
vname_to_offset_CRCM5,
DataManager.SP_MULTIPLIER_MAPPING:
vname_to_multiplier_CRCM5,
DataManager.SP_VARNAME_TO_FILENAME_PREFIX_MAPPING:
vname_to_fname_prefix_CRCM5,
"out_folder":
"lake_effect_analysis_icefix_{}_{}-{}_test1".format(
label, period.start.year, period.end.year)
})])
calculate_lake_effect_snowfall_each_year_in_parallel(
label_to_config=label_to_config, period=period, nprocs_to_use=20)
def enh_lakeffect_snfall_calculator_proc(args):
"""
For multiprocessing
:param args:
"""
data_manager, label, period, out_folder = args
if not isinstance(period, Period):
p = Period(start=period[0], end=period[1])
p.months_of_interest = period[2]
period = p
print("Start calculations for {} ... {}".format(period.start, period.end))
calculate_enh_lakeffect_snowfall_for_a_datasource(data_mngr=data_manager,
label=label,
period=period,
out_folder=out_folder)
print("Finish calculations for {} ... {}".format(period.start, period.end))
def main():
label = "Obs"
period = Period(
datetime(1980, 11, 1), datetime(1981, 2, 1)
)
# should be continuous??
months_of_interest = [11, 12, 1]
period.months_of_interest = months_of_interest
vname_to_level_erai = {
T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID),
U_WE: VerticalLevel(1, level_kinds.HYBRID),
V_SN: VerticalLevel(1, level_kinds.HYBRID),
}
vname_map = {}
vname_map.update(vname_map_CRCM5)
label_to_config = OrderedDict([(
label, {
DataManager.SP_BASE_FOLDER: "/HOME/huziy/skynet3_rech1/obs_data_for_HLES/interploated_to_the_same_grid/GL_0.1_452x260",
DataManager.SP_DATASOURCE_TYPE: data_source_types.ALL_VARS_IN_A_FOLDER_IN_NETCDF_FILES_OPEN_EACH_FILE_SEPARATELY,
DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING: vname_map,
DataManager.SP_LEVEL_MAPPING: vname_to_level_erai,
DataManager.SP_OFFSET_MAPPING: vname_to_offset_CRCM5,
DataManager.SP_MULTIPLIER_MAPPING: vname_to_multiplier_CRCM5,
DataManager.SP_VARNAME_TO_FILENAME_PREFIX_MAPPING: vname_to_fname_prefix_CRCM5,
"out_folder": "lake_effect_analysis_icefix_{}_{}-{}_test1".format(label, period.start.year, period.end.year)
}
)])
calculate_lake_effect_snowfall_each_year_in_parallel(label_to_config=label_to_config, period=period, nprocs_to_use=20)
def calculate_lake_effect_snowfall_each_year_in_parallel(label_to_config, period=None):
"""
:param label_to_config:
:param period: The period of interest defined by the start and the end year of the period (inclusive)
"""
assert hasattr(period, "months_of_interest")
for label, the_config in label_to_config.items():
data_manager = DataManager(store_config=the_config)
print(the_config)
if "out_folder" in the_config:
out_folder = the_config["out_folder"]
else:
out_folder = "."
# Use a fraction of the available processes
nprocs_to_use = max(int(multiprocessing.cpu_count() * 0.75), 1)
nprocs_to_use = min(nprocs_to_use, period.in_years()) # No need for more processes than there is of years
print("Using {} processes for parallelization".format(nprocs_to_use))
pool = Pool(processes=nprocs_to_use)
# Construct the input params for each process
in_data = []
for start in period.range("years"):
p = Period(start=start, end=start.add(months=len(period.months_of_interest)).subtract(seconds=1))
p.months_of_interest = period.months_of_interest
in_data.append([data_manager, label, [p.start, p.end, period.months_of_interest], out_folder])
print(in_data)
pool.map(enh_lakeffect_snfall_calculator_proc, in_data)
def main():
period = Period(
datetime(1979, 12, 1), datetime(1988, 3, 1)
)
# should be consequent
months_of_interest = [12, 1, 2]
period.months_of_interest = months_of_interest
vname_to_level_erai = {
T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID),
U_WE: VerticalLevel(1, level_kinds.HYBRID),
V_SN: VerticalLevel(1, level_kinds.HYBRID),
}
ERAI_label = "ERA-Interim"
label = ERAI_label
label_to_config = OrderedDict(
[ # ERA-Interim
(label,
{
"base_folder": "/RECH/data/Driving_data/Offline/ERA-Interim_0.75/6h_Analysis",
"data_source_type": data_source_types.ALL_VARS_IN_A_FOLDER_OF_RPN_FILES,
"min_dt": timedelta(hours=6),
"varname_mapping": vname_map_CRCM5,
"level_mapping": vname_to_level_erai,
"offset_mapping": vname_to_offset_CRCM5,
"multiplier_mapping": vname_to_multiplier_CRCM5,
"out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year)
}
),
# Add additional sources below
]
)
# calculate_lake_effect_snowfall_each_year_in_parallel(label_to_config=label_to_config, period=period)
# calculate_lake_effect_snowfall(label_to_config=label_to_config, period=period)
label = "CRCM5_NEMO"
label_to_config_CRCM5 = OrderedDict([(
label, {
"base_folder": "/RECH2/huziy/coupling/coupled-GL-NEMO1h_30min/Samples",
"data_source_type": data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT,
"min_dt": timedelta(hours=3),
"varname_mapping": vname_map_CRCM5,
"level_mapping": vname_to_level_erai,
"offset_mapping": vname_to_offset_CRCM5,
"multiplier_mapping": vname_to_multiplier_CRCM5,
"filename_prefix_mapping": vname_to_fname_prefix_CRCM5,
"out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year)
}
)])
# calculate_lake_effect_snowfall(label_to_config=label_to_config_CRCM5, period=period)
# calculate_lake_effect_snowfall_each_year_in_parallel(label_to_config=label_to_config_CRCM5, period=period)
label = "CRCM5_Hostetler"
label_to_config_CRCM5 = OrderedDict([(
label, {
"base_folder": "/RECH2/huziy/coupling/GL_440x260_0.1deg_GL_with_Hostetler/Samples_selected",
"data_source_type": data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT_VNAME_IN_FNAME,
"min_dt": timedelta(hours=3),
"varname_mapping": vname_map_CRCM5,
"level_mapping": vname_to_level_erai,
"offset_mapping": vname_to_offset_CRCM5,
"multiplier_mapping": vname_to_multiplier_CRCM5,
"filename_prefix_mapping": vname_to_fname_prefix_CRCM5,
"out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year)
}
)])
calculate_lake_effect_snowfall_each_year_in_parallel(label_to_config=label_to_config_CRCM5, period=period)
def main():
period = Period(datetime(1980, 12, 1), datetime(2009, 3, 1))
# should be consequent
months_of_interest = [11, 12, 1]
period.months_of_interest = months_of_interest
vname_to_level_erai = {
T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID),
U_WE: VerticalLevel(1, level_kinds.HYBRID),
V_SN: VerticalLevel(1, level_kinds.HYBRID),
}
ERAI_label = "ERA-Interim"
label = ERAI_label
label_to_config = OrderedDict([ # ERA-Interim
(label, {
"base_folder":
"/RECH/data/Driving_data/Offline/ERA-Interim_0.75/6h_Analysis",
"data_source_type":
data_source_types.ALL_VARS_IN_A_FOLDER_OF_RPN_FILES,
"min_dt":
timedelta(hours=6),
"varname_mapping":
vname_map_CRCM5,
"level_mapping":
vname_to_level_erai,
"offset_mapping":
vname_to_offset_CRCM5,
"multiplier_mapping":
vname_to_multiplier_CRCM5,
"out_folder":
"lake_effect_analysis_{}_{}-{}".format(label, period.start.year,
period.end.year)
}),
# Add additional sources below
])
import time
t0 = time.time()
# calculate_lake_effect_snowfall_each_year_in_parallel(label_to_config=label_to_config, period=period, nprocs_to_use=5)
print("Execution time: {} s".format(time.time() - t0))
# calculate_lake_effect_snowfall(label_to_config=label_to_config, period=period)
label = "CRCM5_NEMO_based_on_TT_PR"
vname_map = {}
vname_map.update(vname_map_CRCM5)
vname_map.update({default_varname_mappings.SNOWFALL_RATE: "XX"})
label_to_config_CRCM5 = OrderedDict([(label, {
"base_folder":
"/HOME/huziy/skynet3_rech1/CRCM5_outputs/coupled-GL-NEMO1h/selected_fields",
"data_source_type":
data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT_VNAME_IN_FNAME,
"min_dt":
timedelta(hours=3),
"varname_mapping":
vname_map,
"level_mapping":
vname_to_level_erai,
"offset_mapping":
vname_to_offset_CRCM5,
"multiplier_mapping":
vname_to_multiplier_CRCM5,
"out_folder":
"lake_effect_analysis_{}_{}-{}".format(label, period.start.year,
period.end.year)
})])
calculate_lake_effect_snowfall_each_year_in_parallel(
label_to_config=label_to_config_CRCM5, period=period, nprocs_to_use=16)
label = "CRCM5_Hostetler_based_on_TT_PR"
vname_map = {}
vname_map.update(vname_map_CRCM5)
vname_map.update({default_varname_mappings.SNOWFALL_RATE: "XX"})
label_to_config_CRCM5 = OrderedDict([(label, {
"base_folder":
"/RECH2/huziy/coupling/GL_440x260_0.1deg_GL_with_Hostetler/Samples_selected",
"data_source_type":
data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT_VNAME_IN_FNAME,
"min_dt":
timedelta(hours=3),
"varname_mapping":
vname_map,
"level_mapping":
vname_to_level_erai,
"offset_mapping":
vname_to_offset_CRCM5,
"multiplier_mapping":
vname_to_multiplier_CRCM5,
"filename_prefix_mapping":
vname_to_fname_prefix_CRCM5,
"out_folder":
"lake_effect_analysis_{}_{}-{}".format(label, period.start.year,
period.end.year)
})])
def calculate_lake_effect_snowfall_each_year_in_parallel(label_to_config, period=None, months_of_interest=None,
nprocs_to_use=None):
"""
:param label_to_config:
:param period: The period of interest defined by the start and the end year of the period (inclusive)
"""
if months_of_interest is not None:
period.months_of_interest = months_of_interest
assert hasattr(period, "months_of_interest")
for label, the_config in label_to_config.items():
data_manager = DataManager(store_config=the_config)
print(the_config)
if "out_folder" in the_config:
out_folder = the_config["out_folder"]
else:
out_folder = "."
out_folder = Path(out_folder)
try:
# Try to create the output folder if it does not exist
if not out_folder.exists():
out_folder.mkdir()
print("{}: {} created".format(multiprocessing.current_process().name, out_folder))
except FileExistsError:
print("{}: {} already exists".format(multiprocessing.current_process().name, out_folder))
if nprocs_to_use is None:
# Use a fraction of the available processes
nprocs_to_use = max(int(multiprocessing.cpu_count() * 0.75), 1)
nprocs_to_use = min(nprocs_to_use, period.in_years()) # No need for more processes than there is of years
nprocs_to_use = max(1, nprocs_to_use) # make sure that nprocs_to_use is not 0
print("Using {} processes for parallelization".format(nprocs_to_use))
# Construct the input params for each process
in_data = []
for start in period.range("years"):
end_date = start.add(months=len(period.months_of_interest)).subtract(seconds=1)
end_date = min(end_date, period.end)
p = Period(start=start, end=end_date)
p.months_of_interest = period.months_of_interest
in_data.append([data_manager, label, [p.start, p.end, period.months_of_interest], out_folder])
print(in_data)
if nprocs_to_use > 1:
pool = Pool(processes=nprocs_to_use)
pool.map(enh_lakeffect_snfall_calculator_proc, in_data)
else:
for current_in_data in in_data:
enh_lakeffect_snfall_calculator_proc(current_in_data)
del in_data
del data_manager
def calculate_lake_effect_snowfall_each_year_in_parallel(
label_to_config,
period=None,
months_of_interest=None,
nprocs_to_use=None):
"""
:param label_to_config:
:param period: The period of interest defined by the start and the end year of the period (inclusive)
"""
if months_of_interest is not None:
period.months_of_interest = months_of_interest
assert hasattr(period, "months_of_interest")
for label, the_config in label_to_config.items():
data_manager = DataManager(store_config=the_config)
print(the_config)
if "out_folder" in the_config:
out_folder = the_config["out_folder"]
else:
out_folder = "."
out_folder = Path(out_folder)
try:
# Try to create the output folder if it does not exist
if not out_folder.exists():
out_folder.mkdir()
print("{}: {} created".format(
multiprocessing.current_process().name, out_folder))
except FileExistsError:
print("{}: {} already exists".format(
multiprocessing.current_process().name, out_folder))
if nprocs_to_use is None:
# Use a fraction of the available processes
nprocs_to_use = max(int(multiprocessing.cpu_count() * 0.75), 1)
nprocs_to_use = min(nprocs_to_use, period.in_years(
)) # No need for more processes than there is of years
nprocs_to_use = max(
1, nprocs_to_use) # make sure that nprocs_to_use is not 0
print("Using {} processes for parallelization".format(nprocs_to_use))
# Construct the input params for each process
in_data = []
for start in period.range("years"):
end_date = start.add(
months=len(period.months_of_interest)).subtract(seconds=1)
end_date = min(end_date, period.end)
p = Period(start=start, end=end_date)
p.months_of_interest = period.months_of_interest
in_data.append([
data_manager, label,
[p.start, p.end, period.months_of_interest], out_folder
])
print(in_data)
if nprocs_to_use > 1:
pool = Pool(processes=nprocs_to_use)
pool.map(enh_lakeffect_snfall_calculator_proc, in_data)
else:
for current_in_data in in_data:
enh_lakeffect_snfall_calculator_proc(current_in_data)
del in_data
del data_manager
def main():
# First approximation of the lake-effect snow, by looking at the daily snowfall of more than 1 cm/day
period = Period(
datetime(1994, 12, 1), datetime(1995, 3, 1)
)
# should be consequent
months_of_interest = [12, 1, 2]
period.months_of_interest = months_of_interest
ERAI_label = "ERA-Interim"
vname_to_level_erai = {
default_varname_mappings.T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID),
default_varname_mappings.U_WE: VerticalLevel(1, level_kinds.HYBRID),
default_varname_mappings.V_SN: VerticalLevel(1, level_kinds.HYBRID),
}
label_to_config = OrderedDict(
[ # ERA-Interim
(ERAI_label,
{
"base_folder": "/RECH/data/Driving_data/Offline/ERA-Interim_0.75/6h_Analysis",
"data_source_type": data_source_types.ALL_VARS_IN_A_FOLDER_OF_RPN_FILES,
"min_dt": timedelta(hours=6),
"varname_mapping": default_varname_mappings.vname_map_CRCM5,
"level_mapping": vname_to_level_erai,
"offset_mapping": default_varname_mappings.vname_to_offset_CRCM5,
"multiplier_mapping": default_varname_mappings.vname_to_multiplier_CRCM5
}
),
# Add additional sources below
]
)
label = "ECMWF_CRCM5_FLake_0"
label_to_config_CRCM5 = OrderedDict([(
label, {
"base_folder": "/HOME/huziy/skynet3_rech1/ens_simulations_links_diro/ENSSEASF_NorthAmerica_0.22deg_B1_0",
"data_source_type": data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT,
"min_dt": timedelta(hours=3),
"varname_mapping": default_varname_mappings.vname_map_CRCM5,
"level_mapping": vname_to_level_erai,
"offset_mapping": default_varname_mappings.vname_to_offset_CRCM5,
"multiplier_mapping": default_varname_mappings.vname_to_multiplier_CRCM5,
"filename_prefix_mapping": default_varname_mappings.vname_to_fname_prefix_CRCM5,
"out_folder": "lake_effect_analysis_{}".format(label)
}
)])
# for i in range(1, 9):
# label = "ECMWF_CRCM5_FLake_{}".format(i)
# label_to_config_CRCM5[label] = label_to_config_CRCM5[label0]
# label_to_config_CRCM5[label]["out_folder"] = "lake_effect_analysis_{}".format(label)
# ECMWF GCM ensemble member outputs
label_ECMWF_GCM = "ECMWF_GCM_1"
multiplier_map_ECMWF_GCM = defaultdict(lambda: 1)
multiplier_map_ECMWF_GCM[default_varname_mappings.TOTAL_PREC] = 1.0e-3 / (24.0 * 3600.0) # convert to M/S]
label_to_config_ECMWF_GCM = OrderedDict(
[
(label_ECMWF_GCM, {
"base_folder": "/RESCUE/skynet3_rech1/huziy/ens_simulations_links_diro/ECMWF_GCM/ensm_1",
"data_source_type": data_source_types.ALL_VARS_IN_A_FOLDER_IN_NETCDF_FILES,
"out_folder": "lake_effect_analysis_{}".format(label_ECMWF_GCM),
"varname_mapping": {
default_varname_mappings.T_AIR_2M: "tas",
default_varname_mappings.TOTAL_PREC: "prlr",
default_varname_mappings.U_WE: "uas",
default_varname_mappings.V_SN: "vas",
},
"multiplier_mapping": multiplier_map_ECMWF_GCM,
"offset_mapping": defaultdict(lambda: 0),
"level_mapping": defaultdict(lambda: 0),
}),
]
)
calculate_lake_effect_snowfall(label_to_config=label_to_config_CRCM5, period=period)
def main():
# First approximation of the lake-effect snow, by looking at the daily snowfall of more than 1 cm/day
period = Period(datetime(1994, 12, 1), datetime(1995, 3, 1))
# should be consequent
months_of_interest = [12, 1, 2]
period.months_of_interest = months_of_interest
ERAI_label = "ERA-Interim"
vname_to_level_erai = {
default_varname_mappings.T_AIR_2M:
VerticalLevel(1, level_kinds.HYBRID),
default_varname_mappings.U_WE: VerticalLevel(1, level_kinds.HYBRID),
default_varname_mappings.V_SN: VerticalLevel(1, level_kinds.HYBRID),
}
label_to_config = OrderedDict([ # ERA-Interim
(ERAI_label, {
"base_folder":
"/RECH/data/Driving_data/Offline/ERA-Interim_0.75/6h_Analysis",
"data_source_type":
data_source_types.ALL_VARS_IN_A_FOLDER_OF_RPN_FILES,
"min_dt":
timedelta(hours=6),
"varname_mapping":
default_varname_mappings.vname_map_CRCM5,
"level_mapping":
vname_to_level_erai,
"offset_mapping":
default_varname_mappings.vname_to_offset_CRCM5,
"multiplier_mapping":
default_varname_mappings.vname_to_multiplier_CRCM5
}),
# Add additional sources below
])
label = "ECMWF_CRCM5_FLake_0"
label_to_config_CRCM5 = OrderedDict([(label, {
"base_folder":
"/HOME/huziy/skynet3_rech1/ens_simulations_links_diro/ENSSEASF_NorthAmerica_0.22deg_B1_0",
"data_source_type":
data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT,
"min_dt":
timedelta(hours=3),
"varname_mapping":
default_varname_mappings.vname_map_CRCM5,
"level_mapping":
vname_to_level_erai,
"offset_mapping":
default_varname_mappings.vname_to_offset_CRCM5,
"multiplier_mapping":
default_varname_mappings.vname_to_multiplier_CRCM5,
"filename_prefix_mapping":
default_varname_mappings.vname_to_fname_prefix_CRCM5,
"out_folder":
"lake_effect_analysis_{}".format(label)
})])
# for i in range(1, 9):
# label = "ECMWF_CRCM5_FLake_{}".format(i)
# label_to_config_CRCM5[label] = label_to_config_CRCM5[label0]
# label_to_config_CRCM5[label]["out_folder"] = "lake_effect_analysis_{}".format(label)
# ECMWF GCM ensemble member outputs
label_ECMWF_GCM = "ECMWF_GCM_1"
multiplier_map_ECMWF_GCM = defaultdict(lambda: 1)
multiplier_map_ECMWF_GCM[default_varname_mappings.TOTAL_PREC] = 1.0e-3 / (
24.0 * 3600.0) # convert to M/S]
label_to_config_ECMWF_GCM = OrderedDict([
(label_ECMWF_GCM, {
"base_folder":
"/RESCUE/skynet3_rech1/huziy/ens_simulations_links_diro/ECMWF_GCM/ensm_1",
"data_source_type":
data_source_types.ALL_VARS_IN_A_FOLDER_IN_NETCDF_FILES,
"out_folder": "lake_effect_analysis_{}".format(label_ECMWF_GCM),
"varname_mapping": {
default_varname_mappings.T_AIR_2M: "tas",
default_varname_mappings.TOTAL_PREC: "prlr",
default_varname_mappings.U_WE: "uas",
default_varname_mappings.V_SN: "vas",
},
"multiplier_mapping": multiplier_map_ECMWF_GCM,
"offset_mapping": defaultdict(lambda: 0),
"level_mapping": defaultdict(lambda: 0),
}),
])
calculate_lake_effect_snowfall(label_to_config=label_to_config_CRCM5,
period=period)
def main():
period = Period(
datetime(1980, 12, 1), datetime(1985, 3, 1)
)
# should be consequent
months_of_interest = [11, 12, 1]
period.months_of_interest = months_of_interest
vname_to_level_erai = {
T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID),
U_WE: VerticalLevel(1, level_kinds.HYBRID),
V_SN: VerticalLevel(1, level_kinds.HYBRID),
}
ERAI_label = "ERA-Interim"
label = ERAI_label
label_to_config = OrderedDict(
[ # ERA-Interim
(label,
{
"base_folder": "/RECH/data/Driving_data/Offline/ERA-Interim_0.75/6h_Analysis",
"data_source_type": data_source_types.ALL_VARS_IN_A_FOLDER_OF_RPN_FILES,
"min_dt": timedelta(hours=6),
"varname_mapping": vname_map_CRCM5,
"level_mapping": vname_to_level_erai,
"offset_mapping": vname_to_offset_CRCM5,
"multiplier_mapping": vname_to_multiplier_CRCM5,
"out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year)
}
),
# Add additional sources below
]
)
import time
t0 = time.time()
calculate_lake_effect_snowfall_each_year_in_parallel(label_to_config=label_to_config, period=period, nprocs_to_use=5)
print("Execution time: {} s".format(time.time() - t0))
#calculate_lake_effect_snowfall(label_to_config=label_to_config, period=period)
label = "CRCM5_NEMO"
vname_map = {}
vname_map.update(vname_map_CRCM5)
vname_map.update({
default_varname_mappings.SNOWFALL_RATE: "U3"
})
label_to_config_CRCM5 = OrderedDict([(
label, {
"base_folder": "/HOME/huziy/skynet3_rech1/CRCM5_outputs/coupled-GL-NEMO1h/selected_fields",
"data_source_type": data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT_VNAME_IN_FNAME,
"min_dt": timedelta(hours=3),
"varname_mapping": vname_map,
"level_mapping": vname_to_level_erai,
"offset_mapping": vname_to_offset_CRCM5,
"multiplier_mapping": vname_to_multiplier_CRCM5,
"out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year)
}
)])
# calculate_lake_effect_snowfall_each_year_in_parallel(label_to_config=label_to_config_CRCM5,
# period=period,
# nprocs_to_use=16)
label = "CRCM5_Hostetler"
vname_map = {}
vname_map.update(vname_map_CRCM5)
vname_map.update({
default_varname_mappings.SNOWFALL_RATE: "U3"
})
label_to_config_CRCM5 = OrderedDict([(
label, {
"base_folder": "/RECH2/huziy/coupling/GL_440x260_0.1deg_GL_with_Hostetler/Samples_selected",
"data_source_type": data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT_VNAME_IN_FNAME,
"min_dt": timedelta(hours=3),
"varname_mapping": vname_map,
"level_mapping": vname_to_level_erai,
"offset_mapping": vname_to_offset_CRCM5,
"multiplier_mapping": vname_to_multiplier_CRCM5,
"filename_prefix_mapping": vname_to_fname_prefix_CRCM5,
"out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year)
}
)])
def main_crcm5_nemo():
label = "CRCM5_NEMO"
period = Period(datetime(1980, 1, 1), datetime(2015, 12, 31))
pool = Pool(processes=10)
input_params = []
for month_start in period.range("months"):
month_end = month_start.add(months=1).subtract(seconds=1)
current_month_period = Period(month_start, month_end)
current_month_period.months_of_interest = [
month_start.month,
]
vname_to_level_erai = {
T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID),
U_WE: VerticalLevel(1, level_kinds.HYBRID),
V_SN: VerticalLevel(1, level_kinds.HYBRID),
}
vname_map = {}
vname_map.update(vname_map_CRCM5)
vname_map = {}
vname_map.update(vname_map_CRCM5)
vname_map.update({default_varname_mappings.SNOWFALL_RATE: "SN"})
label_to_config = OrderedDict([(label, {
DataManager.SP_BASE_FOLDER:
"/snow3/huziy/NEI/GL/erai0.75deg_driven/GL_with_NEMO_dtN_1h_and_30min/Samples",
DataManager.SP_DATASOURCE_TYPE:
data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT,
DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING:
vname_map,
DataManager.SP_LEVEL_MAPPING:
vname_to_level_erai,
DataManager.SP_OFFSET_MAPPING:
vname_to_offset_CRCM5,
DataManager.SP_MULTIPLIER_MAPPING:
vname_to_multiplier_CRCM5,
DataManager.SP_VARNAME_TO_FILENAME_PREFIX_MAPPING:
default_varname_mappings.vname_to_fname_prefix_CRCM5,
"out_folder":
"lake_effect_analysis_{}_{}-{}_monthly".format(
label, period.start.year, period.end.year)
})])
kwargs = dict(
label_to_config=label_to_config,
period=current_month_period,
months_of_interest=current_month_period.months_of_interest,
nprocs_to_use=1)
print(current_month_period.months_of_interest)
input_params.append(kwargs)
# execute in parallel
pool.map(monthly_func, input_params)
