def generate_hles_obs_variants(): period = Period(datetime(1980, 11, 1), datetime(2009, 2, 1)) # should be continuous?? months_of_interest = list(range(1, 13)) period.months_of_interest = months_of_interest vname_to_level_erai = { T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID), U_WE: VerticalLevel(1, level_kinds.HYBRID), V_SN: VerticalLevel(1, level_kinds.HYBRID), } vname_map = {} vname_map.update(vname_map_CRCM5) # set the paths to the data label_to_base_folder = OrderedDict() common_dir = Path( "/home/data/big1/obs_data_for_HLES/interploated_to_the_same_grid/") # label_to_base_folder["obs_anuspmaurer_erai"] = common_dir / "obs_anuspmaurer_erai" label_to_base_folder["obs_daymet_erai"] = common_dir / "obs_daymet_erai" # label_to_base_folder["obs_anuspmaurer_narr"] = common_dir / "obs_anuspmaurer_narr" # label_to_base_folder["obs_daymet_narr"] = common_dir / "obs_daymet_narr" # --- for label, base_folder in label_to_base_folder.items(): __obs_case_monthly(period, vname_to_level_erai, vname_map, label, base_folder)
def generate_hles_obs_variants(): period = Period( datetime(1980, 11, 1), datetime(2009, 2, 1) ) # should be continuous?? months_of_interest = list(range(1, 13)) period.months_of_interest = months_of_interest vname_to_level_erai = { T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID), U_WE: VerticalLevel(1, level_kinds.HYBRID), V_SN: VerticalLevel(1, level_kinds.HYBRID), } vname_map = {} vname_map.update(vname_map_CRCM5) # set the paths to the data label_to_base_folder = OrderedDict() common_dir = Path("/HOME/huziy/skynet3_rech1/obs_data_for_HLES/interploated_to_the_same_grid/") label_to_base_folder["obs_anuspmaurer_erai"] = common_dir / "obs_anuspmaurer_erai" label_to_base_folder["obs_daymet_erai"] = common_dir / "obs_daymet_erai" label_to_base_folder["obs_anuspmaurer_narr"] = common_dir / "obs_anuspmaurer_narr" label_to_base_folder["obs_daymet_narr"] = common_dir / "obs_daymet_narr" # --- for label, base_folder in label_to_base_folder.items(): __obs_case(period, vname_to_level_erai, vname_map, label, base_folder)
def main_crcm5_hl(): label = "CRCM5_HL" period = Period(datetime(1980, 1, 1), datetime(2009, 12, 31)) pool = Pool(processes=12) input_params = [] for month_start in period.range("months"): month_end = month_start.add(months=1).subtract(seconds=1) current_month_period = Period(month_start, month_end) current_month_period.months_of_interest = [ month_start.month, ] vname_to_level_erai = { T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID), U_WE: VerticalLevel(1, level_kinds.HYBRID), V_SN: VerticalLevel(1, level_kinds.HYBRID), } vname_map = {} vname_map.update(vname_map_CRCM5) vname_map = {} vname_map.update(vname_map_CRCM5) vname_map.update({default_varname_mappings.SNOWFALL_RATE: "U3"}) label_to_config = OrderedDict([(label, { DataManager.SP_BASE_FOLDER: "/RECH2/huziy/coupling/GL_440x260_0.1deg_GL_with_Hostetler/Samples_selected", DataManager.SP_DATASOURCE_TYPE: data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT_VNAME_IN_FNAME, DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING: vname_map, DataManager.SP_LEVEL_MAPPING: vname_to_level_erai, DataManager.SP_OFFSET_MAPPING: vname_to_offset_CRCM5, DataManager.SP_MULTIPLIER_MAPPING: vname_to_multiplier_CRCM5, "out_folder": "lake_effect_analysis_{}_{}-{}_monthly".format( label, period.start.year, period.end.year) })]) kwargs = dict( label_to_config=label_to_config, period=current_month_period, months_of_interest=current_month_period.months_of_interest, nprocs_to_use=1) print(current_month_period.months_of_interest) input_params.append(kwargs) # execute in parallel pool.map(monthly_func, input_params)
def main_obs(): label = "Obs_monthly_icefix_test2_1proc_speedtest_3" period = Period(datetime(1980, 1, 1), datetime(2010, 12, 31)) pool = Pool(processes=20) input_params = [] for month_start in period.range("months"): month_end = month_start.add(months=1).subtract(seconds=1) current_month_period = Period(month_start, month_end) current_month_period.months_of_interest = [ month_start.month, ] vname_to_level_erai = { T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID), U_WE: VerticalLevel(1, level_kinds.HYBRID), V_SN: VerticalLevel(1, level_kinds.HYBRID), } vname_map = {} vname_map.update(vname_map_CRCM5) label_to_config = OrderedDict([(label, { DataManager.SP_BASE_FOLDER: "/HOME/huziy/skynet3_rech1/obs_data_for_HLES/interploated_to_the_same_grid/GL_0.1_452x260_icefix", DataManager.SP_DATASOURCE_TYPE: data_source_types. ALL_VARS_IN_A_FOLDER_IN_NETCDF_FILES_OPEN_EACH_FILE_SEPARATELY, DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING: vname_map, DataManager.SP_LEVEL_MAPPING: vname_to_level_erai, DataManager.SP_OFFSET_MAPPING: vname_to_offset_CRCM5, DataManager.SP_MULTIPLIER_MAPPING: vname_to_multiplier_CRCM5, DataManager.SP_VARNAME_TO_FILENAME_PREFIX_MAPPING: vname_to_fname_prefix_CRCM5, "out_folder": "lake_effect_analysis_daily_{}_{}-{}".format( label, period.start.year, period.end.year) })]) kwargs = dict( label_to_config=label_to_config, period=current_month_period, months_of_interest=current_month_period.months_of_interest, nprocs_to_use=1) print(current_month_period.months_of_interest) input_params.append(kwargs) # execute in parallel pool.map(monthly_func, input_params)
def __obs_case_monthly(period, vname_to_level: dict, vname_map: dict, label: str, base_dir: str, nprocs: int = 4): """ find hles for each month of interest :param period: :param vname_to_level: :param vname_map: :param label: :param base_dir: """ pool = multiprocessing.Pool(processes=nprocs) input_params = [] for month_start in period.range("months"): month_end = month_start.add(months=1).subtract(seconds=1) current_month_period = Period(month_start, month_end) current_month_period.months_of_interest = [ month_start.month, ] label_to_config = OrderedDict([(label, { DataManager.SP_BASE_FOLDER: base_dir, DataManager.SP_DATASOURCE_TYPE: data_source_types. ALL_VARS_IN_A_FOLDER_IN_NETCDF_FILES_OPEN_EACH_FILE_SEPARATELY, DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING: vname_map, DataManager.SP_LEVEL_MAPPING: vname_to_level, DataManager.SP_OFFSET_MAPPING: vname_to_offset_CRCM5, DataManager.SP_MULTIPLIER_MAPPING: vname_to_multiplier_CRCM5, DataManager.SP_VARNAME_TO_FILENAME_PREFIX_MAPPING: vname_to_fname_prefix_CRCM5, "out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year) })]) kwargs = dict( label_to_config=label_to_config, period=current_month_period, months_of_interest=current_month_period.months_of_interest, nprocs_to_use=1) print(current_month_period.months_of_interest) input_params.append(kwargs) # execute in parallel pool.map(monthly_func, input_params)
def main_crcm5_nemo(): label = "CRCM5_NEMO" period = Period( datetime(1980, 1, 1), datetime(2015, 12, 31) ) pool = Pool(processes=10) input_params = [] for month_start in period.range("months"): month_end = month_start.add(months=1).subtract(seconds=1) current_month_period = Period(month_start, month_end) current_month_period.months_of_interest = [month_start.month, ] vname_to_level_erai = { T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID), U_WE: VerticalLevel(1, level_kinds.HYBRID), V_SN: VerticalLevel(1, level_kinds.HYBRID), } vname_map = {} vname_map.update(vname_map_CRCM5) vname_map = {} vname_map.update(vname_map_CRCM5) vname_map.update({ default_varname_mappings.SNOWFALL_RATE: "SN" }) label_to_config = OrderedDict([( label, { DataManager.SP_BASE_FOLDER: "/snow3/huziy/NEI/GL/erai0.75deg_driven/GL_with_NEMO_dtN_1h_and_30min/Samples", DataManager.SP_DATASOURCE_TYPE: data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT, DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING: vname_map, DataManager.SP_LEVEL_MAPPING: vname_to_level_erai, DataManager.SP_OFFSET_MAPPING: vname_to_offset_CRCM5, DataManager.SP_MULTIPLIER_MAPPING: vname_to_multiplier_CRCM5, DataManager.SP_VARNAME_TO_FILENAME_PREFIX_MAPPING: default_varname_mappings.vname_to_fname_prefix_CRCM5, "out_folder": "lake_effect_analysis_{}_{}-{}_monthly".format(label, period.start.year, period.end.year) } )]) kwargs = dict( label_to_config=label_to_config, period=current_month_period, months_of_interest=current_month_period.months_of_interest, nprocs_to_use=1 ) print(current_month_period.months_of_interest) input_params.append(kwargs) # execute in parallel pool.map(monthly_func, input_params)
def main_crcm5_hl(): label = "CRCM5_HL" period = Period( datetime(1980, 1, 1), datetime(2009, 12, 31) ) pool = Pool(processes=12) input_params = [] for month_start in period.range("months"): month_end = month_start.add(months=1).subtract(seconds=1) current_month_period = Period(month_start, month_end) current_month_period.months_of_interest = [month_start.month, ] vname_to_level_erai = { T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID), U_WE: VerticalLevel(1, level_kinds.HYBRID), V_SN: VerticalLevel(1, level_kinds.HYBRID), } vname_map = {} vname_map.update(vname_map_CRCM5) vname_map = {} vname_map.update(vname_map_CRCM5) vname_map.update({ default_varname_mappings.SNOWFALL_RATE: "U3" }) label_to_config = OrderedDict([( label, { DataManager.SP_BASE_FOLDER: "/RECH2/huziy/coupling/GL_440x260_0.1deg_GL_with_Hostetler/Samples_selected", DataManager.SP_DATASOURCE_TYPE: data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT_VNAME_IN_FNAME, DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING: vname_map, DataManager.SP_LEVEL_MAPPING: vname_to_level_erai, DataManager.SP_OFFSET_MAPPING: vname_to_offset_CRCM5, DataManager.SP_MULTIPLIER_MAPPING: vname_to_multiplier_CRCM5, "out_folder": "lake_effect_analysis_{}_{}-{}_monthly".format(label, period.start.year, period.end.year) } )]) kwargs = dict( label_to_config=label_to_config, period=current_month_period, months_of_interest=current_month_period.months_of_interest, nprocs_to_use=1 ) print(current_month_period.months_of_interest) input_params.append(kwargs) # execute in parallel pool.map(monthly_func, input_params)
def main_future(nprocs=20): period = Period( datetime(2079, 1, 1), datetime(2100, 12, 31) ) label = "CRCM5_NEMO_fix_TT_PR_CanESM2_RCP85_{}-{}_monthly".format(period.start.year, period.end.year) vname_to_level_erai = { T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID), U_WE: VerticalLevel(1, level_kinds.HYBRID), V_SN: VerticalLevel(1, level_kinds.HYBRID), } base_folder = "/scratch/huziy/Output/GL_CC_CanESM2_RCP85/coupled-GL-future_CanESM2/Samples" vname_map = {} vname_map.update(vname_map_CRCM5) # vname_map[default_varname_mappings.SNOWFALL_RATE] = "SN" vname_map[default_varname_mappings.SNOWFALL_RATE] = "XXX" pool = Pool(processes=nprocs) input_params = [] for month_start in period.range("months"): month_end = month_start.add(months=1).subtract(seconds=1) current_month_period = Period(month_start, month_end) current_month_period.months_of_interest = [month_start.month, ] label_to_config = OrderedDict([( label, { # "base_folder": "/HOME/huziy/skynet3_rech1/CRCM5_outputs/cc_canesm2_rcp85_gl/coupled-GL-future_CanESM2/Samples", DataManager.SP_BASE_FOLDER: base_folder, DataManager.SP_DATASOURCE_TYPE: data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT, DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING: vname_map, DataManager.SP_LEVEL_MAPPING: vname_to_level_erai, DataManager.SP_OFFSET_MAPPING: vname_to_offset_CRCM5, DataManager.SP_MULTIPLIER_MAPPING: vname_to_multiplier_CRCM5, DataManager.SP_VARNAME_TO_FILENAME_PREFIX_MAPPING: vname_to_fname_prefix_CRCM5, "out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year) } )]) kwargs = dict( label_to_config=label_to_config, period=current_month_period, months_of_interest=current_month_period.months_of_interest, nprocs_to_use=1 ) print(current_month_period.months_of_interest) input_params.append(kwargs) # execute in parallel pool.map(monthly_func, input_params)
def main_future(nprocs=20): period = Period( datetime(2079, 1, 1), datetime(2100, 12, 31) ) label = "CRCM5_HL_CanESM2_RCP85_{}-{}_monthly".format(period.start.year, period.end.year) vname_to_level_erai = { T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID), U_WE: VerticalLevel(1, level_kinds.HYBRID), V_SN: VerticalLevel(1, level_kinds.HYBRID), } base_folder = "/scratch/huziy/NEI/GL_samples_only/GL_CC_CanESM2_RCP85/HL-GL-current_CanESM2/Samples" vname_map = {} vname_map.update(vname_map_CRCM5) vname_map[default_varname_mappings.SNOWFALL_RATE] = "SN" pool = Pool(processes=nprocs) input_params = [] for month_start in period.range("months"): month_end = month_start.add(months=1).subtract(seconds=1) current_month_period = Period(month_start, month_end) current_month_period.months_of_interest = [month_start.month, ] label_to_config = OrderedDict([( label, { # "base_folder": "/HOME/huziy/skynet3_rech1/CRCM5_outputs/cc_canesm2_rcp85_gl/coupled-GL-future_CanESM2/Samples", DataManager.SP_BASE_FOLDER: base_folder, DataManager.SP_DATASOURCE_TYPE: data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT, DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING: vname_map, DataManager.SP_LEVEL_MAPPING: vname_to_level_erai, DataManager.SP_OFFSET_MAPPING: vname_to_offset_CRCM5, DataManager.SP_MULTIPLIER_MAPPING: vname_to_multiplier_CRCM5, DataManager.SP_VARNAME_TO_FILENAME_PREFIX_MAPPING: vname_to_fname_prefix_CRCM5, "out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year) } )]) kwargs = dict( label_to_config=label_to_config, period=current_month_period, months_of_interest=current_month_period.months_of_interest, nprocs_to_use=1 ) print(current_month_period.months_of_interest) input_params.append(kwargs) # execute in parallel pool.map(monthly_func, input_params)
def main_obs(): label = "Obs_monthly_icefix_test2_1proc_speedtest_3" period = Period( datetime(1980, 1, 1), datetime(2010, 12, 31) ) pool = Pool(processes=20) input_params = [] for month_start in period.range("months"): month_end = month_start.add(months=1).subtract(seconds=1) current_month_period = Period(month_start, month_end) current_month_period.months_of_interest = [month_start.month, ] vname_to_level_erai = { T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID), U_WE: VerticalLevel(1, level_kinds.HYBRID), V_SN: VerticalLevel(1, level_kinds.HYBRID), } vname_map = {} vname_map.update(vname_map_CRCM5) label_to_config = OrderedDict([( label, { DataManager.SP_BASE_FOLDER: "/HOME/huziy/skynet3_rech1/obs_data_for_HLES/interploated_to_the_same_grid/GL_0.1_452x260_icefix", DataManager.SP_DATASOURCE_TYPE: data_source_types.ALL_VARS_IN_A_FOLDER_IN_NETCDF_FILES_OPEN_EACH_FILE_SEPARATELY, DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING: vname_map, DataManager.SP_LEVEL_MAPPING: vname_to_level_erai, DataManager.SP_OFFSET_MAPPING: vname_to_offset_CRCM5, DataManager.SP_MULTIPLIER_MAPPING: vname_to_multiplier_CRCM5, DataManager.SP_VARNAME_TO_FILENAME_PREFIX_MAPPING: vname_to_fname_prefix_CRCM5, "out_folder": "lake_effect_analysis_daily_{}_{}-{}".format(label, period.start.year, period.end.year) } )]) kwargs = dict( label_to_config=label_to_config, period=current_month_period, months_of_interest=current_month_period.months_of_interest, nprocs_to_use=1 ) print(current_month_period.months_of_interest) input_params.append(kwargs) # execute in parallel pool.map(monthly_func, input_params)
def enh_lakeffect_snfall_calculator_proc(args): """ For multiprocessing :param args: """ data_manager, label, period, out_folder = args if not isinstance(period, Period): p = Period(start=period[0], end=period[1]) p.months_of_interest = period[2] period = p print("Start calculations for {} ... {}".format(period.start, period.end)) calculate_enh_lakeffect_snowfall_for_a_datasource(data_mngr=data_manager, label=label, period=period, out_folder=out_folder) print("Finish calculations for {} ... {}".format(period.start, period.end))
def main(): label = "Obs" period = Period(datetime(1980, 11, 1), datetime(1981, 2, 1)) # should be continuous?? months_of_interest = [11, 12, 1] period.months_of_interest = months_of_interest vname_to_level_erai = { T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID), U_WE: VerticalLevel(1, level_kinds.HYBRID), V_SN: VerticalLevel(1, level_kinds.HYBRID), } vname_map = {} vname_map.update(vname_map_CRCM5) label_to_config = OrderedDict([(label, { DataManager.SP_BASE_FOLDER: "/HOME/huziy/skynet3_rech1/obs_data_for_HLES/interploated_to_the_same_grid/GL_0.1_452x260", DataManager.SP_DATASOURCE_TYPE: data_source_types. ALL_VARS_IN_A_FOLDER_IN_NETCDF_FILES_OPEN_EACH_FILE_SEPARATELY, DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING: vname_map, DataManager.SP_LEVEL_MAPPING: vname_to_level_erai, DataManager.SP_OFFSET_MAPPING: vname_to_offset_CRCM5, DataManager.SP_MULTIPLIER_MAPPING: vname_to_multiplier_CRCM5, DataManager.SP_VARNAME_TO_FILENAME_PREFIX_MAPPING: vname_to_fname_prefix_CRCM5, "out_folder": "lake_effect_analysis_icefix_{}_{}-{}_test1".format( label, period.start.year, period.end.year) })]) calculate_lake_effect_snowfall_each_year_in_parallel( label_to_config=label_to_config, period=period, nprocs_to_use=20)
def main(): label = "Obs" period = Period( datetime(1980, 11, 1), datetime(1981, 2, 1) ) # should be continuous?? months_of_interest = [11, 12, 1] period.months_of_interest = months_of_interest vname_to_level_erai = { T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID), U_WE: VerticalLevel(1, level_kinds.HYBRID), V_SN: VerticalLevel(1, level_kinds.HYBRID), } vname_map = {} vname_map.update(vname_map_CRCM5) label_to_config = OrderedDict([( label, { DataManager.SP_BASE_FOLDER: "/HOME/huziy/skynet3_rech1/obs_data_for_HLES/interploated_to_the_same_grid/GL_0.1_452x260", DataManager.SP_DATASOURCE_TYPE: data_source_types.ALL_VARS_IN_A_FOLDER_IN_NETCDF_FILES_OPEN_EACH_FILE_SEPARATELY, DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING: vname_map, DataManager.SP_LEVEL_MAPPING: vname_to_level_erai, DataManager.SP_OFFSET_MAPPING: vname_to_offset_CRCM5, DataManager.SP_MULTIPLIER_MAPPING: vname_to_multiplier_CRCM5, DataManager.SP_VARNAME_TO_FILENAME_PREFIX_MAPPING: vname_to_fname_prefix_CRCM5, "out_folder": "lake_effect_analysis_icefix_{}_{}-{}_test1".format(label, period.start.year, period.end.year) } )]) calculate_lake_effect_snowfall_each_year_in_parallel(label_to_config=label_to_config, period=period, nprocs_to_use=20)
def calculate_lake_effect_snowfall_each_year_in_parallel(label_to_config, period=None): """ :param label_to_config: :param period: The period of interest defined by the start and the end year of the period (inclusive) """ assert hasattr(period, "months_of_interest") for label, the_config in label_to_config.items(): data_manager = DataManager(store_config=the_config) print(the_config) if "out_folder" in the_config: out_folder = the_config["out_folder"] else: out_folder = "." # Use a fraction of the available processes nprocs_to_use = max(int(multiprocessing.cpu_count() * 0.75), 1) nprocs_to_use = min(nprocs_to_use, period.in_years()) # No need for more processes than there is of years print("Using {} processes for parallelization".format(nprocs_to_use)) pool = Pool(processes=nprocs_to_use) # Construct the input params for each process in_data = [] for start in period.range("years"): p = Period(start=start, end=start.add(months=len(period.months_of_interest)).subtract(seconds=1)) p.months_of_interest = period.months_of_interest in_data.append([data_manager, label, [p.start, p.end, period.months_of_interest], out_folder]) print(in_data) pool.map(enh_lakeffect_snfall_calculator_proc, in_data)
def main(): period = Period( datetime(1979, 12, 1), datetime(1988, 3, 1) ) # should be consequent months_of_interest = [12, 1, 2] period.months_of_interest = months_of_interest vname_to_level_erai = { T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID), U_WE: VerticalLevel(1, level_kinds.HYBRID), V_SN: VerticalLevel(1, level_kinds.HYBRID), } ERAI_label = "ERA-Interim" label = ERAI_label label_to_config = OrderedDict( [ # ERA-Interim (label, { "base_folder": "/RECH/data/Driving_data/Offline/ERA-Interim_0.75/6h_Analysis", "data_source_type": data_source_types.ALL_VARS_IN_A_FOLDER_OF_RPN_FILES, "min_dt": timedelta(hours=6), "varname_mapping": vname_map_CRCM5, "level_mapping": vname_to_level_erai, "offset_mapping": vname_to_offset_CRCM5, "multiplier_mapping": vname_to_multiplier_CRCM5, "out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year) } ), # Add additional sources below ] ) # calculate_lake_effect_snowfall_each_year_in_parallel(label_to_config=label_to_config, period=period) # calculate_lake_effect_snowfall(label_to_config=label_to_config, period=period) label = "CRCM5_NEMO" label_to_config_CRCM5 = OrderedDict([( label, { "base_folder": "/RECH2/huziy/coupling/coupled-GL-NEMO1h_30min/Samples", "data_source_type": data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT, "min_dt": timedelta(hours=3), "varname_mapping": vname_map_CRCM5, "level_mapping": vname_to_level_erai, "offset_mapping": vname_to_offset_CRCM5, "multiplier_mapping": vname_to_multiplier_CRCM5, "filename_prefix_mapping": vname_to_fname_prefix_CRCM5, "out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year) } )]) # calculate_lake_effect_snowfall(label_to_config=label_to_config_CRCM5, period=period) # calculate_lake_effect_snowfall_each_year_in_parallel(label_to_config=label_to_config_CRCM5, period=period) label = "CRCM5_Hostetler" label_to_config_CRCM5 = OrderedDict([( label, { "base_folder": "/RECH2/huziy/coupling/GL_440x260_0.1deg_GL_with_Hostetler/Samples_selected", "data_source_type": data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT_VNAME_IN_FNAME, "min_dt": timedelta(hours=3), "varname_mapping": vname_map_CRCM5, "level_mapping": vname_to_level_erai, "offset_mapping": vname_to_offset_CRCM5, "multiplier_mapping": vname_to_multiplier_CRCM5, "filename_prefix_mapping": vname_to_fname_prefix_CRCM5, "out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year) } )]) calculate_lake_effect_snowfall_each_year_in_parallel(label_to_config=label_to_config_CRCM5, period=period)
def main(): period = Period(datetime(1980, 12, 1), datetime(2009, 3, 1)) # should be consequent months_of_interest = [11, 12, 1] period.months_of_interest = months_of_interest vname_to_level_erai = { T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID), U_WE: VerticalLevel(1, level_kinds.HYBRID), V_SN: VerticalLevel(1, level_kinds.HYBRID), } ERAI_label = "ERA-Interim" label = ERAI_label label_to_config = OrderedDict([ # ERA-Interim (label, { "base_folder": "/RECH/data/Driving_data/Offline/ERA-Interim_0.75/6h_Analysis", "data_source_type": data_source_types.ALL_VARS_IN_A_FOLDER_OF_RPN_FILES, "min_dt": timedelta(hours=6), "varname_mapping": vname_map_CRCM5, "level_mapping": vname_to_level_erai, "offset_mapping": vname_to_offset_CRCM5, "multiplier_mapping": vname_to_multiplier_CRCM5, "out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year) }), # Add additional sources below ]) import time t0 = time.time() # calculate_lake_effect_snowfall_each_year_in_parallel(label_to_config=label_to_config, period=period, nprocs_to_use=5) print("Execution time: {} s".format(time.time() - t0)) # calculate_lake_effect_snowfall(label_to_config=label_to_config, period=period) label = "CRCM5_NEMO_based_on_TT_PR" vname_map = {} vname_map.update(vname_map_CRCM5) vname_map.update({default_varname_mappings.SNOWFALL_RATE: "XX"}) label_to_config_CRCM5 = OrderedDict([(label, { "base_folder": "/HOME/huziy/skynet3_rech1/CRCM5_outputs/coupled-GL-NEMO1h/selected_fields", "data_source_type": data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT_VNAME_IN_FNAME, "min_dt": timedelta(hours=3), "varname_mapping": vname_map, "level_mapping": vname_to_level_erai, "offset_mapping": vname_to_offset_CRCM5, "multiplier_mapping": vname_to_multiplier_CRCM5, "out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year) })]) calculate_lake_effect_snowfall_each_year_in_parallel( label_to_config=label_to_config_CRCM5, period=period, nprocs_to_use=16) label = "CRCM5_Hostetler_based_on_TT_PR" vname_map = {} vname_map.update(vname_map_CRCM5) vname_map.update({default_varname_mappings.SNOWFALL_RATE: "XX"}) label_to_config_CRCM5 = OrderedDict([(label, { "base_folder": "/RECH2/huziy/coupling/GL_440x260_0.1deg_GL_with_Hostetler/Samples_selected", "data_source_type": data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT_VNAME_IN_FNAME, "min_dt": timedelta(hours=3), "varname_mapping": vname_map, "level_mapping": vname_to_level_erai, "offset_mapping": vname_to_offset_CRCM5, "multiplier_mapping": vname_to_multiplier_CRCM5, "filename_prefix_mapping": vname_to_fname_prefix_CRCM5, "out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year) })])
def calculate_lake_effect_snowfall_each_year_in_parallel(label_to_config, period=None, months_of_interest=None, nprocs_to_use=None): """ :param label_to_config: :param period: The period of interest defined by the start and the end year of the period (inclusive) """ if months_of_interest is not None: period.months_of_interest = months_of_interest assert hasattr(period, "months_of_interest") for label, the_config in label_to_config.items(): data_manager = DataManager(store_config=the_config) print(the_config) if "out_folder" in the_config: out_folder = the_config["out_folder"] else: out_folder = "." out_folder = Path(out_folder) try: # Try to create the output folder if it does not exist if not out_folder.exists(): out_folder.mkdir() print("{}: {} created".format(multiprocessing.current_process().name, out_folder)) except FileExistsError: print("{}: {} already exists".format(multiprocessing.current_process().name, out_folder)) if nprocs_to_use is None: # Use a fraction of the available processes nprocs_to_use = max(int(multiprocessing.cpu_count() * 0.75), 1) nprocs_to_use = min(nprocs_to_use, period.in_years()) # No need for more processes than there is of years nprocs_to_use = max(1, nprocs_to_use) # make sure that nprocs_to_use is not 0 print("Using {} processes for parallelization".format(nprocs_to_use)) # Construct the input params for each process in_data = [] for start in period.range("years"): end_date = start.add(months=len(period.months_of_interest)).subtract(seconds=1) end_date = min(end_date, period.end) p = Period(start=start, end=end_date) p.months_of_interest = period.months_of_interest in_data.append([data_manager, label, [p.start, p.end, period.months_of_interest], out_folder]) print(in_data) if nprocs_to_use > 1: pool = Pool(processes=nprocs_to_use) pool.map(enh_lakeffect_snfall_calculator_proc, in_data) else: for current_in_data in in_data: enh_lakeffect_snfall_calculator_proc(current_in_data) del in_data del data_manager
def calculate_lake_effect_snowfall_each_year_in_parallel( label_to_config, period=None, months_of_interest=None, nprocs_to_use=None): """ :param label_to_config: :param period: The period of interest defined by the start and the end year of the period (inclusive) """ if months_of_interest is not None: period.months_of_interest = months_of_interest assert hasattr(period, "months_of_interest") for label, the_config in label_to_config.items(): data_manager = DataManager(store_config=the_config) print(the_config) if "out_folder" in the_config: out_folder = the_config["out_folder"] else: out_folder = "." out_folder = Path(out_folder) try: # Try to create the output folder if it does not exist if not out_folder.exists(): out_folder.mkdir() print("{}: {} created".format( multiprocessing.current_process().name, out_folder)) except FileExistsError: print("{}: {} already exists".format( multiprocessing.current_process().name, out_folder)) if nprocs_to_use is None: # Use a fraction of the available processes nprocs_to_use = max(int(multiprocessing.cpu_count() * 0.75), 1) nprocs_to_use = min(nprocs_to_use, period.in_years( )) # No need for more processes than there is of years nprocs_to_use = max( 1, nprocs_to_use) # make sure that nprocs_to_use is not 0 print("Using {} processes for parallelization".format(nprocs_to_use)) # Construct the input params for each process in_data = [] for start in period.range("years"): end_date = start.add( months=len(period.months_of_interest)).subtract(seconds=1) end_date = min(end_date, period.end) p = Period(start=start, end=end_date) p.months_of_interest = period.months_of_interest in_data.append([ data_manager, label, [p.start, p.end, period.months_of_interest], out_folder ]) print(in_data) if nprocs_to_use > 1: pool = Pool(processes=nprocs_to_use) pool.map(enh_lakeffect_snfall_calculator_proc, in_data) else: for current_in_data in in_data: enh_lakeffect_snfall_calculator_proc(current_in_data) del in_data del data_manager
def main(): # First approximation of the lake-effect snow, by looking at the daily snowfall of more than 1 cm/day period = Period( datetime(1994, 12, 1), datetime(1995, 3, 1) ) # should be consequent months_of_interest = [12, 1, 2] period.months_of_interest = months_of_interest ERAI_label = "ERA-Interim" vname_to_level_erai = { default_varname_mappings.T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID), default_varname_mappings.U_WE: VerticalLevel(1, level_kinds.HYBRID), default_varname_mappings.V_SN: VerticalLevel(1, level_kinds.HYBRID), } label_to_config = OrderedDict( [ # ERA-Interim (ERAI_label, { "base_folder": "/RECH/data/Driving_data/Offline/ERA-Interim_0.75/6h_Analysis", "data_source_type": data_source_types.ALL_VARS_IN_A_FOLDER_OF_RPN_FILES, "min_dt": timedelta(hours=6), "varname_mapping": default_varname_mappings.vname_map_CRCM5, "level_mapping": vname_to_level_erai, "offset_mapping": default_varname_mappings.vname_to_offset_CRCM5, "multiplier_mapping": default_varname_mappings.vname_to_multiplier_CRCM5 } ), # Add additional sources below ] ) label = "ECMWF_CRCM5_FLake_0" label_to_config_CRCM5 = OrderedDict([( label, { "base_folder": "/HOME/huziy/skynet3_rech1/ens_simulations_links_diro/ENSSEASF_NorthAmerica_0.22deg_B1_0", "data_source_type": data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT, "min_dt": timedelta(hours=3), "varname_mapping": default_varname_mappings.vname_map_CRCM5, "level_mapping": vname_to_level_erai, "offset_mapping": default_varname_mappings.vname_to_offset_CRCM5, "multiplier_mapping": default_varname_mappings.vname_to_multiplier_CRCM5, "filename_prefix_mapping": default_varname_mappings.vname_to_fname_prefix_CRCM5, "out_folder": "lake_effect_analysis_{}".format(label) } )]) # for i in range(1, 9): # label = "ECMWF_CRCM5_FLake_{}".format(i) # label_to_config_CRCM5[label] = label_to_config_CRCM5[label0] # label_to_config_CRCM5[label]["out_folder"] = "lake_effect_analysis_{}".format(label) # ECMWF GCM ensemble member outputs label_ECMWF_GCM = "ECMWF_GCM_1" multiplier_map_ECMWF_GCM = defaultdict(lambda: 1) multiplier_map_ECMWF_GCM[default_varname_mappings.TOTAL_PREC] = 1.0e-3 / (24.0 * 3600.0) # convert to M/S] label_to_config_ECMWF_GCM = OrderedDict( [ (label_ECMWF_GCM, { "base_folder": "/RESCUE/skynet3_rech1/huziy/ens_simulations_links_diro/ECMWF_GCM/ensm_1", "data_source_type": data_source_types.ALL_VARS_IN_A_FOLDER_IN_NETCDF_FILES, "out_folder": "lake_effect_analysis_{}".format(label_ECMWF_GCM), "varname_mapping": { default_varname_mappings.T_AIR_2M: "tas", default_varname_mappings.TOTAL_PREC: "prlr", default_varname_mappings.U_WE: "uas", default_varname_mappings.V_SN: "vas", }, "multiplier_mapping": multiplier_map_ECMWF_GCM, "offset_mapping": defaultdict(lambda: 0), "level_mapping": defaultdict(lambda: 0), }), ] ) calculate_lake_effect_snowfall(label_to_config=label_to_config_CRCM5, period=period)
def main(): # First approximation of the lake-effect snow, by looking at the daily snowfall of more than 1 cm/day period = Period(datetime(1994, 12, 1), datetime(1995, 3, 1)) # should be consequent months_of_interest = [12, 1, 2] period.months_of_interest = months_of_interest ERAI_label = "ERA-Interim" vname_to_level_erai = { default_varname_mappings.T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID), default_varname_mappings.U_WE: VerticalLevel(1, level_kinds.HYBRID), default_varname_mappings.V_SN: VerticalLevel(1, level_kinds.HYBRID), } label_to_config = OrderedDict([ # ERA-Interim (ERAI_label, { "base_folder": "/RECH/data/Driving_data/Offline/ERA-Interim_0.75/6h_Analysis", "data_source_type": data_source_types.ALL_VARS_IN_A_FOLDER_OF_RPN_FILES, "min_dt": timedelta(hours=6), "varname_mapping": default_varname_mappings.vname_map_CRCM5, "level_mapping": vname_to_level_erai, "offset_mapping": default_varname_mappings.vname_to_offset_CRCM5, "multiplier_mapping": default_varname_mappings.vname_to_multiplier_CRCM5 }), # Add additional sources below ]) label = "ECMWF_CRCM5_FLake_0" label_to_config_CRCM5 = OrderedDict([(label, { "base_folder": "/HOME/huziy/skynet3_rech1/ens_simulations_links_diro/ENSSEASF_NorthAmerica_0.22deg_B1_0", "data_source_type": data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT, "min_dt": timedelta(hours=3), "varname_mapping": default_varname_mappings.vname_map_CRCM5, "level_mapping": vname_to_level_erai, "offset_mapping": default_varname_mappings.vname_to_offset_CRCM5, "multiplier_mapping": default_varname_mappings.vname_to_multiplier_CRCM5, "filename_prefix_mapping": default_varname_mappings.vname_to_fname_prefix_CRCM5, "out_folder": "lake_effect_analysis_{}".format(label) })]) # for i in range(1, 9): # label = "ECMWF_CRCM5_FLake_{}".format(i) # label_to_config_CRCM5[label] = label_to_config_CRCM5[label0] # label_to_config_CRCM5[label]["out_folder"] = "lake_effect_analysis_{}".format(label) # ECMWF GCM ensemble member outputs label_ECMWF_GCM = "ECMWF_GCM_1" multiplier_map_ECMWF_GCM = defaultdict(lambda: 1) multiplier_map_ECMWF_GCM[default_varname_mappings.TOTAL_PREC] = 1.0e-3 / ( 24.0 * 3600.0) # convert to M/S] label_to_config_ECMWF_GCM = OrderedDict([ (label_ECMWF_GCM, { "base_folder": "/RESCUE/skynet3_rech1/huziy/ens_simulations_links_diro/ECMWF_GCM/ensm_1", "data_source_type": data_source_types.ALL_VARS_IN_A_FOLDER_IN_NETCDF_FILES, "out_folder": "lake_effect_analysis_{}".format(label_ECMWF_GCM), "varname_mapping": { default_varname_mappings.T_AIR_2M: "tas", default_varname_mappings.TOTAL_PREC: "prlr", default_varname_mappings.U_WE: "uas", default_varname_mappings.V_SN: "vas", }, "multiplier_mapping": multiplier_map_ECMWF_GCM, "offset_mapping": defaultdict(lambda: 0), "level_mapping": defaultdict(lambda: 0), }), ]) calculate_lake_effect_snowfall(label_to_config=label_to_config_CRCM5, period=period)
def main(): period = Period( datetime(1980, 12, 1), datetime(1985, 3, 1) ) # should be consequent months_of_interest = [11, 12, 1] period.months_of_interest = months_of_interest vname_to_level_erai = { T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID), U_WE: VerticalLevel(1, level_kinds.HYBRID), V_SN: VerticalLevel(1, level_kinds.HYBRID), } ERAI_label = "ERA-Interim" label = ERAI_label label_to_config = OrderedDict( [ # ERA-Interim (label, { "base_folder": "/RECH/data/Driving_data/Offline/ERA-Interim_0.75/6h_Analysis", "data_source_type": data_source_types.ALL_VARS_IN_A_FOLDER_OF_RPN_FILES, "min_dt": timedelta(hours=6), "varname_mapping": vname_map_CRCM5, "level_mapping": vname_to_level_erai, "offset_mapping": vname_to_offset_CRCM5, "multiplier_mapping": vname_to_multiplier_CRCM5, "out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year) } ), # Add additional sources below ] ) import time t0 = time.time() calculate_lake_effect_snowfall_each_year_in_parallel(label_to_config=label_to_config, period=period, nprocs_to_use=5) print("Execution time: {} s".format(time.time() - t0)) #calculate_lake_effect_snowfall(label_to_config=label_to_config, period=period) label = "CRCM5_NEMO" vname_map = {} vname_map.update(vname_map_CRCM5) vname_map.update({ default_varname_mappings.SNOWFALL_RATE: "U3" }) label_to_config_CRCM5 = OrderedDict([( label, { "base_folder": "/HOME/huziy/skynet3_rech1/CRCM5_outputs/coupled-GL-NEMO1h/selected_fields", "data_source_type": data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT_VNAME_IN_FNAME, "min_dt": timedelta(hours=3), "varname_mapping": vname_map, "level_mapping": vname_to_level_erai, "offset_mapping": vname_to_offset_CRCM5, "multiplier_mapping": vname_to_multiplier_CRCM5, "out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year) } )]) # calculate_lake_effect_snowfall_each_year_in_parallel(label_to_config=label_to_config_CRCM5, # period=period, # nprocs_to_use=16) label = "CRCM5_Hostetler" vname_map = {} vname_map.update(vname_map_CRCM5) vname_map.update({ default_varname_mappings.SNOWFALL_RATE: "U3" }) label_to_config_CRCM5 = OrderedDict([( label, { "base_folder": "/RECH2/huziy/coupling/GL_440x260_0.1deg_GL_with_Hostetler/Samples_selected", "data_source_type": data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT_VNAME_IN_FNAME, "min_dt": timedelta(hours=3), "varname_mapping": vname_map, "level_mapping": vname_to_level_erai, "offset_mapping": vname_to_offset_CRCM5, "multiplier_mapping": vname_to_multiplier_CRCM5, "filename_prefix_mapping": vname_to_fname_prefix_CRCM5, "out_folder": "lake_effect_analysis_{}_{}-{}".format(label, period.start.year, period.end.year) } )])
def main_crcm5_nemo(): label = "CRCM5_NEMO" period = Period(datetime(1980, 1, 1), datetime(2015, 12, 31)) pool = Pool(processes=10) input_params = [] for month_start in period.range("months"): month_end = month_start.add(months=1).subtract(seconds=1) current_month_period = Period(month_start, month_end) current_month_period.months_of_interest = [ month_start.month, ] vname_to_level_erai = { T_AIR_2M: VerticalLevel(1, level_kinds.HYBRID), U_WE: VerticalLevel(1, level_kinds.HYBRID), V_SN: VerticalLevel(1, level_kinds.HYBRID), } vname_map = {} vname_map.update(vname_map_CRCM5) vname_map = {} vname_map.update(vname_map_CRCM5) vname_map.update({default_varname_mappings.SNOWFALL_RATE: "SN"}) label_to_config = OrderedDict([(label, { DataManager.SP_BASE_FOLDER: "/snow3/huziy/NEI/GL/erai0.75deg_driven/GL_with_NEMO_dtN_1h_and_30min/Samples", DataManager.SP_DATASOURCE_TYPE: data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT, DataManager.SP_INTERNAL_TO_INPUT_VNAME_MAPPING: vname_map, DataManager.SP_LEVEL_MAPPING: vname_to_level_erai, DataManager.SP_OFFSET_MAPPING: vname_to_offset_CRCM5, DataManager.SP_MULTIPLIER_MAPPING: vname_to_multiplier_CRCM5, DataManager.SP_VARNAME_TO_FILENAME_PREFIX_MAPPING: default_varname_mappings.vname_to_fname_prefix_CRCM5, "out_folder": "lake_effect_analysis_{}_{}-{}_monthly".format( label, period.start.year, period.end.year) })]) kwargs = dict( label_to_config=label_to_config, period=current_month_period, months_of_interest=current_month_period.months_of_interest, nprocs_to_use=1) print(current_month_period.months_of_interest) input_params.append(kwargs) # execute in parallel pool.map(monthly_func, input_params)