def test_generate_joules_inflow(): """ Checks generating inflow file from Joules LSM """ print("TEST 7: TEST GENERATE INFLOW FILE FROM Joules DATA") rapid_output_path = os.path.join(OUTPUT_DATA_PATH, "output", "u-k") #Create testing environment try: os.mkdir(RAPID_DATA_PATH) except OSError: pass try: os.mkdir(os.path.join(OUTPUT_DATA_PATH, "output")) except OSError: pass try: copytree(os.path.join(COMPARE_DATA_PATH, "gis", "u-k"), os.path.join(RAPID_DATA_PATH, "u-k")) except OSError: pass #run main process run_lsm_rapid_process( rapid_executable_location=RAPID_EXE_PATH, cygwin_bin_location=CYGWIN_BIN_PATH, rapid_io_files_location=OUTPUT_DATA_PATH, lsm_data_location=os.path.join(LSM_INPUT_DATA_PATH, 'joules'), simulation_start_datetime=datetime(1980, 1, 1), simulation_end_datetime=datetime(2014, 1, 31), file_datetime_re_pattern=r'\d{8}_\d{2}', file_datetime_pattern="%Y%m%d_%H", generate_rapid_namelist_file=False, run_rapid_simulation=False, use_all_processors=True, ) #CHECK OUTPUT m3_file_name = "m3_riv_bas_met_office_joules_3hr_20080803to20080803.nc" generated_m3_file = os.path.join(rapid_output_path, m3_file_name) generated_m3_file_solution = os.path.join(INFLOW_COMPARE_DATA_PATH, m3_file_name) #check other info in netcdf file d1 = Dataset(generated_m3_file) d2 = Dataset(generated_m3_file_solution) assert_almost_equal(d1.variables['m3_riv'][:], d2.variables['m3_riv'][:], decimal=5) if 'rivid' in d2.variables.keys(): ok_((d1.variables['rivid'][:] == d2.variables['rivid'][:]).all()) if 'lat' in d2.variables.keys(): ok_((d1.variables['lat'][:] == d2.variables['lat'][:]).all()) if 'lon' in d2.variables.keys(): ok_((d1.variables['lon'][:] == d2.variables['lon'][:]).all()) d1.close() d2.close() #cleanup rmtree(os.path.join(OUTPUT_DATA_PATH, "input")) rmtree(os.path.join(OUTPUT_DATA_PATH, "output"))
def test_generate_era20cm_inflow(self): """ Checks generating inflow file from ERA 20CM LSM """ print("TEST 3: TEST GENERATE INFLOW FILE FROM ERA 20CM DATA") rapid_input_path, rapid_output_path = self._setup_automated("x-x") run_lsm_rapid_process( rapid_executable_location=self.RAPID_EXE_PATH, cygwin_bin_location=self.CYGWIN_BIN_PATH, rapid_io_files_location=self.OUTPUT_DATA_PATH, lsm_data_location=os.path.join(self.LSM_INPUT_DATA_PATH, 'era20cm'), simulation_start_datetime=datetime(1980, 1, 1), simulation_end_datetime=datetime(2014, 1, 31), ensemble_list=range(10), generate_rapid_namelist_file=False, run_rapid_simulation=False, use_all_processors=True, ) for i in range(10): #CHECK OUTPUT #m3_riv m3_file_name = "m3_riv_bas_era_20cm_t159_3hr_20000129to20000130_{0}.nc".format( i) generated_m3_file = os.path.join(rapid_output_path, m3_file_name) generated_m3_file_solution = os.path.join( self.INFLOW_COMPARE_DATA_PATH, m3_file_name) self._compare_m3(generated_m3_file, generated_m3_file_solution)
def start_run(self, start_utc_dt, end_utc_dt, init_first_run=False): model_run_dt_list = self.prepare_model_run(start_utc_dt, end_utc_dt) for i in range(len(model_run_dt_list)): model_run_dt = model_run_dt_list[i] print("****************************************Run RAPID {}".format(model_run_dt)) model_run_dt_str = model_run_dt.strftime("%Y%m%d%H") model_run_root = os.path.join(self.target_path, model_run_dt_str) previous_model_run_dt = model_run_dt - self.model_run_interval_tdelta previous_model_run_dt_str = previous_model_run_dt.strftime("%Y%m%d%H") previous_model_run_root = os.path.join(self.target_path, previous_model_run_dt_str) # Run Cosmo run_lsm_rapid_process( rapid_executable_location=self.rapid_executable_path, rapid_io_files_location=model_run_root, lsm_data_location=os.path.join(model_run_root, "data", model_run_dt_str), use_all_processors=False, # defaults to use all processors available num_processors=1, # you can change this number if use_all_processors=False generate_initialization_file=True, timedelta_between_simulations=datetime.timedelta(hours=12), initial_flows_file=os.path.join(previous_model_run_root, "qinit.csv") if i > 0 or init_first_run else None ) print("*************************Done {}".format(model_run_dt)) pass print("****************************All Done")
def test_generate_era20cm_inflow(self): """ Checks generating inflow file from ERA 20CM LSM """ print("TEST 3: TEST GENERATE INFLOW FILE FROM ERA 20CM DATA") rapid_input_path, rapid_output_path = self._setup_automated("x-x") run_lsm_rapid_process( rapid_executable_location=self.RAPID_EXE_PATH, cygwin_bin_location=self.CYGWIN_BIN_PATH, rapid_io_files_location=self.OUTPUT_DATA_PATH, lsm_data_location=os.path.join(self.LSM_INPUT_DATA_PATH, 'era20cm'), simulation_start_datetime=datetime(1980, 1, 1), simulation_end_datetime=datetime(2014, 1, 31), ensemble_list=range(10), generate_rapid_namelist_file=False, run_rapid_simulation=False, use_all_processors=True, ) for i in range(10): #CHECK OUTPUT #m3_riv m3_file_name = "m3_riv_bas_era_20cm_t159_3hr_20000129to20000130_{0}.nc".format(i) generated_m3_file = os.path.join(rapid_output_path, m3_file_name) generated_m3_file_solution = os.path.join(self.INFLOW_COMPARE_DATA_PATH, m3_file_name) self._compare_m3(generated_m3_file,generated_m3_file_solution)
def test_generate_joules_inflow(self): """ Checks generating inflow file from Joules LSM """ print("TEST 7: TEST GENERATE INFLOW FILE FROM Joules DATA") rapid_input_path, rapid_output_path = self._setup_automated("u-k") #run main process run_lsm_rapid_process( rapid_executable_location=self.RAPID_EXE_PATH, cygwin_bin_location=self.CYGWIN_BIN_PATH, rapid_io_files_location=self.OUTPUT_DATA_PATH, lsm_data_location=os.path.join(self.LSM_INPUT_DATA_PATH, 'joules'), simulation_start_datetime=datetime(1980, 1, 1), simulation_end_datetime=datetime(2014, 1, 31), file_datetime_re_pattern=r'\d{8}_\d{2}', file_datetime_pattern="%Y%m%d_%H", generate_rapid_namelist_file=False, run_rapid_simulation=False, use_all_processors=True, ) #CHECK OUTPUT m3_file_name = "m3_riv_bas_met_office_joules_3hr_20080803to20080803.nc" generated_m3_file = os.path.join(rapid_output_path, m3_file_name) generated_m3_file_solution = os.path.join( self.INFLOW_COMPARE_DATA_PATH, m3_file_name) #check other info in netcdf file self._compare_m3(generated_m3_file, generated_m3_file_solution)
def test_generate_joules_inflow(self): """ Checks generating inflow file from Joules LSM """ print("TEST 7: TEST GENERATE INFLOW FILE FROM Joules DATA") rapid_input_path, rapid_output_path = self._setup_automated("u-k") #run main process run_lsm_rapid_process( rapid_executable_location=self.RAPID_EXE_PATH, cygwin_bin_location=self.CYGWIN_BIN_PATH, rapid_io_files_location=self.OUTPUT_DATA_PATH, lsm_data_location=os.path.join(self.LSM_INPUT_DATA_PATH, 'joules'), simulation_start_datetime=datetime(1980, 1, 1), simulation_end_datetime=datetime(2014, 1, 31), file_datetime_re_pattern = r'\d{8}_\d{2}', file_datetime_pattern = "%Y%m%d_%H", generate_rapid_namelist_file=False, run_rapid_simulation=False, use_all_processors=True, ) #CHECK OUTPUT m3_file_name = "m3_riv_bas_met_office_joules_3hr_20080803to20080803.nc" generated_m3_file = os.path.join(rapid_output_path, m3_file_name) generated_m3_file_solution = os.path.join(self.INFLOW_COMPARE_DATA_PATH, m3_file_name) #check other info in netcdf file self._compare_m3(generated_m3_file, generated_m3_file_solution)
def test_run_era_interim_inflow(self): """ Checks generating inflow file from ERA Interim LSM """ print("TEST 1: TEST GENERATE INFLOW FILE FROM ERA INTERIM DATA") rapid_input_path, rapid_output_path = self._setup_automated("x-x") #run main process run_lsm_rapid_process( rapid_executable_location=self.RAPID_EXE_PATH, cygwin_bin_location=self.CYGWIN_BIN_PATH, rapid_io_files_location=self.OUTPUT_DATA_PATH, lsm_data_location=os.path.join(self.LSM_INPUT_DATA_PATH, 'erai3'), simulation_start_datetime=datetime(1980, 1, 1), simulation_end_datetime=datetime(2014, 1, 31), generate_rapid_namelist_file=False, run_rapid_simulation=True, generate_return_periods_file=False, generate_seasonal_initialization_file=False, generate_initialization_file=True, use_all_processors=True, ) #CHECK OUTPUT #m3_riv m3_file_name = "m3_riv_bas_erai_t511_3hr_20030121to20030122.nc" generated_m3_file = os.path.join(rapid_output_path, m3_file_name) generated_m3_file_solution = os.path.join(self.INFLOW_COMPARE_DATA_PATH, m3_file_name) self._compare_m3(generated_m3_file,generated_m3_file_solution) #qout file qout_file_name = "Qout_erai_t511_3hr_20030121to20030122.nc" generated_qout_file = os.path.join(rapid_output_path, qout_file_name) generated_qout_file_solution = os.path.join(self.INFLOW_COMPARE_DATA_PATH, qout_file_name) d1 = Dataset(generated_qout_file) d2 = Dataset(generated_qout_file_solution) assert_almost_equal(d1.variables['Qout'][:], d2.variables['Qout'][:], decimal=5) ok_((d1.variables['rivid'][:] == d2.variables['rivid'][:]).all()) ok_((d1.variables['time'][:] == d2.variables['time'][:]).all()) if 'lat' in d2.variables.keys(): ok_((d1.variables['lat'][:] == d2.variables['lat'][:]).all()) if 'lon' in d2.variables.keys(): ok_((d1.variables['lon'][:] == d2.variables['lon'][:]).all()) d1.close() d2.close() #initialization file qinit_file_name = "qinit_erai_t511_3hr_20030121to20030122.csv" generated_qinit_file = os.path.join(rapid_input_path, qinit_file_name) generated_qinit_file_solution = os.path.join(self.INFLOW_COMPARE_DATA_PATH, qinit_file_name) ok_(compare_csv_decimal_files(generated_qinit_file, generated_qinit_file_solution)) #additional cleanup remove_files(generated_qinit_file)
def run_era5_rapid_simulation(region, rapid_executable_location, lsm_data_location, master_input_location, master_output_location, prev_sim_end_date_str, simulation_start_datetime, simulation_end_datetime): print('Region: ' + region) # Define rapid input and output folder for specific region rapid_input_location = os.path.join(master_input_location, region) print('Rapid Input Folder: ' + rapid_input_location) rapid_output_location = os.path.join(master_output_location, region) if not os.path.exists(rapid_output_location): print('Creating Output Folder...') os.makedirs(rapid_output_location) print('Rapid Output Folder: ' + rapid_output_location) # Check for initial flows file initial_flows_file = '' for file in os.listdir(rapid_input_location): if file.endswith(prev_sim_end_date_str + '.csv') and file.startswith('qinit_era5'): initial_flows_file = os.path.join(rapid_input_location, file) print('Initial FLows File: ' + initial_flows_file) if initial_flows_file == '': print('ERROR: Initial Flows File not found for the region ' + region) raise Exception('Initial Flows File not found for the region ' + region) # Run RAPID print('Starting RAPID process for ' + region + '...') run_lsm_rapid_process( rapid_executable_location=rapid_executable_location, lsm_data_location= lsm_data_location, # folder containing ERA-5 runoff data rapid_input_location=rapid_input_location, rapid_output_location=rapid_output_location, initial_flows_file=initial_flows_file, simulation_start_datetime=simulation_start_datetime, simulation_end_datetime= simulation_end_datetime, # will stop at last date of available runoff grid run_rapid_simulation= True, # if you want to run RAPID after generating inflow file generate_rapid_namelist_file= False, # if you want to run RAPID manually later generate_initialization_file= True, # if you want to generate qinit file from end of RAPID simulation use_all_processors=True, # defaults to use all processors available # num_processors=1 # you can change this number if use_all_processors=False ) print('------------------------------\n')
def _run_automatic(self, lsm_folder_name): """ run for automatic method """ #run main process run_lsm_rapid_process( rapid_executable_location=self.RAPID_EXE_PATH, cygwin_bin_location=self.CYGWIN_BIN_PATH, rapid_io_files_location=self.OUTPUT_DATA_PATH, lsm_data_location=os.path.join(self.LSM_INPUT_DATA_PATH, lsm_folder_name), simulation_start_datetime=datetime(1980, 1, 1), simulation_end_datetime=datetime(2014, 12, 31), generate_rapid_namelist_file=False, run_rapid_simulation=False, use_all_processors=True, )
def main(): #current_utc_dt_str = sys.argv[1] #print (current_utc_dt_str) #current_utc_dt = datetime.datetime.strptime(current_utc_dt_str, "%Y%m%d%H%M") #start_utc_dt = datetime.datetime.utcnow() #end_utc_dt = None start_utc_dt = datetime.datetime(year=2018, month=8, day=8, hour=0) end_utc_dt = datetime.datetime(year=2018, month=8, day=9, hour=0) model_run_dt_list = prepare_model_run(start_utc_dt, end_utc_dt) from RAPIDpy.inflow import run_lsm_rapid_process for model_run_dt in model_run_dt_list: model_run_dt_str = model_run_dt.strftime("%Y%m%d%H") model_run_root = os.path.join(target_path, model_run_dt_str) previous_model_run_dt = model_run_dt - datetime.timedelta(hours=12) previous_model_run_dt_str = previous_model_run_dt.strftime("%Y%m%d%H") previous_model_run_root = os.path.join(target_path, previous_model_run_dt_str) # Run Cosmo run_lsm_rapid_process( rapid_executable_location='/home/sherry/rapid/run/rapid', rapid_io_files_location=model_run_root, lsm_data_location=os.path.join(model_run_root, "data", model_run_dt_str), use_all_processors=False, # defaults to use all processors available num_processors=1, # you can change this number if use_all_processors=False generate_initialization_file=True, timedelta_between_simulations=datetime.timedelta(hours=12), initial_flows_file=os.path.join(previous_model_run_root, "qinit.csv"), # path to folder with LSM data # simulation_start_datetime=datetime(1980, 1, 1), # simulation_end_datetime=datetime(2017, 1, 1), # file_datetime_re_pattern = r'\d{12}', # file_datetime_pattern = "%Y%m%d%H%M", # file_datetime_re_pattern = r'\d{6}', # file_datetime_pattern = "%Y%m", # expected_time_step = "86400", # convert_one_hour_to_three=False ) pass
def test_run_era_interim_inflow_init(self): """ Checks generating inflow file from ERA Interim LSM """ rapid_input_path, rapid_output_path = self._setup_automated("x-x") # initialization file qinit_file_name = "qinit_erai_t511_3hr_20030121to20030122.csv" qinit_file = os.path.join(rapid_input_path, qinit_file_name) copy(os.path.join(self.INFLOW_COMPARE_DATA_PATH, qinit_file_name), qinit_file) # run main process output_file_info = run_lsm_rapid_process( rapid_executable_location=RAPID_EXE_PATH, cygwin_bin_location=self.CYGWIN_BIN_PATH, rapid_io_files_location=self.OUTPUT_DATA_PATH, lsm_data_location=os.path.join(self.LSM_INPUT_DATA_PATH, 'erai3'), generate_rapid_namelist_file=False, run_rapid_simulation=True, generate_initialization_file=True, initial_flows_file=qinit_file, use_all_processors=True, ) # qout file generated_qout_file = os.path.join( rapid_output_path, "Qout_erai_t511_3hr_20030121to20030122.nc") generated_qout_file_solution = os.path.join( self.INFLOW_COMPARE_DATA_PATH, "Qout_erai_t511_3hr_20030121to20030122_init.nc") # check output file info assert output_file_info[0]['x-x']['m3_riv'] == os.path.join( rapid_output_path, "m3_riv_bas_erai_t511_3hr_20030121to20030122.nc") assert output_file_info[0]['x-x']['qout'] == generated_qout_file d1 = Dataset(generated_qout_file) d2 = Dataset(generated_qout_file_solution) assert_almost_equal(d1.variables['Qout'][:], d2.variables['Qout'][:], decimal=0) assert (d1.variables['rivid'][:] == d2.variables['rivid'][:]).all() assert (d1.variables['time'][:] == d2.variables['time'][:]).all() if 'lat' in d2.variables.keys(): assert (d1.variables['lat'][:] == d2.variables['lat'][:]).all() if 'lon' in d2.variables.keys(): assert (d1.variables['lon'][:] == d2.variables['lon'][:]).all() d1.close() d2.close()
def _run_automatic(self, lsm_folder_name, watershed_folder, file_datetime_pattern=None, file_datetime_re_pattern=None, convert_one_hour_to_three=False, expected_time_step=None, single_run=False, filter_dates=True): """ run for automatic method """ rapid_input_path, rapid_output_path = self._setup_automated( watershed_folder) run_input_folder = None run_output_folder = None rapid_io_folder = self.OUTPUT_DATA_PATH if single_run: run_input_folder = rapid_input_path run_output_folder = rapid_output_path rapid_io_folder = None start_datetime = None end_datetime = None if filter_dates: start_datetime = datetime(1980, 1, 1) end_datetime = datetime(2014, 12, 31) # run main process output_file_info = run_lsm_rapid_process( rapid_executable_location=RAPID_EXE_PATH, cygwin_bin_location=self.CYGWIN_BIN_PATH, rapid_io_files_location=rapid_io_folder, rapid_input_location=run_input_folder, rapid_output_location=run_output_folder, lsm_data_location=os.path.join(self.LSM_INPUT_DATA_PATH, lsm_folder_name), simulation_start_datetime=start_datetime, simulation_end_datetime=end_datetime, generate_rapid_namelist_file=False, run_rapid_simulation=False, use_all_processors=True, file_datetime_pattern=file_datetime_pattern, file_datetime_re_pattern=file_datetime_re_pattern, convert_one_hour_to_three=convert_one_hour_to_three, expected_time_step=expected_time_step, ) return rapid_input_path, rapid_output_path, output_file_info
from datetime import datetime, timedelta from RAPIDpy.inflow import run_lsm_rapid_process #------------------------------------------------------------------------------ #main process #------------------------------------------------------------------------------ if __name__ == "__main__": run_lsm_rapid_process( rapid_executable_location='/home/sherry/rapid/run/rapid', rapid_io_files_location='/home/sherry/Downloads/hiwat/', #lsm_data_location='/home/sherry/Downloads/hiwat/mod_files', #path to folder with LSM data lsm_data_location='/home/sherry/Downloads/hiwat/data/merged', # model_run_datetime=datetime(2017, 8, 5, 18), # YYYY, MM, DD, HH # simulation_start_datetime=timedelta(hours=1), # simulation_end_datetime=timedelta(hours=72), use_all_processors=False, num_processors=1, #file_datetime_re_pattern = r'\d{12}', #file_datetime_pattern = "%Y%m%d%H%M", #file_datetime_re_pattern = r'\d{6}', #file_datetime_pattern = "%Y%m", #expected_time_step = "3600", #convert_one_hour_to_three=False )
def test_generate_era20cm_inflow(): """ Checks generating inflow file from ERA 20CM LSM """ print("TEST 3: TEST GENERATE INFLOW FILE FROM ERA 20CM DATA") rapid_output_path = os.path.join(OUTPUT_DATA_PATH, "output", "x-x") #Create testing environment try: os.mkdir(RAPID_DATA_PATH) except OSError: pass try: os.mkdir(os.path.join(OUTPUT_DATA_PATH, "output")) except OSError: pass try: copytree(os.path.join(COMPARE_DATA_PATH, "gis", "x-x"), os.path.join(RAPID_DATA_PATH, "x-x")) except OSError: pass run_lsm_rapid_process( rapid_executable_location=RAPID_EXE_PATH, cygwin_bin_location=CYGWIN_BIN_PATH, rapid_io_files_location=OUTPUT_DATA_PATH, lsm_data_location=os.path.join(LSM_INPUT_DATA_PATH, 'era20cm'), simulation_start_datetime=datetime(1980, 1, 1), simulation_end_datetime=datetime(2014, 1, 31), ensemble_list=range(10), generate_rapid_namelist_file=False, run_rapid_simulation=False, use_all_processors=True, ) for i in range(10): #CHECK OUTPUT #m3_riv m3_file_name = "m3_riv_bas_era_20cm_t159_3hr_20000129to20000130_{0}.nc".format( i) generated_m3_file = os.path.join(rapid_output_path, m3_file_name) generated_m3_file_solution = os.path.join(INFLOW_COMPARE_DATA_PATH, m3_file_name) #check other info in netcdf file d1 = Dataset(generated_m3_file) d2 = Dataset(generated_m3_file_solution) assert_almost_equal(d1.variables['m3_riv'][:], d2.variables['m3_riv'][:], decimal=5) if 'rivid' in d2.variables.keys(): ok_((d1.variables['rivid'][:] == d2.variables['rivid'][:]).all()) if 'lat' in d2.variables.keys(): ok_((d1.variables['lat'][:] == d2.variables['lat'][:]).all()) if 'lon' in d2.variables.keys(): ok_((d1.variables['lon'][:] == d2.variables['lon'][:]).all()) d1.close() d2.close() #cleanup rmtree(os.path.join(OUTPUT_DATA_PATH, "input")) rmtree(os.path.join(OUTPUT_DATA_PATH, "output"))
from datetime import datetime from RAPIDpy.inflow import run_lsm_rapid_process # ------------------------------------------------------------------------------ # main process # ------------------------------------------------------------------------------ if __name__ == "__main__": run_lsm_rapid_process( rapid_executable_location='/root/rapid/run/rapid', rapid_io_files_location='/home/rapid-io', lsm_data_location= '/home/ERA-5_hourly_data', # path to folder with LSM data simulation_start_datetime=datetime(1979, 1, 1), simulation_end_datetime=datetime(2018, 12, 31), generate_rapid_namelist_file= True, # if you want to run RAPID manually later run_rapid_simulation= True, # if you want to run RAPID after generating inflow file generate_return_periods_file= True, # if you want to get return period file from RAPID simulation return_period_method='weibull', generate_seasonal_averages_file=True, generate_seasonal_initialization_file= True, # if you want to get seasonal init file from RAPID simulation generate_initialization_file= False, # if you want to generate qinit file from end of RAPID simulation use_all_processors=True)
from datetime import datetime from RAPIDpy.inflow import run_lsm_rapid_process #------------------------------------------------------------------------------ #main process #------------------------------------------------------------------------------ if __name__ == "__main__": run_lsm_rapid_process( rapid_executable_location='/home/sherry/rapid/run/rapid', rapid_io_files_location='/home/sherry/Downloads/ECMWF/rapid-io', lsm_data_location= '/home/sherry/Downloads/ECMWF/ecmwf/data', #path to folder with LSM data generate_rapid_namelist_file= True, #if you want to run RAPID manually later run_rapid_simulation= True, #if you want to run RAPID after generating inflow file generate_return_periods_file= False, #if you want to get return period file from RAPID simulation generate_seasonal_averages_file=False, generate_seasonal_initialization_file= False, #if you want to get seasonal init file from RAPID simulation generate_initialization_file= False, #if you want to generate qinit file from end of RAPID simulation use_all_processors=True, #defaults to use all processors available # num_processors=1, #you can change this number if use_all_processors=False # cygwin_bin_location="" #if you are using Windows with Cygwin )
def test_run_era_interim_inflow(): """ Checks generating inflow file from ERA Interim LSM """ print("TEST 1: TEST GENERATE INFLOW FILE FROM ERA INTERIM DATA") rapid_input_path = os.path.join(RAPID_DATA_PATH, "x-x") rapid_output_path = os.path.join(OUTPUT_DATA_PATH, "output", "x-x") #Create testing environment try: os.mkdir(RAPID_DATA_PATH) except OSError: pass try: os.mkdir(os.path.join(OUTPUT_DATA_PATH, "output")) except OSError: pass try: copytree(os.path.join(COMPARE_DATA_PATH, "gis", "x-x"), rapid_input_path) except OSError: pass #run main process run_lsm_rapid_process( rapid_executable_location=RAPID_EXE_PATH, cygwin_bin_location=CYGWIN_BIN_PATH, rapid_io_files_location=OUTPUT_DATA_PATH, lsm_data_location=os.path.join(LSM_INPUT_DATA_PATH, 'erai3'), simulation_start_datetime=datetime(1980, 1, 1), simulation_end_datetime=datetime(2014, 1, 31), generate_rapid_namelist_file=False, run_rapid_simulation=True, generate_return_periods_file=False, generate_seasonal_initialization_file=False, generate_initialization_file=True, use_all_processors=True, ) #CHECK OUTPUT #m3_riv m3_file_name = "m3_riv_bas_erai_t511_3hr_20030121to20030122.nc" generated_m3_file = os.path.join(rapid_output_path, m3_file_name) generated_m3_file_solution = os.path.join(INFLOW_COMPARE_DATA_PATH, m3_file_name) #check other info in netcdf file d1 = Dataset(generated_m3_file) d2 = Dataset(generated_m3_file_solution) assert_almost_equal(d1.variables['m3_riv'][:], d2.variables['m3_riv'][:], decimal=5) if 'rivid' in d2.variables.keys(): ok_((d1.variables['rivid'][:] == d2.variables['rivid'][:]).all()) if 'lat' in d2.variables.keys(): ok_((d1.variables['lat'][:] == d2.variables['lat'][:]).all()) if 'lon' in d2.variables.keys(): ok_((d1.variables['lon'][:] == d2.variables['lon'][:]).all()) d1.close() d2.close() #qout file qout_file_name = "Qout_erai_t511_3hr_20030121to20030122.nc" generated_qout_file = os.path.join(rapid_output_path, qout_file_name) generated_qout_file_solution = os.path.join(INFLOW_COMPARE_DATA_PATH, qout_file_name) d1 = Dataset(generated_qout_file) d2 = Dataset(generated_qout_file_solution) assert_almost_equal(d1.variables['Qout'][:], d2.variables['Qout'][:], decimal=5) ok_((d1.variables['rivid'][:] == d2.variables['rivid'][:]).all()) ok_((d1.variables['time'][:] == d2.variables['time'][:]).all()) if 'lat' in d2.variables.keys(): ok_((d1.variables['lat'][:] == d2.variables['lat'][:]).all()) if 'lon' in d2.variables.keys(): ok_((d1.variables['lon'][:] == d2.variables['lon'][:]).all()) d1.close() d2.close() #initialization file qinit_file_name = "qinit_erai_t511_3hr_20030121to20030122.csv" generated_qinit_file = os.path.join(rapid_input_path, qinit_file_name) generated_qinit_file_solution = os.path.join(INFLOW_COMPARE_DATA_PATH, qinit_file_name) ok_( compare_csv_decimal_files(generated_qinit_file, generated_qinit_file_solution)) #cleanup remove_files(generated_qinit_file) rmtree(os.path.join(OUTPUT_DATA_PATH, "input")) rmtree(os.path.join(OUTPUT_DATA_PATH, "output"))
def test_run_era_interim_inflow(self): """ Checks generating inflow file from ERA Interim LSM """ rapid_input_path, rapid_output_path = self._setup_automated("x-x") # run main process output_file_info = run_lsm_rapid_process( rapid_executable_location=RAPID_EXE_PATH, cygwin_bin_location=self.CYGWIN_BIN_PATH, rapid_io_files_location=self.OUTPUT_DATA_PATH, lsm_data_location=os.path.join(self.LSM_INPUT_DATA_PATH, 'erai3'), simulation_start_datetime=datetime(1980, 1, 1), simulation_end_datetime=datetime(2014, 1, 31), generate_rapid_namelist_file=False, run_rapid_simulation=True, generate_return_periods_file=False, generate_seasonal_initialization_file=False, generate_initialization_file=True, use_all_processors=True, ) # CHECK OUTPUT # m3_riv m3_file_name = "m3_riv_bas_erai_t511_3hr_20030121to20030122.nc" generated_m3_file = os.path.join(rapid_output_path, m3_file_name) generated_m3_file_solution = os.path.join( self.INFLOW_COMPARE_DATA_PATH, m3_file_name) self._compare_m3(generated_m3_file, generated_m3_file_solution) # qout file qout_file_name = "Qout_erai_t511_3hr_20030121to20030122.nc" generated_qout_file = os.path.join(rapid_output_path, qout_file_name) generated_qout_file_solution = os.path.join( self.INFLOW_COMPARE_DATA_PATH, qout_file_name) d1 = Dataset(generated_qout_file) d2 = Dataset(generated_qout_file_solution) assert_almost_equal(d1.variables['Qout'][:], d2.variables['Qout'][:], decimal=5) assert (d1.variables['rivid'][:] == d2.variables['rivid'][:]).all() assert (d1.variables['time'][:] == d2.variables['time'][:]).all() if 'lat' in d2.variables.keys(): assert (d1.variables['lat'][:] == d2.variables['lat'][:]).all() if 'lon' in d2.variables.keys(): assert (d1.variables['lon'][:] == d2.variables['lon'][:]).all() d1.close() d2.close() # check output file info assert output_file_info[0]['x-x']['m3_riv'] == generated_m3_file assert output_file_info[0]['x-x']['qout'] == generated_qout_file # initialization file qinit_file_name = "qinit_erai_t511_3hr_20030121to20030122.csv" generated_qinit_file = os.path.join(rapid_input_path, qinit_file_name) generated_qinit_file_solution = os.path.join( self.INFLOW_COMPARE_DATA_PATH, qinit_file_name) assert compare_csv_decimal_files(generated_qinit_file, generated_qinit_file_solution) # additional cleanup remove_files(generated_qinit_file)
import os import sys from datetime import datetime from re import split from RAPIDpy.inflow import run_lsm_rapid_process # home = os.path.expanduser('~') # print(home) # rapid_exec = 'rapid/run/rapid' # era_data = 'host_share/era5_data/era5_runoff_2001to2015' # era_data = 'host_share/era_data' start_date = list(map(int, split("/|-", str(sys.argv[4])))) end_date = list(map(int, split("/|-", str(sys.argv[5])))) run_lsm_rapid_process( rapid_executable_location=str(sys.argv[1]), rapid_io_files_location=str(sys.argv[2]), lsm_data_location=str(sys.argv[3]), simulation_start_datetime=datetime(*start_date), # datetime(2010, 1, 1), simulation_end_datetime=datetime(*end_date), # datetime(2014, 12, 31), generate_rapid_namelist_file=True, run_rapid_simulation=True, generate_return_periods_file=True, return_period_method='weibull', generate_seasonal_averages_file=True, generate_seasonal_initialization_file=True, generate_initialization_file=False, use_all_processors=False, num_processors=1, )
def run_lsm_forecast_process(rapid_executable_location, rapid_io_files_location, lsm_forecast_location, main_log_directory, timedelta_between_forecasts=timedelta(seconds=12 * 3600), historical_data_location="", warning_flow_threshold=None): """ Parameters ---------- rapid_executable_location: str Path to RAPID executable. rapid_io_files_location: str Path ro RAPID input/output directory. lsm_forecast_location: str Path to WRF forecast directory. main_log_directory: str Path to directory to store main logs. timedelta_between_forecasts: :obj:`datetime.timedelta` Time difference between forecasts. historical_data_location: str, optional Path to return period and seasonal data. warning_flow_threshold: float, optional Minimum value for return period in m3/s to generate warning. Default is None. """ time_begin_all = datetime.utcnow() log_file_path = os.path.join( main_log_directory, "spt_compute_lsm_{0}.log".format( time_begin_all.strftime("%y%m%d%H%M%S"))) with CaptureStdOutToLog(log_file_path): clean_main_logs(main_log_directory, prepend="spt_compute_lsm_") # get list of correclty formatted rapid input directories in rapid directory rapid_input_directories = get_valid_watershed_list( os.path.join(rapid_io_files_location, "input")) current_forecast_start_datetime = \ determine_start_end_timestep(sorted(glob(os.path.join(lsm_forecast_location, "*.nc"))))[0] forecast_date_string = current_forecast_start_datetime.strftime( "%Y%m%dt%H") # look for past forecast qinit past_forecast_date_string = ( current_forecast_start_datetime - timedelta_between_forecasts).strftime("%Y%m%dt%H") init_file_name = 'Qinit_{0}.csv'.format(past_forecast_date_string) # PHASE 1: SEASONAL INITIALIZATION ON FIRST RUN if historical_data_location and os.path.exists( historical_data_location): seasonal_init_job_list = [] # iterate over models for rapid_input_directory in rapid_input_directories: seasonal_master_watershed_input_directory = os.path.join( rapid_io_files_location, "input", rapid_input_directory) init_file_path = os.path.join( seasonal_master_watershed_input_directory, init_file_name) historical_watershed_directory = os.path.join( historical_data_location, rapid_input_directory) if os.path.exists(historical_watershed_directory): seasonal_streamflow_file = glob( os.path.join(historical_watershed_directory, "seasonal_average*.nc")) if seasonal_streamflow_file and not os.path.exists( init_file_path): seasonal_init_job_list.append(( seasonal_streamflow_file[0], seasonal_master_watershed_input_directory, init_file_path, )) if seasonal_init_job_list: if len(seasonal_init_job_list) > 1: seasonal_pool = mp_Pool() seasonal_pool.imap( compute_seasonal_average_initial_flows_multiprocess_worker, seasonal_init_job_list, chunksize=1) seasonal_pool.close() seasonal_pool.join() else: compute_seasonal_average_initial_flows_multiprocess_worker( seasonal_init_job_list[0]) # PHASE 2: MAIN RUN for rapid_input_directory in rapid_input_directories: master_watershed_input_directory = os.path.join( rapid_io_files_location, "input", rapid_input_directory) master_watershed_output_directory = os.path.join( rapid_io_files_location, 'output', rapid_input_directory, forecast_date_string) watershed, subbasin = get_watershed_subbasin_from_folder( rapid_input_directory) # PHASE 2.1 RUN RAPID output_file_information = run_lsm_rapid_process( rapid_executable_location=rapid_executable_location, lsm_data_location=lsm_forecast_location, rapid_input_location=master_watershed_input_directory, rapid_output_location=master_watershed_output_directory, initial_flows_file=os.path.join( master_watershed_input_directory, init_file_name), ) forecast_file = output_file_information[0][rapid_input_directory][ 'qout'] m3_riv_file = output_file_information[0][rapid_input_directory][ 'm3_riv'] try: os.remove(m3_riv_file) except OSError: pass # PHASE 2.2: GENERATE WARNINGS forecast_directory = os.path.join(rapid_io_files_location, 'output', rapid_input_directory, forecast_date_string) historical_watershed_directory = os.path.join( historical_data_location, rapid_input_directory) print('Historical watershed dir: ', historical_watershed_directory) if os.path.exists(historical_watershed_directory): return_period_files = glob( os.path.join(historical_watershed_directory, "return_period*.nc")) print('Return periiod files: ', return_period_files) if return_period_files: print("Generating warning points for {0}-{1} from {2}". format(watershed, subbasin, forecast_date_string)) try: generate_lsm_warning_points(forecast_file, return_period_files[0], forecast_directory, warning_flow_threshold) except Exception as ex: print(ex) pass # PHASE 2.3: GENERATE INITIALIZATION FOR NEXT RUN print("Initializing flows for {0}-{1} from {2}".format( watershed, subbasin, forecast_date_string)) try: compute_initial_flows_lsm( forecast_file, master_watershed_input_directory, current_forecast_start_datetime + timedelta_between_forecasts) except Exception as ex: print(ex) pass # print info to user time_end = datetime.utcnow() print("Time Begin: {0}".format(time_begin_all)) print("Time Finish: {0}".format(time_end)) print("TOTAL TIME: {0}".format(time_end - time_begin_all))
from datetime import datetime from RAPIDpy.inflow import run_lsm_rapid_process #------------------------------------------------------------------------------ #main process #------------------------------------------------------------------------------ if __name__ == "__main__": run_lsm_rapid_process( rapid_executable_location='/home/sherry/rapid/run/rapid', rapid_io_files_location='/home/sherry/Downloads/lis', lsm_data_location='/home/sherry/Downloads/lis/data/201808', use_all_processors=False, # defaults to use all processors available num_processors=1, # you can change this number if use_all_processors=False #path to folder with LSM data #simulation_start_datetime=datetime(1980, 1, 1), #simulation_end_datetime=datetime(2017, 1, 1), #file_datetime_re_pattern = r'\d{12}', #file_datetime_pattern = "%Y%m%d%H%M", #file_datetime_re_pattern = r'\d{6}', #file_datetime_pattern = "%Y%m", # expected_time_step = "86400", #convert_one_hour_to_three=False )
from datetime import datetime from RAPIDpy.inflow import run_lsm_rapid_process import os #------------------------------------------------------------------------------ #main process #------------------------------------------------------------------------------ if __name__ == "__main__": run_lsm_rapid_process( rapid_executable_location='/root/rapid/run/rapid', rapid_input_location=os.path.abspath( os.path.join(os.path.dirname(__file__), '..', 'rapid_io_files')), lsm_data_location=os.path.abspath( os.path.join(os.path.dirname(__file__), '..', 'lsm_data')), #path to folder with LSM data rapid_output_location=os.path.abspath( os.path.join(os.path.dirname(__file__), '..', 'streamflow')), use_all_processors=False, #defaults to use all processors available num_processors= 1, #you can change this number if use_all_processors=False )