def test_generate_joules_inflow2(self): """ Checks generating inflow file from Joules LSM manually """ print("TEST 7.1: TEST GENERATE INFLOW FILE FROM Joules DATA MANUALLY") rapid_input_path, rapid_output_path = self._setup_manual("u-k") lsm_file_list = sorted(glob(os.path.join(self.LSM_INPUT_DATA_PATH, 'joules', '*.nc'))) lsm_file_list = [lsm_file_list[nldas_index:nldas_index+3] for nldas_index in range(0, len(lsm_file_list), 3)\ if len(lsm_file_list[nldas_index:nldas_index+3])==3] mp_lock = multiprocessing.Manager().Lock() inf_tool = CreateInflowFileFromLDASRunoff(lat_dim="north_south", lon_dim="east_west", lat_var="north_south", lon_var="east_west", surface_runoff_var="Qs_inst", subsurface_runoff_var="Qsb_inst", time_step_seconds=3*3600) m3_file_name = "m3_riv_bas_met_office_joules_3hr_20080803to20080803.nc" generated_m3_file = os.path.join(rapid_output_path, m3_file_name) inf_tool.generateOutputInflowFile(out_nc=generated_m3_file, start_datetime_utc=datetime(2008,3,3), number_of_timesteps=len(lsm_file_list), simulation_time_step_seconds=3*3600, in_rapid_connect_file=os.path.join(rapid_input_path, 'rapid_connect.csv'), in_rivid_lat_lon_z_file=os.path.join(rapid_input_path, 'comid_lat_lon_z.csv'), land_surface_model_description="RAPID Inflow from Met Office Joules Hourly Runoff", modeling_institution="US Army Engineer Research and Development Center" ) inf_tool.execute(nc_file_list=lsm_file_list, index_list=list(xrange(len(lsm_file_list))), in_weight_table=os.path.join(rapid_input_path, 'weight_joules.csv'), out_nc=generated_m3_file, grid_type='joules', mp_lock=mp_lock) #CHECK OUTPUT generated_m3_file_solution = os.path.join(self.INFLOW_COMPARE_DATA_PATH, m3_file_name) self._compare_m3(generated_m3_file, generated_m3_file_solution)
def test_generate_gldas2_inflow2(self): """ Checks generating inflow file from GLDAS V2 LSM manually """ print("TEST 5.1: TEST GENERATE INFLOW FILE FROM GLDAS V2 DATA MANUALLY") rapid_input_path, rapid_output_path = self._setup_manual("x-x") lsm_file_list = sorted(glob(os.path.join(self.LSM_INPUT_DATA_PATH, 'gldas2', '*.nc4'))) mp_lock = multiprocessing.Manager().Lock() inf_tool = CreateInflowFileFromLDASRunoff(lat_dim="lat", lon_dim="lon", lat_var="lat", lon_var="lon", surface_runoff_var="Qs_acc", subsurface_runoff_var="Qsb_acc", time_step_seconds=3*3600) m3_file_name = "m3_riv_bas_nasa_gldas2_3hr_20101231to20101231.nc" generated_m3_file = os.path.join(rapid_output_path, m3_file_name) inf_tool.generateOutputInflowFile(out_nc=generated_m3_file, start_datetime_utc=datetime(2010,12,31), number_of_timesteps=len(lsm_file_list), simulation_time_step_seconds=3*3600, in_rapid_connect_file=os.path.join(rapid_input_path, 'rapid_connect.csv'), in_rivid_lat_lon_z_file=os.path.join(rapid_input_path, 'comid_lat_lon_z.csv'), land_surface_model_description="RAPID Inflow from GLDAS2.0 LIS land surface model 3 hourly runoff", modeling_institution="US Army Engineer Research and Development Center" ) inf_tool.execute(nc_file_list=lsm_file_list, index_list=list(xrange(len(lsm_file_list))), in_weight_table=os.path.join(rapid_input_path, 'weight_gldas2.csv'), out_nc=generated_m3_file, grid_type='gldas2', mp_lock=mp_lock) #CHECK OUTPUT generated_m3_file_solution = os.path.join(self.INFLOW_COMPARE_DATA_PATH, m3_file_name) self._compare_m3(generated_m3_file,generated_m3_file_solution)
def test_generate_joules_inflow2(self): """ Checks generating inflow file from Joules LSM manually """ print("TEST 7.1: TEST GENERATE INFLOW FILE FROM Joules DATA MANUALLY") rapid_input_path, rapid_output_path = self._setup_manual("u-k") lsm_file_list = sorted( glob(os.path.join(self.LSM_INPUT_DATA_PATH, 'joules', '*.nc'))) lsm_file_list = [lsm_file_list[nldas_index:nldas_index+3] for nldas_index in range(0, len(lsm_file_list), 3)\ if len(lsm_file_list[nldas_index:nldas_index+3])==3] mp_lock = multiprocessing.Manager().Lock() inf_tool = CreateInflowFileFromLDASRunoff( lat_dim="north_south", lon_dim="east_west", lat_var="north_south", lon_var="east_west", surface_runoff_var="Qs_inst", subsurface_runoff_var="Qsb_inst", time_step_seconds=3 * 3600) m3_file_name = "m3_riv_bas_met_office_joules_3hr_20080803to20080803.nc" generated_m3_file = os.path.join(rapid_output_path, m3_file_name) inf_tool.generateOutputInflowFile( out_nc=generated_m3_file, start_datetime_utc=datetime(2008, 3, 3), number_of_timesteps=len(lsm_file_list), simulation_time_step_seconds=3 * 3600, in_rapid_connect_file=os.path.join(rapid_input_path, 'rapid_connect.csv'), in_rivid_lat_lon_z_file=os.path.join(rapid_input_path, 'comid_lat_lon_z.csv'), land_surface_model_description= "RAPID Inflow from Met Office Joules Hourly Runoff", modeling_institution= "US Army Engineer Research and Development Center") inf_tool.execute(nc_file_list=lsm_file_list, index_list=list(xrange(len(lsm_file_list))), in_weight_table=os.path.join(rapid_input_path, 'weight_joules.csv'), out_nc=generated_m3_file, grid_type='joules', mp_lock=mp_lock) #CHECK OUTPUT generated_m3_file_solution = os.path.join( self.INFLOW_COMPARE_DATA_PATH, m3_file_name) self._compare_m3(generated_m3_file, generated_m3_file_solution)
def test_generate_gldas2_inflow2(self): """ Checks generating inflow file from GLDAS V2 LSM manually """ print( "TEST 5.1: TEST GENERATE INFLOW FILE FROM GLDAS V2 DATA MANUALLY") rapid_input_path, rapid_output_path = self._setup_manual("x-x") lsm_file_list = sorted( glob(os.path.join(self.LSM_INPUT_DATA_PATH, 'gldas2', '*.nc4'))) mp_lock = multiprocessing.Manager().Lock() inf_tool = CreateInflowFileFromLDASRunoff( lat_dim="lat", lon_dim="lon", lat_var="lat", lon_var="lon", surface_runoff_var="Qs_acc", subsurface_runoff_var="Qsb_acc", time_step_seconds=3 * 3600) m3_file_name = "m3_riv_bas_nasa_gldas2_3hr_20101231to20101231.nc" generated_m3_file = os.path.join(rapid_output_path, m3_file_name) inf_tool.generateOutputInflowFile( out_nc=generated_m3_file, start_datetime_utc=datetime(2010, 12, 31), number_of_timesteps=len(lsm_file_list), simulation_time_step_seconds=3 * 3600, in_rapid_connect_file=os.path.join(rapid_input_path, 'rapid_connect.csv'), in_rivid_lat_lon_z_file=os.path.join(rapid_input_path, 'comid_lat_lon_z.csv'), land_surface_model_description= "RAPID Inflow from GLDAS2.0 LIS land surface model 3 hourly runoff", modeling_institution= "US Army Engineer Research and Development Center") inf_tool.execute(nc_file_list=lsm_file_list, index_list=list(xrange(len(lsm_file_list))), in_weight_table=os.path.join(rapid_input_path, 'weight_gldas2.csv'), out_nc=generated_m3_file, grid_type='gldas2', mp_lock=mp_lock) #CHECK OUTPUT generated_m3_file_solution = os.path.join( self.INFLOW_COMPARE_DATA_PATH, m3_file_name) self._compare_m3(generated_m3_file, generated_m3_file_solution)
def test_generate_nldas2_inflow2(self): """ Checks generating inflow file from NLDAS V2 LSM manually """ rapid_input_path, rapid_output_path = self._setup_manual("x-x") lsm_file_list = sorted( glob(os.path.join(self.LSM_INPUT_DATA_PATH, 'nldas2', '*.nc'))) lsm_file_list = [lsm_file_list[nldas_index:nldas_index+3] for nldas_index in range(0, len(lsm_file_list), 3)\ if len(lsm_file_list[nldas_index:nldas_index+3])==3] mp_lock = multiprocessing.Manager().Lock() inf_tool = CreateInflowFileFromLDASRunoff( lat_dim="lat_110", lon_dim="lon_110", lat_var="lat_110", lon_var="lon_110", runoff_vars=["SSRUNsfc_110_SFC_ave2h", "BGRUNsfc_110_SFC_ave2h"], ) m3_file_name = "m3_riv_bas_nasa_nldas_3hr_20030121to20030121.nc" generated_m3_file = os.path.join(rapid_output_path, m3_file_name) inf_tool.generateOutputInflowFile( out_nc=generated_m3_file, start_datetime_utc=datetime(2003, 1, 21), number_of_timesteps=len(lsm_file_list), simulation_time_step_seconds=3 * 3600, in_rapid_connect_file=os.path.join(rapid_input_path, 'rapid_connect.csv'), in_rivid_lat_lon_z_file=os.path.join(rapid_input_path, 'comid_lat_lon_z.csv'), land_surface_model_description= "RAPID Inflow from NLDAS Hourly Runoff", modeling_institution= "US Army Engineer Research and Development Center") inf_tool.execute(nc_file_list=lsm_file_list, index_list=list(xrange(len(lsm_file_list))), in_weight_table=os.path.join(rapid_input_path, 'weight_nldas.csv'), out_nc=generated_m3_file, grid_type='nldas', mp_lock=mp_lock) # CHECK OUTPUT generated_m3_file_solution = os.path.join( self.INFLOW_COMPARE_DATA_PATH, m3_file_name) self._compare_m3(generated_m3_file, generated_m3_file_solution)
def test_generate_joules_inflow2(): """ Checks generating inflow file from Joules LSM manually """ print("TEST 7.1: TEST GENERATE INFLOW FILE FROM Joules DATA MANUALLY") rapid_output_path = os.path.join(OUTPUT_DATA_PATH, "output", "u-k") rapid_input_path = os.path.join(RAPID_DATA_PATH, "u-k") #Create testing environment try: os.mkdir(RAPID_DATA_PATH) except OSError: pass try: os.mkdir(os.path.join(OUTPUT_DATA_PATH, "output")) except OSError: pass try: os.mkdir(rapid_output_path) except OSError: pass try: copytree(os.path.join(COMPARE_DATA_PATH, "gis", "u-k"), rapid_input_path) except OSError: pass lsm_file_list = sorted( glob(os.path.join(LSM_INPUT_DATA_PATH, 'joules', '*.nc'))) lsm_file_list = [lsm_file_list[nldas_index:nldas_index+3] for nldas_index in range(0, len(lsm_file_list), 3)\ if len(lsm_file_list[nldas_index:nldas_index+3])==3] mp_lock = multiprocessing.Manager().Lock() inf_tool = CreateInflowFileFromLDASRunoff(lat_dim="north_south", lon_dim="east_west", lat_var="north_south", lon_var="east_west", surface_runoff_var="Qs_inst", subsurface_runoff_var="Qsb_inst", time_step_seconds=3 * 3600) m3_file_name = "m3_riv_bas_met_office_joules_3hr_20080803to20080803.nc" generated_m3_file = os.path.join(rapid_output_path, m3_file_name) inf_tool.generateOutputInflowFile( out_nc=generated_m3_file, start_datetime_utc=datetime(2008, 3, 3), number_of_timesteps=len(lsm_file_list), simulation_time_step_seconds=3 * 3600, in_rapid_connect_file=os.path.join(rapid_input_path, 'rapid_connect.csv'), in_rivid_lat_lon_z_file=os.path.join(rapid_input_path, 'comid_lat_lon_z.csv'), land_surface_model_description= "RAPID Inflow from Met Office Joules Hourly Runoff", modeling_institution="US Army Engineer Research and Development Center" ) inf_tool.execute(nc_file_list=lsm_file_list, index_list=list(xrange(len(lsm_file_list))), in_weight_table=os.path.join(rapid_input_path, 'weight_joules.csv'), out_nc=generated_m3_file, grid_type='joules', mp_lock=mp_lock) #CHECK OUTPUT generated_m3_file_solution = os.path.join(INFLOW_COMPARE_DATA_PATH, m3_file_name) #check other info in netcdf file d1 = Dataset(generated_m3_file) d2 = Dataset(generated_m3_file_solution) assert_almost_equal(d1.variables['m3_riv'][:], d2.variables['m3_riv'][:], decimal=5) if 'rivid' in d2.variables.keys(): ok_((d1.variables['rivid'][:] == d2.variables['rivid'][:]).all()) if 'lat' in d2.variables.keys(): ok_((d1.variables['lat'][:] == d2.variables['lat'][:]).all()) if 'lon' in d2.variables.keys(): ok_((d1.variables['lon'][:] == d2.variables['lon'][:]).all()) d1.close() d2.close() #cleanup rmtree(os.path.join(OUTPUT_DATA_PATH, "input")) rmtree(os.path.join(OUTPUT_DATA_PATH, "output"))