def test_simplest(self): ret_req = get_or_create(RetrievalRequest, requester=self.user, start_year=1000, end_year=3000, id=999999) ret_req.data_request.add(self.dreq1) ret_req.save() class ArgparseNamespace(object): retrieval_id = ret_req.id no_restore = False skip_checksums = True alternative = None incoming = False self.mock_exists.side_effect = [ False, # if os.path.exists(retrieval_dir): True, # if not os.path.exists(extracted_file_path): True, # if not os.path.exists(drs_dir): False # if os.path.exists(dest_file_path): ] ns = ArgparseNamespace() get_tape_url('et:1234', [self.df1], ns) df = match_one(DataFile, name='file_one.nc') self.assertIsNotNone(df) self.mock_rename.assert_called_once_with( '/gws/nopw/j04/primavera5/.et_retrievals/ret_999999/' 'batch_01234/gws/MOHC/MY-MODEL/incoming/v12345678/file_one.nc', '/gws/nopw/j04/primavera5/stream1/CMIP6/HighResMIP/' 'MOHC/MY-MODEL/experiment/r1i1p1f1/my-table/my-var/gn/v12345678/' 'file_one.nc') self.assertTrue(df.online) self.assertEqual( df.directory, '/gws/nopw/j04/primavera5/' 'stream1/CMIP6/HighResMIP/MOHC/' 'MY-MODEL/experiment/r1i1p1f1/' 'my-table/my-var/gn/v12345678')
def _dict_to_object(dict_): """ Convert a dictionary to an object """ if '__class__' in dict_: module = __import__(dict_['__module__'], fromlist=[dict_['__class__']]) klass = getattr(module, dict_['__class__']) if dict_['__class__'] == 'PartialDateTime': inst = klass(**dict_['__kwargs__']) elif dict_['__class__'] in ('ActivityId', 'ClimateModel', 'Experiment', 'Institute', 'Project', 'VariableRequest', 'DataRequest'): inst = match_one(klass, **dict_['__kwargs__']) else: msg = ('Cannot load from JSON files class {}'.format( dict_['__class__'])) raise NotImplementedError(msg) else: inst = dict_ return inst
def _get_cmor_name(var_name, table_name): """ Check the variable request to find the actual CMOR name from a variable name. :param str var_name: The variable name from the first component of the file name. :param str table_name: The table name. :return: The CMOR name :rtype: str """ var_match = match_one(VariableRequest, cmor_name=var_name, table_name=table_name) if var_match: return var_name else: # if cmor_name doesn't match then it may be a variable where out_name # is different to cmor_name so check these var_matches = VariableRequest.objects.filter(var_name=var_name, table_name=table_name) if var_matches.count() == 1: return var_matches[0].cmor_name elif var_matches.count() == 0: msg = ('No variable request found for variable {} {}.'.format( table_name, var_name)) logger.error(msg) else: if table_name == 'E3hrPt': if var_name == 'ua': return 'ua7h' elif var_name == 'va': return 'va7h' else: msg = 'Please hard code a mapping for {} {}'.format( table_name, var_name) logger.error(msg) else: msg = 'Please hard code a mapping for {} {}'.format( table_name, var_name) logger.error(msg)
def main(args): """ Main entry point """ institute_details = { 'id': 'AWI', 'model_ids': ['AWI-CM-1-1-LR', 'AWI-CM-1-1-HR'], 'calendar': CALENDAR_STANDARD } experiments = { 'control-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2050, 1, 1) }, 'highres-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'hist-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'spinup-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(1980, 1, 1) } } variant_label = 'r1i1p1f2' # Experiment new_dreqs = [ 'rsut_E3hr', ] experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: try: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar'], rip_code=variant_label) except django.core.exceptions.MultipleObjectsReturned: logger.error('{}'.format(var_req_obj)) raise else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ # new_vreqs = ['lwsnl_Eday', 'mrros_Eday', 'snm_Eday', 'snd_Eday'] new_dreqs = [ 'mrlsl_6hrPlevPt', 'snw_6hrPlevPt', 'psl_6hrPlevPt', 'rsdt_E3hr', 'prw_E3hr', 'tas_6hrPlevPt', 'clivi_E3hr', 'mrsos_6hrPlevPt', 'ta7h_6hrPlevPt', 'clwvi_E3hr', 'pr_Prim6hr', 'rsdscs_3hr', 'va7h_6hrPlevPt', 'ts_6hrPlevPt', 'ua7h_6hrPlevPt', 'rsut_E3hr', 'clt_Prim6hr', 'ta27_6hrPlevPt', 'tsl_6hrPlevPt', 'rldscs_3hr', 'rlut_E3hr', 'huss_6hrPlevPt', 'hus27_6hrPlevPt', 'rlutcs_E3hr', 'hus7h_6hrPlevPt', 'rsuscs_3hr', 'ps_Prim6hr', 'sfcWind_6hrPlevPt', 'zg27_6hrPlevPt', 'rsds_Prim6hr' ] institute_details = { 'id': 'EC-Earth-Consortium', 'model_ids': ['EC-Earth3-LR', 'EC-Earth3-HR'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'control-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2050, 1, 1) }, 'highres-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'hist-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-present': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'highresSST-LAI': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-smoothed': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-p4K': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-4co2': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'spinup-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(1980, 1, 1) }, } # Experiment experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar']) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ new_vreqs = [ { 'table_name': 'Eday', 'long_name': 'River Discharge', 'units': 'm3 s-1', 'var_name': 'rivo', 'standard_name': 'water_flux_to_downstream', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'rivo', 'modeling_realm': 'land', 'frequency': 'day', 'cell_measures': 'area: areacellr', 'uid': 'd2285b46-4a9f-11e6-b84e-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Water table depth', 'units': 'm', 'var_name': 'wtd', 'standard_name': 'depth_of_soil_moisture_saturation', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'wtd', 'modeling_realm': 'land', 'frequency': 'mon', 'cell_measures': 'area: areacellr', 'uid': '8b81f0ce-4a5b-11e6-9cd2-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Terrestrial Water Storage', 'units': 'kg m-2', 'var_name': 'mrtws', 'standard_name': 'total_water_storage', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'mrtws', 'modeling_realm': 'land', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '6f6a4484-9acb-11e6-b7ee-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Vertically integrated Eastward moisture transport (Mass_weighted_vertical integral of the product of eastward wind by total water mass per unit mass)', 'units': 'kg m-1 s-1', 'var_name': 'intuaw', 'standard_name': 'vertical_integral_eastward_wind_by_total_water', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'intuaw', 'modeling_realm': 'atmos', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '6f690484-9acb-11e6-b7ee-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Vertically integrated Northward moisture transport (Mass_weighted_vertical integral of the product of northward wind by total water mass per unit mass)', 'units': 'kg m-1 s-1', 'var_name': 'intvaw', 'standard_name': 'vertical_integral_northward_wind_by_total_water', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'intvaw', 'modeling_realm': 'atmos', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '6f690b5a-9acb-11e6-b7ee-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Vertically Integrated Eastward Dry Statice Energy Transport', 'units': '1.e6 J m-1 s-1', 'var_name': 'intuadse', 'standard_name': 'vertical_integral_eastward_wind_by_dry_static_energy', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'intuadse', 'modeling_realm': 'atmos', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '6f691104-9acb-11e6-b7ee-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Vertically integrated Northward dry transport (cp.T +zg).v (Mass_weighted_vertical integral of the product of northward wind by dry static_energy per mass unit)', 'units': '1.e6 J m-1 s-1', 'var_name': 'intvadse', 'standard_name': 'vertical_integral_northward_wind_by_dry_static_energy', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'intvadse', 'modeling_realm': 'atmos', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '6f6916a4-9acb-11e6-b7ee-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Heat content of upper 300 meters', 'units': 'm K', 'var_name': 'hcont300', 'standard_name': 'heat_content_of_ocean_layer', 'cell_methods': 'area: mean where sea time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time depth300m', 'cmor_name': 'hcont300', 'modeling_realm': 'ocean', 'frequency': 'mon', 'cell_measures': 'area: areacello', 'uid': '6f69513c-9acb-11e6-b7ee-ac72891c3257' }, { 'table_name': 'AERmon', 'long_name': 'Sulfate Aerosol Optical Depth at 550nm', 'units': '1', 'var_name': 'od550so4', 'standard_name': 'atmosphere_optical_thickness_due_to_sulfate_ambient_aerosol', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'od550so4', 'modeling_realm': 'aerosol', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '19bf19ca-81b1-11e6-92de-ac72891c3257' }, { 'table_name': 'AERmon', 'long_name': 'TOA Outgoing Clear-Sky, Aerosol-Free Shortwave Radiation', 'units': 'W m-2', 'var_name': 'rsutcsaf', 'standard_name': 'toa_outgoing_shortwave_flux_assuming_clear_sky', 'cell_methods': 'area: time: mean', 'positive': 'up', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'rsutcsaf', 'modeling_realm': 'aerosol', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '8feac232-267c-11e7-8933-ac72891c3257' }, { 'table_name': 'SImon', 'long_name': 'Sea-ice area fraction', 'units': '%', 'var_name': 'siconca', 'standard_name': 'sea_ice_area_fraction', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time typesi', 'cmor_name': 'siconca', 'modeling_realm': 'seaIce', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '71190054-faa7-11e6-bfb7-ac72891c3257' }, { 'table_name': 'SImon', 'long_name': 'Sea ice area flux through straits', 'units': 'm2 s-1', 'var_name': 'siareaacrossline', 'standard_name': 'sea_ice_area_transport_across_line', 'cell_methods': 'time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'siline time', 'cmor_name': 'siareaacrossline', 'modeling_realm': 'seaIce', 'frequency': 'mon', 'cell_measures': '', 'uid': '712442ca-faa7-11e6-bfb7-ac72891c3257' }, { 'table_name': 'SImon', 'long_name': 'Snow mass flux through straits', 'units': 'kg s-1', 'var_name': 'snmassacrossline', 'standard_name': 'snow_mass_transport_across_line', 'cell_methods': 'time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'siline time', 'cmor_name': 'snmassacrossline', 'modeling_realm': 'seaIce', 'frequency': 'mon', 'cell_measures': '', 'uid': '712fb3ee-faa7-11e6-bfb7-ac72891c3257' }, ] institute_details = { 'id': 'CNRM-CERFACS', 'model_ids': ['CNRM-CM6-1'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'control-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2050, 1, 1) }, 'highres-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'hist-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, } variant_label = 'r2i1p1f2' # Experiment new_dreqs = [ 'rivo_Eday', 'siforceintstrx_PrimSIday', 'siforceintstry_PrimSIday', 'simassacrossline_PrimSIday', 'hfgeou_Omon', 'htovgyre_Omon', 'htovovrt_Omon', 'sicompstren_SImon', 'siflcondbot_SImon', 'siflcondtop_SImon', 'siflfwdrain_SImon', 'sifllatstop_SImon', 'sifllwutop_SImon', 'siflsensupbot_SImon', 'siflswdtop_SImon', 'siflswutop_SImon', 'simassacrossline_SImon', 'sisnmass_SImon', 'sitempbot_SImon', 'sithick_SImon', 'sltovgyre_Omon', 'sltovovrt_Omon', 'hcont300_Emon', 'intuadse_Emon', 'intuaw_Emon', 'intvadse_Emon', 'intvaw_Emon', 'mrtws_Emon', 'od550so4_AERmon', 'rsutcsaf_AERmon', 'siareaacrossline_SImon', 'siconca_SImon', 'snmassacrossline_SImon', 'wtd_Emon', ] experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the additional variable requests for new_vreq in new_vreqs: _vr = get_or_create(VariableRequest, **new_vreq) # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: try: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar'], rip_code=variant_label) except django.core.exceptions.MultipleObjectsReturned: logger.error('{}'.format(var_req_obj)) raise else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ rip_code = 'r1i1p2f1' new_dreqs = [ 'cl_Amon', 'clt_Amon', 'rlut_day', 'ua_day', 'va_day', 'rsut_CFday', ] institute_details = { 'id': 'MOHC', 'model_ids': ['HadGEM3-GC31-LL'], 'calendar': CALENDAR_360_DAY } experiments = { 'hist-1950': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1)}, } # Experiment experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id'] ) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'. format(clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar'] ), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar'] ), time_units=std_units, calendar=institute_details['calendar'], rip_code=rip_code ) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ activity_id = 'primWP5' new_dreqs = [ 'wfcorr_Omon', ] institute_details = { 'id': 'EC-Earth-Consortium', 'model_ids': ['EC-Earth3P-HR'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'primWP5-amv-neg': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(1960, 1, 1)}, 'primWP5-amv-pos': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(1960, 1, 1)} } variant_labels = ['r{}i1p2f1'.format(i) for i in range(1, 11)] # activity_id ActivityId.objects.get_or_create(short_name=activity_id, full_name=activity_id) # Experiment cache experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id'] ) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'. format(clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: for var_lab in variant_labels: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar'] ), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar'] ), time_units=std_units, calendar=institute_details['calendar'], rip_code = var_lab ) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ cmcc_dreqs = [ 'siconc_SIday', 'simass_SImon', 'sisnmass_SImon', 'sitimefrac_SImon' ] cmcc_details = { 'id': 'CMCC', 'model_ids': ['CMCC-CM2-HR4', 'CMCC-CM2-VHR4'], 'calendar': CALENDAR_STANDARD } experiments = { 'control-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2050, 1, 1) }, 'highres-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'hist-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'spinup-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(1980, 1, 1) }, } # Experiment experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institutes result = match_one(Institute, short_name=cmcc_details['id']) if result: cmcc = result else: msg = 'institute_id {} not found in the database.'.format( cmcc_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later cmcc_model_objs = [] for clim_model in cmcc_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: cmcc_model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in cmcc_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in cmcc_model_objs: _dr = get_or_create( DataRequest, project=project, institute=cmcc, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, cmcc_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, cmcc_details['calendar']), time_units=std_units, calendar=cmcc_details['calendar']) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ new_vreqs = [ { 'table_name': 'Eday', 'long_name': 'Surface Snow Amount', 'units': 'kg m-2', 'var_name': 'snw', 'standard_name': 'surface_snow_amount', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'snw', 'modeling_realm': 'landIce land', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': 'd2288954-4a9f-11e6-b84e-ac72891c3257' }, ] new_dreqs = ['mrso_Primday', 'mrlsl_Primday', 'zg_day', 'snw_Eday'] institute_details = { 'id': 'MPI-M', 'model_ids': ['MPIESM-1-2-HR', 'MPIESM-1-2-XR'], 'calendar': CALENDAR_PROLEPTIC_GREGORIAN } experiments = { 'control-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2050, 1, 1) }, 'highres-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'hist-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-present': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'highresSST-LAI': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-smoothed': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-p4K': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-4co2': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'spinup-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(1980, 1, 1) }, } # Experiment experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the additional variable requests for new_vreq in new_vreqs: _vr = get_or_create(VariableRequest, **new_vreq) # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar']) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ new_vreqs = [{ 'table_name': 'Omon', 'long_name': 'Ocean Meridional Overturning Mass Streamfunction', 'units': 'kg s-1', 'var_name': 'msftmz', 'standard_name': 'ocean_meridional_overturning_mass_streamfunction', 'cell_methods': 'longitude: mean (comment: basin mean[ along zig-zag grid path]) time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'latitude olevel basin time', 'cmor_name': 'msftmz', 'modeling_realm': 'ocean', 'frequency': 'mon', 'cell_measures': '', 'uid': 'baa59d48-e5dd-11e5-8482-ac72891c3257' }] cmcc_dreqs = [ 'msftmz_Omon', 'vso_PrimOmon', 'sos_Omon', 'wo_PrimOmon', 'mlotst_PrimOday', 'htovgyre_Omon', 'vto_PrimOmon', 'wto_PrimOmon', 'sltovgyre_Omon', 'hfbasin_Omon', 'tos_Oday', 'sos_Oday', 'tauuo_Omon', 'zossq_Omon', 'uto_PrimOmon', 'sltovovrt_Omon', 'mlotst_Omon', 'friver_Omon', 'wso_PrimOmon', 'tossq_Oday', 'uso_PrimOmon', 'tauvo_Omon', 'hfds_Omon', 'zos_PrimOday', 'htovovrt_Omon', 'tauuo_PrimOday', 'tauvo_PrimOday' ] cmcc_details = { 'id': 'CMCC', 'model_ids': ['CMCC-CM2-HR4', 'CMCC-CM2-VHR4'], 'calendar': CALENDAR_STANDARD } experiments = { 'control-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2050, 1, 1) }, 'highres-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'hist-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-present': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'highresSST-LAI': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-smoothed': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-p4K': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-4co2': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'spinup-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(1980, 1, 1) }, } # Experiment experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institutes result = match_one(Institute, short_name=cmcc_details['id']) if result: cmcc = result else: msg = 'institute_id {} not found in the database.'.format( cmcc_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later cmcc_model_objs = [] for clim_model in cmcc_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: cmcc_model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the additional variable requests for new_vreq in new_vreqs: _vr = get_or_create(VariableRequest, **new_vreq) # create the new data requests for new_dreq in cmcc_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in cmcc_model_objs: _dr = get_or_create( DataRequest, project=project, institute=cmcc, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, cmcc_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, cmcc_details['calendar']), time_units=std_units, calendar=cmcc_details['calendar']) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ new_vreqs = [ { 'table_name': 'AERmon', 'long_name': 'Convective Cloud Area Fraction', 'units': '%', 'var_name': 'cltc', 'standard_name': 'convective_cloud_area_fraction', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'float', 'dimensions': 'longitude latitude time', 'cmor_name': 'cltc', 'modeling_realm': 'aerosol', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '01d412d0-c792-11e6-aa58-5404a60d96b5' }, { 'table_name': 'AERmon', 'long_name': 'Eastward Wind', 'units': 'm s-1', 'var_name': 'ua', 'standard_name': 'eastward_wind', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'float', 'dimensions': 'longitude latitude alevel time', 'cmor_name': 'ua', 'modeling_realm': 'aerosol', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '19bfc73a-81b1-11e6-92de-ac72891c3257' }, { 'table_name': 'AERmon', 'long_name': 'Northward Wind', 'units': 'm s-1', 'var_name': 'va', 'standard_name': 'northward_wind', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'float', 'dimensions': 'longitude latitude alevel time', 'cmor_name': 'va', 'modeling_realm': 'aerosol', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '19bfc9f6-81b1-11e6-92de-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Specific Humidity', 'units': '1.0', 'var_name': 'hus', 'standard_name': 'specific_humidity', 'cell_methods': 'time: mean', 'positive': '', 'variable_type': '', 'dimensions': 'longitude latitude plev7c time', 'cmor_name': 'hus', 'modeling_realm': 'atmos', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '81d83384-aa6a-11e6-9736-5404a60d96b5' }, { 'table_name': 'E3hr', 'long_name': 'Sea Level Pressure', 'units': 'Pa', 'var_name': 'psl', 'standard_name': 'air_pressure_at_sea_level', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'psl', 'modeling_realm': 'atmos', 'frequency': '3hr', 'cell_measures': 'area: areacella', 'uid': '8bb0d1c8-4a5b-11e6-9cd2-ac72891c3257' }, { 'table_name': 'E3hrPt', 'long_name': 'ISCCP Percentage Cloud Area', 'units': '%', 'var_name': 'clisccp', 'standard_name': 'cloud_area_fraction_in_atmosphere_layer', 'cell_methods': 'area: mean time: point', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude plev7c tau time1', 'cmor_name': 'clisccp', 'modeling_realm': 'atmos', 'frequency': '3hr', 'cell_measures': 'area: areacella', 'uid': '8bb0db0a-4a5b-11e6-9cd2-ac72891c3257' }, { 'table_name': '6hrPlevPt', 'long_name': 'Geopotential Height at 500 hPa', 'units': 'm', 'var_name': 'zg500', 'standard_name': 'geopotential_height', 'cell_methods': 'area: mean time: point', 'positive': '', 'variable_type': 'float', 'dimensions': 'longitude latitude time1', 'cmor_name': 'zg500', 'modeling_realm': 'aerosol', 'frequency': '6hr', 'cell_measures': 'area: areacella', 'uid': '7c70f59e-1ab7-11e7-8dfc-5404a60d96b5' }, { 'table_name': '6hrPlevPt', 'long_name': 'Eastward Wind', 'units': 'm s-1', 'var_name': 'ua', 'standard_name': 'eastward_wind', 'cell_methods': 'area: mean time: point', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude plev3 time1', 'cmor_name': 'ua', 'modeling_realm': 'atmos', 'frequency': '6hr', 'cell_measures': 'area: areacella', 'uid': '8bae55ba-4a5b-11e6-9cd2-ac72891c3257' }, { 'table_name': '6hrPlevPt', 'long_name': 'Northward Wind', 'units': 'm s-1', 'var_name': 'va', 'standard_name': 'northward_wind', 'cell_methods': 'area: mean time: point', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude plev3 time1', 'cmor_name': 'va', 'modeling_realm': 'atmos', 'frequency': '6hr', 'cell_measures': 'area: areacella', 'uid': '8bae5aba-4a5b-11e6-9cd2-ac72891c3257' }, { 'table_name': '6hrPlevPt', 'long_name': 'Air Temperature', 'units': 'K', 'var_name': 'ta', 'standard_name': 'air_temperature', 'cell_methods': 'area: mean time: point', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude plev3 time1', 'cmor_name': 'ta', 'modeling_realm': 'atmos', 'frequency': '6hr', 'cell_measures': 'area: areacella', 'uid': '6a35d178-aa6a-11e6-9736-5404a60d96b5' }, { 'table_name': 'Eday', 'long_name': 'Liquid Water Content of Snow Layer', 'units': 'kg m-2', 'var_name': 'lwsnl', 'standard_name': 'liquid_water_content_of_snow_layer', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'lwsnl', 'modeling_realm': 'landIce land', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': 'd228925a-4a9f-11e6-b84e-ac72891c3257' }, { 'table_name': 'Eday', 'long_name': 'Surface Runoff', 'units': 'kg m-2 s-1', 'var_name': 'mrros', 'standard_name': 'surface_runoff_flux', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'mrros', 'modeling_realm': 'land', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': 'd2284048-4a9f-11e6-b84e-ac72891c3257' }, { 'table_name': 'Eday', 'long_name': 'Surface Snow Melt', 'units': 'kg m-2 s-1', 'var_name': 'snm', 'standard_name': 'surface_snow_melt_flux', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'snm', 'modeling_realm': 'landIce land', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': 'd22848ea-4a9f-11e6-b84e-ac72891c3257' }, { 'table_name': 'Eday', 'long_name': 'Snow Depth', 'units': 'm', 'var_name': 'snd', 'standard_name': 'surface_snow_thickness', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'snd', 'modeling_realm': 'landIce land', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': 'b7ccdf0a-7c00-11e6-bcdf-ac72891c3257' }, { 'table_name': 'Eday', 'long_name': 'Surface Temperature', 'units': 'K', 'var_name': 'ts', 'standard_name': 'surface_temperature', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'ts', 'modeling_realm': 'atmos', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': '8b8fc3de-4a5b-11e6-9cd2-ac72891c3257' }, { 'table_name': 'Eday', 'long_name': 'Eastward Wind', 'units': 'm s-1', 'var_name': 'ua', 'standard_name': 'eastward_wind', 'cell_methods': 'time: mean', 'positive': '', 'variable_type': '', 'dimensions': 'longitude latitude plev19 time', 'cmor_name': 'ua', 'modeling_realm': 'atmos', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': '8bae00ba-4a5b-11e6-9cd2-ac72891c3257' }, { 'table_name': 'Eday', 'long_name': 'Northward Wind', 'units': 'm s-1', 'var_name': 'va', 'standard_name': 'northward_wind', 'cell_methods': 'time: mean', 'positive': '', 'variable_type': '', 'dimensions': 'longitude latitude plev19 time', 'cmor_name': 'va', 'modeling_realm': 'atmos', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': '8bae05ec-4a5b-11e6-9cd2-ac72891c3257' }, { 'table_name': '6hrPlev', 'long_name': 'Specific Humidity', 'units': '1.0', 'var_name': 'hus', 'standard_name': 'specific_humidity', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': '', 'dimensions': 'longitude latitude plev4 time', 'cmor_name': 'hus4', 'modeling_realm': 'atmos', 'frequency': '6hr', 'cell_measures': 'area: areacella', 'uid': '8bae64ba-4a5b-11e6-9cd2-ac72891c3257' }, { 'table_name': 'day', 'long_name': 'Total Soil Moisture Content', 'units': 'kg m-2', 'var_name': 'mrso', 'standard_name': 'soil_moisture_content', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'mrso', 'modeling_realm': 'land', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': '3c641b6c-b89b-11e6-be04-ac72891c3257' }, ] new_dreqs = [ 'psl_E3hr', ] institute_details = { 'id': 'MOHC', 'model_ids': ['HadGEM3-GC31-HM', 'HadGEM3-GC31-MM', 'HadGEM3-GC31-LM'], 'calendar': CALENDAR_360_DAY } experiments = { 'control-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2050, 1, 1) }, 'highres-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'hist-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-present': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'highresSST-LAI': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-smoothed': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-p4K': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-4co2': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'spinup-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(1980, 1, 1) }, } # Experiment experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the additional variable requests for new_vreq in new_vreqs: _vr = get_or_create(VariableRequest, **new_vreq) # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar']) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ activity_id = 'primWP5' new_dreqs = [ 'psl_6hrPlevPt', 'tas_6hrPlevPt', 'uas_6hrPlevPt', 'vas_6hrPlevPt', 'clivi_Amon', 'clt_Amon', 'clwvi_Amon', 'evspsbl_Amon', 'hfls_Amon', 'hfss_Amon', 'hur_Amon', 'hus_Amon', 'pr_Amon', 'prc_Amon', 'prsn_Amon', 'prw_Amon', 'ps_Amon', 'psl_Amon', 'rlds_Amon', 'rldscs_Amon', 'rlus_Amon', 'rlut_Amon', 'rlutcs_Amon', 'rsds_Amon', 'rsdscs_Amon', 'rsdt_Amon', 'rsus_Amon', 'rsuscs_Amon', 'rsut_Amon', 'rsutcs_Amon', 'sfcWind_Amon', 'ta_Amon', 'tas_Amon', 'tasmax_Amon', 'tasmin_Amon', 'tauu_Amon', 'tauv_Amon', 'ts_Amon', 'ua_Amon', 'uas_Amon', 'va_Amon', 'vas_Amon', 'wap_Amon', 'zg_Amon', 'snd_LImon', 'snm_LImon', 'snw_LImon', 'tsn_LImon', 'lai_Lmon', 'mrlsl_Lmon', 'mrro_Lmon', 'mrso_Lmon', 'tsl_Lmon', 'sos_Oday', 'tos_Oday', 'hfds_Omon', 'mlotst_Omon', 'mlotstsq_Omon', 'so_Omon', 'sos_Omon', 'tauuo_Omon', 'tauvo_Omon', 'thetao_Omon', 'tos_Omon', 'tossq_Omon', 'uo_Omon', 'vo_Omon', 'wfo_Omon', 'wo_Omon', 'zos_Omon', 'zossq_Omon', 'clt_Prim6hrPt', 'mlotst_PrimOday', 'tauuo_PrimOday', 'tauvo_PrimOday', 'uo_PrimOday', 'vo_PrimOday', 'zos_PrimOday', 'uso_PrimOmon', 'uto_PrimOmon', 'vso_PrimOmon', 'vto_PrimOmon', 'wso_PrimOmon', 'wto_PrimOmon', 'sistrxdtop_PrimSIday', 'sistrydtop_PrimSIday', 'evspsbl_Primday', 'mrlsl_Primday', 'mrso_Primday', 'ts_Primday', 'uneutrals_Primday', 'vneutrals_Primday', 'siconc_SIday', 'sisnthick_SIday', 'sitemptop_SIday', 'sithick_SIday', 'siu_SIday', 'siv_SIday', 'siconc_SImon', 'sisnthick_SImon', 'sistrxdtop_SImon', 'sistrydtop_SImon', 'sitemptop_SImon', 'sithick_SImon', 'siu_SImon', 'siv_SImon', 'sivol_SImon', 'clt_day', 'hfls_day', 'hfss_day', 'hur_day', 'hus_day', 'pr_day', 'prc_day', 'prsn_day', 'psl_day', 'rlds_day', 'rlus_day', 'rlut_day', 'rsds_day', 'rsus_day', 'sfcWind_day', 'tas_day', 'tasmax_day', 'tasmin_day', 'tslsi_day', 'uas_day', 'vas_day', 'wap_day', ] institute_details = { 'id': 'ECMWF', 'model_ids': ['ECMWF-IFS-LR', 'ECMWF-IFS-HR'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'primWP5-coupledseaice': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(1960, 1, 1) }, } variant_labels = ['r{}i1p1f1'.format(i) for i in range(1, 41)] # activity_id ActivityId.objects.get_or_create(short_name=activity_id, full_name=activity_id) # Experiment create for expt in experiments: Experiment.objects.get_or_create(short_name=expt, full_name=expt) # Experiment cache experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: for var_lab in variant_labels: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar'], rip_code=var_lab) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ if args.processes == 1 and not iris.__version__.startswith('1.'): # if not multiprocessing then limit the number of Dask threads # this can't seem to be limited when using multiprocessing dask.config.set(pool=ThreadPool(2)) submission_dir = os.path.normpath(args.directory) logger.debug('Submission directory: %s', submission_dir) logger.debug('Project: %s', args.mip_era) logger.debug('Processes requested: %s', args.processes) try: if args.input: validated_metadata = read_json_file(args.input) data_sub = _get_submission_object(submission_dir) files_online = False else: files_online = True data_files = list_files(submission_dir) logger.debug('%s files identified', len(data_files)) if not args.validate_only and not args.output: data_sub = _get_submission_object(submission_dir) if data_sub.status != 'ARRIVED': msg = "The submission's status is not ARRIVED." logger.error(msg) raise SubmissionError(msg) try: if not args.no_prepare: run_prepare(data_files, args.processes) validated_metadata = list( identify_and_validate(data_files, args.mip_era, args.processes, args.file_format)) except SubmissionError: if not args.validate_only and not args.output: send_admin_rejection_email(data_sub) raise logger.debug('%s files validated successfully', len(validated_metadata)) if args.validate_only: logger.debug('Data submission not run (-v option specified)') logger.debug('Processing complete') sys.exit(0) if not args.relaxed and len(validated_metadata) != len(data_files): # if not args.output: # rejected_dir = move_rejected_files(submission_dir) # set_status_rejected(data_sub, rejected_dir) # send_user_rejection_email(data_sub) msg = ('Not all files passed validation. Please fix these ' 'errors and then re-run this script.') logger.error(msg) raise SubmissionError(msg) if args.tape_base_url: add_tape_url(validated_metadata, args.tape_base_url, submission_dir) if args.output: write_json_file(validated_metadata, args.output) else: update_database_submission(validated_metadata, data_sub, files_online, args.version_string) logger.debug( '%s files submitted successfully', match_one(DataSubmission, incoming_directory=submission_dir). get_data_files().count()) except SubmissionError: sys.exit(1) logger.debug('Processing complete')
def verify_fk_relationships(metadata): """ Identify the variable_request and data_request objects corresponding to this file. :param dict metadata: Metadata identified for this file. :raises SubmissionError: If there are no existing entries in the database for `Project`, `ClimateModel` or `Experiment`. """ foreign_key_types = [(Project, 'project'), (ClimateModel, 'climate_model'), (Experiment, 'experiment'), (Institute, 'institute'), (ActivityId, 'activity_id')] # get values for each of the foreign key types for object_type, object_str in foreign_key_types: result = match_one(object_type, short_name=metadata[object_str]) if result: metadata[object_str] = result else: msg = ("No {} '{}' found for file: {}. Please create this object " "and resubmit.".format(object_str.replace('_', ' '), metadata[object_str], metadata['basename'])) logger.error(msg) raise SubmissionError(msg) # find the data request dreq_match = match_one(DataRequest, project=metadata['project'], institute=metadata['institute'], climate_model=metadata['climate_model'], experiment=metadata['experiment'], variable_request__table_name=metadata['table'], variable_request__cmor_name=metadata['var_name'], rip_code=metadata['rip_code']) if dreq_match: metadata['data_request'] = dreq_match metadata['variable'] = dreq_match.variable_request else: # if cmor_name doesn't match then it may be a variable where out_name # is different to cmor_name so check these dreq_matches = DataRequest.objects.filter( project=metadata['project'], institute=metadata['institute'], climate_model=metadata['climate_model'], experiment=metadata['experiment'], variable_request__table_name=metadata['table'], variable_request__var_name=metadata['var_name'], rip_code=metadata['rip_code']) if dreq_matches.count() == 0: msg = ('No data request found for file: {}.'.format( metadata['basename'])) logger.error(msg) raise FileValidationError(msg) elif dreq_matches.count() == 1: metadata['data_request'] = dreq_matches[0] metadata['variable'] = dreq_matches[0].variable_request else: try: plev_name = _guess_plev_name(metadata) except Exception: msg = ('Cannot open file to determine plev name: {}.'.format( metadata['basename'])) logger.error(msg) raise FileValidationError(msg) if plev_name: plev_matches = dreq_matches.filter( variable_request__dimensions__icontains=plev_name) if plev_matches.count() == 1: metadata['data_request'] = plev_matches[0] metadata['variable'] = plev_matches[0].variable_request elif plev_matches.count() == 0: msg = ('No data requests found with plev {} for file: {}.'. format(plev_name, metadata['basename'])) logger.error(msg) raise FileValidationError(msg) else: msg = ('Multiple data requests found with plev {} for ' 'file: {}.'.format(plev_name, metadata['basename'])) logger.error(msg) raise FileValidationError(msg) else: msg = ('Unable to determine plev name: {}.'.format( metadata['basename'])) logger.error(msg) raise FileValidationError(msg)
def test_no_match_returns_none(self): self.assertIsNone(dbapi.match_one(models.Institute, full_name='real'))
def main(args): """ Main entry point """ new_vreqs = [ {'table_name': 'Ofx', 'long_name': 'Grid-Cell Area for Ocean Variables', 'units': 'm2', 'var_name': 'areacello', 'standard_name': 'cell_area', 'cell_methods': 'area: sum', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude ', 'cmor_name': 'areacello', 'modeling_realm': 'ocean', 'frequency': 'fx', 'cell_measures': '', 'uid': 'baa3ee94-e5dd-11e5-8482-ac72891c3257'}, {'table_name': 'Ofx', 'long_name': 'Region Selection Index', 'units': '1', 'var_name': 'basin', 'standard_name': 'region', 'cell_methods': 'area: mean', 'positive': '', 'variable_type': 'integer', 'dimensions': 'longitude latitude ', 'cmor_name': 'basin', 'modeling_realm': 'ocean', 'frequency': 'fx', 'cell_measures': 'area: areacello', 'uid': 'baa3f718-e5dd-11e5-8482-ac72891c3257'}, {'table_name': 'Ofx', 'long_name': 'Ocean Model Cell Thickness', 'units': 'm', 'var_name': 'thkcello', 'standard_name': 'cell_thickness', 'cell_methods': 'area: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude olevel ', 'cmor_name': 'thkcello', 'modeling_realm': 'ocean', 'frequency': 'fx', 'cell_measures': 'area: areacello volume: volcello', 'uid': 'bab9bd00-e5dd-11e5-8482-ac72891c3257'}, ] # create the variable requests for new_vreq in new_vreqs: _vr = get_or_create(VariableRequest, **new_vreq) variant_label = 'r1i1p1f1' # The standard reference time std_units = Settings.get_solo().standard_time_units new_dreqs = [ 'areacello_Ofx', 'basin_Ofx', ] institute_id = 'MOHC' calendar = CALENDAR_360_DAY # Institute result = match_one(Institute, short_name=institute_id) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format(institute_id) print(msg) raise ValueError(msg) # Coupled ######### model_ids = ['HadGEM3-GC31-HH', 'HadGEM3-GC31-HM', 'HadGEM3-GC31-MM', 'HadGEM3-GC31-LL'] experiments = { 'spinup-1950': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(1980, 1, 1)}, 'hist-1950': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1)}, 'control-1950': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2050, 1, 1)}, 'highres-future': {'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1)}, } experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in model_ids: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'. format(clim_model)) print(msg) raise ValueError(msg) # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: try: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, calendar ), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, calendar ), time_units=std_units, calendar=calendar, rip_code = variant_label ) except django.core.exceptions.MultipleObjectsReturned: logger.error('{}'.format(var_req_obj)) raise else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg) # Atmosphere only ################# experiments = { 'highresSST-present': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1)}, 'highresSST-future': {'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1)}, } model_ids = ['HadGEM3-GC31-HM', 'HadGEM3-GC31-MM', 'HadGEM3-GC31-LM'] experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in model_ids: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'. format(clim_model)) print(msg) raise ValueError(msg) # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: try: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, calendar ), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, calendar ), time_units=std_units, calendar=calendar, rip_code = variant_label ) except django.core.exceptions.MultipleObjectsReturned: logger.error('{}'.format(var_req_obj)) raise else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ # Delete the data requests that could be duplicates dupe_dreqs = DataRequest.objects.filter( climate_model__short_name='CNRM-CM6-1-HR', experiment__short_name='highres-future', variable_request__table_name='SImon', variable_request__table_name__in = ['simass', 'sisaltmass', 'sisnthick'] ) if dupe_dreqs.count() != 3: logger.error('{} data requests found when expecting 3'. format(dupe_dreqs.count())) sys.exit(1) dupe_dreqs.delete() institute_details = { 'id': 'CNRM-CERFACS', 'model_ids': ['CNRM-CM6-1', 'CNRM-CM6-1-HR'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'highres-future': {'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1)}, } variant_label = 'r1i1p1f2' # Experiment new_dreqs = [ 'cropFracC3_Emon', 'cropFracC4_Emon', 'grassFracC3_Emon', 'grassFracC4_Emon', 'nwdFracLut_Emon', 'ps_AERmon', 'sftgif_LImon', 'sftgrf_LImon', 'sidconcdyn_SImon', 'sidconcth_SImon', 'simass_SImon', 'sisaltmass_SImon', 'sisnthick_SImon', 'sndmassdyn_SImon', 'sndmasssi_SImon', 'vegFrac_Emon', ] experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id'] ) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'. format(clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: try: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar'] ), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar'] ), time_units=std_units, calendar=institute_details['calendar'], rip_code = variant_label ) except django.core.exceptions.MultipleObjectsReturned: logger.error('{}'.format(var_req_obj)) raise else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ institute_details = { 'id': 'CNRM-CERFACS', 'model_ids': ['CNRM-CM6-1', 'CNRM-CM6-1-HR'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'spinup-1950': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(1980, 1, 1)}, } variant_label = 'r1i1p1f2' # Experiment new_dreqs = [ 'od550so4_AERmon', 'rsutcsaf_AERmon', 'hcont300_Emon', 'intuadse_Emon', 'intuaw_Emon', 'intvadse_Emon', 'intvaw_Emon', 'mrtws_Emon', 'wtd_Emon', 'htovgyre_Omon', 'htovovrt_Omon', 'siconca_SImon', 'simass_SImon', 'sisaltmass_SImon', 'sisnmass_SImon', 'sisnthick_SImon', 'sithick_SImon', 'rivo_Eday', 'siforceintstrx_PrimSIday', 'siforceintstry_PrimSIday', 'simassacrossline_PrimSIday', ] experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id'] ) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'. format(clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: try: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar'] ), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar'] ), time_units=std_units, calendar=institute_details['calendar'], rip_code = variant_label ) except django.core.exceptions.MultipleObjectsReturned: logger.error('{}'.format(var_req_obj)) raise else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ mip_era = 'CMIP6' activity_id = 'PRIMAVERA' new_dreqs = [ 'pr_day', 'psl_6hrPlevPt', 'ua_6hrPlevPt', 'uas_6hrPlevPt', 'va_6hrPlevPt', 'vas_6hrPlevPt' ] institute_details = { 'id': 'EC-Earth-Consortium', 'model_ids': ['EC-Earth3'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'Primavera-coupled': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2016, 1, 1) } } variant_labels = ['r1i1p101f1'] # activity_id ActivityId.objects.get_or_create(short_name=activity_id, full_name=activity_id) # Experiment create for expt in experiments: Experiment.objects.get_or_create(short_name=expt, full_name=expt) # Experiment cache experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) logger.error(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) logger.error(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) logger.error(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') project = match_one(Project, short_name=mip_era) var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: for var_lab in variant_labels: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar'], rip_code=var_lab) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) logger.error(msg) raise ValueError(msg)
def main(args): """ Main entry point """ activity_id = 'primWP5' new_dreqs = [ 'uas_3hr', 'vas_3hr', 'pr_6hrPlev', 'psl_6hrPlev', 'tas_6hrPlev', 'uas_6hrPlev', 'vas_6hrPlev', 'zg1000_6hrPlev', 'ta_6hrPlevPt', 'ua_6hrPlevPt', 'va_6hrPlevPt', 'zg27_6hrPlevPt', 'zg500_AERday', 'ptp_AERmon', 'rlutaf_AERmon', 'rlutcsaf_AERmon', 'rsutaf_AERmon', 'rsutcsaf_AERmon', 'toz_AERmon', 'ztp_AERmon', 'clivi_Amon', 'clt_Amon', 'clwvi_Amon', 'evspsbl_Amon', 'hfls_Amon', 'hfss_Amon', 'hur_Amon', 'hurs_Amon', 'hus_Amon', 'huss_Amon', 'o3_Amon', 'pr_Amon', 'prc_Amon', 'prsn_Amon', 'prw_Amon', 'ps_Amon', 'psl_Amon', 'rlds_Amon', 'rldscs_Amon', 'rlus_Amon', 'rlut_Amon', 'rlutcs_Amon', 'rsds_Amon', 'rsdscs_Amon', 'rsdt_Amon', 'rsus_Amon', 'rsuscs_Amon', 'rsut_Amon', 'rsutcs_Amon', 'sbl_Amon', 'sfcWind_Amon', 'ta_Amon', 'tas_Amon', 'tasmax_Amon', 'tasmin_Amon', 'tauu_Amon', 'tauv_Amon', 'ts_Amon', 'ua_Amon', 'uas_Amon', 'va_Amon', 'vas_Amon', 'wap_Amon', 'zg_Amon', 'ps_CFday', 'mlotst_Eday', 'rivo_Eday', 't20d_Eday', 'ta850_Eday', 'ts_Eday', 'sftgrf_Efx', 'cropFracC3_Emon', 'cropFracC4_Emon', 'evspsblpot_Emon', 'grassFracC3_Emon', 'grassFracC4_Emon', 'intuadse_Emon', 'intuaw_Emon', 'intvadse_Emon', 'intvaw_Emon', 'mrtws_Emon', 'nwdFracLut_Emon', 'orog_Emon', 'rls_Emon', 'rss_Emon', 'sfcWindmax_Emon', 't20d_Emon', 'thetaot_Emon', 'thetaot2000_Emon', 'thetaot300_Emon', 'thetaot700_Emon', 'vegFrac_Emon', 'wtd_Emon', 'lwsnl_LImon', 'sftgif_LImon', 'sftgrf_LImon', 'snc_LImon', 'snd_LImon', 'snw_LImon', 'baresoilFrac_Lmon', 'c3PftFrac_Lmon', 'c4PftFrac_Lmon', 'cropFrac_Lmon', 'gpp_Lmon', 'grassFrac_Lmon', 'lai_Lmon', 'mrfso_Lmon', 'mrro_Lmon', 'mrros_Lmon', 'mrso_Lmon', 'mrsos_Lmon', 'npp_Lmon', 'prveg_Lmon', 'ra_Lmon', 'rh_Lmon', 'treeFrac_Lmon', 'tsl_Lmon', 'sos_Oday', 'tos_Oday', 'areacello_Ofx', 'basin_Ofx', 'deptho_Ofx', 'hfgeou_Ofx', 'masscello_Ofx', 'thkcello_Ofx', 'bigthetao_Omon', 'bigthetaoga_Omon', 'ficeberg_Omon', 'friver_Omon', 'hfbasin_Omon', 'hfcorr_Omon', 'hfds_Omon', 'hfx_Omon', 'hfy_Omon', 'htovgyre_Omon', 'htovovrt_Omon', 'mfo_Omon', 'mlotst_Omon', 'mlotstmin_Omon', 'msftyz_Omon', 'rsntds_Omon', 'so_Omon', 'soga_Omon', 'sos_Omon', 'sosga_Omon', 'tauuo_Omon', 'tauvo_Omon', 'thetao_Omon', 'thkcello_Omon', 'tos_Omon', 'umo_Omon', 'uo_Omon', 'vmo_Omon', 'vo_Omon', 'volo_Omon', 'wfo_Omon', 'wmo_Omon', 'wo_Omon', 'zos_Omon', 'zossq_Omon', 'zostoga_Omon', 'rsds_Prim6hr', 'rsdsdiff_Prim6hr', 'mrso_Primday', 'siconc_SIday', 'siconca_SIday', 'sisnthick_SIday', 'sispeed_SIday', 'sitemptop_SIday', 'sithick_SIday', 'sitimefrac_SIday', 'siu_SIday', 'siv_SIday', 'siage_SImon', 'siareaacrossline_SImon', 'siarean_SImon', 'siareas_SImon', 'sicompstren_SImon', 'siconc_SImon', 'siconca_SImon', 'sidconcdyn_SImon', 'sidconcth_SImon', 'sidivvel_SImon', 'sidmassdyn_SImon', 'sidmassevapsubl_SImon', 'sidmassgrowthbot_SImon', 'sidmassgrowthwat_SImon', 'sidmasslat_SImon', 'sidmassmeltbot_SImon', 'sidmassmelttop_SImon', 'sidmasssi_SImon', 'sidmassth_SImon', 'sidmasstranx_SImon', 'sidmasstrany_SImon', 'siextentn_SImon', 'siextents_SImon', 'sifb_SImon', 'siflcondbot_SImon', 'siflcondtop_SImon', 'siflfwbot_SImon', 'siflfwdrain_SImon', 'sifllatstop_SImon', 'sifllwutop_SImon', 'siflsensupbot_SImon', 'siflswdbot_SImon', 'siflswdtop_SImon', 'siflswutop_SImon', 'sihc_SImon', 'simass_SImon', 'simassacrossline_SImon', 'sipr_SImon', 'sisaltmass_SImon', 'sishevel_SImon', 'sisnconc_SImon', 'sisnhc_SImon', 'sisnmass_SImon', 'sisnthick_SImon', 'sispeed_SImon', 'sistrxdtop_SImon', 'sistrxubot_SImon', 'sistrydtop_SImon', 'sistryubot_SImon', 'sitempbot_SImon', 'sitempsnic_SImon', 'sitemptop_SImon', 'sithick_SImon', 'sitimefrac_SImon', 'siu_SImon', 'siv_SImon', 'sivol_SImon', 'sivoln_SImon', 'sivols_SImon', 'sndmassdyn_SImon', 'sndmassmelt_SImon', 'sndmasssi_SImon', 'sndmasssnf_SImon', 'sndmasssubl_SImon', 'snmassacrossline_SImon', 'clt_day', 'hfls_day', 'hfss_day', 'huss_day', 'pr_day', 'psl_day', 'rlut_day', 'rsds_day', 'sfcWindmax_day', 'snc_day', 'ta_day', 'tas_day', 'tasmax_day', 'tasmin_day', 'ua_day', 'uas_day', 'va_day', 'vas_day', 'zg_day', 'areacella_fx', 'areacellr_fx', 'mrsofc_fx', 'orog_fx', 'rootd_fx', 'sftgif_fx', 'sftlf_fx', 'zfull_fx', ] institute_details = { 'id': 'CNRM-CERFACS', 'model_ids': ['CNRM-CM6-1-HR', 'CNRM-CM6-1'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'primWP5-amv-neg': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(1960, 1, 1)}, 'primWP5-amv-pos': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(1960, 1, 1)} } variant_labels = ['r{}i1p1f2'.format(i) for i in range(11, 26)] # activity_id ActivityId.objects.get_or_create(short_name=activity_id, full_name=activity_id) # Experiment cache experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id'] ) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'. format(clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: for var_lab in variant_labels: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar'] ), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar'] ), time_units=std_units, calendar=institute_details['calendar'], rip_code = var_lab ) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ logger.debug('Starting delete_request.py for retrieval {}'.format( args.retrieval_id)) deletion_retrieval = match_one(RetrievalRequest, id=args.retrieval_id) if not deletion_retrieval: logger.error('Unable to find retrieval id {}'.format( args.retrieval_id)) sys.exit(1) if deletion_retrieval.date_deleted: logger.error('Retrieval {} was already deleted, at {}.'.format( deletion_retrieval.id, deletion_retrieval.date_deleted.strftime('%Y-%m-%d %H:%M'))) sys.exit(1) if not deletion_retrieval.data_finished: logger.error('Retrieval {} is not marked as finished.'.format( deletion_retrieval.id)) sys.exit(1) problems_encountered = False directories_found = [] base_output_dir = Settings.get_solo().base_output_dir # loop through all of the data requests in this retrieval for data_req in deletion_retrieval.data_request.all(): online_req_files = data_req.datafile_set.filter( online=True, directory__isnull=False) files_to_delete = date_filter_files(online_req_files, deletion_retrieval.start_year, deletion_retrieval.end_year) if files_to_delete is None: continue if not args.force: # find any other retrieval requests that still need this data other_retrievals = RetrievalRequest.objects.filter( data_request=data_req, data_finished=False) # loop through the retrieval requests that still need this data # request for ret_req in other_retrievals: ret_online_files = data_req.datafile_set.filter( online=True, directory__isnull=False) ret_filtered_files = date_filter_files(ret_online_files, ret_req.start_year, ret_req.end_year) if ret_filtered_files is None: continue # remove from the list of files to delete the ones that we have # just found are still needed files_to_delete = files_to_delete.difference( ret_filtered_files) # list the parts of the data request that are still required logger.debug("{} {} to {} won't be deleted".format( data_req, ret_req.start_year, ret_req.end_year)) # don't (try to) delete anything that's in the CEDA archive files_to_delete.exclude(directory__startswith=CEDA_ARCHIVE) # do the deleting if args.dryrun: logger.debug('{} {} files can be deleted.'.format( data_req, files_to_delete.distinct().count())) else: logger.debug('{} {} files will be deleted.'.format( data_req, files_to_delete.distinct().count())) for data_file in files_to_delete: old_file_dir = data_file.directory try: os.remove(os.path.join(data_file.directory, data_file.name)) except OSError as exc: logger.error(str(exc)) problems_encountered = True else: if data_file.directory not in directories_found: directories_found.append(data_file.directory) data_file.online = False data_file.directory = None data_file.save() # if a symbolic link exists from the base output directory # then delete this too if not old_file_dir.startswith(base_output_dir): sym_link_dir = os.path.join(base_output_dir, construct_drs_path(data_file)) sym_link = os.path.join(sym_link_dir, data_file.name) if not os.path.islink(sym_link): logger.error( "Expected {} to be a link but it isn't. " "Leaving this file in place.".format(sym_link)) problems_encountered = True else: try: os.remove(sym_link) except OSError as exc: logger.error(str(exc)) problems_encountered = True else: if sym_link_dir not in directories_found: directories_found.append(sym_link_dir) if not args.dryrun: # delete any empty directories for directory in directories_found: if not os.listdir(directory): delete_drs_dir(directory) # set date_deleted in the db if not problems_encountered: deletion_retrieval.date_deleted = timezone.now() deletion_retrieval.save() else: logger.error( 'Errors were encountered and so retrieval {} has not ' 'been marked as deleted. All possible files have been ' 'deleted.'.format(args.retrieval_id)) logger.debug('Completed delete_request.py for retrieval {}'.format( args.retrieval_id))
def main(args): """ Main entry point """ mip_era = 'CMIP6' activity_id = 'primWP5' new_dreqs = [ 'psl_6hrPlev', 'zg7h_6hrPlevPt', 'clt_Amon', 'hfls_Amon', 'hfss_Amon', 'hur_Amon', 'hurs_Amon', 'hus_Amon', 'huss_Amon', 'pr_Amon', 'prc_Amon', 'prsn_Amon', 'ps_Amon', 'psl_Amon', 'rlds_Amon', 'rlus_Amon', 'rlut_Amon', 'rlutcs_Amon', 'rsds_Amon', 'rsdt_Amon', 'rsus_Amon', 'rsuscs_Amon', 'rsut_Amon', 'sfcWind_Amon', 'ta_Amon', 'tas_Amon', 'tasmax_Amon', 'tasmin_Amon', 'tauu_Amon', 'tauv_Amon', 'ts_Amon', 'ua_Amon', 'uas_Amon', 'va_Amon', 'vas_Amon', 'zg_Amon', 'ua850_E3hrPt', 'va850_E3hrPt', 'snw_LImon', 'hfcorr_Omon', 'sos_Omon', 'tos_Omon', 'wfcorr_Omon', 'siconc_SImon', 'sithick_SImon', 'ta_day', 'tas_day', 'tasmax_day', 'ua_day', 'va_day', 'zg_day', ] institute_details = { 'id': 'EC-Earth-Consortium', 'model_ids': ['EC-Earth3P'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'primWP5-amv-neg': { 'start_date': datetime(2000, 1, 1), 'end_date': datetime(2154, 1, 1) }, 'primWP5-amv-pos': { 'start_date': datetime(2000, 1, 1), 'end_date': datetime(2154, 1, 1) } } variant_labels = ['r{}i1p2f1'.format(i) for i in range(1, 26)] # activity_id ActivityId.objects.get_or_create(short_name=activity_id, full_name=activity_id) # Experiment cache experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') project = match_one(Project, short_name=mip_era) var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: for var_lab in variant_labels: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar'], rip_code=var_lab) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ institute_details = { 'id': 'CNRM-CERFACS', 'model_ids': ['CNRM-CM6-1', 'CNRM-CM6-1-HR'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'highresSST-present': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, } variant_label = 'r1i1p1f2' new_vreqs = [ { 'table_name': 'EmonZ', 'long_name': 'v-tendency nonorographic gravity wave drag', 'units': 'm s-2', 'var_name': 'vtendnogw', 'standard_name': 'tendency_of_northward_wind_due_to_nonorographic_gravity_wave_drag', 'cell_methods': 'longitude: mean time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'latitude plev39 time', 'cmor_name': 'vtendnogw', 'modeling_realm': 'atmos', 'frequency': 'mon', 'cell_measures': '', 'uid': '8183eac8-f906-11e6-a176-5404a60d96b5' }, { 'table_name': 'Emon', 'long_name': 'C3 grass Area Percentage', 'units': '%', 'var_name': 'grassFracC3', 'standard_name': 'area_fraction', 'cell_methods': 'area: mean where land over all_area_types time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time typec3pft typenatgr', 'cmor_name': 'grassFracC3', 'modeling_realm': 'land', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '8b814764-4a5b-11e6-9cd2-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'C4 grass Area Percentage', 'units': '%', 'var_name': 'grassFracC4', 'standard_name': 'area_fraction', 'cell_methods': 'area: mean where land over all_area_types time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time typec4pft typenatgr', 'cmor_name': 'grassFracC4', 'modeling_realm': 'land', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '8b814cc8-4a5b-11e6-9cd2-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Percentage of land use tile tile that is non-woody vegetation ( e.g. herbaceous crops)', 'units': '%', 'var_name': 'nwdFracLut', 'standard_name': 'missing', 'cell_methods': 'area: mean where land over all_area_types time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude landUse time typenwd', 'cmor_name': 'nwdFracLut', 'modeling_realm': 'land', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': 'd22da9f2-4a9f-11e6-b84e-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Total vegetated fraction', 'units': '%', 'var_name': 'vegFrac', 'standard_name': 'area_fraction', 'cell_methods': 'area: mean where land over all_area_types time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time typeveg', 'cmor_name': 'vegFrac', 'modeling_realm': 'land', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '6f6a57d0-9acb-11e6-b7ee-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Percentage Cover by C3 Crops', 'units': '%', 'var_name': 'cropFracC3', 'standard_name': 'area_fraction', 'cell_methods': 'area: mean where land over all_area_types time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time typec3pft typecrop', 'cmor_name': 'cropFracC3', 'modeling_realm': 'land', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '8b81522c-4a5b-11e6-9cd2-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Percentage Cover by C4 Crops', 'units': '%', 'var_name': 'cropFracC4', 'standard_name': 'area_fraction', 'cell_methods': 'area: mean where land over all_area_types time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time typec4pft typecrop', 'cmor_name': 'cropFracC4', 'modeling_realm': 'land', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '6f6a8ea8-9acb-11e6-b7ee-ac72891c3257' }, { 'table_name': 'LImon', 'long_name': 'Land Ice Area Percentage', 'units': '%', 'var_name': 'sftgif', 'standard_name': 'land_ice_area_fraction', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time typeli', 'cmor_name': 'sftgif', 'modeling_realm': 'land', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': 'b3d6e26a-3578-11e7-8257-5404a60d96b5' }, { 'table_name': 'LImon', 'long_name': 'Grounded Ice Sheet Area Percentage', 'units': '%', 'var_name': 'sftgrf', 'standard_name': 'grounded_ice_sheet_area_fraction', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time typegis', 'cmor_name': 'sftgrf', 'modeling_realm': 'landIce', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '8bbb1520-4a5b-11e6-9cd2-ac72891c3257' }, ] # Experiment new_dreqs = [ 'vtendnogw_EmonZ', 'cropFracC3_Emon', 'cropFracC4_Emon', 'grassFracC3_Emon', 'grassFracC4_Emon', 'nwdFracLut_Emon', 'ps_AERmon', 'sftgif_LImon', 'sftgrf_LImon', 'sidconcdyn_SImon', 'sidconcth_SImon', 'simass_SImon', 'sisaltmass_SImon', 'sisnthick_SImon', 'sndmassdyn_SImon', 'sndmasssi_SImon', 'vegFrac_Emon', ] experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the additional variable requests for new_vreq in new_vreqs: _vr = get_or_create(VariableRequest, **new_vreq) # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: try: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar'], rip_code=variant_label) except django.core.exceptions.MultipleObjectsReturned: logger.error('{}'.format(var_req_obj)) raise else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def make_data_submission(): """ Create a Data Submission and add files to it """ test_dsub = datasets.test_data_submission test_dsub.create_test_files() dsub = get_or_create(DataSubmission, status=STATUS_VALUES.ARRIVED, incoming_directory=test_dsub.INCOMING_DIR, directory=test_dsub.INCOMING_DIR, user='******') for dfile_name in test_dsub.files: path = os.path.join(test_dsub.INCOMING_DIR, dfile_name) metadata = _extract_file_metadata(path) proj = get_or_create( Project, short_name="CMIP6", full_name="Coupled Model Intercomparison Project Phase 6") climate_model = get_or_create(ClimateModel, short_name=metadata["climate_model"], full_name="Really good model") experiment = get_or_create(Experiment, short_name=metadata["experiment"], full_name="Really good experiment") institute = get_or_create(Institute, short_name="MOHC", full_name="Met Office Hadley Centre") var_match = match_one(VariableRequest, cmor_name=metadata['var_id'], table_name=metadata['table']) if var_match: variable = var_match else: msg = ('No variable request found for file: {}. Please create a ' 'variable request and resubmit.'.format(dfile_name)) print('ERROR: ' + msg) raise ValueError(msg) _dfile = DataFile.objects.create( name=dfile_name, incoming_directory=test_dsub.INCOMING_DIR, directory=test_dsub.INCOMING_DIR, size=os.path.getsize(path), project=proj, climate_model=climate_model, institute=institute, experiment=experiment, frequency=metadata["frequency"], rip_code=metadata["ensemble"], variable_request=variable, start_time=metadata["start_time"], end_time=metadata["end_time"], time_units=metadata['time_units'], calendar=metadata['calendar'], data_submission=dsub, online=True) ceda_ds = get_or_create(CEDADataset, catalogue_url='http://www.metoffice.gov.uk', directory=test_dsub.INCOMING_DIR) esgf_ds = get_or_create(ESGFDataset, drs_id='ab.cd.ef.gh', version='v19160519', directory=test_dsub.INCOMING_DIR, data_submission=dsub, ceda_dataset=ceda_ds) for df in dsub.get_data_files(): df.esgf_dataset = esgf_ds df.ceda_dataset = ceda_ds df.ceda_download_url = 'http://browse.ceda.ac.uk/browse/badc/cmip5/' + df.name df.ceda_opendap_url = 'http://dap.ceda.ac.uk/data/badc/cmip5/some/dir/' + df.name df.esgf_opendap_url = 'http://esgf.ceda.ac.uk/data/badc/cmip5/some/dir/' + df.name df.esgf_download_url = 'http://esgf.ceda.ac.uk/browse/badc/cmip5/' + df.name df.save() dsub.status = STATUS_VALUES.ARCHIVED dsub.save()
def main(args): """ Main entry point """ activity_id = 'primWP5' new_vreqs = [ { 'table_name': '6hrPlevPt', 'long_name': 'Eastward Wind', 'units': 'm s-1', 'var_name': 'ua', 'standard_name': 'eastward_wind', 'cell_methods': 'time: point', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time1', 'cmor_name': 'ua850', 'modeling_realm': 'atmos', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': 'prim6hrPlevPt_ua850' }, { 'table_name': '6hrPlevPt', 'long_name': 'Northward Wind', 'units': 'm s-1', 'var_name': 'va', 'standard_name': 'northward_wind', 'cell_methods': 'time: point', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time1', 'cmor_name': 'va850', 'modeling_realm': 'atmos', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': 'prim6hrPlevPt_va850' }, ] # create the additional variable requests for new_vreq in new_vreqs: _vr = get_or_create(VariableRequest, **new_vreq) new_dreqs = [ 'psl_6hrPlevPt', 'ua850_6hrPlevPt', 'va850_6hrPlevPt', 'zg500_6hrPlevPt', 'clt_Amon', 'hfls_Amon', 'hfss_Amon', 'hur_Amon', 'hurs_Amon', 'hus_Amon', 'huss_Amon', 'pr_Amon', 'prc_Amon', 'prsn_Amon', 'ps_Amon', 'psl_Amon', 'rlds_Amon', 'rldscs_Amon', 'rlus_Amon', 'rlut_Amon', 'rlutcs_Amon', 'rsds_Amon', 'rsdscs_Amon', 'rsdt_Amon', 'rsus_Amon', 'rsuscs_Amon', 'rsut_Amon', 'rsutcs_Amon', 'rtmt_Amon', 'sfcWind_Amon', 'ta_Amon', 'tas_Amon', 'tasmax_Amon', 'tasmin_Amon', 'tauu_Amon', 'tauv_Amon', 'ts_Amon', 'ua_Amon', 'uas_Amon', 'va_Amon', 'vas_Amon', 'wap_Amon', 'zg_Amon', 'snw_LImon', 'mrso_Lmon', 'ta_day', 'tas_day', 'tasmax_day', 'ua_day', 'va_day', 'zg_day' ] Institute.objects.get_or_create(short_name='NCAS', full_name='NCAS') ClimateModel.objects.get_or_create(short_name='MetUM-GOML2-LR', full_name='MetUM-GOML2-LR') ClimateModel.objects.get_or_create(short_name='MetUM-GOML2-HR', full_name='MetUM-GOML2-HR') institute_details = { 'id': 'NCAS', 'model_ids': ['MetUM-GOML2-LR', 'MetUM-GOML2-HR'], 'calendar': CALENDAR_360_DAY } experiments = { 'primWP5-amv-pos': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'primWP5-amv-neg': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) } } variant_labels = ['r{}i1p1f1'.format(i) for i in range(1, 16)] # activity_id ActivityId.objects.get_or_create(short_name=activity_id, full_name=activity_id) # Experiment create for expt in experiments: Experiment.objects.get_or_create(short_name=expt, full_name=expt) # Experiment cache experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: for var_lab in variant_labels: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar'], rip_code=var_lab) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ new_dreqs = [ 'siage_SImon', 'sisnthick_SImon', 'sicompstren_SImon', 'sisali_SImon', 'hus1000_Prim6hrPt', 'siconc_SImon', 'sistrxdtop_SImon', 'siv_SImon', 'sivol_SImon', 'sistrydtop_SImon', 'va1000_Prim6hrPt', 'sidmassevapsubl_SImon', 'sithick_SImon', 'sispeed_SImon', 'sndmasssnf_SImon', 'siflswdtop_SImon', 'sitemptop_SImon', 'ua1000_Prim6hrPt', 'siu_SImon' ] institute_details = { 'id': 'EC-Earth-Consortium', 'model_ids': ['EC-Earth3', 'EC-Earth3-HR'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'control-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2050, 1, 1) }, 'highres-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'hist-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-present': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'highresSST-LAI': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-smoothed': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-p4K': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-4co2': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'spinup-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(1980, 1, 1) }, } # Experiment experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar']) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def test_single_match(self): item_matched = dbapi.match_one(models.Institute, short_name='t') self.assertEqual(item_matched.short_name, 't')
def main(args): """ Main entry point """ # new_vreqs = ['lwsnl_Eday', 'mrros_Eday', 'snm_Eday', 'snd_Eday'] new_vreqs = [ {'table_name': 'Eday', 'long_name': 'Liquid Water Content of Snow Layer', 'units': 'kg m-2', 'var_name': 'lwsnl', 'standard_name': 'liquid_water_content_of_snow_layer', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'lwsnl', 'modeling_realm': 'landIce land', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': 'd228925a-4a9f-11e6-b84e-ac72891c3257'}, {'table_name': 'Eday', 'long_name': 'Surface Runoff', 'units': 'kg m-2 s-1', 'var_name': 'mrros', 'standard_name': 'surface_runoff_flux', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'mrros', 'modeling_realm': 'land', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': 'd2284048-4a9f-11e6-b84e-ac72891c3257'}, {'table_name': 'Eday', 'long_name': 'Surface Snow Melt', 'units': 'kg m-2 s-1', 'var_name': 'snm', 'standard_name': 'surface_snow_melt_flux', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'snm', 'modeling_realm': 'landIce land', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': 'd22848ea-4a9f-11e6-b84e-ac72891c3257'}, {'table_name': 'Eday', 'long_name': 'Snow Depth', 'units': 'm', 'var_name': 'snd', 'standard_name': 'surface_snow_thickness', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'snd', 'modeling_realm': 'landIce land', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': 'b7ccdf0a-7c00-11e6-bcdf-ac72891c3257'} ] new_dreqs = [ 'wap_Emon', 'hfdsn_LImon', 'snc_day', 'va27_Emon', 'rsuscs_CFday', 'vt_Emon', 'hursmin_day', 'uv_Emon', 'cl_Amon', 'rldscs_CFday', 'lwp_Primmon', 'uwap_Emon', 'rsdscs_Amon', 'zg_day', 'evspsblsoi_Lmon', 'snc_LImon', 'v2_Emon', 'ta27_Emon', 'tsl_Lmon', 'lwsnl_LImon', 'sbl_LImon', 'wap2_Emon', 'ut_Emon', 'hursmax_day', 't2_Emon', 'rsuscs_Amon', 'rsdscs_CFday', 'mrsos_day', 'hus27_Emon', 'snm_LImon', 'tsn_LImon', 'rldscs_Amon', 'vwap_Emon', 'zg27_Emon', 'twap_Emon', 'u2_Emon', 'ua27_Emon', 'lwsnl_Eday', 'mrros_Eday', 'snm_Eday', 'snd_Eday' ] institute_details = { 'id': 'EC-Earth-Consortium', 'model_ids': ['EC-Earth3-LR', 'EC-Earth3-HR'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'control-1950': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2050, 1, 1)}, 'highres-future': {'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1)}, 'hist-1950': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1)}, 'highresSST-present': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1)}, 'highresSST-future': {'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1)}, 'highresSST-LAI': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1)}, 'highresSST-smoothed': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1)}, 'highresSST-p4K': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1)}, 'highresSST-4co2': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1)}, 'spinup-1950': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(1980, 1, 1)}, } # Experiment experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id'] ) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'. format(clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the additional variable requests for new_vreq in new_vreqs: _vr = get_or_create(VariableRequest, **new_vreq) # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar'] ), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar'] ), time_units=std_units, calendar=institute_details['calendar'] ) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def test_double_match_returns_none(self): self.assertIsNone(dbapi.match_one(models.Institute, full_name='test'))