def test_simplest(self):
ret_req = get_or_create(RetrievalRequest,
requester=self.user,
start_year=1000,
end_year=3000,
id=999999)
ret_req.data_request.add(self.dreq1)
ret_req.save()
class ArgparseNamespace(object):
retrieval_id = ret_req.id
no_restore = False
skip_checksums = True
alternative = None
incoming = False
self.mock_exists.side_effect = [
False, # if os.path.exists(retrieval_dir):
True, # if not os.path.exists(extracted_file_path):
True, # if not os.path.exists(drs_dir):
False # if os.path.exists(dest_file_path):
]
ns = ArgparseNamespace()
get_tape_url('et:1234', [self.df1], ns)
df = match_one(DataFile, name='file_one.nc')
self.assertIsNotNone(df)
self.mock_rename.assert_called_once_with(
'/gws/nopw/j04/primavera5/.et_retrievals/ret_999999/'
'batch_01234/gws/MOHC/MY-MODEL/incoming/v12345678/file_one.nc',
'/gws/nopw/j04/primavera5/stream1/CMIP6/HighResMIP/'
'MOHC/MY-MODEL/experiment/r1i1p1f1/my-table/my-var/gn/v12345678/'
'file_one.nc')
self.assertTrue(df.online)
self.assertEqual(
df.directory, '/gws/nopw/j04/primavera5/'
'stream1/CMIP6/HighResMIP/MOHC/'
'MY-MODEL/experiment/r1i1p1f1/'
'my-table/my-var/gn/v12345678')
def _dict_to_object(dict_):
"""
Convert a dictionary to an object
"""
if '__class__' in dict_:
module = __import__(dict_['__module__'], fromlist=[dict_['__class__']])
klass = getattr(module, dict_['__class__'])
if dict_['__class__'] == 'PartialDateTime':
inst = klass(**dict_['__kwargs__'])
elif dict_['__class__'] in ('ActivityId', 'ClimateModel', 'Experiment',
'Institute', 'Project', 'VariableRequest',
'DataRequest'):
inst = match_one(klass, **dict_['__kwargs__'])
else:
msg = ('Cannot load from JSON files class {}'.format(
dict_['__class__']))
raise NotImplementedError(msg)
else:
inst = dict_
return inst
def _get_cmor_name(var_name, table_name):
"""
Check the variable request to find the actual CMOR name from a variable
name.
:param str var_name: The variable name from the first component of the
file name.
:param str table_name: The table name.
:return: The CMOR name
:rtype: str
"""
var_match = match_one(VariableRequest,
cmor_name=var_name,
table_name=table_name)
if var_match:
return var_name
else:
# if cmor_name doesn't match then it may be a variable where out_name
# is different to cmor_name so check these
var_matches = VariableRequest.objects.filter(var_name=var_name,
table_name=table_name)
if var_matches.count() == 1:
return var_matches[0].cmor_name
elif var_matches.count() == 0:
msg = ('No variable request found for variable {} {}.'.format(
table_name, var_name))
logger.error(msg)
else:
if table_name == 'E3hrPt':
if var_name == 'ua':
return 'ua7h'
elif var_name == 'va':
return 'va7h'
else:
msg = 'Please hard code a mapping for {} {}'.format(
table_name, var_name)
logger.error(msg)
else:
msg = 'Please hard code a mapping for {} {}'.format(
table_name, var_name)
logger.error(msg)
def main(args):
"""
Main entry point
"""
institute_details = {
'id': 'AWI',
'model_ids': ['AWI-CM-1-1-LR', 'AWI-CM-1-1-HR'],
'calendar': CALENDAR_STANDARD
}
experiments = {
'control-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2050, 1, 1)
},
'highres-future': {
'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)
},
'hist-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'spinup-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(1980, 1, 1)
}
}
variant_label = 'r1i1p1f2'
# Experiment
new_dreqs = [
'rsut_E3hr',
]
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Institute
result = match_one(Institute, short_name=institute_details['id'])
if result:
institute = result
else:
msg = 'institute_id {} not found in the database.'.format(
institute_details['id'])
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in institute_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.format(
clim_model))
print(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest,
cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
try:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, institute_details['calendar']),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, institute_details['calendar']),
time_units=std_units,
calendar=institute_details['calendar'],
rip_code=variant_label)
except django.core.exceptions.MultipleObjectsReturned:
logger.error('{}'.format(var_req_obj))
raise
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def main(args):
"""
Main entry point
"""
# new_vreqs = ['lwsnl_Eday', 'mrros_Eday', 'snm_Eday', 'snd_Eday']
new_dreqs = [
'mrlsl_6hrPlevPt', 'snw_6hrPlevPt', 'psl_6hrPlevPt', 'rsdt_E3hr',
'prw_E3hr', 'tas_6hrPlevPt', 'clivi_E3hr', 'mrsos_6hrPlevPt',
'ta7h_6hrPlevPt', 'clwvi_E3hr', 'pr_Prim6hr', 'rsdscs_3hr',
'va7h_6hrPlevPt', 'ts_6hrPlevPt', 'ua7h_6hrPlevPt', 'rsut_E3hr',
'clt_Prim6hr', 'ta27_6hrPlevPt', 'tsl_6hrPlevPt', 'rldscs_3hr',
'rlut_E3hr', 'huss_6hrPlevPt', 'hus27_6hrPlevPt', 'rlutcs_E3hr',
'hus7h_6hrPlevPt', 'rsuscs_3hr', 'ps_Prim6hr', 'sfcWind_6hrPlevPt',
'zg27_6hrPlevPt', 'rsds_Prim6hr'
]
institute_details = {
'id': 'EC-Earth-Consortium',
'model_ids': ['EC-Earth3-LR', 'EC-Earth3-HR'],
'calendar': CALENDAR_GREGORIAN
}
experiments = {
'control-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2050, 1, 1)
},
'highres-future': {
'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)
},
'hist-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-present': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-future': {
'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)
},
'highresSST-LAI': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-smoothed': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-p4K': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-4co2': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'spinup-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(1980, 1, 1)
},
}
# Experiment
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Institute
result = match_one(Institute, short_name=institute_details['id'])
if result:
institute = result
else:
msg = 'institute_id {} not found in the database.'.format(
institute_details['id'])
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in institute_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.format(
clim_model))
print(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest,
cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, institute_details['calendar']),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, institute_details['calendar']),
time_units=std_units,
calendar=institute_details['calendar'])
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def main(args):
"""
Main entry point
"""
new_vreqs = [
{
'table_name': 'Eday',
'long_name': 'River Discharge',
'units': 'm3 s-1',
'var_name': 'rivo',
'standard_name': 'water_flux_to_downstream',
'cell_methods': 'area: mean where land time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'rivo',
'modeling_realm': 'land',
'frequency': 'day',
'cell_measures': 'area: areacellr',
'uid': 'd2285b46-4a9f-11e6-b84e-ac72891c3257'
},
{
'table_name': 'Emon',
'long_name': 'Water table depth',
'units': 'm',
'var_name': 'wtd',
'standard_name': 'depth_of_soil_moisture_saturation',
'cell_methods': 'area: mean where land time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'wtd',
'modeling_realm': 'land',
'frequency': 'mon',
'cell_measures': 'area: areacellr',
'uid': '8b81f0ce-4a5b-11e6-9cd2-ac72891c3257'
},
{
'table_name': 'Emon',
'long_name': 'Terrestrial Water Storage',
'units': 'kg m-2',
'var_name': 'mrtws',
'standard_name': 'total_water_storage',
'cell_methods': 'area: mean where land time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'mrtws',
'modeling_realm': 'land',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': '6f6a4484-9acb-11e6-b7ee-ac72891c3257'
},
{
'table_name': 'Emon',
'long_name':
'Vertically integrated Eastward moisture transport (Mass_weighted_vertical integral of the product of eastward wind by total water mass per unit mass)',
'units': 'kg m-1 s-1',
'var_name': 'intuaw',
'standard_name': 'vertical_integral_eastward_wind_by_total_water',
'cell_methods': 'area: time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'intuaw',
'modeling_realm': 'atmos',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': '6f690484-9acb-11e6-b7ee-ac72891c3257'
},
{
'table_name': 'Emon',
'long_name':
'Vertically integrated Northward moisture transport (Mass_weighted_vertical integral of the product of northward wind by total water mass per unit mass)',
'units': 'kg m-1 s-1',
'var_name': 'intvaw',
'standard_name': 'vertical_integral_northward_wind_by_total_water',
'cell_methods': 'area: time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'intvaw',
'modeling_realm': 'atmos',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': '6f690b5a-9acb-11e6-b7ee-ac72891c3257'
},
{
'table_name': 'Emon',
'long_name':
'Vertically Integrated Eastward Dry Statice Energy Transport',
'units': '1.e6 J m-1 s-1',
'var_name': 'intuadse',
'standard_name':
'vertical_integral_eastward_wind_by_dry_static_energy',
'cell_methods': 'area: time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'intuadse',
'modeling_realm': 'atmos',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': '6f691104-9acb-11e6-b7ee-ac72891c3257'
},
{
'table_name': 'Emon',
'long_name':
'Vertically integrated Northward dry transport (cp.T +zg).v (Mass_weighted_vertical integral of the product of northward wind by dry static_energy per mass unit)',
'units': '1.e6 J m-1 s-1',
'var_name': 'intvadse',
'standard_name':
'vertical_integral_northward_wind_by_dry_static_energy',
'cell_methods': 'area: time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'intvadse',
'modeling_realm': 'atmos',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': '6f6916a4-9acb-11e6-b7ee-ac72891c3257'
},
{
'table_name': 'Emon',
'long_name': 'Heat content of upper 300 meters',
'units': 'm K',
'var_name': 'hcont300',
'standard_name': 'heat_content_of_ocean_layer',
'cell_methods': 'area: mean where sea time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time depth300m',
'cmor_name': 'hcont300',
'modeling_realm': 'ocean',
'frequency': 'mon',
'cell_measures': 'area: areacello',
'uid': '6f69513c-9acb-11e6-b7ee-ac72891c3257'
},
{
'table_name': 'AERmon',
'long_name': 'Sulfate Aerosol Optical Depth at 550nm',
'units': '1',
'var_name': 'od550so4',
'standard_name':
'atmosphere_optical_thickness_due_to_sulfate_ambient_aerosol',
'cell_methods': 'area: time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'od550so4',
'modeling_realm': 'aerosol',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': '19bf19ca-81b1-11e6-92de-ac72891c3257'
},
{
'table_name': 'AERmon',
'long_name':
'TOA Outgoing Clear-Sky, Aerosol-Free Shortwave Radiation',
'units': 'W m-2',
'var_name': 'rsutcsaf',
'standard_name': 'toa_outgoing_shortwave_flux_assuming_clear_sky',
'cell_methods': 'area: time: mean',
'positive': 'up',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'rsutcsaf',
'modeling_realm': 'aerosol',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': '8feac232-267c-11e7-8933-ac72891c3257'
},
{
'table_name': 'SImon',
'long_name': 'Sea-ice area fraction',
'units': '%',
'var_name': 'siconca',
'standard_name': 'sea_ice_area_fraction',
'cell_methods': 'area: time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time typesi',
'cmor_name': 'siconca',
'modeling_realm': 'seaIce',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': '71190054-faa7-11e6-bfb7-ac72891c3257'
},
{
'table_name': 'SImon',
'long_name': 'Sea ice area flux through straits',
'units': 'm2 s-1',
'var_name': 'siareaacrossline',
'standard_name': 'sea_ice_area_transport_across_line',
'cell_methods': 'time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'siline time',
'cmor_name': 'siareaacrossline',
'modeling_realm': 'seaIce',
'frequency': 'mon',
'cell_measures': '',
'uid': '712442ca-faa7-11e6-bfb7-ac72891c3257'
},
{
'table_name': 'SImon',
'long_name': 'Snow mass flux through straits',
'units': 'kg s-1',
'var_name': 'snmassacrossline',
'standard_name': 'snow_mass_transport_across_line',
'cell_methods': 'time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'siline time',
'cmor_name': 'snmassacrossline',
'modeling_realm': 'seaIce',
'frequency': 'mon',
'cell_measures': '',
'uid': '712fb3ee-faa7-11e6-bfb7-ac72891c3257'
},
]
institute_details = {
'id': 'CNRM-CERFACS',
'model_ids': ['CNRM-CM6-1'],
'calendar': CALENDAR_GREGORIAN
}
experiments = {
'control-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2050, 1, 1)
},
'highres-future': {
'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)
},
'hist-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
}
variant_label = 'r2i1p1f2'
# Experiment
new_dreqs = [
'rivo_Eday',
'siforceintstrx_PrimSIday',
'siforceintstry_PrimSIday',
'simassacrossline_PrimSIday',
'hfgeou_Omon',
'htovgyre_Omon',
'htovovrt_Omon',
'sicompstren_SImon',
'siflcondbot_SImon',
'siflcondtop_SImon',
'siflfwdrain_SImon',
'sifllatstop_SImon',
'sifllwutop_SImon',
'siflsensupbot_SImon',
'siflswdtop_SImon',
'siflswutop_SImon',
'simassacrossline_SImon',
'sisnmass_SImon',
'sitempbot_SImon',
'sithick_SImon',
'sltovgyre_Omon',
'sltovovrt_Omon',
'hcont300_Emon',
'intuadse_Emon',
'intuaw_Emon',
'intvadse_Emon',
'intvaw_Emon',
'mrtws_Emon',
'od550so4_AERmon',
'rsutcsaf_AERmon',
'siareaacrossline_SImon',
'siconca_SImon',
'snmassacrossline_SImon',
'wtd_Emon',
]
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Institute
result = match_one(Institute, short_name=institute_details['id'])
if result:
institute = result
else:
msg = 'institute_id {} not found in the database.'.format(
institute_details['id'])
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in institute_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.format(
clim_model))
print(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the additional variable requests
for new_vreq in new_vreqs:
_vr = get_or_create(VariableRequest, **new_vreq)
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest,
cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
try:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, institute_details['calendar']),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, institute_details['calendar']),
time_units=std_units,
calendar=institute_details['calendar'],
rip_code=variant_label)
except django.core.exceptions.MultipleObjectsReturned:
logger.error('{}'.format(var_req_obj))
raise
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def main(args):
"""
Main entry point
"""
rip_code = 'r1i1p2f1'
new_dreqs = [
'cl_Amon',
'clt_Amon',
'rlut_day',
'ua_day',
'va_day',
'rsut_CFday',
]
institute_details = {
'id': 'MOHC',
'model_ids': ['HadGEM3-GC31-LL'],
'calendar': CALENDAR_360_DAY
}
experiments = {
'hist-1950': {'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)},
}
# Experiment
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Institute
result = match_one(Institute, short_name=institute_details['id'])
if result:
institute = result
else:
msg = 'institute_id {} not found in the database.'.format(
institute_details['id']
)
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in institute_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.
format(clim_model))
print(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest, cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, institute_details['calendar']
),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, institute_details['calendar']
),
time_units=std_units,
calendar=institute_details['calendar'],
rip_code=rip_code
)
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def main(args):
"""
Main entry point
"""
activity_id = 'primWP5'
new_dreqs = [
'wfcorr_Omon',
]
institute_details = {
'id': 'EC-Earth-Consortium',
'model_ids': ['EC-Earth3P-HR'],
'calendar': CALENDAR_GREGORIAN
}
experiments = {
'primWP5-amv-neg': {'start_date': datetime(1950, 1, 1),
'end_date': datetime(1960, 1, 1)},
'primWP5-amv-pos': {'start_date': datetime(1950, 1, 1),
'end_date': datetime(1960, 1, 1)}
}
variant_labels = ['r{}i1p2f1'.format(i) for i in range(1, 11)]
# activity_id
ActivityId.objects.get_or_create(short_name=activity_id,
full_name=activity_id)
# Experiment cache
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Institute
result = match_one(Institute, short_name=institute_details['id'])
if result:
institute = result
else:
msg = 'institute_id {} not found in the database.'.format(
institute_details['id']
)
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in institute_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.
format(clim_model))
print(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest, cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
for var_lab in variant_labels:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, institute_details['calendar']
),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, institute_details['calendar']
),
time_units=std_units,
calendar=institute_details['calendar'],
rip_code = var_lab
)
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def main(args):
"""
Main entry point
"""
cmcc_dreqs = [
'siconc_SIday', 'simass_SImon', 'sisnmass_SImon', 'sitimefrac_SImon'
]
cmcc_details = {
'id': 'CMCC',
'model_ids': ['CMCC-CM2-HR4', 'CMCC-CM2-VHR4'],
'calendar': CALENDAR_STANDARD
}
experiments = {
'control-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2050, 1, 1)
},
'highres-future': {
'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)
},
'hist-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'spinup-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(1980, 1, 1)
},
}
# Experiment
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Institutes
result = match_one(Institute, short_name=cmcc_details['id'])
if result:
cmcc = result
else:
msg = 'institute_id {} not found in the database.'.format(
cmcc_details['id'])
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
cmcc_model_objs = []
for clim_model in cmcc_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
cmcc_model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.format(
clim_model))
print(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the new data requests
for new_dreq in cmcc_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest,
cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in cmcc_model_objs:
_dr = get_or_create(
DataRequest,
project=project,
institute=cmcc,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, cmcc_details['calendar']),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, cmcc_details['calendar']),
time_units=std_units,
calendar=cmcc_details['calendar'])
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def main(args):
"""
Main entry point
"""
new_vreqs = [
{
'table_name': 'Eday',
'long_name': 'Surface Snow Amount',
'units': 'kg m-2',
'var_name': 'snw',
'standard_name': 'surface_snow_amount',
'cell_methods': 'area: mean where land time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'snw',
'modeling_realm': 'landIce land',
'frequency': 'day',
'cell_measures': 'area: areacella',
'uid': 'd2288954-4a9f-11e6-b84e-ac72891c3257'
},
]
new_dreqs = ['mrso_Primday', 'mrlsl_Primday', 'zg_day', 'snw_Eday']
institute_details = {
'id': 'MPI-M',
'model_ids': ['MPIESM-1-2-HR', 'MPIESM-1-2-XR'],
'calendar': CALENDAR_PROLEPTIC_GREGORIAN
}
experiments = {
'control-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2050, 1, 1)
},
'highres-future': {
'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)
},
'hist-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-present': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-future': {
'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)
},
'highresSST-LAI': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-smoothed': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-p4K': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-4co2': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'spinup-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(1980, 1, 1)
},
}
# Experiment
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Institute
result = match_one(Institute, short_name=institute_details['id'])
if result:
institute = result
else:
msg = 'institute_id {} not found in the database.'.format(
institute_details['id'])
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in institute_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.format(
clim_model))
print(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the additional variable requests
for new_vreq in new_vreqs:
_vr = get_or_create(VariableRequest, **new_vreq)
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest,
cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, institute_details['calendar']),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, institute_details['calendar']),
time_units=std_units,
calendar=institute_details['calendar'])
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def main(args):
"""
Main entry point
"""
new_vreqs = [{
'table_name': 'Omon',
'long_name': 'Ocean Meridional Overturning Mass Streamfunction',
'units': 'kg s-1',
'var_name': 'msftmz',
'standard_name': 'ocean_meridional_overturning_mass_streamfunction',
'cell_methods':
'longitude: mean (comment: basin mean[ along zig-zag grid path]) time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'latitude olevel basin time',
'cmor_name': 'msftmz',
'modeling_realm': 'ocean',
'frequency': 'mon',
'cell_measures': '',
'uid': 'baa59d48-e5dd-11e5-8482-ac72891c3257'
}]
cmcc_dreqs = [
'msftmz_Omon', 'vso_PrimOmon', 'sos_Omon', 'wo_PrimOmon',
'mlotst_PrimOday', 'htovgyre_Omon', 'vto_PrimOmon', 'wto_PrimOmon',
'sltovgyre_Omon', 'hfbasin_Omon', 'tos_Oday', 'sos_Oday', 'tauuo_Omon',
'zossq_Omon', 'uto_PrimOmon', 'sltovovrt_Omon', 'mlotst_Omon',
'friver_Omon', 'wso_PrimOmon', 'tossq_Oday', 'uso_PrimOmon',
'tauvo_Omon', 'hfds_Omon', 'zos_PrimOday', 'htovovrt_Omon',
'tauuo_PrimOday', 'tauvo_PrimOday'
]
cmcc_details = {
'id': 'CMCC',
'model_ids': ['CMCC-CM2-HR4', 'CMCC-CM2-VHR4'],
'calendar': CALENDAR_STANDARD
}
experiments = {
'control-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2050, 1, 1)
},
'highres-future': {
'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)
},
'hist-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-present': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-future': {
'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)
},
'highresSST-LAI': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-smoothed': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-p4K': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-4co2': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'spinup-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(1980, 1, 1)
},
}
# Experiment
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Institutes
result = match_one(Institute, short_name=cmcc_details['id'])
if result:
cmcc = result
else:
msg = 'institute_id {} not found in the database.'.format(
cmcc_details['id'])
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
cmcc_model_objs = []
for clim_model in cmcc_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
cmcc_model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.format(
clim_model))
print(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the additional variable requests
for new_vreq in new_vreqs:
_vr = get_or_create(VariableRequest, **new_vreq)
# create the new data requests
for new_dreq in cmcc_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest,
cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in cmcc_model_objs:
_dr = get_or_create(
DataRequest,
project=project,
institute=cmcc,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, cmcc_details['calendar']),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, cmcc_details['calendar']),
time_units=std_units,
calendar=cmcc_details['calendar'])
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def main(args):
"""
Main entry point
"""
new_vreqs = [
{
'table_name': 'AERmon',
'long_name': 'Convective Cloud Area Fraction',
'units': '%',
'var_name': 'cltc',
'standard_name': 'convective_cloud_area_fraction',
'cell_methods': 'area: time: mean',
'positive': '',
'variable_type': 'float',
'dimensions': 'longitude latitude time',
'cmor_name': 'cltc',
'modeling_realm': 'aerosol',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': '01d412d0-c792-11e6-aa58-5404a60d96b5'
},
{
'table_name': 'AERmon',
'long_name': 'Eastward Wind',
'units': 'm s-1',
'var_name': 'ua',
'standard_name': 'eastward_wind',
'cell_methods': 'area: time: mean',
'positive': '',
'variable_type': 'float',
'dimensions': 'longitude latitude alevel time',
'cmor_name': 'ua',
'modeling_realm': 'aerosol',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': '19bfc73a-81b1-11e6-92de-ac72891c3257'
},
{
'table_name': 'AERmon',
'long_name': 'Northward Wind',
'units': 'm s-1',
'var_name': 'va',
'standard_name': 'northward_wind',
'cell_methods': 'area: time: mean',
'positive': '',
'variable_type': 'float',
'dimensions': 'longitude latitude alevel time',
'cmor_name': 'va',
'modeling_realm': 'aerosol',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': '19bfc9f6-81b1-11e6-92de-ac72891c3257'
},
{
'table_name': 'Emon',
'long_name': 'Specific Humidity',
'units': '1.0',
'var_name': 'hus',
'standard_name': 'specific_humidity',
'cell_methods': 'time: mean',
'positive': '',
'variable_type': '',
'dimensions': 'longitude latitude plev7c time',
'cmor_name': 'hus',
'modeling_realm': 'atmos',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': '81d83384-aa6a-11e6-9736-5404a60d96b5'
},
{
'table_name': 'E3hr',
'long_name': 'Sea Level Pressure',
'units': 'Pa',
'var_name': 'psl',
'standard_name': 'air_pressure_at_sea_level',
'cell_methods': 'area: time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'psl',
'modeling_realm': 'atmos',
'frequency': '3hr',
'cell_measures': 'area: areacella',
'uid': '8bb0d1c8-4a5b-11e6-9cd2-ac72891c3257'
},
{
'table_name': 'E3hrPt',
'long_name': 'ISCCP Percentage Cloud Area',
'units': '%',
'var_name': 'clisccp',
'standard_name': 'cloud_area_fraction_in_atmosphere_layer',
'cell_methods': 'area: mean time: point',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude plev7c tau time1',
'cmor_name': 'clisccp',
'modeling_realm': 'atmos',
'frequency': '3hr',
'cell_measures': 'area: areacella',
'uid': '8bb0db0a-4a5b-11e6-9cd2-ac72891c3257'
},
{
'table_name': '6hrPlevPt',
'long_name': 'Geopotential Height at 500 hPa',
'units': 'm',
'var_name': 'zg500',
'standard_name': 'geopotential_height',
'cell_methods': 'area: mean time: point',
'positive': '',
'variable_type': 'float',
'dimensions': 'longitude latitude time1',
'cmor_name': 'zg500',
'modeling_realm': 'aerosol',
'frequency': '6hr',
'cell_measures': 'area: areacella',
'uid': '7c70f59e-1ab7-11e7-8dfc-5404a60d96b5'
},
{
'table_name': '6hrPlevPt',
'long_name': 'Eastward Wind',
'units': 'm s-1',
'var_name': 'ua',
'standard_name': 'eastward_wind',
'cell_methods': 'area: mean time: point',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude plev3 time1',
'cmor_name': 'ua',
'modeling_realm': 'atmos',
'frequency': '6hr',
'cell_measures': 'area: areacella',
'uid': '8bae55ba-4a5b-11e6-9cd2-ac72891c3257'
},
{
'table_name': '6hrPlevPt',
'long_name': 'Northward Wind',
'units': 'm s-1',
'var_name': 'va',
'standard_name': 'northward_wind',
'cell_methods': 'area: mean time: point',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude plev3 time1',
'cmor_name': 'va',
'modeling_realm': 'atmos',
'frequency': '6hr',
'cell_measures': 'area: areacella',
'uid': '8bae5aba-4a5b-11e6-9cd2-ac72891c3257'
},
{
'table_name': '6hrPlevPt',
'long_name': 'Air Temperature',
'units': 'K',
'var_name': 'ta',
'standard_name': 'air_temperature',
'cell_methods': 'area: mean time: point',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude plev3 time1',
'cmor_name': 'ta',
'modeling_realm': 'atmos',
'frequency': '6hr',
'cell_measures': 'area: areacella',
'uid': '6a35d178-aa6a-11e6-9736-5404a60d96b5'
},
{
'table_name': 'Eday',
'long_name': 'Liquid Water Content of Snow Layer',
'units': 'kg m-2',
'var_name': 'lwsnl',
'standard_name': 'liquid_water_content_of_snow_layer',
'cell_methods': 'area: mean where land time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'lwsnl',
'modeling_realm': 'landIce land',
'frequency': 'day',
'cell_measures': 'area: areacella',
'uid': 'd228925a-4a9f-11e6-b84e-ac72891c3257'
},
{
'table_name': 'Eday',
'long_name': 'Surface Runoff',
'units': 'kg m-2 s-1',
'var_name': 'mrros',
'standard_name': 'surface_runoff_flux',
'cell_methods': 'area: mean where land time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'mrros',
'modeling_realm': 'land',
'frequency': 'day',
'cell_measures': 'area: areacella',
'uid': 'd2284048-4a9f-11e6-b84e-ac72891c3257'
},
{
'table_name': 'Eday',
'long_name': 'Surface Snow Melt',
'units': 'kg m-2 s-1',
'var_name': 'snm',
'standard_name': 'surface_snow_melt_flux',
'cell_methods': 'area: mean where land time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'snm',
'modeling_realm': 'landIce land',
'frequency': 'day',
'cell_measures': 'area: areacella',
'uid': 'd22848ea-4a9f-11e6-b84e-ac72891c3257'
},
{
'table_name': 'Eday',
'long_name': 'Snow Depth',
'units': 'm',
'var_name': 'snd',
'standard_name': 'surface_snow_thickness',
'cell_methods': 'area: mean where land time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'snd',
'modeling_realm': 'landIce land',
'frequency': 'day',
'cell_measures': 'area: areacella',
'uid': 'b7ccdf0a-7c00-11e6-bcdf-ac72891c3257'
},
{
'table_name': 'Eday',
'long_name': 'Surface Temperature',
'units': 'K',
'var_name': 'ts',
'standard_name': 'surface_temperature',
'cell_methods': 'area: time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'ts',
'modeling_realm': 'atmos',
'frequency': 'day',
'cell_measures': 'area: areacella',
'uid': '8b8fc3de-4a5b-11e6-9cd2-ac72891c3257'
},
{
'table_name': 'Eday',
'long_name': 'Eastward Wind',
'units': 'm s-1',
'var_name': 'ua',
'standard_name': 'eastward_wind',
'cell_methods': 'time: mean',
'positive': '',
'variable_type': '',
'dimensions': 'longitude latitude plev19 time',
'cmor_name': 'ua',
'modeling_realm': 'atmos',
'frequency': 'day',
'cell_measures': 'area: areacella',
'uid': '8bae00ba-4a5b-11e6-9cd2-ac72891c3257'
},
{
'table_name': 'Eday',
'long_name': 'Northward Wind',
'units': 'm s-1',
'var_name': 'va',
'standard_name': 'northward_wind',
'cell_methods': 'time: mean',
'positive': '',
'variable_type': '',
'dimensions': 'longitude latitude plev19 time',
'cmor_name': 'va',
'modeling_realm': 'atmos',
'frequency': 'day',
'cell_measures': 'area: areacella',
'uid': '8bae05ec-4a5b-11e6-9cd2-ac72891c3257'
},
{
'table_name': '6hrPlev',
'long_name': 'Specific Humidity',
'units': '1.0',
'var_name': 'hus',
'standard_name': 'specific_humidity',
'cell_methods': 'area: time: mean',
'positive': '',
'variable_type': '',
'dimensions': 'longitude latitude plev4 time',
'cmor_name': 'hus4',
'modeling_realm': 'atmos',
'frequency': '6hr',
'cell_measures': 'area: areacella',
'uid': '8bae64ba-4a5b-11e6-9cd2-ac72891c3257'
},
{
'table_name': 'day',
'long_name': 'Total Soil Moisture Content',
'units': 'kg m-2',
'var_name': 'mrso',
'standard_name': 'soil_moisture_content',
'cell_methods': 'area: mean where land time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'mrso',
'modeling_realm': 'land',
'frequency': 'day',
'cell_measures': 'area: areacella',
'uid': '3c641b6c-b89b-11e6-be04-ac72891c3257'
},
]
new_dreqs = [
'psl_E3hr',
]
institute_details = {
'id': 'MOHC',
'model_ids': ['HadGEM3-GC31-HM', 'HadGEM3-GC31-MM', 'HadGEM3-GC31-LM'],
'calendar': CALENDAR_360_DAY
}
experiments = {
'control-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2050, 1, 1)
},
'highres-future': {
'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)
},
'hist-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-present': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-future': {
'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)
},
'highresSST-LAI': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-smoothed': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-p4K': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-4co2': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'spinup-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(1980, 1, 1)
},
}
# Experiment
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Institute
result = match_one(Institute, short_name=institute_details['id'])
if result:
institute = result
else:
msg = 'institute_id {} not found in the database.'.format(
institute_details['id'])
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in institute_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.format(
clim_model))
print(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the additional variable requests
for new_vreq in new_vreqs:
_vr = get_or_create(VariableRequest, **new_vreq)
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest,
cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, institute_details['calendar']),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, institute_details['calendar']),
time_units=std_units,
calendar=institute_details['calendar'])
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def main(args):
"""
Main entry point
"""
activity_id = 'primWP5'
new_dreqs = [
'psl_6hrPlevPt',
'tas_6hrPlevPt',
'uas_6hrPlevPt',
'vas_6hrPlevPt',
'clivi_Amon',
'clt_Amon',
'clwvi_Amon',
'evspsbl_Amon',
'hfls_Amon',
'hfss_Amon',
'hur_Amon',
'hus_Amon',
'pr_Amon',
'prc_Amon',
'prsn_Amon',
'prw_Amon',
'ps_Amon',
'psl_Amon',
'rlds_Amon',
'rldscs_Amon',
'rlus_Amon',
'rlut_Amon',
'rlutcs_Amon',
'rsds_Amon',
'rsdscs_Amon',
'rsdt_Amon',
'rsus_Amon',
'rsuscs_Amon',
'rsut_Amon',
'rsutcs_Amon',
'sfcWind_Amon',
'ta_Amon',
'tas_Amon',
'tasmax_Amon',
'tasmin_Amon',
'tauu_Amon',
'tauv_Amon',
'ts_Amon',
'ua_Amon',
'uas_Amon',
'va_Amon',
'vas_Amon',
'wap_Amon',
'zg_Amon',
'snd_LImon',
'snm_LImon',
'snw_LImon',
'tsn_LImon',
'lai_Lmon',
'mrlsl_Lmon',
'mrro_Lmon',
'mrso_Lmon',
'tsl_Lmon',
'sos_Oday',
'tos_Oday',
'hfds_Omon',
'mlotst_Omon',
'mlotstsq_Omon',
'so_Omon',
'sos_Omon',
'tauuo_Omon',
'tauvo_Omon',
'thetao_Omon',
'tos_Omon',
'tossq_Omon',
'uo_Omon',
'vo_Omon',
'wfo_Omon',
'wo_Omon',
'zos_Omon',
'zossq_Omon',
'clt_Prim6hrPt',
'mlotst_PrimOday',
'tauuo_PrimOday',
'tauvo_PrimOday',
'uo_PrimOday',
'vo_PrimOday',
'zos_PrimOday',
'uso_PrimOmon',
'uto_PrimOmon',
'vso_PrimOmon',
'vto_PrimOmon',
'wso_PrimOmon',
'wto_PrimOmon',
'sistrxdtop_PrimSIday',
'sistrydtop_PrimSIday',
'evspsbl_Primday',
'mrlsl_Primday',
'mrso_Primday',
'ts_Primday',
'uneutrals_Primday',
'vneutrals_Primday',
'siconc_SIday',
'sisnthick_SIday',
'sitemptop_SIday',
'sithick_SIday',
'siu_SIday',
'siv_SIday',
'siconc_SImon',
'sisnthick_SImon',
'sistrxdtop_SImon',
'sistrydtop_SImon',
'sitemptop_SImon',
'sithick_SImon',
'siu_SImon',
'siv_SImon',
'sivol_SImon',
'clt_day',
'hfls_day',
'hfss_day',
'hur_day',
'hus_day',
'pr_day',
'prc_day',
'prsn_day',
'psl_day',
'rlds_day',
'rlus_day',
'rlut_day',
'rsds_day',
'rsus_day',
'sfcWind_day',
'tas_day',
'tasmax_day',
'tasmin_day',
'tslsi_day',
'uas_day',
'vas_day',
'wap_day',
]
institute_details = {
'id': 'ECMWF',
'model_ids': ['ECMWF-IFS-LR', 'ECMWF-IFS-HR'],
'calendar': CALENDAR_GREGORIAN
}
experiments = {
'primWP5-coupledseaice': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(1960, 1, 1)
},
}
variant_labels = ['r{}i1p1f1'.format(i) for i in range(1, 41)]
# activity_id
ActivityId.objects.get_or_create(short_name=activity_id,
full_name=activity_id)
# Experiment create
for expt in experiments:
Experiment.objects.get_or_create(short_name=expt, full_name=expt)
# Experiment cache
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Institute
result = match_one(Institute, short_name=institute_details['id'])
if result:
institute = result
else:
msg = 'institute_id {} not found in the database.'.format(
institute_details['id'])
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in institute_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.format(
clim_model))
print(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest,
cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
for var_lab in variant_labels:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, institute_details['calendar']),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, institute_details['calendar']),
time_units=std_units,
calendar=institute_details['calendar'],
rip_code=var_lab)
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def main(args):
"""
Main entry point
"""
if args.processes == 1 and not iris.__version__.startswith('1.'):
# if not multiprocessing then limit the number of Dask threads
# this can't seem to be limited when using multiprocessing
dask.config.set(pool=ThreadPool(2))
submission_dir = os.path.normpath(args.directory)
logger.debug('Submission directory: %s', submission_dir)
logger.debug('Project: %s', args.mip_era)
logger.debug('Processes requested: %s', args.processes)
try:
if args.input:
validated_metadata = read_json_file(args.input)
data_sub = _get_submission_object(submission_dir)
files_online = False
else:
files_online = True
data_files = list_files(submission_dir)
logger.debug('%s files identified', len(data_files))
if not args.validate_only and not args.output:
data_sub = _get_submission_object(submission_dir)
if data_sub.status != 'ARRIVED':
msg = "The submission's status is not ARRIVED."
logger.error(msg)
raise SubmissionError(msg)
try:
if not args.no_prepare:
run_prepare(data_files, args.processes)
validated_metadata = list(
identify_and_validate(data_files, args.mip_era,
args.processes, args.file_format))
except SubmissionError:
if not args.validate_only and not args.output:
send_admin_rejection_email(data_sub)
raise
logger.debug('%s files validated successfully',
len(validated_metadata))
if args.validate_only:
logger.debug('Data submission not run (-v option specified)')
logger.debug('Processing complete')
sys.exit(0)
if not args.relaxed and len(validated_metadata) != len(data_files):
# if not args.output:
# rejected_dir = move_rejected_files(submission_dir)
# set_status_rejected(data_sub, rejected_dir)
# send_user_rejection_email(data_sub)
msg = ('Not all files passed validation. Please fix these '
'errors and then re-run this script.')
logger.error(msg)
raise SubmissionError(msg)
if args.tape_base_url:
add_tape_url(validated_metadata, args.tape_base_url,
submission_dir)
if args.output:
write_json_file(validated_metadata, args.output)
else:
update_database_submission(validated_metadata, data_sub,
files_online, args.version_string)
logger.debug(
'%s files submitted successfully',
match_one(DataSubmission, incoming_directory=submission_dir).
get_data_files().count())
except SubmissionError:
sys.exit(1)
logger.debug('Processing complete')
def verify_fk_relationships(metadata):
"""
Identify the variable_request and data_request objects corresponding to this file.
:param dict metadata: Metadata identified for this file.
:raises SubmissionError: If there are no existing entries in the
database for `Project`, `ClimateModel` or `Experiment`.
"""
foreign_key_types = [(Project, 'project'), (ClimateModel, 'climate_model'),
(Experiment, 'experiment'), (Institute, 'institute'),
(ActivityId, 'activity_id')]
# get values for each of the foreign key types
for object_type, object_str in foreign_key_types:
result = match_one(object_type, short_name=metadata[object_str])
if result:
metadata[object_str] = result
else:
msg = ("No {} '{}' found for file: {}. Please create this object "
"and resubmit.".format(object_str.replace('_', ' '),
metadata[object_str],
metadata['basename']))
logger.error(msg)
raise SubmissionError(msg)
# find the data request
dreq_match = match_one(DataRequest,
project=metadata['project'],
institute=metadata['institute'],
climate_model=metadata['climate_model'],
experiment=metadata['experiment'],
variable_request__table_name=metadata['table'],
variable_request__cmor_name=metadata['var_name'],
rip_code=metadata['rip_code'])
if dreq_match:
metadata['data_request'] = dreq_match
metadata['variable'] = dreq_match.variable_request
else:
# if cmor_name doesn't match then it may be a variable where out_name
# is different to cmor_name so check these
dreq_matches = DataRequest.objects.filter(
project=metadata['project'],
institute=metadata['institute'],
climate_model=metadata['climate_model'],
experiment=metadata['experiment'],
variable_request__table_name=metadata['table'],
variable_request__var_name=metadata['var_name'],
rip_code=metadata['rip_code'])
if dreq_matches.count() == 0:
msg = ('No data request found for file: {}.'.format(
metadata['basename']))
logger.error(msg)
raise FileValidationError(msg)
elif dreq_matches.count() == 1:
metadata['data_request'] = dreq_matches[0]
metadata['variable'] = dreq_matches[0].variable_request
else:
try:
plev_name = _guess_plev_name(metadata)
except Exception:
msg = ('Cannot open file to determine plev name: {}.'.format(
metadata['basename']))
logger.error(msg)
raise FileValidationError(msg)
if plev_name:
plev_matches = dreq_matches.filter(
variable_request__dimensions__icontains=plev_name)
if plev_matches.count() == 1:
metadata['data_request'] = plev_matches[0]
metadata['variable'] = plev_matches[0].variable_request
elif plev_matches.count() == 0:
msg = ('No data requests found with plev {} for file: {}.'.
format(plev_name, metadata['basename']))
logger.error(msg)
raise FileValidationError(msg)
else:
msg = ('Multiple data requests found with plev {} for '
'file: {}.'.format(plev_name, metadata['basename']))
logger.error(msg)
raise FileValidationError(msg)
else:
msg = ('Unable to determine plev name: {}.'.format(
metadata['basename']))
logger.error(msg)
raise FileValidationError(msg)
def test_no_match_returns_none(self):
self.assertIsNone(dbapi.match_one(models.Institute, full_name='real'))
def main(args):
"""
Main entry point
"""
new_vreqs = [
{'table_name': 'Ofx',
'long_name': 'Grid-Cell Area for Ocean Variables',
'units': 'm2',
'var_name': 'areacello',
'standard_name': 'cell_area',
'cell_methods': 'area: sum',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude ',
'cmor_name': 'areacello',
'modeling_realm': 'ocean',
'frequency': 'fx',
'cell_measures': '',
'uid': 'baa3ee94-e5dd-11e5-8482-ac72891c3257'},
{'table_name': 'Ofx',
'long_name': 'Region Selection Index',
'units': '1',
'var_name': 'basin',
'standard_name': 'region',
'cell_methods': 'area: mean',
'positive': '',
'variable_type': 'integer',
'dimensions': 'longitude latitude ',
'cmor_name': 'basin',
'modeling_realm': 'ocean',
'frequency': 'fx',
'cell_measures': 'area: areacello',
'uid': 'baa3f718-e5dd-11e5-8482-ac72891c3257'},
{'table_name': 'Ofx',
'long_name': 'Ocean Model Cell Thickness',
'units': 'm',
'var_name': 'thkcello',
'standard_name': 'cell_thickness',
'cell_methods': 'area: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude olevel ',
'cmor_name': 'thkcello',
'modeling_realm': 'ocean',
'frequency': 'fx',
'cell_measures': 'area: areacello volume: volcello',
'uid': 'bab9bd00-e5dd-11e5-8482-ac72891c3257'},
]
# create the variable requests
for new_vreq in new_vreqs:
_vr = get_or_create(VariableRequest, **new_vreq)
variant_label = 'r1i1p1f1'
# The standard reference time
std_units = Settings.get_solo().standard_time_units
new_dreqs = [
'areacello_Ofx',
'basin_Ofx',
]
institute_id = 'MOHC'
calendar = CALENDAR_360_DAY
# Institute
result = match_one(Institute, short_name=institute_id)
if result:
institute = result
else:
msg = 'institute_id {} not found in the database.'.format(institute_id)
print(msg)
raise ValueError(msg)
# Coupled
#########
model_ids = ['HadGEM3-GC31-HH', 'HadGEM3-GC31-HM', 'HadGEM3-GC31-MM',
'HadGEM3-GC31-LL']
experiments = {
'spinup-1950': {'start_date': datetime(1950, 1, 1),
'end_date': datetime(1980, 1, 1)},
'hist-1950': {'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)},
'control-1950': {'start_date': datetime(1950, 1, 1),
'end_date': datetime(2050, 1, 1)},
'highres-future': {'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)},
}
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in model_ids:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.
format(clim_model))
print(msg)
raise ValueError(msg)
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest, cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
try:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, calendar
),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, calendar
),
time_units=std_units,
calendar=calendar,
rip_code = variant_label
)
except django.core.exceptions.MultipleObjectsReturned:
logger.error('{}'.format(var_req_obj))
raise
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
# Atmosphere only
#################
experiments = {
'highresSST-present': {'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)},
'highresSST-future': {'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)},
}
model_ids = ['HadGEM3-GC31-HM', 'HadGEM3-GC31-MM', 'HadGEM3-GC31-LM']
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in model_ids:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.
format(clim_model))
print(msg)
raise ValueError(msg)
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest, cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
try:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, calendar
),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, calendar
),
time_units=std_units,
calendar=calendar,
rip_code = variant_label
)
except django.core.exceptions.MultipleObjectsReturned:
logger.error('{}'.format(var_req_obj))
raise
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def main(args):
"""
Main entry point
"""
# Delete the data requests that could be duplicates
dupe_dreqs = DataRequest.objects.filter(
climate_model__short_name='CNRM-CM6-1-HR',
experiment__short_name='highres-future',
variable_request__table_name='SImon',
variable_request__table_name__in = ['simass', 'sisaltmass', 'sisnthick']
)
if dupe_dreqs.count() != 3:
logger.error('{} data requests found when expecting 3'.
format(dupe_dreqs.count()))
sys.exit(1)
dupe_dreqs.delete()
institute_details = {
'id': 'CNRM-CERFACS',
'model_ids': ['CNRM-CM6-1', 'CNRM-CM6-1-HR'],
'calendar': CALENDAR_GREGORIAN
}
experiments = {
'highres-future': {'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)},
}
variant_label = 'r1i1p1f2'
# Experiment
new_dreqs = [
'cropFracC3_Emon',
'cropFracC4_Emon',
'grassFracC3_Emon',
'grassFracC4_Emon',
'nwdFracLut_Emon',
'ps_AERmon',
'sftgif_LImon',
'sftgrf_LImon',
'sidconcdyn_SImon',
'sidconcth_SImon',
'simass_SImon',
'sisaltmass_SImon',
'sisnthick_SImon',
'sndmassdyn_SImon',
'sndmasssi_SImon',
'vegFrac_Emon',
]
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Institute
result = match_one(Institute, short_name=institute_details['id'])
if result:
institute = result
else:
msg = 'institute_id {} not found in the database.'.format(
institute_details['id']
)
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in institute_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.
format(clim_model))
print(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest, cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
try:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, institute_details['calendar']
),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, institute_details['calendar']
),
time_units=std_units,
calendar=institute_details['calendar'],
rip_code = variant_label
)
except django.core.exceptions.MultipleObjectsReturned:
logger.error('{}'.format(var_req_obj))
raise
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def main(args):
"""
Main entry point
"""
institute_details = {
'id': 'CNRM-CERFACS',
'model_ids': ['CNRM-CM6-1', 'CNRM-CM6-1-HR'],
'calendar': CALENDAR_GREGORIAN
}
experiments = {
'spinup-1950': {'start_date': datetime(1950, 1, 1),
'end_date': datetime(1980, 1, 1)},
}
variant_label = 'r1i1p1f2'
# Experiment
new_dreqs = [
'od550so4_AERmon',
'rsutcsaf_AERmon',
'hcont300_Emon',
'intuadse_Emon',
'intuaw_Emon',
'intvadse_Emon',
'intvaw_Emon',
'mrtws_Emon',
'wtd_Emon',
'htovgyre_Omon',
'htovovrt_Omon',
'siconca_SImon',
'simass_SImon',
'sisaltmass_SImon',
'sisnmass_SImon',
'sisnthick_SImon',
'sithick_SImon',
'rivo_Eday',
'siforceintstrx_PrimSIday',
'siforceintstry_PrimSIday',
'simassacrossline_PrimSIday',
]
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Institute
result = match_one(Institute, short_name=institute_details['id'])
if result:
institute = result
else:
msg = 'institute_id {} not found in the database.'.format(
institute_details['id']
)
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in institute_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.
format(clim_model))
print(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest, cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
try:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, institute_details['calendar']
),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, institute_details['calendar']
),
time_units=std_units,
calendar=institute_details['calendar'],
rip_code = variant_label
)
except django.core.exceptions.MultipleObjectsReturned:
logger.error('{}'.format(var_req_obj))
raise
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def main(args):
"""
Main entry point
"""
mip_era = 'CMIP6'
activity_id = 'PRIMAVERA'
new_dreqs = [
'pr_day', 'psl_6hrPlevPt', 'ua_6hrPlevPt', 'uas_6hrPlevPt',
'va_6hrPlevPt', 'vas_6hrPlevPt'
]
institute_details = {
'id': 'EC-Earth-Consortium',
'model_ids': ['EC-Earth3'],
'calendar': CALENDAR_GREGORIAN
}
experiments = {
'Primavera-coupled': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2016, 1, 1)
}
}
variant_labels = ['r1i1p101f1']
# activity_id
ActivityId.objects.get_or_create(short_name=activity_id,
full_name=activity_id)
# Experiment create
for expt in experiments:
Experiment.objects.get_or_create(short_name=expt, full_name=expt)
# Experiment cache
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
logger.error(msg)
raise ValueError(msg)
# Institute
result = match_one(Institute, short_name=institute_details['id'])
if result:
institute = result
else:
msg = 'institute_id {} not found in the database.'.format(
institute_details['id'])
logger.error(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in institute_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.format(
clim_model))
logger.error(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
project = match_one(Project, short_name=mip_era)
var_req_obj = match_one(VariableRequest,
cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
for var_lab in variant_labels:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, institute_details['calendar']),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, institute_details['calendar']),
time_units=std_units,
calendar=institute_details['calendar'],
rip_code=var_lab)
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
logger.error(msg)
raise ValueError(msg)
def main(args):
"""
Main entry point
"""
activity_id = 'primWP5'
new_dreqs = [
'uas_3hr',
'vas_3hr',
'pr_6hrPlev',
'psl_6hrPlev',
'tas_6hrPlev',
'uas_6hrPlev',
'vas_6hrPlev',
'zg1000_6hrPlev',
'ta_6hrPlevPt',
'ua_6hrPlevPt',
'va_6hrPlevPt',
'zg27_6hrPlevPt',
'zg500_AERday',
'ptp_AERmon',
'rlutaf_AERmon',
'rlutcsaf_AERmon',
'rsutaf_AERmon',
'rsutcsaf_AERmon',
'toz_AERmon',
'ztp_AERmon',
'clivi_Amon',
'clt_Amon',
'clwvi_Amon',
'evspsbl_Amon',
'hfls_Amon',
'hfss_Amon',
'hur_Amon',
'hurs_Amon',
'hus_Amon',
'huss_Amon',
'o3_Amon',
'pr_Amon',
'prc_Amon',
'prsn_Amon',
'prw_Amon',
'ps_Amon',
'psl_Amon',
'rlds_Amon',
'rldscs_Amon',
'rlus_Amon',
'rlut_Amon',
'rlutcs_Amon',
'rsds_Amon',
'rsdscs_Amon',
'rsdt_Amon',
'rsus_Amon',
'rsuscs_Amon',
'rsut_Amon',
'rsutcs_Amon',
'sbl_Amon',
'sfcWind_Amon',
'ta_Amon',
'tas_Amon',
'tasmax_Amon',
'tasmin_Amon',
'tauu_Amon',
'tauv_Amon',
'ts_Amon',
'ua_Amon',
'uas_Amon',
'va_Amon',
'vas_Amon',
'wap_Amon',
'zg_Amon',
'ps_CFday',
'mlotst_Eday',
'rivo_Eday',
't20d_Eday',
'ta850_Eday',
'ts_Eday',
'sftgrf_Efx',
'cropFracC3_Emon',
'cropFracC4_Emon',
'evspsblpot_Emon',
'grassFracC3_Emon',
'grassFracC4_Emon',
'intuadse_Emon',
'intuaw_Emon',
'intvadse_Emon',
'intvaw_Emon',
'mrtws_Emon',
'nwdFracLut_Emon',
'orog_Emon',
'rls_Emon',
'rss_Emon',
'sfcWindmax_Emon',
't20d_Emon',
'thetaot_Emon',
'thetaot2000_Emon',
'thetaot300_Emon',
'thetaot700_Emon',
'vegFrac_Emon',
'wtd_Emon',
'lwsnl_LImon',
'sftgif_LImon',
'sftgrf_LImon',
'snc_LImon',
'snd_LImon',
'snw_LImon',
'baresoilFrac_Lmon',
'c3PftFrac_Lmon',
'c4PftFrac_Lmon',
'cropFrac_Lmon',
'gpp_Lmon',
'grassFrac_Lmon',
'lai_Lmon',
'mrfso_Lmon',
'mrro_Lmon',
'mrros_Lmon',
'mrso_Lmon',
'mrsos_Lmon',
'npp_Lmon',
'prveg_Lmon',
'ra_Lmon',
'rh_Lmon',
'treeFrac_Lmon',
'tsl_Lmon',
'sos_Oday',
'tos_Oday',
'areacello_Ofx',
'basin_Ofx',
'deptho_Ofx',
'hfgeou_Ofx',
'masscello_Ofx',
'thkcello_Ofx',
'bigthetao_Omon',
'bigthetaoga_Omon',
'ficeberg_Omon',
'friver_Omon',
'hfbasin_Omon',
'hfcorr_Omon',
'hfds_Omon',
'hfx_Omon',
'hfy_Omon',
'htovgyre_Omon',
'htovovrt_Omon',
'mfo_Omon',
'mlotst_Omon',
'mlotstmin_Omon',
'msftyz_Omon',
'rsntds_Omon',
'so_Omon',
'soga_Omon',
'sos_Omon',
'sosga_Omon',
'tauuo_Omon',
'tauvo_Omon',
'thetao_Omon',
'thkcello_Omon',
'tos_Omon',
'umo_Omon',
'uo_Omon',
'vmo_Omon',
'vo_Omon',
'volo_Omon',
'wfo_Omon',
'wmo_Omon',
'wo_Omon',
'zos_Omon',
'zossq_Omon',
'zostoga_Omon',
'rsds_Prim6hr',
'rsdsdiff_Prim6hr',
'mrso_Primday',
'siconc_SIday',
'siconca_SIday',
'sisnthick_SIday',
'sispeed_SIday',
'sitemptop_SIday',
'sithick_SIday',
'sitimefrac_SIday',
'siu_SIday',
'siv_SIday',
'siage_SImon',
'siareaacrossline_SImon',
'siarean_SImon',
'siareas_SImon',
'sicompstren_SImon',
'siconc_SImon',
'siconca_SImon',
'sidconcdyn_SImon',
'sidconcth_SImon',
'sidivvel_SImon',
'sidmassdyn_SImon',
'sidmassevapsubl_SImon',
'sidmassgrowthbot_SImon',
'sidmassgrowthwat_SImon',
'sidmasslat_SImon',
'sidmassmeltbot_SImon',
'sidmassmelttop_SImon',
'sidmasssi_SImon',
'sidmassth_SImon',
'sidmasstranx_SImon',
'sidmasstrany_SImon',
'siextentn_SImon',
'siextents_SImon',
'sifb_SImon',
'siflcondbot_SImon',
'siflcondtop_SImon',
'siflfwbot_SImon',
'siflfwdrain_SImon',
'sifllatstop_SImon',
'sifllwutop_SImon',
'siflsensupbot_SImon',
'siflswdbot_SImon',
'siflswdtop_SImon',
'siflswutop_SImon',
'sihc_SImon',
'simass_SImon',
'simassacrossline_SImon',
'sipr_SImon',
'sisaltmass_SImon',
'sishevel_SImon',
'sisnconc_SImon',
'sisnhc_SImon',
'sisnmass_SImon',
'sisnthick_SImon',
'sispeed_SImon',
'sistrxdtop_SImon',
'sistrxubot_SImon',
'sistrydtop_SImon',
'sistryubot_SImon',
'sitempbot_SImon',
'sitempsnic_SImon',
'sitemptop_SImon',
'sithick_SImon',
'sitimefrac_SImon',
'siu_SImon',
'siv_SImon',
'sivol_SImon',
'sivoln_SImon',
'sivols_SImon',
'sndmassdyn_SImon',
'sndmassmelt_SImon',
'sndmasssi_SImon',
'sndmasssnf_SImon',
'sndmasssubl_SImon',
'snmassacrossline_SImon',
'clt_day',
'hfls_day',
'hfss_day',
'huss_day',
'pr_day',
'psl_day',
'rlut_day',
'rsds_day',
'sfcWindmax_day',
'snc_day',
'ta_day',
'tas_day',
'tasmax_day',
'tasmin_day',
'ua_day',
'uas_day',
'va_day',
'vas_day',
'zg_day',
'areacella_fx',
'areacellr_fx',
'mrsofc_fx',
'orog_fx',
'rootd_fx',
'sftgif_fx',
'sftlf_fx',
'zfull_fx',
]
institute_details = {
'id': 'CNRM-CERFACS',
'model_ids': ['CNRM-CM6-1-HR', 'CNRM-CM6-1'],
'calendar': CALENDAR_GREGORIAN
}
experiments = {
'primWP5-amv-neg': {'start_date': datetime(1950, 1, 1),
'end_date': datetime(1960, 1, 1)},
'primWP5-amv-pos': {'start_date': datetime(1950, 1, 1),
'end_date': datetime(1960, 1, 1)}
}
variant_labels = ['r{}i1p1f2'.format(i) for i in range(11, 26)]
# activity_id
ActivityId.objects.get_or_create(short_name=activity_id,
full_name=activity_id)
# Experiment cache
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Institute
result = match_one(Institute, short_name=institute_details['id'])
if result:
institute = result
else:
msg = 'institute_id {} not found in the database.'.format(
institute_details['id']
)
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in institute_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.
format(clim_model))
print(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest, cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
for var_lab in variant_labels:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, institute_details['calendar']
),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, institute_details['calendar']
),
time_units=std_units,
calendar=institute_details['calendar'],
rip_code = var_lab
)
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def main(args):
"""
Main entry point
"""
logger.debug('Starting delete_request.py for retrieval {}'.format(
args.retrieval_id))
deletion_retrieval = match_one(RetrievalRequest, id=args.retrieval_id)
if not deletion_retrieval:
logger.error('Unable to find retrieval id {}'.format(
args.retrieval_id))
sys.exit(1)
if deletion_retrieval.date_deleted:
logger.error('Retrieval {} was already deleted, at {}.'.format(
deletion_retrieval.id,
deletion_retrieval.date_deleted.strftime('%Y-%m-%d %H:%M')))
sys.exit(1)
if not deletion_retrieval.data_finished:
logger.error('Retrieval {} is not marked as finished.'.format(
deletion_retrieval.id))
sys.exit(1)
problems_encountered = False
directories_found = []
base_output_dir = Settings.get_solo().base_output_dir
# loop through all of the data requests in this retrieval
for data_req in deletion_retrieval.data_request.all():
online_req_files = data_req.datafile_set.filter(
online=True, directory__isnull=False)
files_to_delete = date_filter_files(online_req_files,
deletion_retrieval.start_year,
deletion_retrieval.end_year)
if files_to_delete is None:
continue
if not args.force:
# find any other retrieval requests that still need this data
other_retrievals = RetrievalRequest.objects.filter(
data_request=data_req, data_finished=False)
# loop through the retrieval requests that still need this data
# request
for ret_req in other_retrievals:
ret_online_files = data_req.datafile_set.filter(
online=True, directory__isnull=False)
ret_filtered_files = date_filter_files(ret_online_files,
ret_req.start_year,
ret_req.end_year)
if ret_filtered_files is None:
continue
# remove from the list of files to delete the ones that we have
# just found are still needed
files_to_delete = files_to_delete.difference(
ret_filtered_files)
# list the parts of the data request that are still required
logger.debug("{} {} to {} won't be deleted".format(
data_req, ret_req.start_year, ret_req.end_year))
# don't (try to) delete anything that's in the CEDA archive
files_to_delete.exclude(directory__startswith=CEDA_ARCHIVE)
# do the deleting
if args.dryrun:
logger.debug('{} {} files can be deleted.'.format(
data_req,
files_to_delete.distinct().count()))
else:
logger.debug('{} {} files will be deleted.'.format(
data_req,
files_to_delete.distinct().count()))
for data_file in files_to_delete:
old_file_dir = data_file.directory
try:
os.remove(os.path.join(data_file.directory,
data_file.name))
except OSError as exc:
logger.error(str(exc))
problems_encountered = True
else:
if data_file.directory not in directories_found:
directories_found.append(data_file.directory)
data_file.online = False
data_file.directory = None
data_file.save()
# if a symbolic link exists from the base output directory
# then delete this too
if not old_file_dir.startswith(base_output_dir):
sym_link_dir = os.path.join(base_output_dir,
construct_drs_path(data_file))
sym_link = os.path.join(sym_link_dir, data_file.name)
if not os.path.islink(sym_link):
logger.error(
"Expected {} to be a link but it isn't. "
"Leaving this file in place.".format(sym_link))
problems_encountered = True
else:
try:
os.remove(sym_link)
except OSError as exc:
logger.error(str(exc))
problems_encountered = True
else:
if sym_link_dir not in directories_found:
directories_found.append(sym_link_dir)
if not args.dryrun:
# delete any empty directories
for directory in directories_found:
if not os.listdir(directory):
delete_drs_dir(directory)
# set date_deleted in the db
if not problems_encountered:
deletion_retrieval.date_deleted = timezone.now()
deletion_retrieval.save()
else:
logger.error(
'Errors were encountered and so retrieval {} has not '
'been marked as deleted. All possible files have been '
'deleted.'.format(args.retrieval_id))
logger.debug('Completed delete_request.py for retrieval {}'.format(
args.retrieval_id))
def main(args):
"""
Main entry point
"""
mip_era = 'CMIP6'
activity_id = 'primWP5'
new_dreqs = [
'psl_6hrPlev',
'zg7h_6hrPlevPt',
'clt_Amon',
'hfls_Amon',
'hfss_Amon',
'hur_Amon',
'hurs_Amon',
'hus_Amon',
'huss_Amon',
'pr_Amon',
'prc_Amon',
'prsn_Amon',
'ps_Amon',
'psl_Amon',
'rlds_Amon',
'rlus_Amon',
'rlut_Amon',
'rlutcs_Amon',
'rsds_Amon',
'rsdt_Amon',
'rsus_Amon',
'rsuscs_Amon',
'rsut_Amon',
'sfcWind_Amon',
'ta_Amon',
'tas_Amon',
'tasmax_Amon',
'tasmin_Amon',
'tauu_Amon',
'tauv_Amon',
'ts_Amon',
'ua_Amon',
'uas_Amon',
'va_Amon',
'vas_Amon',
'zg_Amon',
'ua850_E3hrPt',
'va850_E3hrPt',
'snw_LImon',
'hfcorr_Omon',
'sos_Omon',
'tos_Omon',
'wfcorr_Omon',
'siconc_SImon',
'sithick_SImon',
'ta_day',
'tas_day',
'tasmax_day',
'ua_day',
'va_day',
'zg_day',
]
institute_details = {
'id': 'EC-Earth-Consortium',
'model_ids': ['EC-Earth3P'],
'calendar': CALENDAR_GREGORIAN
}
experiments = {
'primWP5-amv-neg': {
'start_date': datetime(2000, 1, 1),
'end_date': datetime(2154, 1, 1)
},
'primWP5-amv-pos': {
'start_date': datetime(2000, 1, 1),
'end_date': datetime(2154, 1, 1)
}
}
variant_labels = ['r{}i1p2f1'.format(i) for i in range(1, 26)]
# activity_id
ActivityId.objects.get_or_create(short_name=activity_id,
full_name=activity_id)
# Experiment cache
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Institute
result = match_one(Institute, short_name=institute_details['id'])
if result:
institute = result
else:
msg = 'institute_id {} not found in the database.'.format(
institute_details['id'])
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in institute_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.format(
clim_model))
print(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
project = match_one(Project, short_name=mip_era)
var_req_obj = match_one(VariableRequest,
cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
for var_lab in variant_labels:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, institute_details['calendar']),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, institute_details['calendar']),
time_units=std_units,
calendar=institute_details['calendar'],
rip_code=var_lab)
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def main(args):
"""
Main entry point
"""
institute_details = {
'id': 'CNRM-CERFACS',
'model_ids': ['CNRM-CM6-1', 'CNRM-CM6-1-HR'],
'calendar': CALENDAR_GREGORIAN
}
experiments = {
'highresSST-present': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-future': {
'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)
},
}
variant_label = 'r1i1p1f2'
new_vreqs = [
{
'table_name': 'EmonZ',
'long_name': 'v-tendency nonorographic gravity wave drag',
'units': 'm s-2',
'var_name': 'vtendnogw',
'standard_name':
'tendency_of_northward_wind_due_to_nonorographic_gravity_wave_drag',
'cell_methods': 'longitude: mean time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'latitude plev39 time',
'cmor_name': 'vtendnogw',
'modeling_realm': 'atmos',
'frequency': 'mon',
'cell_measures': '',
'uid': '8183eac8-f906-11e6-a176-5404a60d96b5'
},
{
'table_name': 'Emon',
'long_name': 'C3 grass Area Percentage',
'units': '%',
'var_name': 'grassFracC3',
'standard_name': 'area_fraction',
'cell_methods':
'area: mean where land over all_area_types time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time typec3pft typenatgr',
'cmor_name': 'grassFracC3',
'modeling_realm': 'land',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': '8b814764-4a5b-11e6-9cd2-ac72891c3257'
},
{
'table_name': 'Emon',
'long_name': 'C4 grass Area Percentage',
'units': '%',
'var_name': 'grassFracC4',
'standard_name': 'area_fraction',
'cell_methods':
'area: mean where land over all_area_types time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time typec4pft typenatgr',
'cmor_name': 'grassFracC4',
'modeling_realm': 'land',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': '8b814cc8-4a5b-11e6-9cd2-ac72891c3257'
},
{
'table_name': 'Emon',
'long_name':
'Percentage of land use tile tile that is non-woody vegetation ( e.g. herbaceous crops)',
'units': '%',
'var_name': 'nwdFracLut',
'standard_name': 'missing',
'cell_methods':
'area: mean where land over all_area_types time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude landUse time typenwd',
'cmor_name': 'nwdFracLut',
'modeling_realm': 'land',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': 'd22da9f2-4a9f-11e6-b84e-ac72891c3257'
},
{
'table_name': 'Emon',
'long_name': 'Total vegetated fraction',
'units': '%',
'var_name': 'vegFrac',
'standard_name': 'area_fraction',
'cell_methods':
'area: mean where land over all_area_types time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time typeveg',
'cmor_name': 'vegFrac',
'modeling_realm': 'land',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': '6f6a57d0-9acb-11e6-b7ee-ac72891c3257'
},
{
'table_name': 'Emon',
'long_name': 'Percentage Cover by C3 Crops',
'units': '%',
'var_name': 'cropFracC3',
'standard_name': 'area_fraction',
'cell_methods':
'area: mean where land over all_area_types time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time typec3pft typecrop',
'cmor_name': 'cropFracC3',
'modeling_realm': 'land',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': '8b81522c-4a5b-11e6-9cd2-ac72891c3257'
},
{
'table_name': 'Emon',
'long_name': 'Percentage Cover by C4 Crops',
'units': '%',
'var_name': 'cropFracC4',
'standard_name': 'area_fraction',
'cell_methods':
'area: mean where land over all_area_types time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time typec4pft typecrop',
'cmor_name': 'cropFracC4',
'modeling_realm': 'land',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': '6f6a8ea8-9acb-11e6-b7ee-ac72891c3257'
},
{
'table_name': 'LImon',
'long_name': 'Land Ice Area Percentage',
'units': '%',
'var_name': 'sftgif',
'standard_name': 'land_ice_area_fraction',
'cell_methods': 'area: time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time typeli',
'cmor_name': 'sftgif',
'modeling_realm': 'land',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': 'b3d6e26a-3578-11e7-8257-5404a60d96b5'
},
{
'table_name': 'LImon',
'long_name': 'Grounded Ice Sheet Area Percentage',
'units': '%',
'var_name': 'sftgrf',
'standard_name': 'grounded_ice_sheet_area_fraction',
'cell_methods': 'area: time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time typegis',
'cmor_name': 'sftgrf',
'modeling_realm': 'landIce',
'frequency': 'mon',
'cell_measures': 'area: areacella',
'uid': '8bbb1520-4a5b-11e6-9cd2-ac72891c3257'
},
]
# Experiment
new_dreqs = [
'vtendnogw_EmonZ',
'cropFracC3_Emon',
'cropFracC4_Emon',
'grassFracC3_Emon',
'grassFracC4_Emon',
'nwdFracLut_Emon',
'ps_AERmon',
'sftgif_LImon',
'sftgrf_LImon',
'sidconcdyn_SImon',
'sidconcth_SImon',
'simass_SImon',
'sisaltmass_SImon',
'sisnthick_SImon',
'sndmassdyn_SImon',
'sndmasssi_SImon',
'vegFrac_Emon',
]
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Institute
result = match_one(Institute, short_name=institute_details['id'])
if result:
institute = result
else:
msg = 'institute_id {} not found in the database.'.format(
institute_details['id'])
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in institute_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.format(
clim_model))
print(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the additional variable requests
for new_vreq in new_vreqs:
_vr = get_or_create(VariableRequest, **new_vreq)
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest,
cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
try:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, institute_details['calendar']),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, institute_details['calendar']),
time_units=std_units,
calendar=institute_details['calendar'],
rip_code=variant_label)
except django.core.exceptions.MultipleObjectsReturned:
logger.error('{}'.format(var_req_obj))
raise
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def make_data_submission():
"""
Create a Data Submission and add files to it
"""
test_dsub = datasets.test_data_submission
test_dsub.create_test_files()
dsub = get_or_create(DataSubmission,
status=STATUS_VALUES.ARRIVED,
incoming_directory=test_dsub.INCOMING_DIR,
directory=test_dsub.INCOMING_DIR,
user='******')
for dfile_name in test_dsub.files:
path = os.path.join(test_dsub.INCOMING_DIR, dfile_name)
metadata = _extract_file_metadata(path)
proj = get_or_create(
Project,
short_name="CMIP6",
full_name="Coupled Model Intercomparison Project Phase 6")
climate_model = get_or_create(ClimateModel,
short_name=metadata["climate_model"],
full_name="Really good model")
experiment = get_or_create(Experiment,
short_name=metadata["experiment"],
full_name="Really good experiment")
institute = get_or_create(Institute,
short_name="MOHC",
full_name="Met Office Hadley Centre")
var_match = match_one(VariableRequest,
cmor_name=metadata['var_id'],
table_name=metadata['table'])
if var_match:
variable = var_match
else:
msg = ('No variable request found for file: {}. Please create a '
'variable request and resubmit.'.format(dfile_name))
print('ERROR: ' + msg)
raise ValueError(msg)
_dfile = DataFile.objects.create(
name=dfile_name,
incoming_directory=test_dsub.INCOMING_DIR,
directory=test_dsub.INCOMING_DIR,
size=os.path.getsize(path),
project=proj,
climate_model=climate_model,
institute=institute,
experiment=experiment,
frequency=metadata["frequency"],
rip_code=metadata["ensemble"],
variable_request=variable,
start_time=metadata["start_time"],
end_time=metadata["end_time"],
time_units=metadata['time_units'],
calendar=metadata['calendar'],
data_submission=dsub,
online=True)
ceda_ds = get_or_create(CEDADataset,
catalogue_url='http://www.metoffice.gov.uk',
directory=test_dsub.INCOMING_DIR)
esgf_ds = get_or_create(ESGFDataset,
drs_id='ab.cd.ef.gh',
version='v19160519',
directory=test_dsub.INCOMING_DIR,
data_submission=dsub,
ceda_dataset=ceda_ds)
for df in dsub.get_data_files():
df.esgf_dataset = esgf_ds
df.ceda_dataset = ceda_ds
df.ceda_download_url = 'http://browse.ceda.ac.uk/browse/badc/cmip5/' + df.name
df.ceda_opendap_url = 'http://dap.ceda.ac.uk/data/badc/cmip5/some/dir/' + df.name
df.esgf_opendap_url = 'http://esgf.ceda.ac.uk/data/badc/cmip5/some/dir/' + df.name
df.esgf_download_url = 'http://esgf.ceda.ac.uk/browse/badc/cmip5/' + df.name
df.save()
dsub.status = STATUS_VALUES.ARCHIVED
dsub.save()
def main(args):
"""
Main entry point
"""
activity_id = 'primWP5'
new_vreqs = [
{
'table_name': '6hrPlevPt',
'long_name': 'Eastward Wind',
'units': 'm s-1',
'var_name': 'ua',
'standard_name': 'eastward_wind',
'cell_methods': 'time: point',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time1',
'cmor_name': 'ua850',
'modeling_realm': 'atmos',
'frequency': 'day',
'cell_measures': 'area: areacella',
'uid': 'prim6hrPlevPt_ua850'
},
{
'table_name': '6hrPlevPt',
'long_name': 'Northward Wind',
'units': 'm s-1',
'var_name': 'va',
'standard_name': 'northward_wind',
'cell_methods': 'time: point',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time1',
'cmor_name': 'va850',
'modeling_realm': 'atmos',
'frequency': 'day',
'cell_measures': 'area: areacella',
'uid': 'prim6hrPlevPt_va850'
},
]
# create the additional variable requests
for new_vreq in new_vreqs:
_vr = get_or_create(VariableRequest, **new_vreq)
new_dreqs = [
'psl_6hrPlevPt', 'ua850_6hrPlevPt', 'va850_6hrPlevPt',
'zg500_6hrPlevPt', 'clt_Amon', 'hfls_Amon', 'hfss_Amon', 'hur_Amon',
'hurs_Amon', 'hus_Amon', 'huss_Amon', 'pr_Amon', 'prc_Amon',
'prsn_Amon', 'ps_Amon', 'psl_Amon', 'rlds_Amon', 'rldscs_Amon',
'rlus_Amon', 'rlut_Amon', 'rlutcs_Amon', 'rsds_Amon', 'rsdscs_Amon',
'rsdt_Amon', 'rsus_Amon', 'rsuscs_Amon', 'rsut_Amon', 'rsutcs_Amon',
'rtmt_Amon', 'sfcWind_Amon', 'ta_Amon', 'tas_Amon', 'tasmax_Amon',
'tasmin_Amon', 'tauu_Amon', 'tauv_Amon', 'ts_Amon', 'ua_Amon',
'uas_Amon', 'va_Amon', 'vas_Amon', 'wap_Amon', 'zg_Amon', 'snw_LImon',
'mrso_Lmon', 'ta_day', 'tas_day', 'tasmax_day', 'ua_day', 'va_day',
'zg_day'
]
Institute.objects.get_or_create(short_name='NCAS', full_name='NCAS')
ClimateModel.objects.get_or_create(short_name='MetUM-GOML2-LR',
full_name='MetUM-GOML2-LR')
ClimateModel.objects.get_or_create(short_name='MetUM-GOML2-HR',
full_name='MetUM-GOML2-HR')
institute_details = {
'id': 'NCAS',
'model_ids': ['MetUM-GOML2-LR', 'MetUM-GOML2-HR'],
'calendar': CALENDAR_360_DAY
}
experiments = {
'primWP5-amv-pos': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'primWP5-amv-neg': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
}
}
variant_labels = ['r{}i1p1f1'.format(i) for i in range(1, 16)]
# activity_id
ActivityId.objects.get_or_create(short_name=activity_id,
full_name=activity_id)
# Experiment create
for expt in experiments:
Experiment.objects.get_or_create(short_name=expt, full_name=expt)
# Experiment cache
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Institute
result = match_one(Institute, short_name=institute_details['id'])
if result:
institute = result
else:
msg = 'institute_id {} not found in the database.'.format(
institute_details['id'])
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in institute_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.format(
clim_model))
print(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest,
cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
for var_lab in variant_labels:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, institute_details['calendar']),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, institute_details['calendar']),
time_units=std_units,
calendar=institute_details['calendar'],
rip_code=var_lab)
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def main(args):
"""
Main entry point
"""
new_dreqs = [
'siage_SImon', 'sisnthick_SImon', 'sicompstren_SImon', 'sisali_SImon',
'hus1000_Prim6hrPt', 'siconc_SImon', 'sistrxdtop_SImon', 'siv_SImon',
'sivol_SImon', 'sistrydtop_SImon', 'va1000_Prim6hrPt',
'sidmassevapsubl_SImon', 'sithick_SImon', 'sispeed_SImon',
'sndmasssnf_SImon', 'siflswdtop_SImon', 'sitemptop_SImon',
'ua1000_Prim6hrPt', 'siu_SImon'
]
institute_details = {
'id': 'EC-Earth-Consortium',
'model_ids': ['EC-Earth3', 'EC-Earth3-HR'],
'calendar': CALENDAR_GREGORIAN
}
experiments = {
'control-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2050, 1, 1)
},
'highres-future': {
'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)
},
'hist-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-present': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-future': {
'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)
},
'highresSST-LAI': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-smoothed': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-p4K': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'highresSST-4co2': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)
},
'spinup-1950': {
'start_date': datetime(1950, 1, 1),
'end_date': datetime(1980, 1, 1)
},
}
# Experiment
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Institute
result = match_one(Institute, short_name=institute_details['id'])
if result:
institute = result
else:
msg = 'institute_id {} not found in the database.'.format(
institute_details['id'])
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in institute_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.format(
clim_model))
print(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest,
cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, institute_details['calendar']),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, institute_details['calendar']),
time_units=std_units,
calendar=institute_details['calendar'])
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def test_single_match(self):
item_matched = dbapi.match_one(models.Institute, short_name='t')
self.assertEqual(item_matched.short_name, 't')
def main(args):
"""
Main entry point
"""
# new_vreqs = ['lwsnl_Eday', 'mrros_Eday', 'snm_Eday', 'snd_Eday']
new_vreqs = [
{'table_name': 'Eday',
'long_name': 'Liquid Water Content of Snow Layer',
'units': 'kg m-2',
'var_name': 'lwsnl',
'standard_name': 'liquid_water_content_of_snow_layer',
'cell_methods': 'area: mean where land time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'lwsnl',
'modeling_realm': 'landIce land',
'frequency': 'day',
'cell_measures': 'area: areacella',
'uid': 'd228925a-4a9f-11e6-b84e-ac72891c3257'},
{'table_name': 'Eday',
'long_name': 'Surface Runoff',
'units': 'kg m-2 s-1',
'var_name': 'mrros',
'standard_name': 'surface_runoff_flux',
'cell_methods': 'area: mean where land time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'mrros',
'modeling_realm': 'land',
'frequency': 'day',
'cell_measures': 'area: areacella',
'uid': 'd2284048-4a9f-11e6-b84e-ac72891c3257'},
{'table_name': 'Eday',
'long_name': 'Surface Snow Melt',
'units': 'kg m-2 s-1',
'var_name': 'snm',
'standard_name': 'surface_snow_melt_flux',
'cell_methods': 'area: mean where land time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'snm',
'modeling_realm': 'landIce land',
'frequency': 'day',
'cell_measures': 'area: areacella',
'uid': 'd22848ea-4a9f-11e6-b84e-ac72891c3257'},
{'table_name': 'Eday',
'long_name': 'Snow Depth',
'units': 'm',
'var_name': 'snd',
'standard_name': 'surface_snow_thickness',
'cell_methods': 'area: mean where land time: mean',
'positive': '',
'variable_type': 'real',
'dimensions': 'longitude latitude time',
'cmor_name': 'snd',
'modeling_realm': 'landIce land',
'frequency': 'day',
'cell_measures': 'area: areacella',
'uid': 'b7ccdf0a-7c00-11e6-bcdf-ac72891c3257'}
]
new_dreqs = [
'wap_Emon',
'hfdsn_LImon',
'snc_day',
'va27_Emon',
'rsuscs_CFday',
'vt_Emon',
'hursmin_day',
'uv_Emon',
'cl_Amon',
'rldscs_CFday',
'lwp_Primmon',
'uwap_Emon',
'rsdscs_Amon',
'zg_day',
'evspsblsoi_Lmon',
'snc_LImon',
'v2_Emon',
'ta27_Emon',
'tsl_Lmon',
'lwsnl_LImon',
'sbl_LImon',
'wap2_Emon',
'ut_Emon',
'hursmax_day',
't2_Emon',
'rsuscs_Amon',
'rsdscs_CFday',
'mrsos_day',
'hus27_Emon',
'snm_LImon',
'tsn_LImon',
'rldscs_Amon',
'vwap_Emon',
'zg27_Emon',
'twap_Emon',
'u2_Emon',
'ua27_Emon',
'lwsnl_Eday',
'mrros_Eday',
'snm_Eday',
'snd_Eday'
]
institute_details = {
'id': 'EC-Earth-Consortium',
'model_ids': ['EC-Earth3-LR', 'EC-Earth3-HR'],
'calendar': CALENDAR_GREGORIAN
}
experiments = {
'control-1950': {'start_date': datetime(1950, 1, 1),
'end_date': datetime(2050, 1, 1)},
'highres-future': {'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)},
'hist-1950': {'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)},
'highresSST-present': {'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)},
'highresSST-future': {'start_date': datetime(2015, 1, 1),
'end_date': datetime(2051, 1, 1)},
'highresSST-LAI': {'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)},
'highresSST-smoothed': {'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)},
'highresSST-p4K': {'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)},
'highresSST-4co2': {'start_date': datetime(1950, 1, 1),
'end_date': datetime(2015, 1, 1)},
'spinup-1950': {'start_date': datetime(1950, 1, 1),
'end_date': datetime(1980, 1, 1)},
}
# Experiment
experiment_objs = []
for expt in experiments:
expt_obj = match_one(Experiment, short_name=expt)
if expt_obj:
experiment_objs.append(expt_obj)
else:
msg = 'experiment {} not found in the database.'.format(expt)
print(msg)
raise ValueError(msg)
# Institute
result = match_one(Institute, short_name=institute_details['id'])
if result:
institute = result
else:
msg = 'institute_id {} not found in the database.'.format(
institute_details['id']
)
print(msg)
raise ValueError(msg)
# Look up the ClimateModel object for each institute_id and save the
# results to a dictionary for quick look up later
model_objs = []
for clim_model in institute_details['model_ids']:
result = match_one(ClimateModel, short_name=clim_model)
if result:
model_objs.append(result)
else:
msg = ('climate_model {} not found in the database.'.
format(clim_model))
print(msg)
raise ValueError(msg)
# The standard reference time
std_units = Settings.get_solo().standard_time_units
# create the additional variable requests
for new_vreq in new_vreqs:
_vr = get_or_create(VariableRequest, **new_vreq)
# create the new data requests
for new_dreq in new_dreqs:
cmor_name, table_name = new_dreq.split('_')
if table_name.startswith('Prim'):
project = match_one(Project, short_name='PRIMAVERA')
else:
project = match_one(Project, short_name='CMIP6')
var_req_obj = match_one(VariableRequest, cmor_name=cmor_name,
table_name=table_name)
if var_req_obj:
for expt in experiment_objs:
for clim_model in model_objs:
_dr = get_or_create(
DataRequest,
project=project,
institute=institute,
climate_model=clim_model,
experiment=expt,
variable_request=var_req_obj,
request_start_time=date2num(
experiments[expt.short_name]['start_date'],
std_units, institute_details['calendar']
),
request_end_time=date2num(
experiments[expt.short_name]['end_date'],
std_units, institute_details['calendar']
),
time_units=std_units,
calendar=institute_details['calendar']
)
else:
msg = ('Unable to find variable request matching '
'cmor_name {} and table_name {} in the '
'database.'.format(cmor_name, table_name))
print(msg)
raise ValueError(msg)
def test_double_match_returns_none(self):
self.assertIsNone(dbapi.match_one(models.Institute, full_name='test'))
