def _email_admin_failure(retrieval): """ Send an email to the admin user advising them that the retrieval failed. :param pdata_app.models.RetrievalRequest retrieval: the retrieval object """ contact_user_id = Settings.get_solo().contact_user_id contact_user = User.objects.get(username=contact_user_id) msg = ( 'Dear {},\n' '\n' 'Retrieval request number {} failed.\n' '\n' 'Thanks,\n' '\n' '{}'.format(contact_user.first_name, retrieval.id, contact_user.first_name) ) _email = EmailQueue.objects.create( recipient=contact_user, subject=('[PRIMAVERA_DMT] Retrieval Request {} Failed'. format(retrieval.id)), message=msg )
def _email_user_success(retrieval): """ Send an email to request's creator advising them that their data's been successfully restored. :param pdata_app.models.RetrievalRequest retrieval: the retrieval object """ contact_user_id = Settings.get_solo().contact_user_id contact_user = User.objects.get(username=contact_user_id) msg = ( 'Dear {},\n' '\n' 'Your retrieval request number {} has now been restored from ' 'tape to group workspace. The data will be available in the DRS ' 'directory structure at {}.\n' '\n' 'To free up disk space on the group workspaces we would be grateful ' 'if this data could be marked as finished at ' 'https://prima-dm.ceda.ac.uk/retrieval_requests/ as soon as you have ' 'finished analysing it.\n' '\n' 'Thanks,\n' '\n' '{}'.format(retrieval.requester.first_name, retrieval.id, BASE_OUTPUT_DIR, contact_user.first_name) ) _email = EmailQueue.objects.create( recipient=retrieval.requester, subject=('[PRIMAVERA_DMT] Retrieval Request {} Complete'. format(retrieval.id)), message=msg )
def main(args): """ Main entry point """ base_output_dir = Settings.get_solo().base_output_dir ceda_base = '/badc/cmip6/data' for ed in ESGFDataset.objects.filter(status='PUBLISHED'): for df in ed.data_request.datafile_set.order_by('name'): ceda_dir = os.path.join(ceda_base, construct_drs_path(df)) ceda_path = os.path.join(ceda_dir, df.name) if df.directory: logger.error('Directory given {}'.format( os.path.join(df.directory, df.name))) if os.path.exists(ceda_path): df.online = True df.directory = ceda_dir df.save() else: logger.error('Not in archive {}'.format(ceda_path)) continue base_dir = os.path.join(base_output_dir, construct_drs_path(df)) base_path = os.path.join(base_dir, df.name) if os.path.exists(base_path): os.remove(base_path) logger.error('Deleted {}'.format(base_path)) if not os.path.exists(base_dir): os.makedirs(base_dir) os.symlink(ceda_path, base_path)
def main(args): """ Main entry point """ base_output_dir = Settings.get_solo().base_output_dir for data_file in DataFile.objects.filter(online=True): gws_pattern = r'^/group_workspaces/jasmin2/primavera(\d)/(\S*)' gws = re.match(gws_pattern, data_file.directory) if not gws: logger.error('No GWS match for {}'.format(data_file.name)) continue new_gws = '/gws/nopw/j04/primavera' + gws.group(1) new_dir = os.path.join(new_gws, gws.group(2)) new_path = os.path.join(new_dir, data_file.name) if not os.path.exists(new_path): logger.error('Cannot find {}'.format(new_path)) continue data_file.directory = new_dir data_file.save() if not is_same_gws(data_file.directory, base_output_dir): link_path = os.path.join(base_output_dir, construct_drs_path(data_file), data_file.name) # it's got to be a link but check anyway if os.path.islink(link_path): os.remove(link_path) os.symlink(os.path.join(data_file.directory, data_file.name), link_path) else: logger.error('Expected a link but found a file at {}'. format(link_path))
def send_admin_rejection_email(data_sub): """ Send the admin user an email warning them that a submission failed due to a server problem (missing data request, etc). :param pdata_app.models.DataSubmission data_sub: """ admin_user_id = Settings.get_solo().contact_user_id admin_user = User.objects.get(username=admin_user_id) msg = ( 'Data submission {} from incoming directory {} failed validation due ' 'to a SubmissionError being raised. Please run the validation script ' 'manually on this submission and correct the error.\n' '\n' 'Thanks,\n' '\n' '{}'.format(data_sub.id, data_sub.incoming_directory, admin_user.first_name)) _email = EmailQueue.objects.create( recipient=admin_user, subject=('[PRIMAVERA_DMT] Submission {} failed validation'.format( data_sub.id)), message=msg)
def _email_user_split(orig_retrieval, new_retrieval): """ Send an email to request's creator advising them that their request's been split. :param pdata_app.models.RetrievalRequest orig_retrieval: the retrieval object """ contact_user_id = Settings.get_solo().contact_user_id contact_user = User.objects.get(username=contact_user_id) msg = ('Dear {},\n' '\n' 'Thank you for your recent data retrieval request.\n' '\n' 'Due to temporary restrictions on the tape servers at JASMIN ' 'Your request has been split into two separate requests, so that ' 'Met Office and NERC data are in a request of their own.\n' '\n' 'Therefore your data will not be fully restored until you have ' 'received emails to say that requests {} and {} have both been ' 'restored.\n' '\n' 'Thanks,\n' '\n' '{}'.format(orig_retrieval.requester.first_name, orig_retrieval.id, new_retrieval.id, contact_user.first_name)) _email = EmailQueue.objects.create( recipient=orig_retrieval.requester, subject=('[PRIMAVERA_DMT] Retrieval Request {}'.format( orig_retrieval.id)), message=msg)
def main(args): """ Main entry point """ dss = DataSubmission.objects.filter(incoming_directory__in=[ '/gws/nopw/j04/primavera4/upload/EC-Earth-Consortium/EC-Earth3P-HR/' 'control-1950/r3i1p2f1/1950-1952/SeaIce', '/gws/nopw/j04/primavera4/upload/EC-Earth-Consortium/EC-Earth3P-HR/' 'control-1950/r3i1p2f1/1950-1952/Rest' ]) base_output_dir = Settings.get_solo().base_output_dir for ds in dss: delete_files(ds.datafile_set.all(), base_output_dir) replace_files(ds.datafile_set.all())
def main(args): """ Main entry point """ bad_sets = ESGFDataset.objects.filter( status__in=['CREATED', 'NOT_ON_DISK'], data_request__climate_model__short_name='HadGEM3-GC31-LL', data_request__experiment__short_name='control-1950', data_request__rip_code='r1i1p1f1', ).distinct() logger.debug(f'{bad_sets.count()} data sets found') base_output_dir = Settings.get_solo().base_output_dir for bad_set in bad_sets: dr = bad_set.data_request delete_files(dr.datafile_set.all(), base_output_dir)
def main(args): """Main entry point""" base_dir = Settings.get_solo().base_output_dir for extracted_file in ilist_files(args.top_dir): found_name = os.path.basename(extracted_file) try: data_file = DataFile.objects.get(name=found_name) except django.core.exceptions.ObjectDoesNotExist: logger.warning('Cannot find DMT entry. Skipping {}'. format(extracted_file)) continue found_checksum = adler32(extracted_file) if not found_checksum == data_file.checksum_set.first().checksum_value: logger.warning("Checksum doesn't match. Skipping {}". format(found_name)) continue dest_dir = os.path.join(get_gws_any_dir(extracted_file), 'stream1', construct_drs_path(data_file)) dest_path = os.path.join(dest_dir, found_name) if os.path.exists(dest_path): logger.warning('Skipping {} as it already exists at {}'. format(found_name, dest_path)) continue # create the directory if it doesn't exist if not os.path.exists(dest_dir): os.makedirs(dest_dir) os.rename(extracted_file, dest_path) # create a link from the base dir if not is_same_gws(dest_path, base_dir): link_dir = os.path.join(base_dir, construct_drs_path(data_file)) link_path = os.path.join(link_dir, data_file.name) if not os.path.exists(link_dir): os.makedirs(link_dir) os.symlink(dest_path, link_path) data_file.online = True data_file.directory = dest_dir data_file.save()
def send_user_rejection_email(data_sub): """ Send an email to the submission's creator warning them of validation failure. :param pdata_app.models.DataSubmission data_sub: """ val_tool_url = ('http://proj.badc.rl.ac.uk/primavera-private/wiki/JASMIN/' 'HowTo#SoftwarepackagesinstalledonthePRIMAVERAworkspace') contact_user_id = Settings.get_solo().contact_user_id contact_user = User.objects.get(username=contact_user_id) contact_string = '{} {} ({})'.format(contact_user.first_name, contact_user.last_name, contact_user.email) msg = ('Dear {first_name} {surname},\n' '\n' 'Your data submission in {incoming_dir} has failed validation and ' 'has been moved to {rejected_dir}.\n' '\n' 'Please run the validation tool ({val_tool_url}) to check why this ' 'submission failed validation. Once the data is passing validation ' 'then please resubmit the corrected data.\n' '\n' 'Please contact {contact_person} if you ' 'have any questions.\n' '\n' 'Thanks,\n' '\n' '{friendly_name}'.format(first_name=data_sub.user.first_name, surname=data_sub.user.last_name, incoming_dir=data_sub.incoming_directory, rejected_dir=data_sub.directory, val_tool_url=val_tool_url, contact_person=contact_string, friendly_name=contact_user.first_name)) _email = EmailQueue.objects.create( recipient=data_sub.user, subject='[PRIMAVERA_DMT] Data submission failed validation', message=msg)
def main(args): """ Main entry point Task names in the JSON file are in the form: <climate-model>_<experiment>_<variant-label>_<table>_<variable> e.g.: HadGEM3-GC31-LM_highresSST-present_r1i1p1f1_Amon_psl """ with open(args.json_file) as fh: task_names = json.load(fh) logger.debug('{} task names loaded'.format(len(task_names))) system_user = User.objects.get( username=Settings.get_solo().contact_user_id) ret_req = RetrievalRequest.objects.create(requester=system_user, start_year=1900, end_year=2100) time_zone = datetime.timezone(datetime.timedelta()) ret_req.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone) ret_req.save() for task_name in task_names: task_cmpts = parse_rose_stream_name(task_name) add_data_request(task_cmpts, debug_req_found=False) ret_req.data_request.add(task_cmpts['data_req']) logger.debug('Request id {} created'.format(ret_req.id))
def main(args): """ Main entry point """ new_vreqs = [ { 'table_name': 'Eday', 'long_name': 'River Discharge', 'units': 'm3 s-1', 'var_name': 'rivo', 'standard_name': 'water_flux_to_downstream', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'rivo', 'modeling_realm': 'land', 'frequency': 'day', 'cell_measures': 'area: areacellr', 'uid': 'd2285b46-4a9f-11e6-b84e-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Water table depth', 'units': 'm', 'var_name': 'wtd', 'standard_name': 'depth_of_soil_moisture_saturation', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'wtd', 'modeling_realm': 'land', 'frequency': 'mon', 'cell_measures': 'area: areacellr', 'uid': '8b81f0ce-4a5b-11e6-9cd2-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Terrestrial Water Storage', 'units': 'kg m-2', 'var_name': 'mrtws', 'standard_name': 'total_water_storage', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'mrtws', 'modeling_realm': 'land', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '6f6a4484-9acb-11e6-b7ee-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Vertically integrated Eastward moisture transport (Mass_weighted_vertical integral of the product of eastward wind by total water mass per unit mass)', 'units': 'kg m-1 s-1', 'var_name': 'intuaw', 'standard_name': 'vertical_integral_eastward_wind_by_total_water', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'intuaw', 'modeling_realm': 'atmos', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '6f690484-9acb-11e6-b7ee-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Vertically integrated Northward moisture transport (Mass_weighted_vertical integral of the product of northward wind by total water mass per unit mass)', 'units': 'kg m-1 s-1', 'var_name': 'intvaw', 'standard_name': 'vertical_integral_northward_wind_by_total_water', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'intvaw', 'modeling_realm': 'atmos', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '6f690b5a-9acb-11e6-b7ee-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Vertically Integrated Eastward Dry Statice Energy Transport', 'units': '1.e6 J m-1 s-1', 'var_name': 'intuadse', 'standard_name': 'vertical_integral_eastward_wind_by_dry_static_energy', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'intuadse', 'modeling_realm': 'atmos', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '6f691104-9acb-11e6-b7ee-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Vertically integrated Northward dry transport (cp.T +zg).v (Mass_weighted_vertical integral of the product of northward wind by dry static_energy per mass unit)', 'units': '1.e6 J m-1 s-1', 'var_name': 'intvadse', 'standard_name': 'vertical_integral_northward_wind_by_dry_static_energy', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'intvadse', 'modeling_realm': 'atmos', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '6f6916a4-9acb-11e6-b7ee-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Heat content of upper 300 meters', 'units': 'm K', 'var_name': 'hcont300', 'standard_name': 'heat_content_of_ocean_layer', 'cell_methods': 'area: mean where sea time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time depth300m', 'cmor_name': 'hcont300', 'modeling_realm': 'ocean', 'frequency': 'mon', 'cell_measures': 'area: areacello', 'uid': '6f69513c-9acb-11e6-b7ee-ac72891c3257' }, { 'table_name': 'AERmon', 'long_name': 'Sulfate Aerosol Optical Depth at 550nm', 'units': '1', 'var_name': 'od550so4', 'standard_name': 'atmosphere_optical_thickness_due_to_sulfate_ambient_aerosol', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'od550so4', 'modeling_realm': 'aerosol', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '19bf19ca-81b1-11e6-92de-ac72891c3257' }, { 'table_name': 'AERmon', 'long_name': 'TOA Outgoing Clear-Sky, Aerosol-Free Shortwave Radiation', 'units': 'W m-2', 'var_name': 'rsutcsaf', 'standard_name': 'toa_outgoing_shortwave_flux_assuming_clear_sky', 'cell_methods': 'area: time: mean', 'positive': 'up', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'rsutcsaf', 'modeling_realm': 'aerosol', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '8feac232-267c-11e7-8933-ac72891c3257' }, { 'table_name': 'SImon', 'long_name': 'Sea-ice area fraction', 'units': '%', 'var_name': 'siconca', 'standard_name': 'sea_ice_area_fraction', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time typesi', 'cmor_name': 'siconca', 'modeling_realm': 'seaIce', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '71190054-faa7-11e6-bfb7-ac72891c3257' }, { 'table_name': 'SImon', 'long_name': 'Sea ice area flux through straits', 'units': 'm2 s-1', 'var_name': 'siareaacrossline', 'standard_name': 'sea_ice_area_transport_across_line', 'cell_methods': 'time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'siline time', 'cmor_name': 'siareaacrossline', 'modeling_realm': 'seaIce', 'frequency': 'mon', 'cell_measures': '', 'uid': '712442ca-faa7-11e6-bfb7-ac72891c3257' }, { 'table_name': 'SImon', 'long_name': 'Snow mass flux through straits', 'units': 'kg s-1', 'var_name': 'snmassacrossline', 'standard_name': 'snow_mass_transport_across_line', 'cell_methods': 'time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'siline time', 'cmor_name': 'snmassacrossline', 'modeling_realm': 'seaIce', 'frequency': 'mon', 'cell_measures': '', 'uid': '712fb3ee-faa7-11e6-bfb7-ac72891c3257' }, ] institute_details = { 'id': 'CNRM-CERFACS', 'model_ids': ['CNRM-CM6-1'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'control-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2050, 1, 1) }, 'highres-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'hist-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, } variant_label = 'r2i1p1f2' # Experiment new_dreqs = [ 'rivo_Eday', 'siforceintstrx_PrimSIday', 'siforceintstry_PrimSIday', 'simassacrossline_PrimSIday', 'hfgeou_Omon', 'htovgyre_Omon', 'htovovrt_Omon', 'sicompstren_SImon', 'siflcondbot_SImon', 'siflcondtop_SImon', 'siflfwdrain_SImon', 'sifllatstop_SImon', 'sifllwutop_SImon', 'siflsensupbot_SImon', 'siflswdtop_SImon', 'siflswutop_SImon', 'simassacrossline_SImon', 'sisnmass_SImon', 'sitempbot_SImon', 'sithick_SImon', 'sltovgyre_Omon', 'sltovovrt_Omon', 'hcont300_Emon', 'intuadse_Emon', 'intuaw_Emon', 'intvadse_Emon', 'intvaw_Emon', 'mrtws_Emon', 'od550so4_AERmon', 'rsutcsaf_AERmon', 'siareaacrossline_SImon', 'siconca_SImon', 'snmassacrossline_SImon', 'wtd_Emon', ] experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the additional variable requests for new_vreq in new_vreqs: _vr = get_or_create(VariableRequest, **new_vreq) # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: try: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar'], rip_code=variant_label) except django.core.exceptions.MultipleObjectsReturned: logger.error('{}'.format(var_req_obj)) raise else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ # new_vreqs = ['lwsnl_Eday', 'mrros_Eday', 'snm_Eday', 'snd_Eday'] new_dreqs = [ 'mrlsl_6hrPlevPt', 'snw_6hrPlevPt', 'psl_6hrPlevPt', 'rsdt_E3hr', 'prw_E3hr', 'tas_6hrPlevPt', 'clivi_E3hr', 'mrsos_6hrPlevPt', 'ta7h_6hrPlevPt', 'clwvi_E3hr', 'pr_Prim6hr', 'rsdscs_3hr', 'va7h_6hrPlevPt', 'ts_6hrPlevPt', 'ua7h_6hrPlevPt', 'rsut_E3hr', 'clt_Prim6hr', 'ta27_6hrPlevPt', 'tsl_6hrPlevPt', 'rldscs_3hr', 'rlut_E3hr', 'huss_6hrPlevPt', 'hus27_6hrPlevPt', 'rlutcs_E3hr', 'hus7h_6hrPlevPt', 'rsuscs_3hr', 'ps_Prim6hr', 'sfcWind_6hrPlevPt', 'zg27_6hrPlevPt', 'rsds_Prim6hr' ] institute_details = { 'id': 'EC-Earth-Consortium', 'model_ids': ['EC-Earth3-LR', 'EC-Earth3-HR'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'control-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2050, 1, 1) }, 'highres-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'hist-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-present': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'highresSST-LAI': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-smoothed': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-p4K': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-4co2': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'spinup-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(1980, 1, 1) }, } # Experiment experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar']) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ mip_era = 'CMIP6' activity_id = 'primWP5' new_dreqs = [ 'psl_6hrPlev', 'zg7h_6hrPlevPt', 'clt_Amon', 'hfls_Amon', 'hfss_Amon', 'hur_Amon', 'hurs_Amon', 'hus_Amon', 'huss_Amon', 'pr_Amon', 'prc_Amon', 'prsn_Amon', 'ps_Amon', 'psl_Amon', 'rlds_Amon', 'rlus_Amon', 'rlut_Amon', 'rlutcs_Amon', 'rsds_Amon', 'rsdt_Amon', 'rsus_Amon', 'rsuscs_Amon', 'rsut_Amon', 'sfcWind_Amon', 'ta_Amon', 'tas_Amon', 'tasmax_Amon', 'tasmin_Amon', 'tauu_Amon', 'tauv_Amon', 'ts_Amon', 'ua_Amon', 'uas_Amon', 'va_Amon', 'vas_Amon', 'zg_Amon', 'ua850_E3hrPt', 'va850_E3hrPt', 'snw_LImon', 'hfcorr_Omon', 'sos_Omon', 'tos_Omon', 'wfcorr_Omon', 'siconc_SImon', 'sithick_SImon', 'ta_day', 'tas_day', 'tasmax_day', 'ua_day', 'va_day', 'zg_day', ] institute_details = { 'id': 'EC-Earth-Consortium', 'model_ids': ['EC-Earth3P'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'primWP5-amv-neg': { 'start_date': datetime(2000, 1, 1), 'end_date': datetime(2154, 1, 1) }, 'primWP5-amv-pos': { 'start_date': datetime(2000, 1, 1), 'end_date': datetime(2154, 1, 1) } } variant_labels = ['r{}i1p2f1'.format(i) for i in range(1, 26)] # activity_id ActivityId.objects.get_or_create(short_name=activity_id, full_name=activity_id) # Experiment cache experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') project = match_one(Project, short_name=mip_era) var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: for var_lab in variant_labels: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar'], rip_code=var_lab) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def _make_test_data_objects(self): """ Create the test DB objects """ proj = get_or_create(Project, short_name="CMIP6", full_name="6th " "Coupled Model Intercomparison Project") climate_model = get_or_create(ClimateModel, short_name="MY-MODEL", full_name="Really good model") institute = get_or_create(Institute, short_name='MOHC', full_name='Met Office Hadley Centre') act_id = get_or_create( ActivityId, short_name='HighResMIP', full_name='High Resolution Model Intercomparison Project') experiment = get_or_create(Experiment, short_name="experiment", full_name="Really good experiment") incoming_directory = '/gws/MOHC/MY-MODEL/incoming/v12345678' var1 = get_or_create(VariableRequest, table_name='my-table', long_name='very descriptive', units='1', var_name='my-var', standard_name='var-name', cell_methods='time: mean', positive='optimistic', variable_type=VARIABLE_TYPES['real'], dimensions='massive', cmor_name='my-var', modeling_realm='atmos', frequency=FREQUENCY_VALUES['ann'], cell_measures='', uid='123abc') var2 = get_or_create(VariableRequest, table_name='your-table', long_name='very descriptive', units='1', var_name='your-var', standard_name='var-name', cell_methods='time: mean', positive='optimistic', variable_type=VARIABLE_TYPES['real'], dimensions='massive', cmor_name='your-var', modeling_realm='atmos', frequency=FREQUENCY_VALUES['ann'], cell_measures='', uid='123abc') self.dreq1 = get_or_create(DataRequest, project=proj, institute=institute, climate_model=climate_model, experiment=experiment, variable_request=var1, rip_code='r1i1p1f1', request_start_time=0.0, request_end_time=23400.0, time_units='days since 1950-01-01', calendar='360_day') self.dreq2 = get_or_create(DataRequest, project=proj, institute=institute, climate_model=climate_model, experiment=experiment, variable_request=var2, rip_code='r1i1p1f1', request_start_time=0.0, request_end_time=23400.0, time_units='days since 1950-01-01', calendar='360_day') self.user = get_or_create(User, username=Settings.get_solo().contact_user_id) dsub = get_or_create(DataSubmission, status=STATUS_VALUES['VALIDATED'], incoming_directory=incoming_directory, directory=incoming_directory, user=self.user) df1 = get_or_create(DataFile, name='file_one.nc', incoming_directory=incoming_directory, directory=None, size=1, project=proj, climate_model=climate_model, experiment=experiment, institute=institute, variable_request=var1, data_request=self.dreq1, frequency=FREQUENCY_VALUES['ann'], activity_id=act_id, rip_code='r1i1p1f1', online=False, start_time=0., end_time=359., time_units='days since 1950-01-01', calendar=CALENDARS['360_day'], grid='gn', version='v12345678', tape_url='et:1234', data_submission=dsub) df2 = get_or_create(DataFile, name='file_two.nc', incoming_directory=incoming_directory, directory=None, size=1, project=proj, climate_model=climate_model, experiment=experiment, institute=institute, variable_request=var2, data_request=self.dreq2, frequency=FREQUENCY_VALUES['ann'], activity_id=act_id, rip_code='r1i1p1f1', online=False, start_time=0., end_time=359., time_units='days since 1950-01-01', calendar=CALENDARS['360_day'], grid='gn', version='v12345678', tape_url='et:5678', data_submission=dsub) df3 = get_or_create(DataFile, name='file_three.nc', incoming_directory=incoming_directory, directory=None, size=1, project=proj, climate_model=climate_model, experiment=experiment, institute=institute, variable_request=var2, data_request=self.dreq2, frequency=FREQUENCY_VALUES['ann'], activity_id=act_id, rip_code='r1i1p1f1', online=False, start_time=360., end_time=719., time_units='days since 1950-01-01', calendar=CALENDARS['360_day'], grid='gn', version='v12345678', tape_url='et:8765', data_submission=dsub) rf1 = get_or_create(ReplacedFile, name='file_four.nc', incoming_directory=incoming_directory, size=1, project=proj, climate_model=climate_model, experiment=experiment, institute=institute, variable_request=var1, data_request=self.dreq1, frequency=FREQUENCY_VALUES['ann'], activity_id=act_id, rip_code='r1i1p1f1', start_time=0., end_time=359., time_units='days since 1950-01-01', calendar=CALENDARS['360_day'], grid='gn', version='v12345678', tape_url='et:1234', data_submission=dsub, checksum_value='01234567', checksum_type='ADLER32') rf2 = get_or_create(ReplacedFile, name='file_five.nc', incoming_directory=incoming_directory, size=2, project=proj, climate_model=climate_model, experiment=experiment, institute=institute, variable_request=var1, data_request=self.dreq1, frequency=FREQUENCY_VALUES['ann'], activity_id=act_id, rip_code='r1i1p1f1', start_time=360., end_time=719., time_units='days since 1950-01-01', calendar=CALENDARS['360_day'], grid='gn', version='v87654321', tape_url='et:4321', data_submission=dsub, checksum_value='76543210', checksum_type='ADLER32')
def main(args): """ Main entry point """ activity_id = 'primWP5' new_dreqs = [ 'uas_3hr', 'vas_3hr', 'pr_6hrPlev', 'psl_6hrPlev', 'tas_6hrPlev', 'uas_6hrPlev', 'vas_6hrPlev', 'zg1000_6hrPlev', 'ta_6hrPlevPt', 'ua_6hrPlevPt', 'va_6hrPlevPt', 'zg27_6hrPlevPt', 'zg500_AERday', 'ptp_AERmon', 'rlutaf_AERmon', 'rlutcsaf_AERmon', 'rsutaf_AERmon', 'rsutcsaf_AERmon', 'toz_AERmon', 'ztp_AERmon', 'clivi_Amon', 'clt_Amon', 'clwvi_Amon', 'evspsbl_Amon', 'hfls_Amon', 'hfss_Amon', 'hur_Amon', 'hurs_Amon', 'hus_Amon', 'huss_Amon', 'o3_Amon', 'pr_Amon', 'prc_Amon', 'prsn_Amon', 'prw_Amon', 'ps_Amon', 'psl_Amon', 'rlds_Amon', 'rldscs_Amon', 'rlus_Amon', 'rlut_Amon', 'rlutcs_Amon', 'rsds_Amon', 'rsdscs_Amon', 'rsdt_Amon', 'rsus_Amon', 'rsuscs_Amon', 'rsut_Amon', 'rsutcs_Amon', 'sbl_Amon', 'sfcWind_Amon', 'ta_Amon', 'tas_Amon', 'tasmax_Amon', 'tasmin_Amon', 'tauu_Amon', 'tauv_Amon', 'ts_Amon', 'ua_Amon', 'uas_Amon', 'va_Amon', 'vas_Amon', 'wap_Amon', 'zg_Amon', 'ps_CFday', 'mlotst_Eday', 'rivo_Eday', 't20d_Eday', 'ta850_Eday', 'ts_Eday', 'sftgrf_Efx', 'cropFracC3_Emon', 'cropFracC4_Emon', 'evspsblpot_Emon', 'grassFracC3_Emon', 'grassFracC4_Emon', 'intuadse_Emon', 'intuaw_Emon', 'intvadse_Emon', 'intvaw_Emon', 'mrtws_Emon', 'nwdFracLut_Emon', 'orog_Emon', 'rls_Emon', 'rss_Emon', 'sfcWindmax_Emon', 't20d_Emon', 'thetaot_Emon', 'thetaot2000_Emon', 'thetaot300_Emon', 'thetaot700_Emon', 'vegFrac_Emon', 'wtd_Emon', 'lwsnl_LImon', 'sftgif_LImon', 'sftgrf_LImon', 'snc_LImon', 'snd_LImon', 'snw_LImon', 'baresoilFrac_Lmon', 'c3PftFrac_Lmon', 'c4PftFrac_Lmon', 'cropFrac_Lmon', 'gpp_Lmon', 'grassFrac_Lmon', 'lai_Lmon', 'mrfso_Lmon', 'mrro_Lmon', 'mrros_Lmon', 'mrso_Lmon', 'mrsos_Lmon', 'npp_Lmon', 'prveg_Lmon', 'ra_Lmon', 'rh_Lmon', 'treeFrac_Lmon', 'tsl_Lmon', 'sos_Oday', 'tos_Oday', 'areacello_Ofx', 'basin_Ofx', 'deptho_Ofx', 'hfgeou_Ofx', 'masscello_Ofx', 'thkcello_Ofx', 'bigthetao_Omon', 'bigthetaoga_Omon', 'ficeberg_Omon', 'friver_Omon', 'hfbasin_Omon', 'hfcorr_Omon', 'hfds_Omon', 'hfx_Omon', 'hfy_Omon', 'htovgyre_Omon', 'htovovrt_Omon', 'mfo_Omon', 'mlotst_Omon', 'mlotstmin_Omon', 'msftyz_Omon', 'rsntds_Omon', 'so_Omon', 'soga_Omon', 'sos_Omon', 'sosga_Omon', 'tauuo_Omon', 'tauvo_Omon', 'thetao_Omon', 'thkcello_Omon', 'tos_Omon', 'umo_Omon', 'uo_Omon', 'vmo_Omon', 'vo_Omon', 'volo_Omon', 'wfo_Omon', 'wmo_Omon', 'wo_Omon', 'zos_Omon', 'zossq_Omon', 'zostoga_Omon', 'rsds_Prim6hr', 'rsdsdiff_Prim6hr', 'mrso_Primday', 'siconc_SIday', 'siconca_SIday', 'sisnthick_SIday', 'sispeed_SIday', 'sitemptop_SIday', 'sithick_SIday', 'sitimefrac_SIday', 'siu_SIday', 'siv_SIday', 'siage_SImon', 'siareaacrossline_SImon', 'siarean_SImon', 'siareas_SImon', 'sicompstren_SImon', 'siconc_SImon', 'siconca_SImon', 'sidconcdyn_SImon', 'sidconcth_SImon', 'sidivvel_SImon', 'sidmassdyn_SImon', 'sidmassevapsubl_SImon', 'sidmassgrowthbot_SImon', 'sidmassgrowthwat_SImon', 'sidmasslat_SImon', 'sidmassmeltbot_SImon', 'sidmassmelttop_SImon', 'sidmasssi_SImon', 'sidmassth_SImon', 'sidmasstranx_SImon', 'sidmasstrany_SImon', 'siextentn_SImon', 'siextents_SImon', 'sifb_SImon', 'siflcondbot_SImon', 'siflcondtop_SImon', 'siflfwbot_SImon', 'siflfwdrain_SImon', 'sifllatstop_SImon', 'sifllwutop_SImon', 'siflsensupbot_SImon', 'siflswdbot_SImon', 'siflswdtop_SImon', 'siflswutop_SImon', 'sihc_SImon', 'simass_SImon', 'simassacrossline_SImon', 'sipr_SImon', 'sisaltmass_SImon', 'sishevel_SImon', 'sisnconc_SImon', 'sisnhc_SImon', 'sisnmass_SImon', 'sisnthick_SImon', 'sispeed_SImon', 'sistrxdtop_SImon', 'sistrxubot_SImon', 'sistrydtop_SImon', 'sistryubot_SImon', 'sitempbot_SImon', 'sitempsnic_SImon', 'sitemptop_SImon', 'sithick_SImon', 'sitimefrac_SImon', 'siu_SImon', 'siv_SImon', 'sivol_SImon', 'sivoln_SImon', 'sivols_SImon', 'sndmassdyn_SImon', 'sndmassmelt_SImon', 'sndmasssi_SImon', 'sndmasssnf_SImon', 'sndmasssubl_SImon', 'snmassacrossline_SImon', 'clt_day', 'hfls_day', 'hfss_day', 'huss_day', 'pr_day', 'psl_day', 'rlut_day', 'rsds_day', 'sfcWindmax_day', 'snc_day', 'ta_day', 'tas_day', 'tasmax_day', 'tasmin_day', 'ua_day', 'uas_day', 'va_day', 'vas_day', 'zg_day', 'areacella_fx', 'areacellr_fx', 'mrsofc_fx', 'orog_fx', 'rootd_fx', 'sftgif_fx', 'sftlf_fx', 'zfull_fx', ] institute_details = { 'id': 'CNRM-CERFACS', 'model_ids': ['CNRM-CM6-1-HR', 'CNRM-CM6-1'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'primWP5-amv-neg': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(1960, 1, 1)}, 'primWP5-amv-pos': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(1960, 1, 1)} } variant_labels = ['r{}i1p1f2'.format(i) for i in range(11, 26)] # activity_id ActivityId.objects.get_or_create(short_name=activity_id, full_name=activity_id) # Experiment cache experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id'] ) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'. format(clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: for var_lab in variant_labels: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar'] ), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar'] ), time_units=std_units, calendar=institute_details['calendar'], rip_code = var_lab ) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ rip_code = 'r1i1p2f1' new_dreqs = [ 'cl_Amon', 'clt_Amon', 'rlut_day', 'ua_day', 'va_day', 'rsut_CFday', ] institute_details = { 'id': 'MOHC', 'model_ids': ['HadGEM3-GC31-LL'], 'calendar': CALENDAR_360_DAY } experiments = { 'hist-1950': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1)}, } # Experiment experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id'] ) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'. format(clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar'] ), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar'] ), time_units=std_units, calendar=institute_details['calendar'], rip_code=rip_code ) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ institute_details = { 'id': 'CNRM-CERFACS', 'model_ids': ['CNRM-CM6-1', 'CNRM-CM6-1-HR'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'spinup-1950': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(1980, 1, 1)}, } variant_label = 'r1i1p1f2' # Experiment new_dreqs = [ 'od550so4_AERmon', 'rsutcsaf_AERmon', 'hcont300_Emon', 'intuadse_Emon', 'intuaw_Emon', 'intvadse_Emon', 'intvaw_Emon', 'mrtws_Emon', 'wtd_Emon', 'htovgyre_Omon', 'htovovrt_Omon', 'siconca_SImon', 'simass_SImon', 'sisaltmass_SImon', 'sisnmass_SImon', 'sisnthick_SImon', 'sithick_SImon', 'rivo_Eday', 'siforceintstrx_PrimSIday', 'siforceintstry_PrimSIday', 'simassacrossline_PrimSIday', ] experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id'] ) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'. format(clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: try: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar'] ), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar'] ), time_units=std_units, calendar=institute_details['calendar'], rip_code = variant_label ) except django.core.exceptions.MultipleObjectsReturned: logger.error('{}'.format(var_req_obj)) raise else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ activity_id = 'primWP5' new_dreqs = [ 'wfcorr_Omon', ] institute_details = { 'id': 'EC-Earth-Consortium', 'model_ids': ['EC-Earth3P-HR'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'primWP5-amv-neg': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(1960, 1, 1)}, 'primWP5-amv-pos': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(1960, 1, 1)} } variant_labels = ['r{}i1p2f1'.format(i) for i in range(1, 11)] # activity_id ActivityId.objects.get_or_create(short_name=activity_id, full_name=activity_id) # Experiment cache experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id'] ) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'. format(clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: for var_lab in variant_labels: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar'] ), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar'] ), time_units=std_units, calendar=institute_details['calendar'], rip_code = var_lab ) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
from pdata_app.models import DataRequest, Settings from pdata_app.utils.common import (adler32, construct_drs_path, delete_drs_dir, is_same_gws, directories_spanned) __version__ = '0.1.0b' DEFAULT_LOG_LEVEL = logging.WARNING DEFAULT_LOG_FORMAT = '%(levelname)s: %(message)s' logger = logging.getLogger(__name__) # The common prefix of all of our GWSs COMMON_GWS_NAME = '/gws/nopw/j04/primavera' # The top-level directory to write output data to BASE_OUTPUT_DIR = Settings.get_solo().base_output_dir def move_dirs(data_req, new_gws): """ Move the files :param pdata_app.models.DataRequest data_req: the data request to move :param int new_gws: the number of the gws to move to """ single_dir = '{}{}'.format(COMMON_GWS_NAME, new_gws) existing_dirs = data_req.directories() # ignore data that is offline if None in existing_dirs: existing_dirs.remove(None) use_single_dir = False
def main(args): """ Main entry point """ # Delete the data requests that could be duplicates dupe_dreqs = DataRequest.objects.filter( climate_model__short_name='CNRM-CM6-1-HR', experiment__short_name='highres-future', variable_request__table_name='SImon', variable_request__table_name__in = ['simass', 'sisaltmass', 'sisnthick'] ) if dupe_dreqs.count() != 3: logger.error('{} data requests found when expecting 3'. format(dupe_dreqs.count())) sys.exit(1) dupe_dreqs.delete() institute_details = { 'id': 'CNRM-CERFACS', 'model_ids': ['CNRM-CM6-1', 'CNRM-CM6-1-HR'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'highres-future': {'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1)}, } variant_label = 'r1i1p1f2' # Experiment new_dreqs = [ 'cropFracC3_Emon', 'cropFracC4_Emon', 'grassFracC3_Emon', 'grassFracC4_Emon', 'nwdFracLut_Emon', 'ps_AERmon', 'sftgif_LImon', 'sftgrf_LImon', 'sidconcdyn_SImon', 'sidconcth_SImon', 'simass_SImon', 'sisaltmass_SImon', 'sisnthick_SImon', 'sndmassdyn_SImon', 'sndmasssi_SImon', 'vegFrac_Emon', ] experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id'] ) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'. format(clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: try: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar'] ), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar'] ), time_units=std_units, calendar=institute_details['calendar'], rip_code = variant_label ) except django.core.exceptions.MultipleObjectsReturned: logger.error('{}'.format(var_req_obj)) raise else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ new_dreqs = [ 'siage_SImon', 'sisnthick_SImon', 'sicompstren_SImon', 'sisali_SImon', 'hus1000_Prim6hrPt', 'siconc_SImon', 'sistrxdtop_SImon', 'siv_SImon', 'sivol_SImon', 'sistrydtop_SImon', 'va1000_Prim6hrPt', 'sidmassevapsubl_SImon', 'sithick_SImon', 'sispeed_SImon', 'sndmasssnf_SImon', 'siflswdtop_SImon', 'sitemptop_SImon', 'ua1000_Prim6hrPt', 'siu_SImon' ] institute_details = { 'id': 'EC-Earth-Consortium', 'model_ids': ['EC-Earth3', 'EC-Earth3-HR'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'control-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2050, 1, 1) }, 'highres-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'hist-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-present': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'highresSST-LAI': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-smoothed': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-p4K': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-4co2': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'spinup-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(1980, 1, 1) }, } # Experiment experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar']) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ activity_id = 'primWP5' new_vreqs = [ { 'table_name': '6hrPlevPt', 'long_name': 'Eastward Wind', 'units': 'm s-1', 'var_name': 'ua', 'standard_name': 'eastward_wind', 'cell_methods': 'time: point', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time1', 'cmor_name': 'ua850', 'modeling_realm': 'atmos', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': 'prim6hrPlevPt_ua850' }, { 'table_name': '6hrPlevPt', 'long_name': 'Northward Wind', 'units': 'm s-1', 'var_name': 'va', 'standard_name': 'northward_wind', 'cell_methods': 'time: point', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time1', 'cmor_name': 'va850', 'modeling_realm': 'atmos', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': 'prim6hrPlevPt_va850' }, ] # create the additional variable requests for new_vreq in new_vreqs: _vr = get_or_create(VariableRequest, **new_vreq) new_dreqs = [ 'psl_6hrPlevPt', 'ua850_6hrPlevPt', 'va850_6hrPlevPt', 'zg500_6hrPlevPt', 'clt_Amon', 'hfls_Amon', 'hfss_Amon', 'hur_Amon', 'hurs_Amon', 'hus_Amon', 'huss_Amon', 'pr_Amon', 'prc_Amon', 'prsn_Amon', 'ps_Amon', 'psl_Amon', 'rlds_Amon', 'rldscs_Amon', 'rlus_Amon', 'rlut_Amon', 'rlutcs_Amon', 'rsds_Amon', 'rsdscs_Amon', 'rsdt_Amon', 'rsus_Amon', 'rsuscs_Amon', 'rsut_Amon', 'rsutcs_Amon', 'rtmt_Amon', 'sfcWind_Amon', 'ta_Amon', 'tas_Amon', 'tasmax_Amon', 'tasmin_Amon', 'tauu_Amon', 'tauv_Amon', 'ts_Amon', 'ua_Amon', 'uas_Amon', 'va_Amon', 'vas_Amon', 'wap_Amon', 'zg_Amon', 'snw_LImon', 'mrso_Lmon', 'ta_day', 'tas_day', 'tasmax_day', 'ua_day', 'va_day', 'zg_day' ] Institute.objects.get_or_create(short_name='NCAS', full_name='NCAS') ClimateModel.objects.get_or_create(short_name='MetUM-GOML2-LR', full_name='MetUM-GOML2-LR') ClimateModel.objects.get_or_create(short_name='MetUM-GOML2-HR', full_name='MetUM-GOML2-HR') institute_details = { 'id': 'NCAS', 'model_ids': ['MetUM-GOML2-LR', 'MetUM-GOML2-HR'], 'calendar': CALENDAR_360_DAY } experiments = { 'primWP5-amv-pos': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'primWP5-amv-neg': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) } } variant_labels = ['r{}i1p1f1'.format(i) for i in range(1, 16)] # activity_id ActivityId.objects.get_or_create(short_name=activity_id, full_name=activity_id) # Experiment create for expt in experiments: Experiment.objects.get_or_create(short_name=expt, full_name=expt) # Experiment cache experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: for var_lab in variant_labels: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar'], rip_code=var_lab) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ institute_details = { 'id': 'CNRM-CERFACS', 'model_ids': ['CNRM-CM6-1', 'CNRM-CM6-1-HR'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'highresSST-present': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'highresSST-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, } variant_label = 'r1i1p1f2' new_vreqs = [ { 'table_name': 'EmonZ', 'long_name': 'v-tendency nonorographic gravity wave drag', 'units': 'm s-2', 'var_name': 'vtendnogw', 'standard_name': 'tendency_of_northward_wind_due_to_nonorographic_gravity_wave_drag', 'cell_methods': 'longitude: mean time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'latitude plev39 time', 'cmor_name': 'vtendnogw', 'modeling_realm': 'atmos', 'frequency': 'mon', 'cell_measures': '', 'uid': '8183eac8-f906-11e6-a176-5404a60d96b5' }, { 'table_name': 'Emon', 'long_name': 'C3 grass Area Percentage', 'units': '%', 'var_name': 'grassFracC3', 'standard_name': 'area_fraction', 'cell_methods': 'area: mean where land over all_area_types time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time typec3pft typenatgr', 'cmor_name': 'grassFracC3', 'modeling_realm': 'land', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '8b814764-4a5b-11e6-9cd2-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'C4 grass Area Percentage', 'units': '%', 'var_name': 'grassFracC4', 'standard_name': 'area_fraction', 'cell_methods': 'area: mean where land over all_area_types time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time typec4pft typenatgr', 'cmor_name': 'grassFracC4', 'modeling_realm': 'land', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '8b814cc8-4a5b-11e6-9cd2-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Percentage of land use tile tile that is non-woody vegetation ( e.g. herbaceous crops)', 'units': '%', 'var_name': 'nwdFracLut', 'standard_name': 'missing', 'cell_methods': 'area: mean where land over all_area_types time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude landUse time typenwd', 'cmor_name': 'nwdFracLut', 'modeling_realm': 'land', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': 'd22da9f2-4a9f-11e6-b84e-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Total vegetated fraction', 'units': '%', 'var_name': 'vegFrac', 'standard_name': 'area_fraction', 'cell_methods': 'area: mean where land over all_area_types time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time typeveg', 'cmor_name': 'vegFrac', 'modeling_realm': 'land', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '6f6a57d0-9acb-11e6-b7ee-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Percentage Cover by C3 Crops', 'units': '%', 'var_name': 'cropFracC3', 'standard_name': 'area_fraction', 'cell_methods': 'area: mean where land over all_area_types time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time typec3pft typecrop', 'cmor_name': 'cropFracC3', 'modeling_realm': 'land', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '8b81522c-4a5b-11e6-9cd2-ac72891c3257' }, { 'table_name': 'Emon', 'long_name': 'Percentage Cover by C4 Crops', 'units': '%', 'var_name': 'cropFracC4', 'standard_name': 'area_fraction', 'cell_methods': 'area: mean where land over all_area_types time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time typec4pft typecrop', 'cmor_name': 'cropFracC4', 'modeling_realm': 'land', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '6f6a8ea8-9acb-11e6-b7ee-ac72891c3257' }, { 'table_name': 'LImon', 'long_name': 'Land Ice Area Percentage', 'units': '%', 'var_name': 'sftgif', 'standard_name': 'land_ice_area_fraction', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time typeli', 'cmor_name': 'sftgif', 'modeling_realm': 'land', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': 'b3d6e26a-3578-11e7-8257-5404a60d96b5' }, { 'table_name': 'LImon', 'long_name': 'Grounded Ice Sheet Area Percentage', 'units': '%', 'var_name': 'sftgrf', 'standard_name': 'grounded_ice_sheet_area_fraction', 'cell_methods': 'area: time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time typegis', 'cmor_name': 'sftgrf', 'modeling_realm': 'landIce', 'frequency': 'mon', 'cell_measures': 'area: areacella', 'uid': '8bbb1520-4a5b-11e6-9cd2-ac72891c3257' }, ] # Experiment new_dreqs = [ 'vtendnogw_EmonZ', 'cropFracC3_Emon', 'cropFracC4_Emon', 'grassFracC3_Emon', 'grassFracC4_Emon', 'nwdFracLut_Emon', 'ps_AERmon', 'sftgif_LImon', 'sftgrf_LImon', 'sidconcdyn_SImon', 'sidconcth_SImon', 'simass_SImon', 'sisaltmass_SImon', 'sisnthick_SImon', 'sndmassdyn_SImon', 'sndmasssi_SImon', 'vegFrac_Emon', ] experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the additional variable requests for new_vreq in new_vreqs: _vr = get_or_create(VariableRequest, **new_vreq) # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: try: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar'], rip_code=variant_label) except django.core.exceptions.MultipleObjectsReturned: logger.error('{}'.format(var_req_obj)) raise else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ logger.debug('Starting delete_request.py for retrieval {}'.format( args.retrieval_id)) deletion_retrieval = match_one(RetrievalRequest, id=args.retrieval_id) if not deletion_retrieval: logger.error('Unable to find retrieval id {}'.format( args.retrieval_id)) sys.exit(1) if deletion_retrieval.date_deleted: logger.error('Retrieval {} was already deleted, at {}.'.format( deletion_retrieval.id, deletion_retrieval.date_deleted.strftime('%Y-%m-%d %H:%M'))) sys.exit(1) if not deletion_retrieval.data_finished: logger.error('Retrieval {} is not marked as finished.'.format( deletion_retrieval.id)) sys.exit(1) problems_encountered = False directories_found = [] base_output_dir = Settings.get_solo().base_output_dir # loop through all of the data requests in this retrieval for data_req in deletion_retrieval.data_request.all(): online_req_files = data_req.datafile_set.filter( online=True, directory__isnull=False) files_to_delete = date_filter_files(online_req_files, deletion_retrieval.start_year, deletion_retrieval.end_year) if files_to_delete is None: continue if not args.force: # find any other retrieval requests that still need this data other_retrievals = RetrievalRequest.objects.filter( data_request=data_req, data_finished=False) # loop through the retrieval requests that still need this data # request for ret_req in other_retrievals: ret_online_files = data_req.datafile_set.filter( online=True, directory__isnull=False) ret_filtered_files = date_filter_files(ret_online_files, ret_req.start_year, ret_req.end_year) if ret_filtered_files is None: continue # remove from the list of files to delete the ones that we have # just found are still needed files_to_delete = files_to_delete.difference( ret_filtered_files) # list the parts of the data request that are still required logger.debug("{} {} to {} won't be deleted".format( data_req, ret_req.start_year, ret_req.end_year)) # don't (try to) delete anything that's in the CEDA archive files_to_delete.exclude(directory__startswith=CEDA_ARCHIVE) # do the deleting if args.dryrun: logger.debug('{} {} files can be deleted.'.format( data_req, files_to_delete.distinct().count())) else: logger.debug('{} {} files will be deleted.'.format( data_req, files_to_delete.distinct().count())) for data_file in files_to_delete: old_file_dir = data_file.directory try: os.remove(os.path.join(data_file.directory, data_file.name)) except OSError as exc: logger.error(str(exc)) problems_encountered = True else: if data_file.directory not in directories_found: directories_found.append(data_file.directory) data_file.online = False data_file.directory = None data_file.save() # if a symbolic link exists from the base output directory # then delete this too if not old_file_dir.startswith(base_output_dir): sym_link_dir = os.path.join(base_output_dir, construct_drs_path(data_file)) sym_link = os.path.join(sym_link_dir, data_file.name) if not os.path.islink(sym_link): logger.error( "Expected {} to be a link but it isn't. " "Leaving this file in place.".format(sym_link)) problems_encountered = True else: try: os.remove(sym_link) except OSError as exc: logger.error(str(exc)) problems_encountered = True else: if sym_link_dir not in directories_found: directories_found.append(sym_link_dir) if not args.dryrun: # delete any empty directories for directory in directories_found: if not os.listdir(directory): delete_drs_dir(directory) # set date_deleted in the db if not problems_encountered: deletion_retrieval.date_deleted = timezone.now() deletion_retrieval.save() else: logger.error( 'Errors were encountered and so retrieval {} has not ' 'been marked as deleted. All possible files have been ' 'deleted.'.format(args.retrieval_id)) logger.debug('Completed delete_request.py for retrieval {}'.format( args.retrieval_id))
def main(args): """ Main entry point """ mip_era = 'CMIP6' activity_id = 'PRIMAVERA' new_dreqs = [ 'pr_day', 'psl_6hrPlevPt', 'ua_6hrPlevPt', 'uas_6hrPlevPt', 'va_6hrPlevPt', 'vas_6hrPlevPt' ] institute_details = { 'id': 'EC-Earth-Consortium', 'model_ids': ['EC-Earth3'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'Primavera-coupled': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2016, 1, 1) } } variant_labels = ['r1i1p101f1'] # activity_id ActivityId.objects.get_or_create(short_name=activity_id, full_name=activity_id) # Experiment create for expt in experiments: Experiment.objects.get_or_create(short_name=expt, full_name=expt) # Experiment cache experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) logger.error(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) logger.error(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) logger.error(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') project = match_one(Project, short_name=mip_era) var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: for var_lab in variant_labels: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar'], rip_code=var_lab) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) logger.error(msg) raise ValueError(msg)
def main(args): """ Main entry point """ institute_details = { 'id': 'AWI', 'model_ids': ['AWI-CM-1-1-LR', 'AWI-CM-1-1-HR'], 'calendar': CALENDAR_STANDARD } experiments = { 'control-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2050, 1, 1) }, 'highres-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'hist-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'spinup-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(1980, 1, 1) } } variant_label = 'r1i1p1f2' # Experiment new_dreqs = [ 'rsut_E3hr', ] experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: try: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar']), time_units=std_units, calendar=institute_details['calendar'], rip_code=variant_label) except django.core.exceptions.MultipleObjectsReturned: logger.error('{}'.format(var_req_obj)) raise else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ new_vreqs = [ {'table_name': 'Ofx', 'long_name': 'Grid-Cell Area for Ocean Variables', 'units': 'm2', 'var_name': 'areacello', 'standard_name': 'cell_area', 'cell_methods': 'area: sum', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude ', 'cmor_name': 'areacello', 'modeling_realm': 'ocean', 'frequency': 'fx', 'cell_measures': '', 'uid': 'baa3ee94-e5dd-11e5-8482-ac72891c3257'}, {'table_name': 'Ofx', 'long_name': 'Region Selection Index', 'units': '1', 'var_name': 'basin', 'standard_name': 'region', 'cell_methods': 'area: mean', 'positive': '', 'variable_type': 'integer', 'dimensions': 'longitude latitude ', 'cmor_name': 'basin', 'modeling_realm': 'ocean', 'frequency': 'fx', 'cell_measures': 'area: areacello', 'uid': 'baa3f718-e5dd-11e5-8482-ac72891c3257'}, {'table_name': 'Ofx', 'long_name': 'Ocean Model Cell Thickness', 'units': 'm', 'var_name': 'thkcello', 'standard_name': 'cell_thickness', 'cell_methods': 'area: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude olevel ', 'cmor_name': 'thkcello', 'modeling_realm': 'ocean', 'frequency': 'fx', 'cell_measures': 'area: areacello volume: volcello', 'uid': 'bab9bd00-e5dd-11e5-8482-ac72891c3257'}, ] # create the variable requests for new_vreq in new_vreqs: _vr = get_or_create(VariableRequest, **new_vreq) variant_label = 'r1i1p1f1' # The standard reference time std_units = Settings.get_solo().standard_time_units new_dreqs = [ 'areacello_Ofx', 'basin_Ofx', ] institute_id = 'MOHC' calendar = CALENDAR_360_DAY # Institute result = match_one(Institute, short_name=institute_id) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format(institute_id) print(msg) raise ValueError(msg) # Coupled ######### model_ids = ['HadGEM3-GC31-HH', 'HadGEM3-GC31-HM', 'HadGEM3-GC31-MM', 'HadGEM3-GC31-LL'] experiments = { 'spinup-1950': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(1980, 1, 1)}, 'hist-1950': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1)}, 'control-1950': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2050, 1, 1)}, 'highres-future': {'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1)}, } experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in model_ids: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'. format(clim_model)) print(msg) raise ValueError(msg) # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: try: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, calendar ), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, calendar ), time_units=std_units, calendar=calendar, rip_code = variant_label ) except django.core.exceptions.MultipleObjectsReturned: logger.error('{}'.format(var_req_obj)) raise else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg) # Atmosphere only ################# experiments = { 'highresSST-present': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1)}, 'highresSST-future': {'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1)}, } model_ids = ['HadGEM3-GC31-HM', 'HadGEM3-GC31-MM', 'HadGEM3-GC31-LM'] experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in model_ids: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'. format(clim_model)) print(msg) raise ValueError(msg) # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: try: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, calendar ), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, calendar ), time_units=std_units, calendar=calendar, rip_code = variant_label ) except django.core.exceptions.MultipleObjectsReturned: logger.error('{}'.format(var_req_obj)) raise else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ # new_vreqs = ['lwsnl_Eday', 'mrros_Eday', 'snm_Eday', 'snd_Eday'] new_vreqs = [ {'table_name': 'Eday', 'long_name': 'Liquid Water Content of Snow Layer', 'units': 'kg m-2', 'var_name': 'lwsnl', 'standard_name': 'liquid_water_content_of_snow_layer', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'lwsnl', 'modeling_realm': 'landIce land', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': 'd228925a-4a9f-11e6-b84e-ac72891c3257'}, {'table_name': 'Eday', 'long_name': 'Surface Runoff', 'units': 'kg m-2 s-1', 'var_name': 'mrros', 'standard_name': 'surface_runoff_flux', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'mrros', 'modeling_realm': 'land', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': 'd2284048-4a9f-11e6-b84e-ac72891c3257'}, {'table_name': 'Eday', 'long_name': 'Surface Snow Melt', 'units': 'kg m-2 s-1', 'var_name': 'snm', 'standard_name': 'surface_snow_melt_flux', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'snm', 'modeling_realm': 'landIce land', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': 'd22848ea-4a9f-11e6-b84e-ac72891c3257'}, {'table_name': 'Eday', 'long_name': 'Snow Depth', 'units': 'm', 'var_name': 'snd', 'standard_name': 'surface_snow_thickness', 'cell_methods': 'area: mean where land time: mean', 'positive': '', 'variable_type': 'real', 'dimensions': 'longitude latitude time', 'cmor_name': 'snd', 'modeling_realm': 'landIce land', 'frequency': 'day', 'cell_measures': 'area: areacella', 'uid': 'b7ccdf0a-7c00-11e6-bcdf-ac72891c3257'} ] new_dreqs = [ 'wap_Emon', 'hfdsn_LImon', 'snc_day', 'va27_Emon', 'rsuscs_CFday', 'vt_Emon', 'hursmin_day', 'uv_Emon', 'cl_Amon', 'rldscs_CFday', 'lwp_Primmon', 'uwap_Emon', 'rsdscs_Amon', 'zg_day', 'evspsblsoi_Lmon', 'snc_LImon', 'v2_Emon', 'ta27_Emon', 'tsl_Lmon', 'lwsnl_LImon', 'sbl_LImon', 'wap2_Emon', 'ut_Emon', 'hursmax_day', 't2_Emon', 'rsuscs_Amon', 'rsdscs_CFday', 'mrsos_day', 'hus27_Emon', 'snm_LImon', 'tsn_LImon', 'rldscs_Amon', 'vwap_Emon', 'zg27_Emon', 'twap_Emon', 'u2_Emon', 'ua27_Emon', 'lwsnl_Eday', 'mrros_Eday', 'snm_Eday', 'snd_Eday' ] institute_details = { 'id': 'EC-Earth-Consortium', 'model_ids': ['EC-Earth3-LR', 'EC-Earth3-HR'], 'calendar': CALENDAR_GREGORIAN } experiments = { 'control-1950': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2050, 1, 1)}, 'highres-future': {'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1)}, 'hist-1950': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1)}, 'highresSST-present': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1)}, 'highresSST-future': {'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1)}, 'highresSST-LAI': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1)}, 'highresSST-smoothed': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1)}, 'highresSST-p4K': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1)}, 'highresSST-4co2': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1)}, 'spinup-1950': {'start_date': datetime(1950, 1, 1), 'end_date': datetime(1980, 1, 1)}, } # Experiment experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institute result = match_one(Institute, short_name=institute_details['id']) if result: institute = result else: msg = 'institute_id {} not found in the database.'.format( institute_details['id'] ) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later model_objs = [] for clim_model in institute_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: model_objs.append(result) else: msg = ('climate_model {} not found in the database.'. format(clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the additional variable requests for new_vreq in new_vreqs: _vr = get_or_create(VariableRequest, **new_vreq) # create the new data requests for new_dreq in new_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in model_objs: _dr = get_or_create( DataRequest, project=project, institute=institute, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, institute_details['calendar'] ), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, institute_details['calendar'] ), time_units=std_units, calendar=institute_details['calendar'] ) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)
def main(args): """ Main entry point """ cmcc_dreqs = [ 'siconc_SIday', 'simass_SImon', 'sisnmass_SImon', 'sitimefrac_SImon' ] cmcc_details = { 'id': 'CMCC', 'model_ids': ['CMCC-CM2-HR4', 'CMCC-CM2-VHR4'], 'calendar': CALENDAR_STANDARD } experiments = { 'control-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2050, 1, 1) }, 'highres-future': { 'start_date': datetime(2015, 1, 1), 'end_date': datetime(2051, 1, 1) }, 'hist-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(2015, 1, 1) }, 'spinup-1950': { 'start_date': datetime(1950, 1, 1), 'end_date': datetime(1980, 1, 1) }, } # Experiment experiment_objs = [] for expt in experiments: expt_obj = match_one(Experiment, short_name=expt) if expt_obj: experiment_objs.append(expt_obj) else: msg = 'experiment {} not found in the database.'.format(expt) print(msg) raise ValueError(msg) # Institutes result = match_one(Institute, short_name=cmcc_details['id']) if result: cmcc = result else: msg = 'institute_id {} not found in the database.'.format( cmcc_details['id']) print(msg) raise ValueError(msg) # Look up the ClimateModel object for each institute_id and save the # results to a dictionary for quick look up later cmcc_model_objs = [] for clim_model in cmcc_details['model_ids']: result = match_one(ClimateModel, short_name=clim_model) if result: cmcc_model_objs.append(result) else: msg = ('climate_model {} not found in the database.'.format( clim_model)) print(msg) raise ValueError(msg) # The standard reference time std_units = Settings.get_solo().standard_time_units # create the new data requests for new_dreq in cmcc_dreqs: cmor_name, table_name = new_dreq.split('_') if table_name.startswith('Prim'): project = match_one(Project, short_name='PRIMAVERA') else: project = match_one(Project, short_name='CMIP6') var_req_obj = match_one(VariableRequest, cmor_name=cmor_name, table_name=table_name) if var_req_obj: for expt in experiment_objs: for clim_model in cmcc_model_objs: _dr = get_or_create( DataRequest, project=project, institute=cmcc, climate_model=clim_model, experiment=expt, variable_request=var_req_obj, request_start_time=date2num( experiments[expt.short_name]['start_date'], std_units, cmcc_details['calendar']), request_end_time=date2num( experiments[expt.short_name]['end_date'], std_units, cmcc_details['calendar']), time_units=std_units, calendar=cmcc_details['calendar']) else: msg = ('Unable to find variable request matching ' 'cmor_name {} and table_name {} in the ' 'database.'.format(cmor_name, table_name)) print(msg) raise ValueError(msg)