def main(): """ Main entry point """ alevhalf = DataRequest.objects.filter( institute__short_name='EC-Earth-Consortium', variable_request__dimensions__contains='alevhalf', datafile__isnull=False).distinct() alevel = DataRequest.objects.filter( institute__short_name='EC-Earth-Consortium', variable_request__dimensions__contains='alevel', datafile__isnull=False).distinct() dreqs = alevhalf | alevel logger.debug(f'{dreqs.count()} data requests found') for dreq in dreqs.order_by('climate_model__short_name', 'experiment__short_name', 'rip_code', 'variable_request__table_name', 'variable_request__cmor_name'): if dreq.online_status() in ['online', 'partial']: logger.debug(dreq) delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR, skip_badc=True)
def main(args): """ Main entry point """ dfs = DataFile.objects.filter( climate_model__short_name='EC-Earth3P', experiment__short_name='control-1950', rip_code='r1i1p2f1', variable_request__table_name='3hr', variable_request__cmor_name='ps' ) num_files = dfs.count() if num_files != 1200: logger.error(f'{num_files} found but was expecting 1200') sys.exit(1) delete_files(dfs, '/gws/nopw/j04/primavera5/stream1') replace_files(dfs) dfs = DataFile.objects.filter( climate_model__short_name='EC-Earth3P-HR', experiment__short_name='highres-future', rip_code='r1i1p1f1', variable_request__table_name='3hr', variable_request__cmor_name='ps' ) num_files = dfs.count() if num_files != 432: logger.error(f'{num_files} found but was expecting 432') sys.exit(1) delete_files(dfs, '/gws/nopw/j04/primavera5/stream1') replace_files(dfs)
def main(args): """ Main entry point """ model, expt, var_lab, table, var = args.request_id.split('_') dreq = DataRequest.objects.get( climate_model__short_name=model, experiment__short_name=expt, rip_code=var_lab, variable_request__table_name=table, variable_request__cmor_name=var ) logger.debug('DataRequest is {}'.format(dreq)) logger.debug('Checking checksums') checksum_mismatch = 0 for data_file in dreq.datafile_set.order_by('name'): logger.debug('Checking {}'.format(data_file.name)) full_path = os.path.join(data_file.directory, data_file.name) actual = adler32(full_path) expected = data_file.checksum_set.first().checksum_value if actual != expected: logger.error(f'Checksum mismatch for {full_path}') checksum_mismatch += 1 dfs = DataFile.objects.filter(name=data_file.name) if dfs.count() != 1: logger.error(f'Unable to select file for deletion {full_path}') else: delete_files(dfs.all(), BASE_OUTPUT_DIR) if checksum_mismatch: logger.error(f'Exiting due to {checksum_mismatch} checksum failures.') logger.error(f'Data request is in {dreq.directories()}') sys.exit(1)
def main(): """ Main entry point """ fixable = DataRequest.objects.filter( climate_model__short_name__contains='EC-Earth3P', experiment__short_name='highresSST-present', variable_request__cmor_name__regex='r[ls]u[ts]*', rip_code='r1i1p1f1', datafile__isnull=False ).distinct() broken = DataRequest.objects.filter( climate_model__short_name__contains='EC-Earth3P', experiment__short_name='highresSST-present', variable_request__cmor_name__in=['rsdscs', 'rsuscs'], rip_code='r1i1p1f1', datafile__isnull=False ).distinct() dreqs = broken | fixable num_dreqs = dreqs.distinct().count() if num_dreqs != 47: logger.error(f'{num_dreqs} affected data requests found') sys.exit(1) for dreq in dreqs: delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR, skip_badc=True) replace_files(dreq.datafile_set.all())
def main(args): """ Main entry point """ jon = User.objects.get(username='******') long_txt = ( "A bug has been discovered in HadGEM3's implementation of COSP. All " "cltcalipso and parasolRefl variables from all versions of HadGEM3 " "are affected and the data should not be used. These variables have " "been removed from PRIMAVERA and from ESGF.") cosp_issue, _created = DataIssue.objects.get_or_create(issue=long_txt, reporter=jon) affected_files = DataFile.objects.filter( climate_model__short_name__startswith='HadGEM3', variable_request__cmor_name__in=['cltcalipso', 'parasolRefl']) num_files = affected_files.count() logger.debug(f'{num_files} affected files found') cosp_issue.data_file.add(*affected_files) delete_files(affected_files, '/gws/nopw/j04/primavera5/stream1', skip_badc=True) replace_files(affected_files)
def main(): """ Main entry point """ dreqs = DataRequest.objects.filter( institute__short_name='CMCC', variable_request__cmor_name__in=['tasmax', 'tasmin'], datafile__isnull=False).distinct() num_dreqs = dreqs.count() expected_dreqs = 40 if num_dreqs != expected_dreqs: logger.error(f'Found {num_dreqs} but was expecting {expected_dreqs}.') sys.exit(1) daniele = User.objects.get(username='******') long_txt = ( "CMCC tasmax and tasmin contain errors and must be withdrawn. Please" "see https://errata.es-doc.org/static/view.html?uid=9b40a054-21a7-5ae7" "-a3eb-8c373c5adddc.") tas_issue, _created = DataIssue.objects.get_or_create(issue=long_txt, reporter=daniele) for dreq in dreqs: logger.info(dreq) tas_issue.data_file.add(*dreq.datafile_set.all()) delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR, skip_badc=True) replace_files(dreq.datafile_set.all())
def main(args): """ Main entry point """ di = DataIssue.objects.get(id=23) for ds_num in di.data_file.values_list('data_submission', flat=True).distinct(): ds = DataSubmission.objects.get(id=ds_num) affected_files = ds.datafile_set.all() delete_files(affected_files, '/gws/nopw/j04/primavera5/stream1') replace_files(affected_files)
def main(args): """ Main entry point """ new_files = list_files(NEW_SUBMISSION) logger.debug(f'{len(new_files)} files found in the submission') dfs = DataFile.objects.filter(name__in=map(os.path.basename, new_files)) logger.debug(f'{dfs.count()} files found in the DMT') delete_files(dfs, '/gws/nopw/j04/primavera5/stream1') replace_files(dfs)
def main(args): """ Main entry point """ dreqs = DataRequest.objects.filter( climate_model__short_name__startswith='HadGEM3', variable_request__cmor_name='epfz', datafile__isnull=False).distinct() logger.debug(f'{dreqs.count()} data requests found') for dreq in dreqs.order_by('climate_model__short_name', 'experiment__short_name', 'rip_code'): logger.debug(str(dreq)) delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR, skip_badc=True) replace_files(dreq.datafile_set.all())
def main(args): """ Main entry point """ dfs = DataFile.objects.filter(climate_model__short_name='EC-Earth3P', experiment__short_name='highresSST-present', rip_code='r2i1p1f1', name__contains='_195') num_files = dfs.count() if num_files != 1920: logger.error(f'{num_files} found but was expecting 1920') sys.exit(1) delete_files(dfs, '/gws/nopw/j04/primavera5/stream1') replace_files(dfs)
def main(args): """ Main entry point """ dss = DataSubmission.objects.filter(incoming_directory__in=[ '/gws/nopw/j04/primavera4/upload/EC-Earth-Consortium/EC-Earth3P-HR/' 'control-1950/r3i1p2f1/1950-1952/SeaIce', '/gws/nopw/j04/primavera4/upload/EC-Earth-Consortium/EC-Earth3P-HR/' 'control-1950/r3i1p2f1/1950-1952/Rest' ]) base_output_dir = Settings.get_solo().base_output_dir for ds in dss: delete_files(ds.datafile_set.all(), base_output_dir) replace_files(ds.datafile_set.all())
def main(args): """ Main entry point """ dreqs = DataRequest.objects.filter( institute__short_name='MPI-M', experiment__short_name__in=['control-1950', 'hist-1950'], variable_request__cmor_name='tos', datafile__isnull=False).distinct() logger.debug(f'Found {dreqs.count()} datasets') for dreq in dreqs: if dreq.esgfdataset_set.all(): # ESGF dataset's been created... esgf = dreq.esgfdataset_set.first() if esgf.status == 'PUBLISHED': # ... and published so the data's in the CEDA archive # and symlinked from the PRIMAVERA data structure # All sym links will be in one directory set_dir = os.path.join( BASE_OUTPUT_DIR, construct_drs_path(dreq.datafile_set.first())) for df in dreq.datafile_set.all(): file_path = os.path.join(set_dir, df.name) if not os.path.islink(file_path): logger.warning(f'Expected a sym link {file_path}') continue try: os.remove(file_path) except OSError as exc: logger.error(str(exc)) df.online = False df.directory = None df.save() delete_drs_dir(set_dir) logger.debug(f'Removed files for ESGFDataset {esgf}') esgf.status = 'CREATED' esgf.save() continue # The data's not been published so delete the files and their sym links delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR) logger.debug(f'Removed files for DataRequest {dreq}') dreq.datafile_set.update(directory=None, online=False) for dreq in dreqs: dreq.datafile_set.update(version='v20191129')
def main(args): """ Main entry point """ bad_sets = ESGFDataset.objects.filter( status__in=['CREATED', 'NOT_ON_DISK'], data_request__climate_model__short_name='HadGEM3-GC31-LL', data_request__experiment__short_name='control-1950', data_request__rip_code='r1i1p1f1', ).distinct() logger.debug(f'{bad_sets.count()} data sets found') base_output_dir = Settings.get_solo().base_output_dir for bad_set in bad_sets: dr = bad_set.data_request delete_files(dr.datafile_set.all(), base_output_dir)
def main(args): """ Main entry point """ affected_files = DataFile.objects.filter( climate_model__short_name='HadGEM3-GC31-HM', experiment__short_name='control-1950', rip_code='r1i1p1f1', variable_request__table_name='Amon', name__contains='_203201-') if affected_files.count() != 48: logger.error('{} affected files found'.format(affected_files.count())) sys.exit(1) else: logger.debug('{} affected files found'.format(affected_files.count())) delete_files(affected_files, '/gws/nopw/j04/primavera5/stream1') replace_files(affected_files)
def main(args): """ Main entry point """ affected_files = DataFile.objects.filter( climate_model__short_name='HadGEM3-GC31-HH', experiment__short_name='highres-future', variable_request__table_name__in=[ 'Omon', 'Oday', 'SImon', 'SIday', 'PrimOmon', 'PrimOday', 'PrimSIday' ], version='v20191105') num_files = affected_files.count() logger.debug(f'{num_files} affected files found') delete_files(affected_files, '/gws/nopw/j04/primavera5/stream1', skip_badc=True) replace_files(affected_files)
def main(args): """ Main entry point """ affected_files = [ 'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199101010000-199101311800.nc', 'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199102010000-199102281800.nc', 'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199103010000-199103311800.nc', 'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199104010000-199104301800.nc', 'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199107010000-199107311800.nc', 'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199106010000-199106301800.nc', 'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199105010000-199105311800.nc', 'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199108010000-199108311800.nc', 'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199110010000-199110311800.nc', 'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199109010000-199109301800.nc', 'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199111010000-199111301800.nc', 'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199112010000-199112311800.nc', 'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19910101-19910131.nc', 'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19910201-19910228.nc', 'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19910301-19910331.nc', 'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19910401-19910430.nc', 'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19910501-19910531.nc', 'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19910601-19910630.nc', 'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19910701-19910731.nc', 'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19910801-19910831.nc', 'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19910901-19910930.nc', 'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19911001-19911031.nc', 'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19911101-19911130.nc', 'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19911201-19911231.nc' ] dfs = DataFile.objects.filter(name__in=affected_files) num_files = dfs.count() num_files_expected = 24 if num_files != num_files_expected: logger.error(f'{num_files} found but was expecting ' f'{num_files_expected}') sys.exit(1) delete_files(dfs, '/gws/nopw/j04/primavera5/stream1') replace_files(dfs)
def main(args): """ Main entry point """ common_queries = { 'climate_model__short_name': 'HadGEM3-GC31-HH', 'experiment__short_name__in': ['control-1950', 'highres-future', 'hist-1950'], 'version': 'v20200514' } omon = DataFile.objects.filter(**common_queries, variable_request__table_name='Omon', variable_request__cmor_name__in=[ 'tauuo', 'tauvo', 'umo', 'uo', 'vmo', 'vo', 'wmo' ]) primomon = DataFile.objects.filter(**common_queries, variable_request__table_name='PrimOmon', variable_request__cmor_name__in=[ 'u2o', 'uso', 'uto', 'v2o', 'vso', 'vto', 'wo' ]) primoday = DataFile.objects.filter(**common_queries, variable_request__table_name='PrimOday', variable_request__cmor_name__in=[ 'tauuo', 'tauvo', 'uo', 'vo' ]) affected_files = omon | primomon | primoday num_files = affected_files.count() logger.debug(f'{num_files} affected files found') delete_files(affected_files, '/gws/nopw/j04/primavera5/stream1', skip_badc=True) replace_files(affected_files)
def main(args): """ Main entry point """ copy_dir = '/gws/nopw/j04/primavera5/upload/CMCC' dreqs = DataRequest.objects.filter( institute__short_name='CMCC', variable_request__cmor_name__in=['tauu', 'tauv'], datafile__isnull=False).distinct() num_dreqs = dreqs.count() expected_dreqs = 20 if num_dreqs != expected_dreqs: logger.error(f'Found {num_dreqs} but was expecting {expected_dreqs}.') sys.exit(1) for dreq in dreqs: logger.debug(dreq) delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR, skip_badc=True) replace_files(dreq.datafile_set.all())
def main(): """ Main entry point """ dreqs = DataRequest.objects.filter( institute__short_name='EC-Earth-Consortium', variable_request__table_name='E3hrPt', variable_request__cmor_name__in=['psl', 'ua850', 'va850'], datafile__isnull=False ).distinct() num_dreqs = dreqs.count() expected_dreqs = 16 if num_dreqs != expected_dreqs: logger.error(f'Found {num_dreqs} but was expecting {expected_dreqs}.') sys.exit(1) for dreq in dreqs: logger.info(dreq) delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR) replace_files(dreq.datafile_set.all())
def main(args): """ Main entry point """ dfs = DataFile.objects.filter( climate_model__short_name='EC-Earth3P-HR', experiment__short_name='primWP5-amv-neg', rip_code__in=['r1i1p2f1', 'r2i1p2f1', 'r3i1p2f1', 'r4i1p2f1', 'r12i1p2f1', 'r13i1p2f1', 'r14i1p2f1', 'r15i1p2f1', 'r16i1p2f1', 'r17i1p2f1',] ) num_files = dfs.count() num_files_expected = 49900 if num_files != num_files_expected: logger.error(f'{num_files} found but was expecting ' f'{num_files_expected}') sys.exit(1) delete_files(dfs, '/gws/nopw/j04/primavera5/stream1') replace_files(dfs)
def main(args): """ Main entry point """ dfs = DataFile.objects.filter( climate_model__short_name='EC-Earth3P-HR', experiment__short_name__in=['hist-1950', 'control-1950'], rip_code='r1i1p1f1', variable_request__table_name='3hr', variable_request__cmor_name='ps') num_files = dfs.count() if num_files != 1968: logger.error(f'{num_files} found but was expecting 1968') sys.exit(1) delete_files(dfs, '/gws/nopw/j04/primavera5/stream1') for df in dfs: df.incoming_directory = df.incoming_directory + '_1' df.save() replace_files(dfs)
def main(args): """ Main entry point """ lr = DataFile.objects.filter( climate_model__short_name='EC-Earth3P', experiment__short_name='highres-future', rip_code='r1i1p1f1', variable_request__table_name='Amon', variable_request__cmor_name__in=['hfls', 'hfss'] ) if lr.count() != 840: raise ValueError('Not 420 LR files found') hr_3hr = DataFile.objects.filter( climate_model__short_name='EC-Earth3P-HR', experiment__short_name='highres-future', rip_code='r1i1p1f1', variable_request__table_name='3hr', variable_request__cmor_name__in=['hfls', 'hfss'] ) if hr_3hr.count() != 864: raise ValueError('Not 420 HR 3hr files found') hr_day = DataFile.objects.filter( climate_model__short_name='EC-Earth3P-HR', experiment__short_name='highres-future', rip_code='r1i1p1f1', variable_request__table_name='day', variable_request__cmor_name__in=['hfls', 'hfss'] ) if hr_day.count() != 864: raise ValueError('Not 420 HR day files found') affected_files = lr | hr_3hr | hr_day logger.debug('{} affected files found'.format(affected_files.count())) delete_files(affected_files, BASE_OUTPUT_DIR) replace_files(affected_files)
def main(args): """ Main entry point Example task_name: crepp_submission_HadGEM3-GC31-LL_hist-1950_r1i7p1f1_Omon_so """ if args.create: start_year = 1948 end_year = 2051 jon = User.objects.get(username='******') rr = RetrievalRequest.objects.create(requester=jon, start_year=start_year, end_year=end_year) time_zone = datetime.timezone(datetime.timedelta()) rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone) rr.save() conn = sqlite3.connect('/home/users/jseddon/cylc-run/db_u-bs020.db') c = conn.cursor() for task_name in c.execute('SELECT "name" FROM "task_jobs" WHERE ' '"name" LIKE "crepp_submission_%" AND ' '"run_signal" IS "SIGUSR2";'): model, expt, var_label, table, var_name = task_name[0].split('_')[2:] dreq = DataRequest.objects.get(climate_model__short_name=model, experiment__short_name=expt, rip_code=var_label, variable_request__table_name=table, variable_request__cmor_name=var_name) logger.debug(f'{task_name[0]} ' f'{dreq.datafile_set.filter(online=True).count()}') if args.create: try: delete_files(dreq.datafile_set.all(), '/gws/nopw/j04/primavera5/stream1') except Exception as exc: logger.error(str(exc)) rr.data_request.add(dreq)
def main(): """ Main entry point """ eday_snw = DataRequest.objects.filter(variable_request__table_name='Eday', variable_request__cmor_name='snw', datafile__isnull=False) lmon_mrlsl = DataRequest.objects.filter( variable_request__table_name='Lmon', variable_request__cmor_name='mrlsl', datafile__isnull=False) omon_msftmyz = DataRequest.objects.filter( variable_request__table_name='Omon', variable_request__cmor_name='msftmyz', datafile__isnull=False) primsiday_siuv = DataRequest.objects.filter( variable_request__table_name='PrimSIday', variable_request__cmor_name__in=['siu', 'siv'], datafile__isnull=False) simon_siflsaltbot = DataRequest.objects.filter( variable_request__table_name='SImon', variable_request__cmor_name='siflsaltbot', datafile__isnull=False) awi = DataRequest.objects.filter(institute__short_name='AWI', datafile__isnull=False) dreqs = (eday_snw | lmon_mrlsl | omon_msftmyz | primsiday_siuv | simon_siflsaltbot | awi) logger.info(f'{dreqs.count()} data requests to delete') for dreq in dreqs: logger.debug(dreq) delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR, skip_badc=True)
def main(args): """ Main entry point """ dfs = DataFile.objects.filter( climate_model__short_name__in=['EC-Earth3P', 'EC-Earth3P-HR'], experiment__short_name='control-1950', rip_code='r1i1p1f1', variable_request__table_name='E3hrPt', variable_request__cmor_name__in=['ua850', 'va850'] ) num_files = dfs.count() num_files_expected = 4633 if num_files != num_files_expected: logger.error(f'{num_files} found but was expecting ' f'{num_files_expected}') sys.exit(1) delete_files(dfs, '/gws/nopw/j04/primavera5/stream1') replace_files(dfs) dfs = DataFile.objects.filter( climate_model__short_name='EC-Earth3P-HR', experiment__short_name='hist-1950', rip_code='r1i1p1f1', variable_request__table_name='E3hrPt', variable_request__cmor_name__in=['ua850', 'va850'] ) num_files = dfs.count() num_files_expected = 1508 if num_files != num_files_expected: logger.error(f'{num_files} found but was expecting ' f'{num_files_expected}') sys.exit(1) delete_files(dfs, '/gws/nopw/j04/primavera5/stream1') replace_files(dfs)
def main(): """ Main entry point """ dreqs = DataRequest.objects.filter( institute__short_name='CMCC', experiment__short_name__in=['highres-future', 'highresSST-future', 'hist-1950'], variable_request__cmor_name__in=['rlut', 'rlutcs', 'rsutcs'], datafile__isnull=False ).distinct() num_dreqs = dreqs.count() expected_dreqs = 18 if num_dreqs != expected_dreqs: logger.error(f'Found {num_dreqs} but was expecting {expected_dreqs}.') sys.exit(1) for dreq in dreqs: logger.info(dreq) delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR, skip_badc=True) replace_files(dreq.datafile_set.all())
def main(): """ Main entry point """ hh = DataRequest.objects.filter( climate_model__short_name='HadGEM3-GC31-HH').distinct() lower_res = DataRequest.objects.filter( climate_model__short_name__startswith='HadGEM3-GC31').exclude( climate_model__short_name='HadGEM3-GC31-HH').exclude( rip_code='r1i1p1f1').distinct() all_dreqs = hh | lower_res dreqs = exclude_hadgem_stream2(all_dreqs) logger.info(f'{dreqs.count()} data requests to delete') for dreq in dreqs.order_by('climate_model__short_name', 'experiment__short_name', 'rip_code', 'variable_request__table_name', 'variable_request__cmor_name'): logger.debug(dreq) delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR, skip_badc=True)
def main(args): """ Main entry point """ dreqs = DataRequest.objects.filter( climate_model__short_name = 'EC-Earth3P', experiment__short_name__in = ['primWP5-amv-pos','primWP5-amv-neg'], variable_request__cmor_name__in = ['rsus', 'rlus'] ).distinct() num_dreqs = dreqs.count() if num_dreqs != 100: logger.error(f'{num_dreqs} affected data requests found') sys.exit(1) for dreq in dreqs: for df in dreq.datafile_set.all(): new_dir = os.path.join(BASE_GWS, construct_drs_path(df)) new_path = os.path.join(new_dir, df.name) old_path = os.path.join(df.directory, df.name) if not os.path.exists(new_dir): os.makedirs(new_dir) shutil.copy(old_path, new_path) delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR)
def main(args): """ Main entry point """ # EC-Earth3P-HR_hist-1950_r1i1p1f1 dfs = DataFile.objects.filter(climate_model__short_name='EC-Earth3P-HR', experiment__short_name='hist-1950', rip_code='r1i1p1f1', variable_request__cmor_name='rsuscs') num_files = dfs.count() num_files_expected = 1540 if num_files != num_files_expected: logger.error(f'{num_files} found but was expecting ' f'{num_files_expected}') sys.exit(1) delete_files(dfs, '/gws/nopw/j04/primavera5/stream1') replace_files(dfs) # EC-Earth3P-HR_control-1950_r1i1p1f1 dfs = DataFile.objects.filter(climate_model__short_name='EC-Earth3P-HR', experiment__short_name='control-1950', rip_code='r1i1p1f1', variable_request__cmor_name='rsuscs') num_files = dfs.count() num_files_expected = 2489 if num_files != num_files_expected: logger.error(f'{num_files} found but was expecting ' f'{num_files_expected}') sys.exit(1) delete_files(dfs, '/gws/nopw/j04/primavera5/stream1') replace_files(dfs) # EC-Earth3P_control-1950_r1i1p2f1 dfs = DataFile.objects.filter(climate_model__short_name='EC-Earth3P', experiment__short_name='control-1950', rip_code='r1i1p2f1', variable_request__cmor_name='rsuscs') num_files = dfs.count() num_files_expected = 50 if num_files != num_files_expected: logger.error(f'{num_files} found but was expecting ' f'{num_files_expected}') sys.exit(1) delete_files(dfs, '/gws/nopw/j04/primavera5/stream1') replace_files(dfs)
def main(args): """ Main entry point """ affected_files = [ 'psl_6hrPlev_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198910010600-198910311800.nc', 'pr_3hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230300.nc', 'rsds_3hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230300.nc', 'tas_3hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc', 'uas_3hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc', 'vas_3hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc', 'psl_6hrPlev_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc', 'tas_6hrPlevPt_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc', 'uas_6hrPlevPt_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc', 'vas_6hrPlevPt_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc', 'zg_6hrPlevPt_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc', 'ps_CFday_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'clt_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'hfls_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'hfss_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'hur_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'hurs_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'hursmax_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'hursmin_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'hus_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'huss_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'mrsos_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'prc_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'pr_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'prsn_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'psl_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'rlds_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'rlus_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'rlut_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'rsds_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'rsus_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'sfcWind_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'sfcWindmax_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'ta_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'tasmax_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'tasmin_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'tslsi_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'ua_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'uas_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'va_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'vas_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'zg_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'tauu_Eday_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'tauv_Eday_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'sfcWind_Prim3hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc', 'sfcWindmax_Prim3hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc', 'pr_Prim6hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230000.nc', 'rsds_Prim6hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230000.nc', 'sfcWindmax_Prim6hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230000.nc', 'wsgmax_Prim6hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230000.nc', 'mrlsl_Primday_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'mrso_Primday_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'prmax_Primday_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'prmin_Primday_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc', 'ua_PrimdayPt_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890922.nc', 'va_PrimdayPt_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890922.nc', 'ts_Primday_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc' ] dfs = DataFile.objects.filter(name__in=affected_files) num_files = dfs.count() num_files_expected = 57 if num_files != num_files_expected: logger.error(f'{num_files} found but was expecting ' f'{num_files_expected}') sys.exit(1) delete_files(dfs, '/gws/nopw/j04/primavera5/stream1') replace_files(dfs)