def main():
"""
Main entry point
"""
alevhalf = DataRequest.objects.filter(
institute__short_name='EC-Earth-Consortium',
variable_request__dimensions__contains='alevhalf',
datafile__isnull=False).distinct()
alevel = DataRequest.objects.filter(
institute__short_name='EC-Earth-Consortium',
variable_request__dimensions__contains='alevel',
datafile__isnull=False).distinct()
dreqs = alevhalf | alevel
logger.debug(f'{dreqs.count()} data requests found')
for dreq in dreqs.order_by('climate_model__short_name',
'experiment__short_name', 'rip_code',
'variable_request__table_name',
'variable_request__cmor_name'):
if dreq.online_status() in ['online', 'partial']:
logger.debug(dreq)
delete_files(dreq.datafile_set.all(),
BASE_OUTPUT_DIR,
skip_badc=True)
def main(args):
"""
Main entry point
"""
dfs = DataFile.objects.filter(
climate_model__short_name='EC-Earth3P',
experiment__short_name='control-1950',
rip_code='r1i1p2f1',
variable_request__table_name='3hr',
variable_request__cmor_name='ps'
)
num_files = dfs.count()
if num_files != 1200:
logger.error(f'{num_files} found but was expecting 1200')
sys.exit(1)
delete_files(dfs, '/gws/nopw/j04/primavera5/stream1')
replace_files(dfs)
dfs = DataFile.objects.filter(
climate_model__short_name='EC-Earth3P-HR',
experiment__short_name='highres-future',
rip_code='r1i1p1f1',
variable_request__table_name='3hr',
variable_request__cmor_name='ps'
)
num_files = dfs.count()
if num_files != 432:
logger.error(f'{num_files} found but was expecting 432')
sys.exit(1)
delete_files(dfs, '/gws/nopw/j04/primavera5/stream1')
replace_files(dfs)
def main(args):
"""
Main entry point
"""
model, expt, var_lab, table, var = args.request_id.split('_')
dreq = DataRequest.objects.get(
climate_model__short_name=model,
experiment__short_name=expt,
rip_code=var_lab,
variable_request__table_name=table,
variable_request__cmor_name=var
)
logger.debug('DataRequest is {}'.format(dreq))
logger.debug('Checking checksums')
checksum_mismatch = 0
for data_file in dreq.datafile_set.order_by('name'):
logger.debug('Checking {}'.format(data_file.name))
full_path = os.path.join(data_file.directory, data_file.name)
actual = adler32(full_path)
expected = data_file.checksum_set.first().checksum_value
if actual != expected:
logger.error(f'Checksum mismatch for {full_path}')
checksum_mismatch += 1
dfs = DataFile.objects.filter(name=data_file.name)
if dfs.count() != 1:
logger.error(f'Unable to select file for deletion {full_path}')
else:
delete_files(dfs.all(), BASE_OUTPUT_DIR)
if checksum_mismatch:
logger.error(f'Exiting due to {checksum_mismatch} checksum failures.')
logger.error(f'Data request is in {dreq.directories()}')
sys.exit(1)
def main():
"""
Main entry point
"""
fixable = DataRequest.objects.filter(
climate_model__short_name__contains='EC-Earth3P',
experiment__short_name='highresSST-present',
variable_request__cmor_name__regex='r[ls]u[ts]*',
rip_code='r1i1p1f1',
datafile__isnull=False
).distinct()
broken = DataRequest.objects.filter(
climate_model__short_name__contains='EC-Earth3P',
experiment__short_name='highresSST-present',
variable_request__cmor_name__in=['rsdscs', 'rsuscs'],
rip_code='r1i1p1f1',
datafile__isnull=False
).distinct()
dreqs = broken | fixable
num_dreqs = dreqs.distinct().count()
if num_dreqs != 47:
logger.error(f'{num_dreqs} affected data requests found')
sys.exit(1)
for dreq in dreqs:
delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR, skip_badc=True)
replace_files(dreq.datafile_set.all())
def main(args):
"""
Main entry point
"""
jon = User.objects.get(username='******')
long_txt = (
"A bug has been discovered in HadGEM3's implementation of COSP. All "
"cltcalipso and parasolRefl variables from all versions of HadGEM3 "
"are affected and the data should not be used. These variables have "
"been removed from PRIMAVERA and from ESGF.")
cosp_issue, _created = DataIssue.objects.get_or_create(issue=long_txt,
reporter=jon)
affected_files = DataFile.objects.filter(
climate_model__short_name__startswith='HadGEM3',
variable_request__cmor_name__in=['cltcalipso', 'parasolRefl'])
num_files = affected_files.count()
logger.debug(f'{num_files} affected files found')
cosp_issue.data_file.add(*affected_files)
delete_files(affected_files,
'/gws/nopw/j04/primavera5/stream1',
skip_badc=True)
replace_files(affected_files)
def main():
"""
Main entry point
"""
dreqs = DataRequest.objects.filter(
institute__short_name='CMCC',
variable_request__cmor_name__in=['tasmax', 'tasmin'],
datafile__isnull=False).distinct()
num_dreqs = dreqs.count()
expected_dreqs = 40
if num_dreqs != expected_dreqs:
logger.error(f'Found {num_dreqs} but was expecting {expected_dreqs}.')
sys.exit(1)
daniele = User.objects.get(username='******')
long_txt = (
"CMCC tasmax and tasmin contain errors and must be withdrawn. Please"
"see https://errata.es-doc.org/static/view.html?uid=9b40a054-21a7-5ae7"
"-a3eb-8c373c5adddc.")
tas_issue, _created = DataIssue.objects.get_or_create(issue=long_txt,
reporter=daniele)
for dreq in dreqs:
logger.info(dreq)
tas_issue.data_file.add(*dreq.datafile_set.all())
delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR, skip_badc=True)
replace_files(dreq.datafile_set.all())
def main(args):
"""
Main entry point
"""
di = DataIssue.objects.get(id=23)
for ds_num in di.data_file.values_list('data_submission',
flat=True).distinct():
ds = DataSubmission.objects.get(id=ds_num)
affected_files = ds.datafile_set.all()
delete_files(affected_files, '/gws/nopw/j04/primavera5/stream1')
replace_files(affected_files)
def main(args):
"""
Main entry point
"""
new_files = list_files(NEW_SUBMISSION)
logger.debug(f'{len(new_files)} files found in the submission')
dfs = DataFile.objects.filter(name__in=map(os.path.basename, new_files))
logger.debug(f'{dfs.count()} files found in the DMT')
delete_files(dfs, '/gws/nopw/j04/primavera5/stream1')
replace_files(dfs)
def main(args):
"""
Main entry point
"""
dreqs = DataRequest.objects.filter(
climate_model__short_name__startswith='HadGEM3',
variable_request__cmor_name='epfz',
datafile__isnull=False).distinct()
logger.debug(f'{dreqs.count()} data requests found')
for dreq in dreqs.order_by('climate_model__short_name',
'experiment__short_name', 'rip_code'):
logger.debug(str(dreq))
delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR, skip_badc=True)
replace_files(dreq.datafile_set.all())
def main(args):
"""
Main entry point
"""
dfs = DataFile.objects.filter(climate_model__short_name='EC-Earth3P',
experiment__short_name='highresSST-present',
rip_code='r2i1p1f1',
name__contains='_195')
num_files = dfs.count()
if num_files != 1920:
logger.error(f'{num_files} found but was expecting 1920')
sys.exit(1)
delete_files(dfs, '/gws/nopw/j04/primavera5/stream1')
replace_files(dfs)
def main(args):
"""
Main entry point
"""
dss = DataSubmission.objects.filter(incoming_directory__in=[
'/gws/nopw/j04/primavera4/upload/EC-Earth-Consortium/EC-Earth3P-HR/'
'control-1950/r3i1p2f1/1950-1952/SeaIce',
'/gws/nopw/j04/primavera4/upload/EC-Earth-Consortium/EC-Earth3P-HR/'
'control-1950/r3i1p2f1/1950-1952/Rest'
])
base_output_dir = Settings.get_solo().base_output_dir
for ds in dss:
delete_files(ds.datafile_set.all(), base_output_dir)
replace_files(ds.datafile_set.all())
def main(args):
"""
Main entry point
"""
dreqs = DataRequest.objects.filter(
institute__short_name='MPI-M',
experiment__short_name__in=['control-1950', 'hist-1950'],
variable_request__cmor_name='tos',
datafile__isnull=False).distinct()
logger.debug(f'Found {dreqs.count()} datasets')
for dreq in dreqs:
if dreq.esgfdataset_set.all():
# ESGF dataset's been created...
esgf = dreq.esgfdataset_set.first()
if esgf.status == 'PUBLISHED':
# ... and published so the data's in the CEDA archive
# and symlinked from the PRIMAVERA data structure
# All sym links will be in one directory
set_dir = os.path.join(
BASE_OUTPUT_DIR,
construct_drs_path(dreq.datafile_set.first()))
for df in dreq.datafile_set.all():
file_path = os.path.join(set_dir, df.name)
if not os.path.islink(file_path):
logger.warning(f'Expected a sym link {file_path}')
continue
try:
os.remove(file_path)
except OSError as exc:
logger.error(str(exc))
df.online = False
df.directory = None
df.save()
delete_drs_dir(set_dir)
logger.debug(f'Removed files for ESGFDataset {esgf}')
esgf.status = 'CREATED'
esgf.save()
continue
# The data's not been published so delete the files and their sym links
delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR)
logger.debug(f'Removed files for DataRequest {dreq}')
dreq.datafile_set.update(directory=None, online=False)
for dreq in dreqs:
dreq.datafile_set.update(version='v20191129')
def main(args):
"""
Main entry point
"""
bad_sets = ESGFDataset.objects.filter(
status__in=['CREATED', 'NOT_ON_DISK'],
data_request__climate_model__short_name='HadGEM3-GC31-LL',
data_request__experiment__short_name='control-1950',
data_request__rip_code='r1i1p1f1',
).distinct()
logger.debug(f'{bad_sets.count()} data sets found')
base_output_dir = Settings.get_solo().base_output_dir
for bad_set in bad_sets:
dr = bad_set.data_request
delete_files(dr.datafile_set.all(), base_output_dir)
def main(args):
"""
Main entry point
"""
affected_files = DataFile.objects.filter(
climate_model__short_name='HadGEM3-GC31-HM',
experiment__short_name='control-1950',
rip_code='r1i1p1f1',
variable_request__table_name='Amon',
name__contains='_203201-')
if affected_files.count() != 48:
logger.error('{} affected files found'.format(affected_files.count()))
sys.exit(1)
else:
logger.debug('{} affected files found'.format(affected_files.count()))
delete_files(affected_files, '/gws/nopw/j04/primavera5/stream1')
replace_files(affected_files)
def main(args):
"""
Main entry point
"""
affected_files = DataFile.objects.filter(
climate_model__short_name='HadGEM3-GC31-HH',
experiment__short_name='highres-future',
variable_request__table_name__in=[
'Omon', 'Oday', 'SImon', 'SIday', 'PrimOmon', 'PrimOday',
'PrimSIday'
],
version='v20191105')
num_files = affected_files.count()
logger.debug(f'{num_files} affected files found')
delete_files(affected_files,
'/gws/nopw/j04/primavera5/stream1',
skip_badc=True)
replace_files(affected_files)
def main(args):
"""
Main entry point
"""
affected_files = [
'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199101010000-199101311800.nc',
'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199102010000-199102281800.nc',
'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199103010000-199103311800.nc',
'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199104010000-199104301800.nc',
'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199107010000-199107311800.nc',
'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199106010000-199106301800.nc',
'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199105010000-199105311800.nc',
'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199108010000-199108311800.nc',
'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199110010000-199110311800.nc',
'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199109010000-199109301800.nc',
'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199111010000-199111301800.nc',
'va1000_Prim6hrPt_EC-Earth3P_hist-1950_r1i1p1f1_gr_199112010000-199112311800.nc',
'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19910101-19910131.nc',
'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19910201-19910228.nc',
'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19910301-19910331.nc',
'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19910401-19910430.nc',
'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19910501-19910531.nc',
'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19910601-19910630.nc',
'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19910701-19910731.nc',
'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19910801-19910831.nc',
'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19910901-19910930.nc',
'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19911001-19911031.nc',
'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19911101-19911130.nc',
'zg_day_EC-Earth3P_hist-1950_r1i1p1f1_gr_19911201-19911231.nc'
]
dfs = DataFile.objects.filter(name__in=affected_files)
num_files = dfs.count()
num_files_expected = 24
if num_files != num_files_expected:
logger.error(f'{num_files} found but was expecting '
f'{num_files_expected}')
sys.exit(1)
delete_files(dfs, '/gws/nopw/j04/primavera5/stream1')
replace_files(dfs)
def main(args):
"""
Main entry point
"""
common_queries = {
'climate_model__short_name':
'HadGEM3-GC31-HH',
'experiment__short_name__in':
['control-1950', 'highres-future', 'hist-1950'],
'version':
'v20200514'
}
omon = DataFile.objects.filter(**common_queries,
variable_request__table_name='Omon',
variable_request__cmor_name__in=[
'tauuo', 'tauvo', 'umo', 'uo', 'vmo',
'vo', 'wmo'
])
primomon = DataFile.objects.filter(**common_queries,
variable_request__table_name='PrimOmon',
variable_request__cmor_name__in=[
'u2o', 'uso', 'uto', 'v2o', 'vso',
'vto', 'wo'
])
primoday = DataFile.objects.filter(**common_queries,
variable_request__table_name='PrimOday',
variable_request__cmor_name__in=[
'tauuo', 'tauvo', 'uo', 'vo'
])
affected_files = omon | primomon | primoday
num_files = affected_files.count()
logger.debug(f'{num_files} affected files found')
delete_files(affected_files,
'/gws/nopw/j04/primavera5/stream1',
skip_badc=True)
replace_files(affected_files)
def main(args):
"""
Main entry point
"""
copy_dir = '/gws/nopw/j04/primavera5/upload/CMCC'
dreqs = DataRequest.objects.filter(
institute__short_name='CMCC',
variable_request__cmor_name__in=['tauu', 'tauv'],
datafile__isnull=False).distinct()
num_dreqs = dreqs.count()
expected_dreqs = 20
if num_dreqs != expected_dreqs:
logger.error(f'Found {num_dreqs} but was expecting {expected_dreqs}.')
sys.exit(1)
for dreq in dreqs:
logger.debug(dreq)
delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR, skip_badc=True)
replace_files(dreq.datafile_set.all())
def main():
"""
Main entry point
"""
dreqs = DataRequest.objects.filter(
institute__short_name='EC-Earth-Consortium',
variable_request__table_name='E3hrPt',
variable_request__cmor_name__in=['psl', 'ua850', 'va850'],
datafile__isnull=False
).distinct()
num_dreqs = dreqs.count()
expected_dreqs = 16
if num_dreqs != expected_dreqs:
logger.error(f'Found {num_dreqs} but was expecting {expected_dreqs}.')
sys.exit(1)
for dreq in dreqs:
logger.info(dreq)
delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR)
replace_files(dreq.datafile_set.all())
def main(args):
"""
Main entry point
"""
dfs = DataFile.objects.filter(
climate_model__short_name='EC-Earth3P-HR',
experiment__short_name='primWP5-amv-neg',
rip_code__in=['r1i1p2f1', 'r2i1p2f1', 'r3i1p2f1', 'r4i1p2f1',
'r12i1p2f1', 'r13i1p2f1', 'r14i1p2f1', 'r15i1p2f1',
'r16i1p2f1', 'r17i1p2f1',]
)
num_files = dfs.count()
num_files_expected = 49900
if num_files != num_files_expected:
logger.error(f'{num_files} found but was expecting '
f'{num_files_expected}')
sys.exit(1)
delete_files(dfs, '/gws/nopw/j04/primavera5/stream1')
replace_files(dfs)
def main(args):
"""
Main entry point
"""
dfs = DataFile.objects.filter(
climate_model__short_name='EC-Earth3P-HR',
experiment__short_name__in=['hist-1950', 'control-1950'],
rip_code='r1i1p1f1',
variable_request__table_name='3hr',
variable_request__cmor_name='ps')
num_files = dfs.count()
if num_files != 1968:
logger.error(f'{num_files} found but was expecting 1968')
sys.exit(1)
delete_files(dfs, '/gws/nopw/j04/primavera5/stream1')
for df in dfs:
df.incoming_directory = df.incoming_directory + '_1'
df.save()
replace_files(dfs)
def main(args):
"""
Main entry point
"""
lr = DataFile.objects.filter(
climate_model__short_name='EC-Earth3P',
experiment__short_name='highres-future',
rip_code='r1i1p1f1',
variable_request__table_name='Amon',
variable_request__cmor_name__in=['hfls', 'hfss']
)
if lr.count() != 840:
raise ValueError('Not 420 LR files found')
hr_3hr = DataFile.objects.filter(
climate_model__short_name='EC-Earth3P-HR',
experiment__short_name='highres-future',
rip_code='r1i1p1f1',
variable_request__table_name='3hr',
variable_request__cmor_name__in=['hfls', 'hfss']
)
if hr_3hr.count() != 864:
raise ValueError('Not 420 HR 3hr files found')
hr_day = DataFile.objects.filter(
climate_model__short_name='EC-Earth3P-HR',
experiment__short_name='highres-future',
rip_code='r1i1p1f1',
variable_request__table_name='day',
variable_request__cmor_name__in=['hfls', 'hfss']
)
if hr_day.count() != 864:
raise ValueError('Not 420 HR day files found')
affected_files = lr | hr_3hr | hr_day
logger.debug('{} affected files found'.format(affected_files.count()))
delete_files(affected_files, BASE_OUTPUT_DIR)
replace_files(affected_files)
def main(args):
"""
Main entry point
Example task_name:
crepp_submission_HadGEM3-GC31-LL_hist-1950_r1i7p1f1_Omon_so
"""
if args.create:
start_year = 1948
end_year = 2051
jon = User.objects.get(username='******')
rr = RetrievalRequest.objects.create(requester=jon,
start_year=start_year,
end_year=end_year)
time_zone = datetime.timezone(datetime.timedelta())
rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone)
rr.save()
conn = sqlite3.connect('/home/users/jseddon/cylc-run/db_u-bs020.db')
c = conn.cursor()
for task_name in c.execute('SELECT "name" FROM "task_jobs" WHERE '
'"name" LIKE "crepp_submission_%" AND '
'"run_signal" IS "SIGUSR2";'):
model, expt, var_label, table, var_name = task_name[0].split('_')[2:]
dreq = DataRequest.objects.get(climate_model__short_name=model,
experiment__short_name=expt,
rip_code=var_label,
variable_request__table_name=table,
variable_request__cmor_name=var_name)
logger.debug(f'{task_name[0]} '
f'{dreq.datafile_set.filter(online=True).count()}')
if args.create:
try:
delete_files(dreq.datafile_set.all(),
'/gws/nopw/j04/primavera5/stream1')
except Exception as exc:
logger.error(str(exc))
rr.data_request.add(dreq)
def main():
"""
Main entry point
"""
eday_snw = DataRequest.objects.filter(variable_request__table_name='Eday',
variable_request__cmor_name='snw',
datafile__isnull=False)
lmon_mrlsl = DataRequest.objects.filter(
variable_request__table_name='Lmon',
variable_request__cmor_name='mrlsl',
datafile__isnull=False)
omon_msftmyz = DataRequest.objects.filter(
variable_request__table_name='Omon',
variable_request__cmor_name='msftmyz',
datafile__isnull=False)
primsiday_siuv = DataRequest.objects.filter(
variable_request__table_name='PrimSIday',
variable_request__cmor_name__in=['siu', 'siv'],
datafile__isnull=False)
simon_siflsaltbot = DataRequest.objects.filter(
variable_request__table_name='SImon',
variable_request__cmor_name='siflsaltbot',
datafile__isnull=False)
awi = DataRequest.objects.filter(institute__short_name='AWI',
datafile__isnull=False)
dreqs = (eday_snw | lmon_mrlsl | omon_msftmyz | primsiday_siuv
| simon_siflsaltbot | awi)
logger.info(f'{dreqs.count()} data requests to delete')
for dreq in dreqs:
logger.debug(dreq)
delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR, skip_badc=True)
def main(args):
"""
Main entry point
"""
dfs = DataFile.objects.filter(
climate_model__short_name__in=['EC-Earth3P', 'EC-Earth3P-HR'],
experiment__short_name='control-1950',
rip_code='r1i1p1f1',
variable_request__table_name='E3hrPt',
variable_request__cmor_name__in=['ua850', 'va850']
)
num_files = dfs.count()
num_files_expected = 4633
if num_files != num_files_expected:
logger.error(f'{num_files} found but was expecting '
f'{num_files_expected}')
sys.exit(1)
delete_files(dfs, '/gws/nopw/j04/primavera5/stream1')
replace_files(dfs)
dfs = DataFile.objects.filter(
climate_model__short_name='EC-Earth3P-HR',
experiment__short_name='hist-1950',
rip_code='r1i1p1f1',
variable_request__table_name='E3hrPt',
variable_request__cmor_name__in=['ua850', 'va850']
)
num_files = dfs.count()
num_files_expected = 1508
if num_files != num_files_expected:
logger.error(f'{num_files} found but was expecting '
f'{num_files_expected}')
sys.exit(1)
delete_files(dfs, '/gws/nopw/j04/primavera5/stream1')
replace_files(dfs)
def main():
"""
Main entry point
"""
dreqs = DataRequest.objects.filter(
institute__short_name='CMCC',
experiment__short_name__in=['highres-future', 'highresSST-future',
'hist-1950'],
variable_request__cmor_name__in=['rlut', 'rlutcs', 'rsutcs'],
datafile__isnull=False
).distinct()
num_dreqs = dreqs.count()
expected_dreqs = 18
if num_dreqs != expected_dreqs:
logger.error(f'Found {num_dreqs} but was expecting {expected_dreqs}.')
sys.exit(1)
for dreq in dreqs:
logger.info(dreq)
delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR, skip_badc=True)
replace_files(dreq.datafile_set.all())
def main():
"""
Main entry point
"""
hh = DataRequest.objects.filter(
climate_model__short_name='HadGEM3-GC31-HH').distinct()
lower_res = DataRequest.objects.filter(
climate_model__short_name__startswith='HadGEM3-GC31').exclude(
climate_model__short_name='HadGEM3-GC31-HH').exclude(
rip_code='r1i1p1f1').distinct()
all_dreqs = hh | lower_res
dreqs = exclude_hadgem_stream2(all_dreqs)
logger.info(f'{dreqs.count()} data requests to delete')
for dreq in dreqs.order_by('climate_model__short_name',
'experiment__short_name', 'rip_code',
'variable_request__table_name',
'variable_request__cmor_name'):
logger.debug(dreq)
delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR, skip_badc=True)
def main(args):
"""
Main entry point
"""
dreqs = DataRequest.objects.filter(
climate_model__short_name = 'EC-Earth3P',
experiment__short_name__in = ['primWP5-amv-pos','primWP5-amv-neg'],
variable_request__cmor_name__in = ['rsus', 'rlus']
).distinct()
num_dreqs = dreqs.count()
if num_dreqs != 100:
logger.error(f'{num_dreqs} affected data requests found')
sys.exit(1)
for dreq in dreqs:
for df in dreq.datafile_set.all():
new_dir = os.path.join(BASE_GWS, construct_drs_path(df))
new_path = os.path.join(new_dir, df.name)
old_path = os.path.join(df.directory, df.name)
if not os.path.exists(new_dir):
os.makedirs(new_dir)
shutil.copy(old_path, new_path)
delete_files(dreq.datafile_set.all(), BASE_OUTPUT_DIR)
def main(args):
"""
Main entry point
"""
# EC-Earth3P-HR_hist-1950_r1i1p1f1
dfs = DataFile.objects.filter(climate_model__short_name='EC-Earth3P-HR',
experiment__short_name='hist-1950',
rip_code='r1i1p1f1',
variable_request__cmor_name='rsuscs')
num_files = dfs.count()
num_files_expected = 1540
if num_files != num_files_expected:
logger.error(f'{num_files} found but was expecting '
f'{num_files_expected}')
sys.exit(1)
delete_files(dfs, '/gws/nopw/j04/primavera5/stream1')
replace_files(dfs)
# EC-Earth3P-HR_control-1950_r1i1p1f1
dfs = DataFile.objects.filter(climate_model__short_name='EC-Earth3P-HR',
experiment__short_name='control-1950',
rip_code='r1i1p1f1',
variable_request__cmor_name='rsuscs')
num_files = dfs.count()
num_files_expected = 2489
if num_files != num_files_expected:
logger.error(f'{num_files} found but was expecting '
f'{num_files_expected}')
sys.exit(1)
delete_files(dfs, '/gws/nopw/j04/primavera5/stream1')
replace_files(dfs)
# EC-Earth3P_control-1950_r1i1p2f1
dfs = DataFile.objects.filter(climate_model__short_name='EC-Earth3P',
experiment__short_name='control-1950',
rip_code='r1i1p2f1',
variable_request__cmor_name='rsuscs')
num_files = dfs.count()
num_files_expected = 50
if num_files != num_files_expected:
logger.error(f'{num_files} found but was expecting '
f'{num_files_expected}')
sys.exit(1)
delete_files(dfs, '/gws/nopw/j04/primavera5/stream1')
replace_files(dfs)
def main(args):
"""
Main entry point
"""
affected_files = [
'psl_6hrPlev_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198910010600-198910311800.nc',
'pr_3hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230300.nc',
'rsds_3hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230300.nc',
'tas_3hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc',
'uas_3hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc',
'vas_3hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc',
'psl_6hrPlev_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc',
'tas_6hrPlevPt_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc',
'uas_6hrPlevPt_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc',
'vas_6hrPlevPt_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc',
'zg_6hrPlevPt_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc',
'ps_CFday_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'clt_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'hfls_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'hfss_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'hur_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'hurs_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'hursmax_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'hursmin_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'hus_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'huss_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'mrsos_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'prc_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'pr_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'prsn_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'psl_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'rlds_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'rlus_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'rlut_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'rsds_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'rsus_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'sfcWind_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'sfcWindmax_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'ta_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'tasmax_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'tasmin_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'tslsi_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'ua_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'uas_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'va_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'vas_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'zg_day_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'tauu_Eday_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'tauv_Eday_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'sfcWind_Prim3hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc',
'sfcWindmax_Prim3hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230600.nc',
'pr_Prim6hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230000.nc',
'rsds_Prim6hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230000.nc',
'sfcWindmax_Prim6hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230000.nc',
'wsgmax_Prim6hr_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_198909010000-198909230000.nc',
'mrlsl_Primday_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'mrso_Primday_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'prmax_Primday_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'prmin_Primday_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc',
'ua_PrimdayPt_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890922.nc',
'va_PrimdayPt_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890922.nc',
'ts_Primday_EC-Earth3P-HR_control-1950_r3i1p2f1_gr_19890901-19890923.nc'
]
dfs = DataFile.objects.filter(name__in=affected_files)
num_files = dfs.count()
num_files_expected = 57
if num_files != num_files_expected:
logger.error(f'{num_files} found but was expecting '
f'{num_files_expected}')
sys.exit(1)
delete_files(dfs, '/gws/nopw/j04/primavera5/stream1')
replace_files(dfs)
