def main(args):
"""
Main entry point
"""
start_year = 1948
end_year = 2051
data_reqs = DataRequest.objects.filter(
climate_model__short_name='ECMWF-IFS-MR',
datafile__isnull=False).exclude(
variable_request__table_name__startswith='Prim').distinct()
logger.debug('Total data volume: {} Volume to restore: {}'.format(
filesizeformat(get_request_size(data_reqs, start_year,
end_year)).replace('\xa0', ' '),
filesizeformat(
get_request_size(data_reqs, start_year, end_year,
offline=True)).replace('\xa0', ' '),
))
if args.create:
jon = User.objects.get(username='******')
rr = RetrievalRequest.objects.create(requester=jon,
start_year=start_year,
end_year=end_year)
time_zone = datetime.timezone(datetime.timedelta())
rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone)
rr.save()
rr.data_request.add(*data_reqs)
logger.debug('Retrieval request {} created.'.format(rr.id))
def main(args):
"""
Main entry point
"""
start_year = 1948
end_year = 2101
# data_reqs = DataRequest.objects.filter(
# institute__short_name='ECMWF',
# experiment__short_name__startswith='primWP5-amv',
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# institute__short_name='NCAS',
# experiment__short_name__startswith='primWP5-amv',
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# institute__short_name='MPI-M',
# experiment__short_name__startswith='dcppc-amv',
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# institute__short_name='CNRM-CERFACS',
# experiment__short_name__startswith='primWP5-amv',
# datafile__isnull=False
# ).distinct()
data_reqs = DataRequest.objects.filter(
institute__short_name='EC-Earth-Consortium',
experiment__short_name__startswith='primWP5-amv',
datafile__isnull=False
).distinct()
logger.debug('Total data volume: {} Volume to restore: {}'.format(
filesizeformat(get_request_size(data_reqs, start_year, end_year)).
replace('\xa0', ' '),
filesizeformat(get_request_size(data_reqs, start_year, end_year,
offline=True)).replace('\xa0', ' '),
))
if args.create:
jon = User.objects.get(username='******')
rr = RetrievalRequest.objects.create(requester=jon, start_year=start_year,
end_year=end_year)
time_zone = datetime.timezone(datetime.timedelta())
rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone)
rr.save()
rr.data_request.add(*data_reqs)
logger.debug('Retrieval request {} created.'.format(rr.id))
def main(args):
"""
Main entry point
"""
start_year = 1948
end_year = 2051
# data_reqs = DataRequest.objects.filter(
# institute__short_name='CNRM-CERFACS',
# experiment__short_name__startswith='highresSST',
# variable_request__table_name__startswith='Prim',
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# institute__short_name='CNRM-CERFACS',
# experiment__short_name='hist-1950',
# variable_request__table_name__startswith='Prim',
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# institute__short_name='CNRM-CERFACS',
# experiment__short_name='control-1950',
# variable_request__table_name__startswith='Prim',
# datafile__isnull=False
# ).distinct()
data_reqs = DataRequest.objects.filter(
institute__short_name='CNRM-CERFACS',
experiment__short_name='highres-future',
variable_request__table_name__startswith='Prim',
datafile__isnull=False).distinct()
logger.debug('Total data volume: {} Volume to restore: {}'.format(
filesizeformat(get_request_size(data_reqs, start_year,
end_year)).replace('\xa0', ' '),
filesizeformat(
get_request_size(data_reqs, start_year, end_year,
offline=True)).replace('\xa0', ' '),
))
if args.create:
jon = User.objects.get(username='******')
rr = RetrievalRequest.objects.create(requester=jon,
start_year=start_year,
end_year=end_year)
time_zone = datetime.timezone(datetime.timedelta())
rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone)
rr.save()
rr.data_request.add(*data_reqs)
logger.debug('Retrieval request {} created.'.format(rr.id))
def main(args):
"""
Main entry point
"""
start_year = 1948
end_year = 2100
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='EC-Earth3P',
# experiment__short_name='control-1950',
# rip_code='r1i1p2f1',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).exclude(
# variable_request__dimensions__contains='alevhalf'
# ).exclude(
# variable_request__dimensions__contains='alevel'
# ).distinct()
data_reqs = DataRequest.objects.filter(
climate_model__short_name='EC-Earth3P-HR',
experiment__short_name='control-1950',
rip_code='r3i1p2f1',
# variable_request__frequency__in=['mon', 'day'],
# variable_request__table_name='3hr',
datafile__isnull=False
).exclude(variable_request__table_name__startswith='Prim').exclude(
variable_request__dimensions__contains='alevhalf').exclude(
variable_request__dimensions__contains='alevel').distinct()
logger.debug('Total data volume: {} Volume to restore: {}'.format(
filesizeformat(get_request_size(data_reqs, start_year,
end_year)).replace('\xa0', ' '),
filesizeformat(
get_request_size(data_reqs, start_year, end_year,
offline=True)).replace('\xa0', ' '),
))
if args.create:
jon = User.objects.get(username='******')
rr = RetrievalRequest.objects.create(requester=jon,
start_year=start_year,
end_year=end_year)
time_zone = datetime.timezone(datetime.timedelta())
rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone)
rr.save()
rr.data_request.add(*data_reqs)
logger.debug('Retrieval request {} created.'.format(rr.id))
def run_retrieve_request(retrieval_id):
"""
Run retrieve_request.py in a subprocess to fetch the appropriate data
from tape to disk
:param int retrieval_id:
"""
retrieval_request = RetrievalRequest.objects.get(id=retrieval_id)
if get_request_size(retrieval_request.data_request.all(),
retrieval_request.start_year,
retrieval_request.end_year) > TWO_TEBIBYTES:
logger.warning('Skipping retrieval {} as it is bigger than {}.'.format(
retrieval_id,
filesizeformat(TWO_TEBIBYTES).encode('utf-8')))
return
cmd = ('{} {} -l debug --skip_checksums -a {} {}'.format(
sys.executable,
os.path.abspath(
os.path.join(os.path.dirname(__file__), 'retrieve_request.py')),
STREAM1_DIR, retrieval_id))
try:
subprocess.check_output(cmd, shell=True).decode('utf-8')
except OSError as exc:
logger.error('Unable to run command:\n{}\n{}'.format(
cmd, exc.strerror))
sys.exit(1)
except subprocess.CalledProcessError as exc:
logger.error('Retrieval failed: {}\n{}'.format(cmd, exc.output))
else:
logger.debug('Retrieved id {}'.format(retrieval_id))
def main(args):
"""
Main entry point
"""
start_year = 1948
end_year = 2051
# data_reqs = filter_hadgem_stream2(DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-HH',
# experiment__short_name='highres-future',
# # variable_request__frequency__in=['3hr'],
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='O'
# ).exclude(
# variable_request__table_name__startswith='SI'
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).distinct())
data_reqs = filter_hadgem_stream2(
DataRequest.objects.filter(
climate_model__short_name='HadGEM3-GC31-MH',
experiment__short_name='spinup-1950',
variable_request__table_name__in=['SImon', 'SIday', 'PrimSIday'],
datafile__isnull=False).distinct())
logger.info('Total data volume: {} Volume to restore: {}'.format(
filesizeformat(get_request_size(data_reqs, start_year,
end_year)).replace('\xa0', ' '),
filesizeformat(
get_request_size(data_reqs, start_year, end_year,
offline=True)).replace('\xa0', ' '),
))
if args.create:
jon = User.objects.get(username='******')
rr = RetrievalRequest.objects.create(requester=jon,
start_year=start_year,
end_year=end_year)
time_zone = datetime.timezone(datetime.timedelta())
rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone)
rr.save()
rr.data_request.add(*data_reqs)
logger.info('Retrieval request {} created.'.format(rr.id))
def test_dates(self):
rreq = dbapi.get_or_create(models.RetrievalRequest,
requester=self.user,
start_year=1950,
end_year=1975)
rreq.data_request.add(self.dreq1)
rreq.save()
rreq.data_request.add(self.dreq2)
rreq.save()
self.assertEqual(
7,
get_request_size(rreq.data_request.all(), rreq.start_year,
rreq.end_year))
def main(args):
"""
Main entry point
"""
start_year = 1948
end_year = 2051
data_reqs = filter_hadgem_stream2(
DataRequest.objects.filter(
climate_model__short_name='HadGEM3-GC31-LL',
experiment__short_name='hist-1950',
# rip_code__in=[f'r1i{i}p1f1' for i in range(2,9)],
rip_code='r1i1p1f1',
variable_request__table_name__startswith='Prim',
datafile__isnull=False).distinct())
logger.debug('Total data volume: {} Volume to restore: {}'.format(
filesizeformat(get_request_size(data_reqs, start_year,
end_year)).replace('\xa0', ' '),
filesizeformat(
get_request_size(data_reqs, start_year, end_year,
offline=True)).replace('\xa0', ' '),
))
if args.create:
jon = User.objects.get(username='******')
rr = RetrievalRequest.objects.create(requester=jon,
start_year=start_year,
end_year=end_year)
time_zone = datetime.timezone(datetime.timedelta())
rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone)
rr.save()
rr.data_request.add(*data_reqs)
logger.debug('Retrieval request {} created.'.format(rr.id))
def test_no_files(self):
rreq = dbapi.get_or_create(models.RetrievalRequest,
requester=self.user,
start_year=2001,
end_year=2014)
rreq.data_request.add(self.dreq1)
rreq.save()
rreq.data_request.add(self.dreq2)
rreq.save()
self.assertEqual(
0,
get_request_size(rreq.data_request.all(),
rreq.start_year,
rreq.end_year,
offline=True))
def render_retrieval_size(self, record):
return filesizeformat(
get_request_size(record.data_request.all(),
record.start_year,
record.end_year,
offline=True))
def render_req_size(self, record):
return filesizeformat(
get_request_size(record.data_request.all(), record.start_year,
record.end_year))
def main(args):
"""
Main entry point
"""
start_year = 1948
end_year = 2051
# data_reqs = filter_hadgem_stream2(DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-LL',
# experiment__short_name='hist-1950',
# variable_request__table_name__startswith='Prim',
# rip_code__in=[f'r1i{i}p1f1' for i in range(2,9)],
# datafile__isnull=False
# ).distinct())
# data_reqs = filter_hadgem_stream2(DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-MM',
# experiment__short_name='hist-1950',
# variable_request__table_name__startswith='Prim',
# rip_code__in=[f'r1i{i}p1f1' for i in range(1, 4)],
# datafile__isnull=False
# ).distinct())
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-MM',
# experiment__short_name='hist-1950',
# variable_request__table_name__startswith='Prim',
# rip_code='r1i1p1f1',
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-HM',
# experiment__short_name='hist-1950',
# variable_request__table_name__startswith='Prim',
# rip_code='r1i1p1f1',
# datafile__isnull=False
# ).distinct()
# data_reqs = filter_hadgem_stream2(DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-HM',
# experiment__short_name='hist-1950',
# variable_request__table_name__startswith='Prim',
# rip_code='r1i2p1f1',
# datafile__isnull=False
# ).distinct())
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-LL',
# experiment__short_name__in=['control-1950', 'spinup-1950'],
# variable_request__table_name__startswith='Prim',
# rip_code='r1i1p1f1',
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-MM',
# experiment__short_name__in=['control-1950', 'spinup-1950'],
# variable_request__table_name__startswith='Prim',
# rip_code='r1i1p1f1',
# datafile__isnull=False
# ).distinct()
# data_reqs = filter_hadgem_stream2(DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-HM',
# experiment__short_name='hist-1950',
# variable_request__table_name__startswith='Prim',
# rip_code='r1i3p1f1',
# datafile__isnull=False
# ).distinct())
# data_reqs = filter_hadgem_stream2(DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-HM',
# experiment__short_name='hist-1950',
# variable_request__table_name__startswith='Prim',
# rip_code='r1i3p1f1',
# datafile__isnull=False
# ).distinct())
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-HH',
# experiment__short_name__in=['control-1950'], # 'spinup-1950'],
# variable_request__table_name='PrimSIday',
# rip_code='r1i1p1f1',
# datafile__isnull=False
# ).distinct()
# data_reqs = filter_hadgem_stream2(DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-HH',
# experiment__short_name__in=['hist-1950'],
# variable_request__table_name__in=['PrimOmon'], # 'PrimOmon'],
# rip_code='r1i1p1f1',
# datafile__isnull=False
# ).distinct())
# data_reqs = filter_hadgem_stream2(DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-HH',
# experiment__short_name__in=['highres-future'],
# variable_request__table_name='PrimOmon',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='PrimSI'
# ).distinct())
# Done LM, MM, HM AMIP
# s1 = DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-HM',
# experiment__short_name='highresSST-present',
# rip_code='r1i1p1f1',
# variable_request__table_name__startswith='Prim',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='PrimSI'
# ).distinct()
# s2 = filter_hadgem_stream2(DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-HM',
# experiment__short_name='highresSST-present',
# rip_code__in=['r1i3p1f1'], # , 'r1i3p1f1'],
# variable_request__table_name__startswith='Prim',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='PrimSI'
# ).distinct())
# data_reqs = s2 # s1 # | s2
# Done LM
# s1 = DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-HM',
# experiment__short_name='highresSST-future',
# rip_code='r1i1p1f1',
# variable_request__table_name__startswith='Prim',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='PrimSI'
# ).distinct()
# s2 = filter_hadgem_stream2(DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-HM',
# experiment__short_name='highresSST-future',
# rip_code__in=['r1i2p1f1', 'r1i3p1f1'],
# variable_request__table_name__startswith='Prim',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='PrimSI'
# ).distinct())
# data_reqs = s1 | s2
# data_reqs = filter_hadgem_stream2(DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-LM',
# experiment__short_name='highresSST-future',
# rip_code__in=['r1i14p1f1', 'r1i15p1f1'],
# variable_request__table_name__startswith='Prim',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='PrimSI'
# ).distinct())
# s1 = DataRequest.objects.filter(
# climate_model__short_name__in=['HadGEM3-GC31-LL'], # , 'HadGEM3-GC31-MM', 'HadGEM3-GC31-HM'],
# experiment__short_name='highres-future',
# rip_code='r1i1p1f1',
# variable_request__table_name__startswith='Prim',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='PrimSI'
# ).distinct()
# s2 = filter_hadgem_stream2(DataRequest.objects.filter(
# climate_model__short_name__in=['HadGEM3-GC31-LL'], # , 'HadGEM3-GC31-MM', 'HadGEM3-GC31-HM'],
# experiment__short_name='highres-future',
# rip_code__in=['r1i2p1f1', 'r1i3p1f1', 'r1i4p1f1'],
# variable_request__table_name__startswith='Prim',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='PrimSI'
# ).distinct())
# data_reqs = s1 | s2
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name__startswith='HadGEM3-GC31',
# experiment__short_name='spinup-1950',
# rip_code='r1i1p1f1',
# variable_request__table_name__startswith='Prim',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='PrimSI'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name__startswith='HadGEM3-GC31',
# # experiment__short_name='spinup-1950',
# # rip_code='r1i1p1f1',
# variable_request__table_name__in=['PrimSIday', 'SIday', 'SImon'],
# datafile__isnull=False
# ).distinct()
data_reqs = DataRequest.objects.filter(
climate_model__short_name__startswith='HadGEM3-GC31',
variable_request__table_name__in=['fx', 'Ofx'],
datafile__isnull=False
).distinct()
logger.debug('Total data volume: {} Volume to restore: {}'.format(
filesizeformat(get_request_size(data_reqs, start_year, end_year)).
replace('\xa0', ' '),
filesizeformat(get_request_size(data_reqs, start_year, end_year,
offline=True)).replace('\xa0', ' '),
))
if args.create:
jon = User.objects.get(username='******')
rr = RetrievalRequest.objects.create(requester=jon, start_year=start_year,
end_year=end_year)
time_zone = datetime.timezone(datetime.timedelta())
rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone)
rr.save()
rr.data_request.add(*data_reqs)
logger.debug('Retrieval request {} created.'.format(rr.id))
def main(args):
"""
Main entry point
"""
start_year = 1948
end_year = 2051
# data_reqs = filter_hadgem_stream2(DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-LL',
# experiment__short_name='highres-future',
# rip_code__in=['r1i2p1f1', 'r1i3p1f1', 'r1i4p1f1'],
# variable_request__table_name__in=[
# '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon',
# 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday',
# 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day'
# ],
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).distinct())
data_reqs = filter_hadgem_stream2(
DataRequest.objects.filter(
climate_model__short_name='HadGEM3-GC31-MM',
experiment__short_name='highres-future',
rip_code__in=['r1i2p1f1', 'r1i3p1f1'],
variable_request__table_name__in=[
'3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon',
'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday',
'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day'
],
datafile__isnull=False).exclude(
variable_request__table_name__startswith='Prim').distinct())
# data_reqs = filter_hadgem_stream2(DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-HM',
# experiment__short_name='highres-future',
# rip_code='r1i3p1f1',
# variable_request__table_name__in=[
# '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon',
# 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday',
# 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day'
# ],
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).distinct())
logger.debug('Total data volume: {} Volume to restore: {}'.format(
filesizeformat(get_request_size(data_reqs, start_year,
end_year)).replace('\xa0', ' '),
filesizeformat(
get_request_size(data_reqs, start_year, end_year,
offline=True)).replace('\xa0', ' '),
))
if args.create:
jon = User.objects.get(username='******')
rr = RetrievalRequest.objects.create(requester=jon,
start_year=start_year,
end_year=end_year)
time_zone = datetime.timezone(datetime.timedelta())
rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone)
rr.save()
rr.data_request.add(*data_reqs)
logger.debug('Retrieval request {} created.'.format(rr.id))
def main(args):
"""
Main entry point
"""
start_year = 1948
end_year = 2051
# data_reqs = DataRequest.objects.filter(
# institute__short_name='ECMWF',
# experiment__short_name='hist-1950',
# variable_request__table_name__startswith='Prim',
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# institute__short_name='ECMWF',
# climate_model__short_name__in=['ECMWF-IFS-LR', 'ECMWF-IFS-MR'],
# experiment__short_name='hist-1950',
# variable_request__table_name__startswith='Prim',
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# institute__short_name='ECMWF',
# climate_model__short_name='ECMWF-IFS-HR',
# experiment__short_name='hist-1950',
# rip_code__in=[f'r{r}i1p1f1' for r in range(4, 7)],
# variable_request__table_name__startswith='Prim',
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# institute__short_name='ECMWF',
# # climate_model__short_name='ECMWF-IFS-HR',
# experiment__short_name='control-1950',
# variable_request__table_name__startswith='Prim',
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# institute__short_name='ECMWF',
# experiment__short_name='highresSST-present',
# variable_request__table_name__startswith='Prim',
# datafile__isnull=False
# ).distinct()
data_reqs = DataRequest.objects.filter(
institute__short_name='ECMWF',
experiment__short_name='spinup-1950',
variable_request__table_name__startswith='Prim',
datafile__isnull=False
).distinct()
logger.debug('Total data volume: {} Volume to restore: {}'.format(
filesizeformat(get_request_size(data_reqs, start_year, end_year)).
replace('\xa0', ' '),
filesizeformat(get_request_size(data_reqs, start_year, end_year,
offline=True)).replace('\xa0', ' '),
))
if args.create:
jon = User.objects.get(username='******')
rr = RetrievalRequest.objects.create(requester=jon, start_year=start_year,
end_year=end_year)
time_zone = datetime.timezone(datetime.timedelta())
rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone)
rr.save()
rr.data_request.add(*data_reqs)
logger.debug('Retrieval request {} created.'.format(rr.id))
def test_manual_list(self):
self.assertEqual(
15, get_request_size([self.dreq1, self.dreq2], 1950, 2000))
def main(args):
"""
Main entry point
"""
start_year = 1948
end_year = 2051
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='EC-Earth3P',
# experiment__short_name='hist-1950',
# rip_code='r1i1p2f1',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).exclude(
# variable_request__dimensions__contains='alevhalf'
# ).exclude(
# variable_request__dimensions__contains='alevel'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='EC-Earth3P',
# experiment__short_name='hist-1950',
# rip_code='r2i1p2f1',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).exclude(
# variable_request__dimensions__contains='alevhalf'
# ).exclude(
# variable_request__dimensions__contains='alevel'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='EC-Earth3P',
# experiment__short_name='hist-1950',
# rip_code='r3i1p2f1',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).exclude(
# variable_request__dimensions__contains='alevhalf'
# ).exclude(
# variable_request__dimensions__contains='alevel'
# ).distinct()
# mon_reqs = DataRequest.objects.filter(
# climate_model__short_name='EC-Earth3P-HR',
# experiment__short_name='hist-1950',
# rip_code__in=[f'r{mem_num}i1p2f1' for mem_num in range(1, 4)],
# variable_request__frequency='mon',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).exclude(
# variable_request__dimensions__contains='alevhalf'
# ).exclude(
# variable_request__dimensions__contains='alevel'
# ).distinct()
#
# day_reqs = DataRequest.objects.filter(
# climate_model__short_name='EC-Earth3P-HR',
# experiment__short_name='hist-1950',
# rip_code='r1i1p2f1',
# variable_request__frequency='day',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).exclude(
# variable_request__dimensions__contains='alevhalf'
# ).exclude(
# variable_request__dimensions__contains='alevel'
# ).distinct()
#
# data_reqs = mon_reqs | day_reqs
data_reqs = DataRequest.objects.filter(
climate_model__short_name='EC-Earth3P-HR',
experiment__short_name='hist-1950',
rip_code='r3i1p2f1',
# variable_request__table_name='6hrPlevPt',
# variable_request__cmor_name__in=['uas', 'vas', 'vortmean', 'snw', 'tas', 'psl',
# 'mrsos', 'ts', 'huss', 'sfcWind', 'tsl'],
variable_request__frequency='day',
datafile__isnull=False
).exclude(variable_request__table_name__startswith='Prim').exclude(
variable_request__dimensions__contains='alevhalf').exclude(
variable_request__dimensions__contains='alevel').distinct()
logger.debug('Total data volume: {} Volume to restore: {}'.format(
filesizeformat(get_request_size(data_reqs, start_year,
end_year)).replace('\xa0', ' '),
filesizeformat(
get_request_size(data_reqs, start_year, end_year,
offline=True)).replace('\xa0', ' '),
))
if args.create:
jon = User.objects.get(username='******')
rr = RetrievalRequest.objects.create(requester=jon,
start_year=start_year,
end_year=end_year)
time_zone = datetime.timezone(datetime.timedelta())
rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone)
rr.save()
rr.data_request.add(*data_reqs)
logger.debug('Retrieval request {} created.'.format(rr.id))
def main(args):
"""
Main entry point
"""
start_year = 1948
end_year = 2051
# data_reqs = DataRequest.objects.filter(
# institute__short_name='MOHC',
# climate_model__short_name='HadGEM3-GC31-MM',
# experiment__short_name='highresSST-present',
# rip_code='r1i1p1f1',
# )
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name__in=['ECMWF-IFS-LR', 'ECMWF-IFS-HR'],
# experiment__short_name='highresSST-present',
# rip_code='r1i1p1f1',
# ).exclude(
# variable_request__table_name__startswith='Prim'
# )
# data_reqs = DataRequest.objects.filter(
# institute__short_name='MOHC',
# climate_model__short_name='HadGEM3-GC31-MM',
# experiment__short_name='highresSST-present',
# rip_code='r1i1p1f1',
# )
# data_reqs = DataRequest.objects.filter(
# institute__short_name='MOHC',
# climate_model__short_name='HadGEM3-GC31-HM',
# experiment__short_name='highresSST-present',
# rip_code='r1i1p1f1',
# variable_request__frequency__in=['day']
# ).exclude(
# variable_request__table_name__startswith='Prim'
# )# .exclude(
# variable_request__table_name='CFday'
# )
# data_reqs = DataRequest.objects.filter(
# institute__short_name='CMCC',
# experiment__short_name='highresSST-present',
# rip_code='r1i1p1f1',
# ).exclude(
# variable_request__table_name__startswith='Prim'
# )
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='CNRM-CM6-1',
# experiment__short_name='highresSST-present',
# rip_code='r1i1p1f2',
# ).exclude(
# variable_request__table_name__startswith='Prim'
# )
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='CNRM-CM6-1-HR',
# experiment__short_name='highresSST-present',
# rip_code='r1i1p1f2',
# variable_request__frequency__in=['mon', 'day', '6hr', '3hr'],
# datafile__isnull=False
#).exclude(
# variable_request__table_name__startswith='Prim'
#).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-LL',
# experiment__short_name='hist-1950',
# rip_code='r1i1p1f1',
# variable_request__table_name__in=[
# '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon',
# 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday',
# 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day'
# ],
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-LL',
# experiment__short_name='control-1950',
# rip_code='r1i1p1f1',
# variable_request__table_name__in=[
# '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon',
# 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday',
# 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day'
# ],
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name__in=['ECMWF-IFS-LR'],
# experiment__short_name='hist-1950',
# rip_code='r1i1p1f1',
# ).exclude(
# variable_request__table_name__startswith='Prim'
# )
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-HM',
# experiment__short_name='control-1950',
# rip_code='r1i1p1f1',
# variable_request__table_name__in=[
# '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon',
# 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday',
# 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day'
# ],
# # variable_request__frequency__in=['6hr', '3hr'],# 'mon', 'day', '6hr', '3hr', '1hr'],
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-MM',
# experiment__short_name='control-1950',
# rip_code='r1i1p1f1',
# variable_request__table_name__in=[
# '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon',
# 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday',
# 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day'
# ],
# # variable_request__frequency__in=['mon', 'day', '6hr'],
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name__in=['ECMWF-IFS-HR'],
# experiment__short_name='control-1950',
# rip_code='r1i1p1f1',
# ).exclude(
# variable_request__table_name__startswith='Prim'
# )
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='CNRM-CM6-1-HR',
# experiment__short_name='hist-1950',
# rip_code='r1i1p1f2',
# # variable_request__frequency__in=['mon', 'day', '6hr'],
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='CNRM-CM6-1',
# experiment__short_name='control-1950',
# rip_code='r1i1p1f2',
# # variable_request__frequency__in=['mon', 'day', '6hr'],
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# institute__short_name='MPI-M',
# experiment__short_name='highresSST-present',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-LM',
# experiment__short_name='highresSST-future',
# rip_code='r1i1p1f1',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-MM',
# experiment__short_name='highresSST-future',
# rip_code='r1i1p1f1',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-HM',
# experiment__short_name='highresSST-future',
# rip_code='r1i1p1f1',
# variable_request__table_name='CFday',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='CNRM-CM6-1',
# experiment__short_name='highresSST-future',
# rip_code='r1i1p1f2',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='CNRM-CM6-1-HR',
# experiment__short_name='highresSST-future',
# rip_code='r1i1p1f2',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='CNRM-CM6-1-HR',
# experiment__short_name='highres-future',
# rip_code='r1i1p1f2',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name__in=['MPI-ESM1-2-HR'],
# experiment__short_name__in=['hist-1950'],
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name__in=['MPI-ESM1-2-HR', 'MPI-ESM1-2-XR'],
# experiment__short_name__in=['hist-1950', 'control-1950'],
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).exclude(
# climate_model__short_name='MPI-ESM1-2-HR',
# experiment__short_name='hist-1950'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name__in=['ECMWF-IFS-LR', 'ECMWF-IFS-HR'],
# experiment__short_name='spinup-1950',
# rip_code='r1i1p1f1',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='CMCC-CM2-VHR4',
# experiment__short_name='control-1950',
# rip_code='r1i1p1f1',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-LL',
# experiment__short_name='highres-future',
# rip_code='r1i1p1f1',
# variable_request__table_name__in=[
# '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon',
# 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday',
# 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day'
# ],
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-MM',
# experiment__short_name='highres-future',
# rip_code='r1i1p1f1',
# variable_request__table_name__in=[
# '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon',
# 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday',
# 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day'
# ],
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-HM',
# experiment__short_name='highres-future',
# rip_code='r1i1p1f1',
# variable_request__table_name__in=[
# '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon',
# 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday',
# 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day'
# ],
# datafile__isnull=False
# ).filter(
# variable_request__table_name='CFday'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name__in=['HadGEM3-GC31-LL', 'HadGEM3-GC31-MM'],
# experiment__short_name='spinup-1950',
# rip_code='r1i1p1f1',
# variable_request__table_name__in=[
# '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon',
# 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday',
# 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day'
# ],
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='HadGEM3-GC31-MH',
# experiment__short_name='spinup-1950',
# rip_code='r1i1p1f1',
# variable_request__table_name__in=[
# '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon',
# 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday',
# 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day'
# ],
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name='CMCC-CM2-VHR4',
# experiment__short_name='hist-1950',
# rip_code='r1i1p1f1',
# variable_request__frequency__in=['6hr'],
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name__in=['CMCC-CM2-HR4', 'CMCC-CM2-VHR4'],
# experiment__short_name='highresSST-future',
# rip_code='r1i1p1f1',
# datafile__isnull=False
# ).exclude(
# variable_request__table_name__startswith='Prim'
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# climate_model__short_name__in=['HadGEM3-GC31-LL', 'HadGEM3-GC31-MM', 'HadGEM3-GC31-HM'],
# experiment__short_name='highres-future',
# # rip_code__in=[f'r1i{i}p1f1' for i in range(1, 4)],
# variable_request__table_name__in=['Omon', 'Oday'],
# datafile__isnull=False
# ).distinct()
# data_reqs = DataRequest.objects.filter(
# institute__short_name='CNRM-CERFACS',
# variable_request__table_name='SImon',
# datafile__isnull=False
# ).exclude(
# esgfdataset__status='PUBLISHED'
# ).exclude(
# experiment__short_name__startswith='primWP5'
# ).distinct()
data_reqs = DataRequest.objects.filter(
climate_model__short_name__in=['HadGEM3-GC31-LL', 'HadGEM3-GC31-MM'],
experiment__short_name='spinup-1950',
variable_request__table_name__in=['Omon', 'Oday'],
datafile__isnull=False).distinct()
logger.debug('Total data volume: {} Volume to restore: {}'.format(
filesizeformat(get_request_size(data_reqs, start_year,
end_year)).replace('\xa0', ' '),
filesizeformat(
get_request_size(data_reqs, start_year, end_year,
offline=True)).replace('\xa0', ' '),
))
if args.create:
jon = User.objects.get(username='******')
rr = RetrievalRequest.objects.create(requester=jon,
start_year=start_year,
end_year=end_year)
time_zone = datetime.timezone(datetime.timedelta())
rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone)
rr.save()
rr.data_request.add(*data_reqs)
logger.debug('Retrieval request {} created.'.format(rr.id))
