def main(args): """ Main entry point """ start_year = 1948 end_year = 2051 data_reqs = DataRequest.objects.filter( climate_model__short_name='ECMWF-IFS-MR', datafile__isnull=False).exclude( variable_request__table_name__startswith='Prim').distinct() logger.debug('Total data volume: {} Volume to restore: {}'.format( filesizeformat(get_request_size(data_reqs, start_year, end_year)).replace('\xa0', ' '), filesizeformat( get_request_size(data_reqs, start_year, end_year, offline=True)).replace('\xa0', ' '), )) if args.create: jon = User.objects.get(username='******') rr = RetrievalRequest.objects.create(requester=jon, start_year=start_year, end_year=end_year) time_zone = datetime.timezone(datetime.timedelta()) rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone) rr.save() rr.data_request.add(*data_reqs) logger.debug('Retrieval request {} created.'.format(rr.id))
def main(args): """ Main entry point """ start_year = 1948 end_year = 2101 # data_reqs = DataRequest.objects.filter( # institute__short_name='ECMWF', # experiment__short_name__startswith='primWP5-amv', # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # institute__short_name='NCAS', # experiment__short_name__startswith='primWP5-amv', # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # institute__short_name='MPI-M', # experiment__short_name__startswith='dcppc-amv', # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # institute__short_name='CNRM-CERFACS', # experiment__short_name__startswith='primWP5-amv', # datafile__isnull=False # ).distinct() data_reqs = DataRequest.objects.filter( institute__short_name='EC-Earth-Consortium', experiment__short_name__startswith='primWP5-amv', datafile__isnull=False ).distinct() logger.debug('Total data volume: {} Volume to restore: {}'.format( filesizeformat(get_request_size(data_reqs, start_year, end_year)). replace('\xa0', ' '), filesizeformat(get_request_size(data_reqs, start_year, end_year, offline=True)).replace('\xa0', ' '), )) if args.create: jon = User.objects.get(username='******') rr = RetrievalRequest.objects.create(requester=jon, start_year=start_year, end_year=end_year) time_zone = datetime.timezone(datetime.timedelta()) rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone) rr.save() rr.data_request.add(*data_reqs) logger.debug('Retrieval request {} created.'.format(rr.id))
def main(args): """ Main entry point """ start_year = 1948 end_year = 2051 # data_reqs = DataRequest.objects.filter( # institute__short_name='CNRM-CERFACS', # experiment__short_name__startswith='highresSST', # variable_request__table_name__startswith='Prim', # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # institute__short_name='CNRM-CERFACS', # experiment__short_name='hist-1950', # variable_request__table_name__startswith='Prim', # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # institute__short_name='CNRM-CERFACS', # experiment__short_name='control-1950', # variable_request__table_name__startswith='Prim', # datafile__isnull=False # ).distinct() data_reqs = DataRequest.objects.filter( institute__short_name='CNRM-CERFACS', experiment__short_name='highres-future', variable_request__table_name__startswith='Prim', datafile__isnull=False).distinct() logger.debug('Total data volume: {} Volume to restore: {}'.format( filesizeformat(get_request_size(data_reqs, start_year, end_year)).replace('\xa0', ' '), filesizeformat( get_request_size(data_reqs, start_year, end_year, offline=True)).replace('\xa0', ' '), )) if args.create: jon = User.objects.get(username='******') rr = RetrievalRequest.objects.create(requester=jon, start_year=start_year, end_year=end_year) time_zone = datetime.timezone(datetime.timedelta()) rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone) rr.save() rr.data_request.add(*data_reqs) logger.debug('Retrieval request {} created.'.format(rr.id))
def main(args): """ Main entry point """ start_year = 1948 end_year = 2100 # data_reqs = DataRequest.objects.filter( # climate_model__short_name='EC-Earth3P', # experiment__short_name='control-1950', # rip_code='r1i1p2f1', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).exclude( # variable_request__dimensions__contains='alevhalf' # ).exclude( # variable_request__dimensions__contains='alevel' # ).distinct() data_reqs = DataRequest.objects.filter( climate_model__short_name='EC-Earth3P-HR', experiment__short_name='control-1950', rip_code='r3i1p2f1', # variable_request__frequency__in=['mon', 'day'], # variable_request__table_name='3hr', datafile__isnull=False ).exclude(variable_request__table_name__startswith='Prim').exclude( variable_request__dimensions__contains='alevhalf').exclude( variable_request__dimensions__contains='alevel').distinct() logger.debug('Total data volume: {} Volume to restore: {}'.format( filesizeformat(get_request_size(data_reqs, start_year, end_year)).replace('\xa0', ' '), filesizeformat( get_request_size(data_reqs, start_year, end_year, offline=True)).replace('\xa0', ' '), )) if args.create: jon = User.objects.get(username='******') rr = RetrievalRequest.objects.create(requester=jon, start_year=start_year, end_year=end_year) time_zone = datetime.timezone(datetime.timedelta()) rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone) rr.save() rr.data_request.add(*data_reqs) logger.debug('Retrieval request {} created.'.format(rr.id))
def run_retrieve_request(retrieval_id): """ Run retrieve_request.py in a subprocess to fetch the appropriate data from tape to disk :param int retrieval_id: """ retrieval_request = RetrievalRequest.objects.get(id=retrieval_id) if get_request_size(retrieval_request.data_request.all(), retrieval_request.start_year, retrieval_request.end_year) > TWO_TEBIBYTES: logger.warning('Skipping retrieval {} as it is bigger than {}.'.format( retrieval_id, filesizeformat(TWO_TEBIBYTES).encode('utf-8'))) return cmd = ('{} {} -l debug --skip_checksums -a {} {}'.format( sys.executable, os.path.abspath( os.path.join(os.path.dirname(__file__), 'retrieve_request.py')), STREAM1_DIR, retrieval_id)) try: subprocess.check_output(cmd, shell=True).decode('utf-8') except OSError as exc: logger.error('Unable to run command:\n{}\n{}'.format( cmd, exc.strerror)) sys.exit(1) except subprocess.CalledProcessError as exc: logger.error('Retrieval failed: {}\n{}'.format(cmd, exc.output)) else: logger.debug('Retrieved id {}'.format(retrieval_id))
def main(args): """ Main entry point """ start_year = 1948 end_year = 2051 # data_reqs = filter_hadgem_stream2(DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-HH', # experiment__short_name='highres-future', # # variable_request__frequency__in=['3hr'], # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='O' # ).exclude( # variable_request__table_name__startswith='SI' # ).exclude( # variable_request__table_name__startswith='Prim' # ).distinct()) data_reqs = filter_hadgem_stream2( DataRequest.objects.filter( climate_model__short_name='HadGEM3-GC31-MH', experiment__short_name='spinup-1950', variable_request__table_name__in=['SImon', 'SIday', 'PrimSIday'], datafile__isnull=False).distinct()) logger.info('Total data volume: {} Volume to restore: {}'.format( filesizeformat(get_request_size(data_reqs, start_year, end_year)).replace('\xa0', ' '), filesizeformat( get_request_size(data_reqs, start_year, end_year, offline=True)).replace('\xa0', ' '), )) if args.create: jon = User.objects.get(username='******') rr = RetrievalRequest.objects.create(requester=jon, start_year=start_year, end_year=end_year) time_zone = datetime.timezone(datetime.timedelta()) rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone) rr.save() rr.data_request.add(*data_reqs) logger.info('Retrieval request {} created.'.format(rr.id))
def test_dates(self): rreq = dbapi.get_or_create(models.RetrievalRequest, requester=self.user, start_year=1950, end_year=1975) rreq.data_request.add(self.dreq1) rreq.save() rreq.data_request.add(self.dreq2) rreq.save() self.assertEqual( 7, get_request_size(rreq.data_request.all(), rreq.start_year, rreq.end_year))
def main(args): """ Main entry point """ start_year = 1948 end_year = 2051 data_reqs = filter_hadgem_stream2( DataRequest.objects.filter( climate_model__short_name='HadGEM3-GC31-LL', experiment__short_name='hist-1950', # rip_code__in=[f'r1i{i}p1f1' for i in range(2,9)], rip_code='r1i1p1f1', variable_request__table_name__startswith='Prim', datafile__isnull=False).distinct()) logger.debug('Total data volume: {} Volume to restore: {}'.format( filesizeformat(get_request_size(data_reqs, start_year, end_year)).replace('\xa0', ' '), filesizeformat( get_request_size(data_reqs, start_year, end_year, offline=True)).replace('\xa0', ' '), )) if args.create: jon = User.objects.get(username='******') rr = RetrievalRequest.objects.create(requester=jon, start_year=start_year, end_year=end_year) time_zone = datetime.timezone(datetime.timedelta()) rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone) rr.save() rr.data_request.add(*data_reqs) logger.debug('Retrieval request {} created.'.format(rr.id))
def test_no_files(self): rreq = dbapi.get_or_create(models.RetrievalRequest, requester=self.user, start_year=2001, end_year=2014) rreq.data_request.add(self.dreq1) rreq.save() rreq.data_request.add(self.dreq2) rreq.save() self.assertEqual( 0, get_request_size(rreq.data_request.all(), rreq.start_year, rreq.end_year, offline=True))
def render_retrieval_size(self, record): return filesizeformat( get_request_size(record.data_request.all(), record.start_year, record.end_year, offline=True))
def render_req_size(self, record): return filesizeformat( get_request_size(record.data_request.all(), record.start_year, record.end_year))
def main(args): """ Main entry point """ start_year = 1948 end_year = 2051 # data_reqs = filter_hadgem_stream2(DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-LL', # experiment__short_name='hist-1950', # variable_request__table_name__startswith='Prim', # rip_code__in=[f'r1i{i}p1f1' for i in range(2,9)], # datafile__isnull=False # ).distinct()) # data_reqs = filter_hadgem_stream2(DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-MM', # experiment__short_name='hist-1950', # variable_request__table_name__startswith='Prim', # rip_code__in=[f'r1i{i}p1f1' for i in range(1, 4)], # datafile__isnull=False # ).distinct()) # data_reqs = DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-MM', # experiment__short_name='hist-1950', # variable_request__table_name__startswith='Prim', # rip_code='r1i1p1f1', # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-HM', # experiment__short_name='hist-1950', # variable_request__table_name__startswith='Prim', # rip_code='r1i1p1f1', # datafile__isnull=False # ).distinct() # data_reqs = filter_hadgem_stream2(DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-HM', # experiment__short_name='hist-1950', # variable_request__table_name__startswith='Prim', # rip_code='r1i2p1f1', # datafile__isnull=False # ).distinct()) # data_reqs = DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-LL', # experiment__short_name__in=['control-1950', 'spinup-1950'], # variable_request__table_name__startswith='Prim', # rip_code='r1i1p1f1', # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-MM', # experiment__short_name__in=['control-1950', 'spinup-1950'], # variable_request__table_name__startswith='Prim', # rip_code='r1i1p1f1', # datafile__isnull=False # ).distinct() # data_reqs = filter_hadgem_stream2(DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-HM', # experiment__short_name='hist-1950', # variable_request__table_name__startswith='Prim', # rip_code='r1i3p1f1', # datafile__isnull=False # ).distinct()) # data_reqs = filter_hadgem_stream2(DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-HM', # experiment__short_name='hist-1950', # variable_request__table_name__startswith='Prim', # rip_code='r1i3p1f1', # datafile__isnull=False # ).distinct()) # data_reqs = DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-HH', # experiment__short_name__in=['control-1950'], # 'spinup-1950'], # variable_request__table_name='PrimSIday', # rip_code='r1i1p1f1', # datafile__isnull=False # ).distinct() # data_reqs = filter_hadgem_stream2(DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-HH', # experiment__short_name__in=['hist-1950'], # variable_request__table_name__in=['PrimOmon'], # 'PrimOmon'], # rip_code='r1i1p1f1', # datafile__isnull=False # ).distinct()) # data_reqs = filter_hadgem_stream2(DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-HH', # experiment__short_name__in=['highres-future'], # variable_request__table_name='PrimOmon', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='PrimSI' # ).distinct()) # Done LM, MM, HM AMIP # s1 = DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-HM', # experiment__short_name='highresSST-present', # rip_code='r1i1p1f1', # variable_request__table_name__startswith='Prim', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='PrimSI' # ).distinct() # s2 = filter_hadgem_stream2(DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-HM', # experiment__short_name='highresSST-present', # rip_code__in=['r1i3p1f1'], # , 'r1i3p1f1'], # variable_request__table_name__startswith='Prim', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='PrimSI' # ).distinct()) # data_reqs = s2 # s1 # | s2 # Done LM # s1 = DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-HM', # experiment__short_name='highresSST-future', # rip_code='r1i1p1f1', # variable_request__table_name__startswith='Prim', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='PrimSI' # ).distinct() # s2 = filter_hadgem_stream2(DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-HM', # experiment__short_name='highresSST-future', # rip_code__in=['r1i2p1f1', 'r1i3p1f1'], # variable_request__table_name__startswith='Prim', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='PrimSI' # ).distinct()) # data_reqs = s1 | s2 # data_reqs = filter_hadgem_stream2(DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-LM', # experiment__short_name='highresSST-future', # rip_code__in=['r1i14p1f1', 'r1i15p1f1'], # variable_request__table_name__startswith='Prim', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='PrimSI' # ).distinct()) # s1 = DataRequest.objects.filter( # climate_model__short_name__in=['HadGEM3-GC31-LL'], # , 'HadGEM3-GC31-MM', 'HadGEM3-GC31-HM'], # experiment__short_name='highres-future', # rip_code='r1i1p1f1', # variable_request__table_name__startswith='Prim', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='PrimSI' # ).distinct() # s2 = filter_hadgem_stream2(DataRequest.objects.filter( # climate_model__short_name__in=['HadGEM3-GC31-LL'], # , 'HadGEM3-GC31-MM', 'HadGEM3-GC31-HM'], # experiment__short_name='highres-future', # rip_code__in=['r1i2p1f1', 'r1i3p1f1', 'r1i4p1f1'], # variable_request__table_name__startswith='Prim', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='PrimSI' # ).distinct()) # data_reqs = s1 | s2 # data_reqs = DataRequest.objects.filter( # climate_model__short_name__startswith='HadGEM3-GC31', # experiment__short_name='spinup-1950', # rip_code='r1i1p1f1', # variable_request__table_name__startswith='Prim', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='PrimSI' # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name__startswith='HadGEM3-GC31', # # experiment__short_name='spinup-1950', # # rip_code='r1i1p1f1', # variable_request__table_name__in=['PrimSIday', 'SIday', 'SImon'], # datafile__isnull=False # ).distinct() data_reqs = DataRequest.objects.filter( climate_model__short_name__startswith='HadGEM3-GC31', variable_request__table_name__in=['fx', 'Ofx'], datafile__isnull=False ).distinct() logger.debug('Total data volume: {} Volume to restore: {}'.format( filesizeformat(get_request_size(data_reqs, start_year, end_year)). replace('\xa0', ' '), filesizeformat(get_request_size(data_reqs, start_year, end_year, offline=True)).replace('\xa0', ' '), )) if args.create: jon = User.objects.get(username='******') rr = RetrievalRequest.objects.create(requester=jon, start_year=start_year, end_year=end_year) time_zone = datetime.timezone(datetime.timedelta()) rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone) rr.save() rr.data_request.add(*data_reqs) logger.debug('Retrieval request {} created.'.format(rr.id))
def main(args): """ Main entry point """ start_year = 1948 end_year = 2051 # data_reqs = filter_hadgem_stream2(DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-LL', # experiment__short_name='highres-future', # rip_code__in=['r1i2p1f1', 'r1i3p1f1', 'r1i4p1f1'], # variable_request__table_name__in=[ # '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon', # 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday', # 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day' # ], # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).distinct()) data_reqs = filter_hadgem_stream2( DataRequest.objects.filter( climate_model__short_name='HadGEM3-GC31-MM', experiment__short_name='highres-future', rip_code__in=['r1i2p1f1', 'r1i3p1f1'], variable_request__table_name__in=[ '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon', 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday', 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day' ], datafile__isnull=False).exclude( variable_request__table_name__startswith='Prim').distinct()) # data_reqs = filter_hadgem_stream2(DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-HM', # experiment__short_name='highres-future', # rip_code='r1i3p1f1', # variable_request__table_name__in=[ # '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon', # 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday', # 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day' # ], # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).distinct()) logger.debug('Total data volume: {} Volume to restore: {}'.format( filesizeformat(get_request_size(data_reqs, start_year, end_year)).replace('\xa0', ' '), filesizeformat( get_request_size(data_reqs, start_year, end_year, offline=True)).replace('\xa0', ' '), )) if args.create: jon = User.objects.get(username='******') rr = RetrievalRequest.objects.create(requester=jon, start_year=start_year, end_year=end_year) time_zone = datetime.timezone(datetime.timedelta()) rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone) rr.save() rr.data_request.add(*data_reqs) logger.debug('Retrieval request {} created.'.format(rr.id))
def main(args): """ Main entry point """ start_year = 1948 end_year = 2051 # data_reqs = DataRequest.objects.filter( # institute__short_name='ECMWF', # experiment__short_name='hist-1950', # variable_request__table_name__startswith='Prim', # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # institute__short_name='ECMWF', # climate_model__short_name__in=['ECMWF-IFS-LR', 'ECMWF-IFS-MR'], # experiment__short_name='hist-1950', # variable_request__table_name__startswith='Prim', # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # institute__short_name='ECMWF', # climate_model__short_name='ECMWF-IFS-HR', # experiment__short_name='hist-1950', # rip_code__in=[f'r{r}i1p1f1' for r in range(4, 7)], # variable_request__table_name__startswith='Prim', # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # institute__short_name='ECMWF', # # climate_model__short_name='ECMWF-IFS-HR', # experiment__short_name='control-1950', # variable_request__table_name__startswith='Prim', # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # institute__short_name='ECMWF', # experiment__short_name='highresSST-present', # variable_request__table_name__startswith='Prim', # datafile__isnull=False # ).distinct() data_reqs = DataRequest.objects.filter( institute__short_name='ECMWF', experiment__short_name='spinup-1950', variable_request__table_name__startswith='Prim', datafile__isnull=False ).distinct() logger.debug('Total data volume: {} Volume to restore: {}'.format( filesizeformat(get_request_size(data_reqs, start_year, end_year)). replace('\xa0', ' '), filesizeformat(get_request_size(data_reqs, start_year, end_year, offline=True)).replace('\xa0', ' '), )) if args.create: jon = User.objects.get(username='******') rr = RetrievalRequest.objects.create(requester=jon, start_year=start_year, end_year=end_year) time_zone = datetime.timezone(datetime.timedelta()) rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone) rr.save() rr.data_request.add(*data_reqs) logger.debug('Retrieval request {} created.'.format(rr.id))
def test_manual_list(self): self.assertEqual( 15, get_request_size([self.dreq1, self.dreq2], 1950, 2000))
def main(args): """ Main entry point """ start_year = 1948 end_year = 2051 # data_reqs = DataRequest.objects.filter( # climate_model__short_name='EC-Earth3P', # experiment__short_name='hist-1950', # rip_code='r1i1p2f1', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).exclude( # variable_request__dimensions__contains='alevhalf' # ).exclude( # variable_request__dimensions__contains='alevel' # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='EC-Earth3P', # experiment__short_name='hist-1950', # rip_code='r2i1p2f1', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).exclude( # variable_request__dimensions__contains='alevhalf' # ).exclude( # variable_request__dimensions__contains='alevel' # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='EC-Earth3P', # experiment__short_name='hist-1950', # rip_code='r3i1p2f1', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).exclude( # variable_request__dimensions__contains='alevhalf' # ).exclude( # variable_request__dimensions__contains='alevel' # ).distinct() # mon_reqs = DataRequest.objects.filter( # climate_model__short_name='EC-Earth3P-HR', # experiment__short_name='hist-1950', # rip_code__in=[f'r{mem_num}i1p2f1' for mem_num in range(1, 4)], # variable_request__frequency='mon', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).exclude( # variable_request__dimensions__contains='alevhalf' # ).exclude( # variable_request__dimensions__contains='alevel' # ).distinct() # # day_reqs = DataRequest.objects.filter( # climate_model__short_name='EC-Earth3P-HR', # experiment__short_name='hist-1950', # rip_code='r1i1p2f1', # variable_request__frequency='day', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).exclude( # variable_request__dimensions__contains='alevhalf' # ).exclude( # variable_request__dimensions__contains='alevel' # ).distinct() # # data_reqs = mon_reqs | day_reqs data_reqs = DataRequest.objects.filter( climate_model__short_name='EC-Earth3P-HR', experiment__short_name='hist-1950', rip_code='r3i1p2f1', # variable_request__table_name='6hrPlevPt', # variable_request__cmor_name__in=['uas', 'vas', 'vortmean', 'snw', 'tas', 'psl', # 'mrsos', 'ts', 'huss', 'sfcWind', 'tsl'], variable_request__frequency='day', datafile__isnull=False ).exclude(variable_request__table_name__startswith='Prim').exclude( variable_request__dimensions__contains='alevhalf').exclude( variable_request__dimensions__contains='alevel').distinct() logger.debug('Total data volume: {} Volume to restore: {}'.format( filesizeformat(get_request_size(data_reqs, start_year, end_year)).replace('\xa0', ' '), filesizeformat( get_request_size(data_reqs, start_year, end_year, offline=True)).replace('\xa0', ' '), )) if args.create: jon = User.objects.get(username='******') rr = RetrievalRequest.objects.create(requester=jon, start_year=start_year, end_year=end_year) time_zone = datetime.timezone(datetime.timedelta()) rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone) rr.save() rr.data_request.add(*data_reqs) logger.debug('Retrieval request {} created.'.format(rr.id))
def main(args): """ Main entry point """ start_year = 1948 end_year = 2051 # data_reqs = DataRequest.objects.filter( # institute__short_name='MOHC', # climate_model__short_name='HadGEM3-GC31-MM', # experiment__short_name='highresSST-present', # rip_code='r1i1p1f1', # ) # data_reqs = DataRequest.objects.filter( # climate_model__short_name__in=['ECMWF-IFS-LR', 'ECMWF-IFS-HR'], # experiment__short_name='highresSST-present', # rip_code='r1i1p1f1', # ).exclude( # variable_request__table_name__startswith='Prim' # ) # data_reqs = DataRequest.objects.filter( # institute__short_name='MOHC', # climate_model__short_name='HadGEM3-GC31-MM', # experiment__short_name='highresSST-present', # rip_code='r1i1p1f1', # ) # data_reqs = DataRequest.objects.filter( # institute__short_name='MOHC', # climate_model__short_name='HadGEM3-GC31-HM', # experiment__short_name='highresSST-present', # rip_code='r1i1p1f1', # variable_request__frequency__in=['day'] # ).exclude( # variable_request__table_name__startswith='Prim' # )# .exclude( # variable_request__table_name='CFday' # ) # data_reqs = DataRequest.objects.filter( # institute__short_name='CMCC', # experiment__short_name='highresSST-present', # rip_code='r1i1p1f1', # ).exclude( # variable_request__table_name__startswith='Prim' # ) # data_reqs = DataRequest.objects.filter( # climate_model__short_name='CNRM-CM6-1', # experiment__short_name='highresSST-present', # rip_code='r1i1p1f2', # ).exclude( # variable_request__table_name__startswith='Prim' # ) # data_reqs = DataRequest.objects.filter( # climate_model__short_name='CNRM-CM6-1-HR', # experiment__short_name='highresSST-present', # rip_code='r1i1p1f2', # variable_request__frequency__in=['mon', 'day', '6hr', '3hr'], # datafile__isnull=False #).exclude( # variable_request__table_name__startswith='Prim' #).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-LL', # experiment__short_name='hist-1950', # rip_code='r1i1p1f1', # variable_request__table_name__in=[ # '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon', # 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday', # 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day' # ], # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-LL', # experiment__short_name='control-1950', # rip_code='r1i1p1f1', # variable_request__table_name__in=[ # '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon', # 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday', # 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day' # ], # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name__in=['ECMWF-IFS-LR'], # experiment__short_name='hist-1950', # rip_code='r1i1p1f1', # ).exclude( # variable_request__table_name__startswith='Prim' # ) # data_reqs = DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-HM', # experiment__short_name='control-1950', # rip_code='r1i1p1f1', # variable_request__table_name__in=[ # '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon', # 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday', # 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day' # ], # # variable_request__frequency__in=['6hr', '3hr'],# 'mon', 'day', '6hr', '3hr', '1hr'], # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-MM', # experiment__short_name='control-1950', # rip_code='r1i1p1f1', # variable_request__table_name__in=[ # '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon', # 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday', # 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day' # ], # # variable_request__frequency__in=['mon', 'day', '6hr'], # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name__in=['ECMWF-IFS-HR'], # experiment__short_name='control-1950', # rip_code='r1i1p1f1', # ).exclude( # variable_request__table_name__startswith='Prim' # ) # data_reqs = DataRequest.objects.filter( # climate_model__short_name='CNRM-CM6-1-HR', # experiment__short_name='hist-1950', # rip_code='r1i1p1f2', # # variable_request__frequency__in=['mon', 'day', '6hr'], # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='CNRM-CM6-1', # experiment__short_name='control-1950', # rip_code='r1i1p1f2', # # variable_request__frequency__in=['mon', 'day', '6hr'], # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).distinct() # data_reqs = DataRequest.objects.filter( # institute__short_name='MPI-M', # experiment__short_name='highresSST-present', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-LM', # experiment__short_name='highresSST-future', # rip_code='r1i1p1f1', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-MM', # experiment__short_name='highresSST-future', # rip_code='r1i1p1f1', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-HM', # experiment__short_name='highresSST-future', # rip_code='r1i1p1f1', # variable_request__table_name='CFday', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='CNRM-CM6-1', # experiment__short_name='highresSST-future', # rip_code='r1i1p1f2', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='CNRM-CM6-1-HR', # experiment__short_name='highresSST-future', # rip_code='r1i1p1f2', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='CNRM-CM6-1-HR', # experiment__short_name='highres-future', # rip_code='r1i1p1f2', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name__in=['MPI-ESM1-2-HR'], # experiment__short_name__in=['hist-1950'], # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name__in=['MPI-ESM1-2-HR', 'MPI-ESM1-2-XR'], # experiment__short_name__in=['hist-1950', 'control-1950'], # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).exclude( # climate_model__short_name='MPI-ESM1-2-HR', # experiment__short_name='hist-1950' # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name__in=['ECMWF-IFS-LR', 'ECMWF-IFS-HR'], # experiment__short_name='spinup-1950', # rip_code='r1i1p1f1', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='CMCC-CM2-VHR4', # experiment__short_name='control-1950', # rip_code='r1i1p1f1', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-LL', # experiment__short_name='highres-future', # rip_code='r1i1p1f1', # variable_request__table_name__in=[ # '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon', # 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday', # 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day' # ], # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-MM', # experiment__short_name='highres-future', # rip_code='r1i1p1f1', # variable_request__table_name__in=[ # '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon', # 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday', # 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day' # ], # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-HM', # experiment__short_name='highres-future', # rip_code='r1i1p1f1', # variable_request__table_name__in=[ # '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon', # 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday', # 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day' # ], # datafile__isnull=False # ).filter( # variable_request__table_name='CFday' # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name__in=['HadGEM3-GC31-LL', 'HadGEM3-GC31-MM'], # experiment__short_name='spinup-1950', # rip_code='r1i1p1f1', # variable_request__table_name__in=[ # '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon', # 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday', # 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day' # ], # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='HadGEM3-GC31-MH', # experiment__short_name='spinup-1950', # rip_code='r1i1p1f1', # variable_request__table_name__in=[ # '3hr', '6hrPlev', '6hrPlevPt', 'AERday', 'AERmon', 'Amon', # 'CF3hr', 'CFday', 'CFmon', 'E1hr', 'E3hr', 'E3hrPt', 'Eday', # 'EdayZ', 'Emon', 'EmonZ', 'Esubhr', 'LImon', 'Lmon', 'day' # ], # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name='CMCC-CM2-VHR4', # experiment__short_name='hist-1950', # rip_code='r1i1p1f1', # variable_request__frequency__in=['6hr'], # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name__in=['CMCC-CM2-HR4', 'CMCC-CM2-VHR4'], # experiment__short_name='highresSST-future', # rip_code='r1i1p1f1', # datafile__isnull=False # ).exclude( # variable_request__table_name__startswith='Prim' # ).distinct() # data_reqs = DataRequest.objects.filter( # climate_model__short_name__in=['HadGEM3-GC31-LL', 'HadGEM3-GC31-MM', 'HadGEM3-GC31-HM'], # experiment__short_name='highres-future', # # rip_code__in=[f'r1i{i}p1f1' for i in range(1, 4)], # variable_request__table_name__in=['Omon', 'Oday'], # datafile__isnull=False # ).distinct() # data_reqs = DataRequest.objects.filter( # institute__short_name='CNRM-CERFACS', # variable_request__table_name='SImon', # datafile__isnull=False # ).exclude( # esgfdataset__status='PUBLISHED' # ).exclude( # experiment__short_name__startswith='primWP5' # ).distinct() data_reqs = DataRequest.objects.filter( climate_model__short_name__in=['HadGEM3-GC31-LL', 'HadGEM3-GC31-MM'], experiment__short_name='spinup-1950', variable_request__table_name__in=['Omon', 'Oday'], datafile__isnull=False).distinct() logger.debug('Total data volume: {} Volume to restore: {}'.format( filesizeformat(get_request_size(data_reqs, start_year, end_year)).replace('\xa0', ' '), filesizeformat( get_request_size(data_reqs, start_year, end_year, offline=True)).replace('\xa0', ' '), )) if args.create: jon = User.objects.get(username='******') rr = RetrievalRequest.objects.create(requester=jon, start_year=start_year, end_year=end_year) time_zone = datetime.timezone(datetime.timedelta()) rr.date_created = datetime.datetime(2000, 1, 1, 0, 0, tzinfo=time_zone) rr.save() rr.data_request.add(*data_reqs) logger.debug('Retrieval request {} created.'.format(rr.id))