def fetch_files(self, context, files):
"""Downloads `files` as needed relative to `context` nominally used to identify
the observatory to the server. If the CRDS cache is currently configured as READONLY,
prints an estimate about which files will download and the total size. The estimate
does not (currently) include the nested mappings associated with the closure of `files`,
but the dominant costs of downloads are reference files.
"""
files = set([os.path.basename(_file) for _file in files])
if config.get_cache_readonly():
log.info("READONLY CACHE estimating required downloads.")
if not self.args.ignore_cache:
already_have = (set(rmap.list_references("*", self.observatory)) |
set(rmap.list_mappings("*", self.observatory)))
else:
already_have = set()
fetched = [ x for x in sorted(files - already_have) if not x.startswith("NOT FOUND") ]
for _file in files:
if _file in already_have:
log.verbose("File", repr(_file), "is already in the CRDS cache.", verbosity=55)
else:
log.verbose("File", repr(_file), "would be downloaded.", verbosity=55)
if fetched:
with log.info_on_exception("File size information not available."):
info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"])
total_bytes = api.get_total_bytes(info_map)
log.info("READONLY CACHE would download", len(fetched), "files totalling",
utils.human_format_number(total_bytes).strip(), "bytes.")
else:
log.info("READONLY CACHE no reference downloads expected.")
else:
self.dump_files(context, files, self.args.ignore_cache)
def fetch_files(self, context, files):
"""Downloads `files` as needed relative to `context` nominally used to identify
the observatory to the server. If the CRDS cache is currently configured as READONLY,
prints an estimate about which files will download and the total size. The estimate
does not (currently) include the nested mappings associated with the closure of `files`,
but the dominant costs of downloads are reference files.
"""
files = set([os.path.basename(_file) for _file in files])
if config.get_cache_readonly():
log.info("READONLY CACHE estimating required downloads.")
if not self.args.ignore_cache:
already_have = (set(rmap.list_references("*", self.observatory)) |
set(rmap.list_mappings("*", self.observatory)))
else:
already_have = set()
fetched = [ x for x in sorted(files - already_have) if not x.startswith("NOT FOUND") ]
for _file in files:
if _file in already_have:
log.verbose("File", repr(_file), "is already in the CRDS cache.", verbosity=55)
else:
log.verbose("File", repr(_file), "would be downloaded.", verbosity=55)
if fetched:
with log.info_on_exception("File size information not available."):
info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"])
total_bytes = api.get_total_bytes(info_map)
log.info("READONLY CACHE would download", len(fetched), "files totalling",
utils.human_format_number(total_bytes).strip(), "bytes.")
else:
log.info("READONLY CACHE no reference downloads expected.")
else:
self.dump_files(context, files, self.args.ignore_cache)
def get_sqlite_db(observatory):
"""Download the CRDS database as a SQLite database."""
assert not config.get_cache_readonly(), "Readonly cache, updating the SQLite database cannot be done."""
encoded_compressed_data = S.get_sqlite_db(observatory)
data = zlib.decompress(base64.b64decode(encoded_compressed_data))
path = config.get_sqlite3_db_path(observatory)
utils.ensure_dir_exists(path)
with open(path, "wb+") as db_out:
db_out.write(data)
return path
def get_sqlite_db(observatory):
"""Download the CRDS database as a SQLite database."""
assert not config.get_cache_readonly(), "Readonly cache, updating the SQLite database cannot be done."""
encoded_compressed_data = S.get_sqlite_db(observatory)
data = zlib.decompress(base64.b64decode(encoded_compressed_data))
path = config.get_sqlite3_db_path(observatory)
utils.ensure_dir_exists(path)
with open(path, "wb+") as db_out:
db_out.write(data)
return path
def download(self, name, localpath):
"""Download a single file."""
# This code is complicated by the desire to blow away failed downloads. For the specific
# case of KeyboardInterrupt, the file needs to be blown away, but the interrupt should not
# be re-characterized so it is still un-trapped elsewhere under normal idioms which try *not*
# to trap KeyboardInterrupt.
assert not config.get_cache_readonly(), "Readonly cache, cannot download files " + repr(name)
try:
utils.ensure_dir_exists(localpath)
return proxy.apply_with_retries(self.download_core, name, localpath)
except Exception as exc:
self.remove_file(localpath)
raise CrdsDownloadError(
"Error fetching data for", srepr(name),
"at CRDS server", srepr(get_crds_server()),
"with mode", srepr(config.get_download_mode()),
":", str(exc)) from exc
except: # mainly for control-c, catch it and throw it.
self.remove_file(localpath)
raise
def download(self, name, localpath):
"""Download a single file."""
# This code is complicated by the desire to blow away failed downloads. For the specific
# case of KeyboardInterrupt, the file needs to be blown away, but the interrupt should not
# be re-characterized so it is still un-trapped elsewhere under normal idioms which try *not*
# to trap KeyboardInterrupt.
assert not config.get_cache_readonly(), "Readonly cache, cannot download files " + repr(name)
try:
utils.ensure_dir_exists(localpath)
return proxy.apply_with_retries(self.download_core, name, localpath)
except Exception as exc:
self.remove_file(localpath)
raise CrdsDownloadError(
"Error fetching data for", srepr(name),
"at CRDS server", srepr(get_crds_server()),
"with mode", srepr(config.get_download_mode()),
":", str(exc)) from exc
except: # mainly for control-c, catch it and throw it.
self.remove_file(localpath)
raise