def fetch_files(self, context, files):
"""Downloads `files` as needed relative to `context` nominally used to identify
the observatory to the server. If the CRDS cache is currently configured as READONLY,
prints an estimate about which files will download and the total size. The estimate
does not (currently) include the nested mappings associated with the closure of `files`,
but the dominant costs of downloads are reference files.
"""
files = set([os.path.basename(_file) for _file in files])
if config.get_cache_readonly():
log.info("READONLY CACHE estimating required downloads.")
if not self.args.ignore_cache:
already_have = (set(rmap.list_references("*", self.observatory)) |
set(rmap.list_mappings("*", self.observatory)))
else:
already_have = set()
fetched = [ x for x in sorted(files - already_have) if not x.startswith("NOT FOUND") ]
for _file in files:
if _file in already_have:
log.verbose("File", repr(_file), "is already in the CRDS cache.", verbosity=55)
else:
log.verbose("File", repr(_file), "would be downloaded.", verbosity=55)
if fetched:
with log.info_on_exception("File size information not available."):
info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"])
total_bytes = api.get_total_bytes(info_map)
log.info("READONLY CACHE would download", len(fetched), "files totalling",
utils.human_format_number(total_bytes).strip(), "bytes.")
else:
log.info("READONLY CACHE no reference downloads expected.")
else:
self.dump_files(context, files, self.args.ignore_cache)
def cat_files(self):
"""Print out the files listed after --cat or implied by a combination of
explicitly specified contexts and --mappings or --references.
--files is not allowed.
"""
# --cat files... specifying *no* files still triggers --cat logic
# --contexts context-specifiers [including --all --last --range...]
# context specifiers can be symbolic and will be resolved.
# --cat @file is allowed
catted_files = self.get_words(self.args.cat) + self.implied_files
try:
self._file_info = api.get_file_info_map(
self.observatory,
files=[
os.path.basename(filename) for filename in catted_files
])
except Exception:
log.verbose_warning(
"Failed retrieving CRDS server catalog information. May need to set CRDS_SERVER_URL."
)
# This could be expanded to include the closure of mappings or references
for name in catted_files:
with log.error_on_exception("Failed dumping:", repr(name)):
path = self.locate_file(name)
self._cat_file(path)
def fetch_files(self, context, files):
"""Downloads `files` as needed relative to `context` nominally used to identify
the observatory to the server. If the CRDS cache is currently configured as READONLY,
prints an estimate about which files will download and the total size. The estimate
does not (currently) include the nested mappings associated with the closure of `files`,
but the dominant costs of downloads are reference files.
"""
files = set([os.path.basename(_file) for _file in files])
if config.get_cache_readonly():
log.info("READONLY CACHE estimating required downloads.")
if not self.args.ignore_cache:
already_have = (set(rmap.list_references("*", self.observatory)) |
set(rmap.list_mappings("*", self.observatory)))
else:
already_have = set()
fetched = [ x for x in sorted(files - already_have) if not x.startswith("NOT FOUND") ]
for _file in files:
if _file in already_have:
log.verbose("File", repr(_file), "is already in the CRDS cache.", verbosity=55)
else:
log.verbose("File", repr(_file), "would be downloaded.", verbosity=55)
if fetched:
with log.info_on_exception("File size information not available."):
info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"])
total_bytes = api.get_total_bytes(info_map)
log.info("READONLY CACHE would download", len(fetched), "files totalling",
utils.human_format_number(total_bytes).strip(), "bytes.")
else:
log.info("READONLY CACHE no reference downloads expected.")
else:
self.dump_files(context, files, self.args.ignore_cache)
def verify_files(self, files):
"""Check `files` against the CRDS server database to ensure integrity and check reject status."""
basenames = [os.path.basename(file) for file in files]
try:
log.verbose("Downloading verification info for",
len(basenames),
"files.",
verbosity=10)
infos = api.get_file_info_map(
observatory=self.observatory,
files=basenames,
fields=["size", "rejected", "blacklisted", "state", "sha1sum"])
except Exception as exc:
log.error(
"Failed getting file info. CACHE VERIFICATION FAILED. Exception: ",
repr(str(exc)))
return
bytes_so_far = 0
total_bytes = api.get_total_bytes(infos)
for nth_file, file in enumerate(files):
bfile = os.path.basename(file)
if infos[bfile] == "NOT FOUND":
log.error("CRDS has no record of file", repr(bfile))
else:
self.verify_file(file, infos[bfile], bytes_so_far, total_bytes,
nth_file, len(files))
bytes_so_far += int(infos[bfile]["size"])
def main(self):
"""Check files for availability from the archive."""
self.require_server_connection()
log.info("Mapping URL:", repr(self.mapping_url))
log.info("Reference URL:", repr(self.reference_url))
stats = utils.TimingStats()
self.file_info = api.get_file_info_map(self.observatory, self.files, fields=["size", "sha1sum"])
for filename in self.files:
self.verify_archive_file(filename)
stats.increment("files")
self.print_files()
stats.report_stat("files")
log.standard_status()
def main(self):
"""Check files for availability from the archive."""
self.require_server_connection()
log.info("Mapping URL:", repr(self.mapping_url))
log.info("Reference URL:", repr(self.reference_url))
stats = utils.TimingStats()
self.file_info = api.get_file_info_map(self.observatory,
self.files,
fields=["size", "sha1sum"])
for filename in self.files:
self.verify_archive_file(filename)
stats.increment("files")
self.print_files()
stats.report_stat("files")
log.standard_status()
def verify_files(self, files):
"""Check `files` against the CRDS server database to ensure integrity and check reject status."""
basenames = [os.path.basename(file) for file in files]
try:
log.verbose("Downloading verification info for", len(basenames), "files.", verbosity=10)
infos = api.get_file_info_map(observatory=self.observatory, files=basenames,
fields=["size","rejected","blacklisted","state","sha1sum"])
except Exception as exc:
log.error("Failed getting file info. CACHE VERIFICATION FAILED. Exception: ", repr(str(exc)))
return
bytes_so_far = 0
total_bytes = api.get_total_bytes(infos)
for nth_file, file in enumerate(files):
bfile = os.path.basename(file)
if infos[bfile] == "NOT FOUND":
log.error("CRDS has no record of file", repr(bfile))
else:
self.verify_file(file, infos[bfile], bytes_so_far, total_bytes, nth_file, len(files))
bytes_so_far += int(infos[bfile]["size"])
def fetch_references(self, references):
"""Gets all references required to support `only_contexts`. Removes
all references from the CRDS reference cache which are not required for
`only_contexts`.
"""
if not self.contexts:
return
if self.args.readonly_cache:
already_have = set(rmap.list_references("*", self.observatory))
fetched = [ x for x in sorted(set(references)-set(already_have)) if not x.startswith("NOT FOUND") ]
if fetched:
log.info("READONLY CACHE would fetch references:", repr(fetched))
with log.info_on_exception("Reference size information not available."):
info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"])
total_bytes = api.get_total_bytes(info_map)
log.info("READONLY CACHE would download", len(fetched), "references totaling",
utils.human_format_number(total_bytes).strip(), "bytes.")
else:
self.dump_files(self.contexts[0], references)
def fetch_references(self, references):
"""Gets all references required to support `only_contexts`. Removes
all references from the CRDS reference cache which are not required for
`only_contexts`.
"""
if not self.contexts:
return
if self.args.readonly_cache:
already_have = set(rmap.list_references("*", self.observatory))
fetched = [ x for x in sorted(set(references)-set(already_have)) if not x.startswith("NOT FOUND") ]
if fetched:
log.info("READONLY CACHE would fetch references:", repr(fetched))
with log.info_on_exception("Reference size information not available."):
info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"])
total_bytes = api.get_total_bytes(info_map)
log.info("READONLY CACHE would download", len(fetched), "references totaling",
utils.human_format_number(total_bytes).strip(), "bytes.")
else:
self.dump_files(self.contexts[0], references)
def cat_files(self):
"""Print out the files listed after --cat or implied by a combination of
explicitly specified contexts and --mappings or --references.
--files is not allowed.
"""
# --cat files... specifying *no* files still triggers --cat logic
# --contexts context-specifiers [including --all --last --range...]
# context specifiers can be symbolic and will be resolved.
# --cat @file is allowed
catted_files = self.get_words(self.args.cat) + self.implied_files
try:
self._file_info = api.get_file_info_map(
self.observatory, files=[os.path.basename(filename) for filename in catted_files])
except Exception:
log.verbose_warning("Failed retrieving CRDS server catalog information. May need to set CRDS_SERVER_URL.")
# This could be expanded to include the closure of mappings or references
for name in catted_files:
with log.error_on_exception("Failed dumping:", repr(name)):
path = self.locate_file(name)
self._cat_file(path)
def newer(name1, name2):
"""Determine if `name1` is a more recent file than `name2` accounting for
limited differences in naming conventions. Official CDBS and CRDS names are
comparable using a simple text comparison, just not to each other.
>>> newer("s7g1700gl_dead.fits", "hst_cos_deadtab_0001.fits")
False
>>> newer("hst_cos_deadtab_0001.fits", "s7g1700gl_dead.fits")
True
>>> newer("s7g1700gl_dead.fits", "bbbbb.fits")
Traceback (most recent call last):
...
NameComparisonError: Failed to classify name 'bbbbb.fits' for determining time order.
>>> newer("bbbbb.fits", "s7g1700gl_dead.fits")
Traceback (most recent call last):
...
NameComparisonError: Failed to classify name 'bbbbb.fits' for determining time order.
>>> newer("hst_cos_deadtab_0001.rmap", "hst_cos_deadtab_0002.rmap")
False
>>> newer("hst_cos_deadtab_0002.rmap", "hst_cos_deadtab_0001.rmap")
True
>>> newer("hst_cos_deadtab_0001.asdf", "hst_cos_deadtab_0050.fits")
True
>>> newer("hst_cos_deadtab_0051.fits", "hst_cos_deadtab_0050.asdf")
False
>>> newer("hst_cos_deadtab_0001.fits", "hst_cos_deadtab_99991.fits")
False
>>> newer("hst_cos_deadtab_99991.fits", "hst_cos_deadtab_0001.fits")
True
>>> newer("07g1700gl_dead.fits", "s7g1700gl_dead.fits")
True
>>> newer("s7g1700gl_dead.fits", "07g1700gl_dead.fits")
False
>>> newer("N/A", "s7g1700gl_dead.fits")
False
>>> newer("07g1700gl_dead.fits", "N/A")
True
>>> newer("N/A", "hst_cos_deadtab_0002.rmap")
False
>>> newer("hst_cos_deadtab_0002.rmap", "N/A")
True
>>> newer("hst_cos_deadtab_0001.fits", "17g1700gl_dead.fits")
Traceback (most recent call last):
...
NameComparisonError: Unhandled name comparison case: ('crds', 'newcdbs')
>>> newer("17g1700gl_dead.fits", "hst_cos_deadtab_0001.fits")
Traceback (most recent call last):
...
NameComparisonError: Unhandled name comparison case: ('newcdbs', 'crds')
"""
cases = {
("crds", "crds") : "compare_crds",
("oldcdbs", "oldcdbs") : "compare",
("newcdbs", "newcdbs") : "compare",
("crds", "oldcdbs") : True,
("oldcdbs", "crds") : False,
("newcdbs", "oldcdbs") : True,
("oldcdbs", "newcdbs") : False,
("crds", "newcdbs") : "raise",
("newcdbs", "crds") : "raise",
}
name1, name2 = crds_basename(name1), crds_basename(name2)
class1 = classify_name(name1)
class2 = classify_name(name2)
case = cases[(class1, class2)]
if name1 == "N/A":
return False
elif name2 =="N/A":
result = True
elif case == "compare_crds":
if extension_rank(name1) == extension_rank(name2):
serial1, serial2 = newstyle_serial(name1), newstyle_serial(name2)
result = serial1 > serial2 # same extension compares by counter
else:
result = extension_rank(name1) > extension_rank(name2)
elif case == "compare":
result = name1 > name2
elif case in [True, False]:
result = case
elif case == "query":
result = True
with log.warn_on_exception("Failed obtaining file activation dates for files",
repr(name1), "and", repr(name2),
"from server. can't determine time order."):
info_map = api.get_file_info_map("hst", [name1, name2], fields=["activation_date"])
result = info_map[name1]["activation_date"] > info_map[name2]["activation_date"]
else:
raise NameComparisonError("Unhandled name comparison case: ", repr((class1, class2)))
log.verbose("Comparing filename time order:", repr(name1), ">", repr(name2), "-->", result)
return result
elif case == "compare_crds":
if extension_rank(name1) == extension_rank(name2):
serial1, serial2 = newstyle_serial(name1), newstyle_serial(name2)
result = serial1 > serial2 # same extension compares by counter
else:
result = extension_rank(name1) > extension_rank(name2)
elif case == "compare":
result = name1 > name2
elif case in [True, False]:
result = case
elif case == "query":
result = True
with log.warn_on_exception("Failed obtaining file activation dates for files",
repr(name1), "and", repr(name2),
"from server. can't determine time order."):
info_map = api.get_file_info_map("hst", [name1, name2], fields=["activation_date"])
result = info_map[name1]["activation_date"] > info_map[name2]["activation_date"]
else:
raise NameComparisonError("Unhandled name comparison case: ", repr((class1, class2)))
log.verbose("Comparing filename time order:", repr(name1), ">", repr(name2), "-->", result)
return result
def crds_basename(name):
"""basename() accounting for N/A pass thru."""
if name == "N/A":
return "N/A"
else:
return os.path.basename(name)
def classify_name(name):
"""Classify filename `name` as "crds", "oldcdbs", or "newcdbs".
def init_files(self, files):
self.file_info = api.get_file_info_map(self.observatory, files, fields=["size", "sha1sum"])