def fetch_files(self, context, files): """Downloads `files` as needed relative to `context` nominally used to identify the observatory to the server. If the CRDS cache is currently configured as READONLY, prints an estimate about which files will download and the total size. The estimate does not (currently) include the nested mappings associated with the closure of `files`, but the dominant costs of downloads are reference files. """ files = set([os.path.basename(_file) for _file in files]) if config.get_cache_readonly(): log.info("READONLY CACHE estimating required downloads.") if not self.args.ignore_cache: already_have = (set(rmap.list_references("*", self.observatory)) | set(rmap.list_mappings("*", self.observatory))) else: already_have = set() fetched = [ x for x in sorted(files - already_have) if not x.startswith("NOT FOUND") ] for _file in files: if _file in already_have: log.verbose("File", repr(_file), "is already in the CRDS cache.", verbosity=55) else: log.verbose("File", repr(_file), "would be downloaded.", verbosity=55) if fetched: with log.info_on_exception("File size information not available."): info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"]) total_bytes = api.get_total_bytes(info_map) log.info("READONLY CACHE would download", len(fetched), "files totalling", utils.human_format_number(total_bytes).strip(), "bytes.") else: log.info("READONLY CACHE no reference downloads expected.") else: self.dump_files(context, files, self.args.ignore_cache)
def cat_files(self): """Print out the files listed after --cat or implied by a combination of explicitly specified contexts and --mappings or --references. --files is not allowed. """ # --cat files... specifying *no* files still triggers --cat logic # --contexts context-specifiers [including --all --last --range...] # context specifiers can be symbolic and will be resolved. # --cat @file is allowed catted_files = self.get_words(self.args.cat) + self.implied_files try: self._file_info = api.get_file_info_map( self.observatory, files=[ os.path.basename(filename) for filename in catted_files ]) except Exception: log.verbose_warning( "Failed retrieving CRDS server catalog information. May need to set CRDS_SERVER_URL." ) # This could be expanded to include the closure of mappings or references for name in catted_files: with log.error_on_exception("Failed dumping:", repr(name)): path = self.locate_file(name) self._cat_file(path)
def fetch_files(self, context, files): """Downloads `files` as needed relative to `context` nominally used to identify the observatory to the server. If the CRDS cache is currently configured as READONLY, prints an estimate about which files will download and the total size. The estimate does not (currently) include the nested mappings associated with the closure of `files`, but the dominant costs of downloads are reference files. """ files = set([os.path.basename(_file) for _file in files]) if config.get_cache_readonly(): log.info("READONLY CACHE estimating required downloads.") if not self.args.ignore_cache: already_have = (set(rmap.list_references("*", self.observatory)) | set(rmap.list_mappings("*", self.observatory))) else: already_have = set() fetched = [ x for x in sorted(files - already_have) if not x.startswith("NOT FOUND") ] for _file in files: if _file in already_have: log.verbose("File", repr(_file), "is already in the CRDS cache.", verbosity=55) else: log.verbose("File", repr(_file), "would be downloaded.", verbosity=55) if fetched: with log.info_on_exception("File size information not available."): info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"]) total_bytes = api.get_total_bytes(info_map) log.info("READONLY CACHE would download", len(fetched), "files totalling", utils.human_format_number(total_bytes).strip(), "bytes.") else: log.info("READONLY CACHE no reference downloads expected.") else: self.dump_files(context, files, self.args.ignore_cache)
def verify_files(self, files): """Check `files` against the CRDS server database to ensure integrity and check reject status.""" basenames = [os.path.basename(file) for file in files] try: log.verbose("Downloading verification info for", len(basenames), "files.", verbosity=10) infos = api.get_file_info_map( observatory=self.observatory, files=basenames, fields=["size", "rejected", "blacklisted", "state", "sha1sum"]) except Exception as exc: log.error( "Failed getting file info. CACHE VERIFICATION FAILED. Exception: ", repr(str(exc))) return bytes_so_far = 0 total_bytes = api.get_total_bytes(infos) for nth_file, file in enumerate(files): bfile = os.path.basename(file) if infos[bfile] == "NOT FOUND": log.error("CRDS has no record of file", repr(bfile)) else: self.verify_file(file, infos[bfile], bytes_so_far, total_bytes, nth_file, len(files)) bytes_so_far += int(infos[bfile]["size"])
def main(self): """Check files for availability from the archive.""" self.require_server_connection() log.info("Mapping URL:", repr(self.mapping_url)) log.info("Reference URL:", repr(self.reference_url)) stats = utils.TimingStats() self.file_info = api.get_file_info_map(self.observatory, self.files, fields=["size", "sha1sum"]) for filename in self.files: self.verify_archive_file(filename) stats.increment("files") self.print_files() stats.report_stat("files") log.standard_status()
def main(self): """Check files for availability from the archive.""" self.require_server_connection() log.info("Mapping URL:", repr(self.mapping_url)) log.info("Reference URL:", repr(self.reference_url)) stats = utils.TimingStats() self.file_info = api.get_file_info_map(self.observatory, self.files, fields=["size", "sha1sum"]) for filename in self.files: self.verify_archive_file(filename) stats.increment("files") self.print_files() stats.report_stat("files") log.standard_status()
def verify_files(self, files): """Check `files` against the CRDS server database to ensure integrity and check reject status.""" basenames = [os.path.basename(file) for file in files] try: log.verbose("Downloading verification info for", len(basenames), "files.", verbosity=10) infos = api.get_file_info_map(observatory=self.observatory, files=basenames, fields=["size","rejected","blacklisted","state","sha1sum"]) except Exception as exc: log.error("Failed getting file info. CACHE VERIFICATION FAILED. Exception: ", repr(str(exc))) return bytes_so_far = 0 total_bytes = api.get_total_bytes(infos) for nth_file, file in enumerate(files): bfile = os.path.basename(file) if infos[bfile] == "NOT FOUND": log.error("CRDS has no record of file", repr(bfile)) else: self.verify_file(file, infos[bfile], bytes_so_far, total_bytes, nth_file, len(files)) bytes_so_far += int(infos[bfile]["size"])
def fetch_references(self, references): """Gets all references required to support `only_contexts`. Removes all references from the CRDS reference cache which are not required for `only_contexts`. """ if not self.contexts: return if self.args.readonly_cache: already_have = set(rmap.list_references("*", self.observatory)) fetched = [ x for x in sorted(set(references)-set(already_have)) if not x.startswith("NOT FOUND") ] if fetched: log.info("READONLY CACHE would fetch references:", repr(fetched)) with log.info_on_exception("Reference size information not available."): info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"]) total_bytes = api.get_total_bytes(info_map) log.info("READONLY CACHE would download", len(fetched), "references totaling", utils.human_format_number(total_bytes).strip(), "bytes.") else: self.dump_files(self.contexts[0], references)
def fetch_references(self, references): """Gets all references required to support `only_contexts`. Removes all references from the CRDS reference cache which are not required for `only_contexts`. """ if not self.contexts: return if self.args.readonly_cache: already_have = set(rmap.list_references("*", self.observatory)) fetched = [ x for x in sorted(set(references)-set(already_have)) if not x.startswith("NOT FOUND") ] if fetched: log.info("READONLY CACHE would fetch references:", repr(fetched)) with log.info_on_exception("Reference size information not available."): info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"]) total_bytes = api.get_total_bytes(info_map) log.info("READONLY CACHE would download", len(fetched), "references totaling", utils.human_format_number(total_bytes).strip(), "bytes.") else: self.dump_files(self.contexts[0], references)
def cat_files(self): """Print out the files listed after --cat or implied by a combination of explicitly specified contexts and --mappings or --references. --files is not allowed. """ # --cat files... specifying *no* files still triggers --cat logic # --contexts context-specifiers [including --all --last --range...] # context specifiers can be symbolic and will be resolved. # --cat @file is allowed catted_files = self.get_words(self.args.cat) + self.implied_files try: self._file_info = api.get_file_info_map( self.observatory, files=[os.path.basename(filename) for filename in catted_files]) except Exception: log.verbose_warning("Failed retrieving CRDS server catalog information. May need to set CRDS_SERVER_URL.") # This could be expanded to include the closure of mappings or references for name in catted_files: with log.error_on_exception("Failed dumping:", repr(name)): path = self.locate_file(name) self._cat_file(path)
def newer(name1, name2): """Determine if `name1` is a more recent file than `name2` accounting for limited differences in naming conventions. Official CDBS and CRDS names are comparable using a simple text comparison, just not to each other. >>> newer("s7g1700gl_dead.fits", "hst_cos_deadtab_0001.fits") False >>> newer("hst_cos_deadtab_0001.fits", "s7g1700gl_dead.fits") True >>> newer("s7g1700gl_dead.fits", "bbbbb.fits") Traceback (most recent call last): ... NameComparisonError: Failed to classify name 'bbbbb.fits' for determining time order. >>> newer("bbbbb.fits", "s7g1700gl_dead.fits") Traceback (most recent call last): ... NameComparisonError: Failed to classify name 'bbbbb.fits' for determining time order. >>> newer("hst_cos_deadtab_0001.rmap", "hst_cos_deadtab_0002.rmap") False >>> newer("hst_cos_deadtab_0002.rmap", "hst_cos_deadtab_0001.rmap") True >>> newer("hst_cos_deadtab_0001.asdf", "hst_cos_deadtab_0050.fits") True >>> newer("hst_cos_deadtab_0051.fits", "hst_cos_deadtab_0050.asdf") False >>> newer("hst_cos_deadtab_0001.fits", "hst_cos_deadtab_99991.fits") False >>> newer("hst_cos_deadtab_99991.fits", "hst_cos_deadtab_0001.fits") True >>> newer("07g1700gl_dead.fits", "s7g1700gl_dead.fits") True >>> newer("s7g1700gl_dead.fits", "07g1700gl_dead.fits") False >>> newer("N/A", "s7g1700gl_dead.fits") False >>> newer("07g1700gl_dead.fits", "N/A") True >>> newer("N/A", "hst_cos_deadtab_0002.rmap") False >>> newer("hst_cos_deadtab_0002.rmap", "N/A") True >>> newer("hst_cos_deadtab_0001.fits", "17g1700gl_dead.fits") Traceback (most recent call last): ... NameComparisonError: Unhandled name comparison case: ('crds', 'newcdbs') >>> newer("17g1700gl_dead.fits", "hst_cos_deadtab_0001.fits") Traceback (most recent call last): ... NameComparisonError: Unhandled name comparison case: ('newcdbs', 'crds') """ cases = { ("crds", "crds") : "compare_crds", ("oldcdbs", "oldcdbs") : "compare", ("newcdbs", "newcdbs") : "compare", ("crds", "oldcdbs") : True, ("oldcdbs", "crds") : False, ("newcdbs", "oldcdbs") : True, ("oldcdbs", "newcdbs") : False, ("crds", "newcdbs") : "raise", ("newcdbs", "crds") : "raise", } name1, name2 = crds_basename(name1), crds_basename(name2) class1 = classify_name(name1) class2 = classify_name(name2) case = cases[(class1, class2)] if name1 == "N/A": return False elif name2 =="N/A": result = True elif case == "compare_crds": if extension_rank(name1) == extension_rank(name2): serial1, serial2 = newstyle_serial(name1), newstyle_serial(name2) result = serial1 > serial2 # same extension compares by counter else: result = extension_rank(name1) > extension_rank(name2) elif case == "compare": result = name1 > name2 elif case in [True, False]: result = case elif case == "query": result = True with log.warn_on_exception("Failed obtaining file activation dates for files", repr(name1), "and", repr(name2), "from server. can't determine time order."): info_map = api.get_file_info_map("hst", [name1, name2], fields=["activation_date"]) result = info_map[name1]["activation_date"] > info_map[name2]["activation_date"] else: raise NameComparisonError("Unhandled name comparison case: ", repr((class1, class2))) log.verbose("Comparing filename time order:", repr(name1), ">", repr(name2), "-->", result) return result
elif case == "compare_crds": if extension_rank(name1) == extension_rank(name2): serial1, serial2 = newstyle_serial(name1), newstyle_serial(name2) result = serial1 > serial2 # same extension compares by counter else: result = extension_rank(name1) > extension_rank(name2) elif case == "compare": result = name1 > name2 elif case in [True, False]: result = case elif case == "query": result = True with log.warn_on_exception("Failed obtaining file activation dates for files", repr(name1), "and", repr(name2), "from server. can't determine time order."): info_map = api.get_file_info_map("hst", [name1, name2], fields=["activation_date"]) result = info_map[name1]["activation_date"] > info_map[name2]["activation_date"] else: raise NameComparisonError("Unhandled name comparison case: ", repr((class1, class2))) log.verbose("Comparing filename time order:", repr(name1), ">", repr(name2), "-->", result) return result def crds_basename(name): """basename() accounting for N/A pass thru.""" if name == "N/A": return "N/A" else: return os.path.basename(name) def classify_name(name): """Classify filename `name` as "crds", "oldcdbs", or "newcdbs".
def init_files(self, files): self.file_info = api.get_file_info_map(self.observatory, files, fields=["size", "sha1sum"])