def verify_file(self, file, info, bytes_so_far, total_bytes, nth_file, total_files):
"""Check one `file` against the provided CRDS database `info` dictionary."""
path = rmap.locate_file(file, observatory=self.observatory)
base = os.path.basename(file)
n_bytes = int(info["size"])
log.verbose(api.file_progress("Verifying", base, path, n_bytes, bytes_so_far, total_bytes, nth_file, total_files),
verbosity=10)
if not os.path.exists(path):
log.error("File", repr(base), "doesn't exist at", repr(path))
return
size = os.stat(path).st_size
if int(info["size"]) != size:
self.error_and_repair(path, "File", repr(base), "length mismatch LOCAL size=" + srepr(size),
"CRDS size=" + srepr(info["size"]))
elif self.args.check_sha1sum:
log.verbose("Computing checksum for", repr(base), "of size", repr(size), verbosity=100)
sha1sum = utils.checksum(path)
if info["sha1sum"] == "none":
log.warning("CRDS doesn't know the checksum for", repr(base))
elif info["sha1sum"] != sha1sum:
self.error_and_repair(path, "File", repr(base), "checksum mismatch CRDS=" + repr(info["sha1sum"]),
"LOCAL=" + repr(sha1sum))
if info["state"] not in ["archived", "operational"]:
log.warning("File", repr(base), "has an unusual CRDS file state", repr(info["state"]))
if info["rejected"] != "false":
log.warning("File", repr(base), "has been explicitly rejected.")
if self.args.purge_rejected:
self.remove_files([path], "files")
return
if info["blacklisted"] != "false":
log.warning("File", repr(base), "has been blacklisted or is dependent on a blacklisted file.")
if self.args.purge_blacklisted:
self.remove_files([path], "files")
return
return
def sync_datasets(self):
"""Sync mappings and references for datasets with respect to `self.contexts`."""
if not self.contexts:
log.error("Define --contexts under which references are fetched for --dataset-files or --dataset-ids.""")
sys.exit(-1)
active_references = []
for context in self.contexts:
if self.args.dataset_ids:
if len(self.args.dataset_ids) == 1 and self.args.dataset_ids[0].startswith("@"):
with open(self.args.dataset_ids[0][1:]) as pfile:
self.args.dataset_ids = pfile.read().splitlines()
with log.error_on_exception("Failed to get matching parameters for", self.args.dataset_ids):
id_headers = api.get_dataset_headers_by_id(context, self.args.dataset_ids)
for dataset in self.args.dataset_files or self.args.dataset_ids:
log.info("Syncing context '%s' dataset '%s'." % (context, dataset))
with log.error_on_exception("Failed to get matching parameters from", repr(dataset)):
if self.args.dataset_files:
headers = { dataset : data_file.get_conditioned_header(dataset, observatory=self.observatory) }
else:
headers = { dataset_id : header for (dataset_id, header) in id_headers.items() if
dataset.upper() in dataset_id }
for assc_dataset, header in headers.items():
with log.error_on_exception("Failed syncing references for dataset", repr(assc_dataset),
"under context", repr(context)):
bestrefs = crds.getrecommendations(header, context=context, observatory=self.observatory,
ignore_cache=self.args.ignore_cache)
log.verbose("Best references for", repr(assc_dataset), "are", bestrefs)
active_references.extend(bestrefs.values())
active_references = [ ref for ref in active_references if not ref.startswith("NOT FOUND") ]
log.verbose("Syncing references:", repr(active_references))
return list(set(active_references))
def sync_datasets(self):
"""Sync mappings and references for datasets with respect to `self.contexts`."""
if not self.contexts:
log.error("Define --contexts under which references are fetched for --dataset-files or --dataset-ids.""")
sys.exit(-1)
active_references = []
for context in self.contexts:
if self.args.dataset_ids:
if len(self.args.dataset_ids) == 1 and self.args.dataset_ids[0].startswith("@"):
with open(self.args.dataset_ids[0][1:]) as pfile:
self.args.dataset_ids = pfile.read().splitlines()
with log.error_on_exception("Failed to get matching parameters for", self.args.dataset_ids):
id_headers = api.get_dataset_headers_by_id(context, self.args.dataset_ids)
for dataset in self.args.dataset_files or self.args.dataset_ids:
log.info("Syncing context '%s' dataset '%s'." % (context, dataset))
with log.error_on_exception("Failed to get matching parameters from", repr(dataset)):
if self.args.dataset_files:
headers = { dataset : data_file.get_conditioned_header(dataset, observatory=self.observatory) }
else:
headers = { dataset_id : header for (dataset_id, header) in id_headers.items() if
dataset.upper() in dataset_id }
for assc_dataset, header in headers.items():
with log.error_on_exception("Failed syncing references for dataset", repr(assc_dataset),
"under context", repr(context)):
bestrefs = crds.getrecommendations(header, context=context, observatory=self.observatory,
ignore_cache=self.args.ignore_cache)
log.verbose("Best references for", repr(assc_dataset), "are", bestrefs)
active_references.extend(bestrefs.values())
active_references = [ ref for ref in active_references if not ref.startswith("NOT FOUND") ]
log.verbose("Syncing references:", repr(active_references))
return list(set(active_references))
def verify_file(self, file, info, bytes_so_far, total_bytes, nth_file, total_files):
"""Check one `file` against the provided CRDS database `info` dictionary."""
path = rmap.locate_file(file, observatory=self.observatory)
base = os.path.basename(file)
n_bytes = int(info["size"])
log.verbose(api.file_progress("Verifying", base, path, n_bytes, bytes_so_far, total_bytes, nth_file, total_files),
verbosity=10)
if not os.path.exists(path):
log.error("File", repr(base), "doesn't exist at", repr(path))
return
size = os.stat(path).st_size
if int(info["size"]) != size:
self.error_and_repair(path, "File", repr(base), "length mismatch LOCAL size=" + srepr(size),
"CRDS size=" + srepr(info["size"]))
elif self.args.check_sha1sum:
log.verbose("Computing checksum for", repr(base), "of size", repr(size), verbosity=100)
sha1sum = utils.checksum(path)
if info["sha1sum"] == "none":
log.warning("CRDS doesn't know the checksum for", repr(base))
elif info["sha1sum"] != sha1sum:
self.error_and_repair(path, "File", repr(base), "checksum mismatch CRDS=" + repr(info["sha1sum"]),
"LOCAL=" + repr(sha1sum))
if info["state"] not in ["archived", "operational"]:
log.warning("File", repr(base), "has an unusual CRDS file state", repr(info["state"]))
if info["rejected"] != "false":
log.warning("File", repr(base), "has been explicitly rejected.")
if self.args.purge_rejected:
self.remove_files([path], "files")
return
if info["blacklisted"] != "false":
log.warning("File", repr(base), "has been blacklisted or is dependent on a blacklisted file.")
if self.args.purge_blacklisted:
self.remove_files([path], "files")
return
return
def verify_context_change(self, old_context):
"""Verify that the starting and post-sync contexts are different, or issue an error."""
new_context = heavy_client.load_server_info(self.observatory).operational_context
if old_context == new_context:
log.error("Expected operational context switch but starting and post-sync contexts are both", repr(old_context))
else:
log.info("Operational context updated from", repr(old_context), "to", repr(new_context))
def list_dataset_headers(self):
"""List dataset header info for self.args.dataset_headers with respect to self.args.context"""
for context in self.contexts:
with log.error_on_exception("Failed fetching dataset parameters with repect to", repr(context),
"for", repr(self.args.dataset_headers)):
pars = api.get_dataset_headers_by_id(context, self.args.dataset_headers)
pmap = crds.get_cached_mapping(context)
for requested_id in self.args.dataset_headers:
for returned_id in sorted(pars.keys()):
if requested_id.upper() in returned_id.upper():
header = pars[returned_id]
if isinstance(header, python23.string_types):
log.error("No header for", repr(returned_id), ":", repr(header)) # header is reason
continue
if self.args.id_expansions_only:
print(returned_id, context if len(self.contexts) > 1 else "")
else:
if self.args.minimize_headers:
header2 = pmap.minimize_header(header)
else:
header2 = dict(header)
header2.pop("REFTYPE", None)
header2["dataset_id"] = returned_id
log.info("Dataset pars for", repr(returned_id), "with respect to", repr(context) + ":\n",
log.PP(header2))
if self.args.first_id_expansion_only:
break
def rmap_check_modifications(old_rmap, new_rmap, old_ref, new_ref, expected=("add",)):
"""Check the differences between `old_rmap` and `new_rmap` and make sure they're
limited to the types listed in `expected`.
expected should be "add" or "replace".
Returns as_expected, True IFF all rmap modifications match `expected`.
"""
diffs = diff.mapping_diffs(old_rmap, new_rmap)
as_expected = True
for difference in diffs:
actual = diff.diff_action(difference)
if actual in expected:
pass # white-list so it will fail when expected is bogus.
else:
log.error("Expected one of", repr(expected), "but got", repr(actual),
"from change", repr(difference))
as_expected = False
with open(old_rmap) as pfile:
old_count = len([line for line in pfile.readlines() if os.path.basename(old_ref) in line])
with open(new_rmap) as pfile:
new_count = len([line for line in pfile.readlines() if os.path.basename(new_ref) in line])
if "replace" in expected and old_count != new_count:
log.error("Replacement COUNT DIFFERENCE replacing", repr(old_ref), "with", repr(new_ref), "in", repr(old_rmap),
old_count, "vs.", new_count)
as_expected = False
return as_expected
def rmap_check_modifications(old_rmap, new_rmap, old_ref, new_ref, expected=("add",)):
"""Check the differences between `old_rmap` and `new_rmap` and make sure they're
limited to the types listed in `expected`.
expected should be "add" or "replace".
Returns as_expected, True IFF all rmap modifications match `expected`.
"""
diffs = diff.mapping_diffs(old_rmap, new_rmap)
as_expected = True
for difference in diffs:
actual = diff.diff_action(difference)
if actual in expected:
pass # white-list so it will fail when expected is bogus.
else:
log.error("Expected one of", srepr(expected), "but got", srepr(actual),
"from change", srepr(difference))
as_expected = False
with open(old_rmap) as pfile:
old_count = len([line for line in pfile.readlines() if os.path.basename(old_ref) in line])
with open(new_rmap) as pfile:
new_count = len([line for line in pfile.readlines() if os.path.basename(new_ref) in line])
if "replace" in expected and old_count != new_count:
log.error("Replacement COUNT DIFFERENCE replacing", srepr(old_ref), "with",
srepr(new_ref), "in", srepr(old_rmap),
old_count, "vs.", new_count)
as_expected = False
return as_expected
def get_total_bytes(info_map):
"""Return the total byte count of file info map `info_map`."""
try:
return sum([long(info_map[name]["size"]) for name in info_map if "NOT FOUND" not in info_map[name]])
except Exception as exc:
log.error("Error computing total byte count: ", str(exc))
return -1
def list_dataset_headers(self):
"""List dataset header info for self.args.dataset_headers with respect to self.args.context"""
for context in self.contexts:
with log.error_on_exception("Failed fetching dataset parameters with repect to", repr(context),
"for", repr(self.args.dataset_headers)):
pars = api.get_dataset_headers_by_id(context, self.args.dataset_headers)
pmap = crds.get_cached_mapping(context)
for requested_id in self.args.dataset_headers:
for returned_id in sorted(pars.keys()):
if requested_id.upper() in returned_id.upper():
header = pars[returned_id]
if isinstance(header, python23.string_types):
log.error("No header for", repr(returned_id), ":", repr(header)) # header is reason
continue
if self.args.id_expansions_only:
print(returned_id, context if len(self.contexts) > 1 else "")
else:
if self.args.minimize_headers:
header2 = pmap.minimize_header(header)
else:
header2 = dict(header)
header2.pop("REFTYPE", None)
header2["dataset_id"] = returned_id
log.info("Dataset pars for", repr(returned_id), "with respect to", repr(context) + ":\n",
log.PP(header2))
if self.args.first_id_expansion_only:
break
def _get_cache_filelist_and_report_errors(bestrefs):
"""Compute the list of files to download based on the `bestrefs` dictionary,
skimming off and reporting errors, and raising an exception on the last error seen.
Return the list of files to download, collapsing complex return types like tuples
and dictionaries into a list of simple filenames.
"""
wanted = []
last_error = None
for filetype, refname in bestrefs.items():
if isinstance(refname, tuple):
wanted.extend(list(refname))
elif isinstance(refname, dict):
wanted.extend(refname.values())
elif isinstance(refname, python23.string_types):
if "NOT FOUND" in refname:
if "n/a" in refname.lower():
log.verbose("Reference type", repr(filetype),
"NOT FOUND. Skipping reference caching/download.")
else:
last_error = CrdsLookupError("Error determining best reference for",
repr(str(filetype)), " = ", str(refname)[len("NOT FOUND"):])
log.error(str(last_error))
else:
wanted.append(refname)
else:
last_error = CrdsLookupError("Unhandled bestrefs return value type for " + repr(str(filetype)))
log.error(str(last_error))
if last_error is not None:
raise last_error
return wanted
def verify_context_change(self, old_context):
"""Verify that the starting and post-sync contexts are different, or issue an error."""
new_context = heavy_client.load_server_info(self.observatory).operational_context
if old_context == new_context:
log.error("Expected operational context switch but starting and post-sync contexts are both", repr(old_context))
else:
log.info("Operational context updated from", repr(old_context), "to", repr(new_context))
def error_and_repair(self, file, *args, **keys):
"""Issue an error message and repair `file` if requested by command line args."""
log.error(*args, **keys)
if self.args.repair_files:
if config.writable_cache_or_info("Skipping remove and dump of", repr(file)):
log.info("Repairing file", repr(file))
utils.remove(file, observatory=self.observatory)
self.dump_files(self.default_context, [file])
def error_and_repair(self, file, *args, **keys):
"""Issue an error message and repair `file` if requested by command line args."""
log.error(*args, **keys)
if self.args.repair_files:
if config.writable_cache_or_info("Skipping remove and dump of", repr(file)):
log.info("Repairing file", repr(file))
utils.remove(file, observatory=self.observatory)
self.dump_files(self.default_context, [file])
def verify_archive_file(self, filename):
"""Verify the likely presence of `filename` on the archive web server. Issue an ERROR if absent."""
url = self.archive_url(filename)
response = requests.head(url)
if response.status_code in [200,]:
log.verbose("File", repr(filename), "is available from", repr(url))
self.check_length(filename, response)
else:
log.error("File", repr(filename), "failed HTTP HEAD with code =", response.status_code, "from", repr(url))
self.missing_files.append(filename)
def check_length(self, filename, response):
"""Check the content-length reported by HEAD against the CRDS database's file size."""
archive_size = python23.long(response.headers["content-length"])
crds_size = python23.long(self.file_info[filename]["size"])
if archive_size != crds_size:
log.error("File", repr(filename), "available but length bad. crds size:", crds_size,
"archive size:", archive_size)
self.bad_length_files.append(filename)
else:
log.verbose("File", repr(filename), "lengths agree:", crds_size)
def _check_error(self, response, xpath_spec, error_prefix):
"""Extract the `xpath_spec` text from `response`, if present call fatal_error() with
`error_prefix` and the response `xpath_spec` text.
"""
error_msg_parse = html.fromstring(response.text).xpath(xpath_spec)
error_message = error_msg_parse and error_msg_parse[0].text.strip()
if error_message:
if error_message.startswith("ERROR: "):
error_message = error_message[len("ERROR: "):]
log.error(error_prefix, error_message)
def _check_error(self, response, xpath_spec, error_prefix):
"""Extract the `xpath_spec` text from `response`, if present call fatal_error() with
`error_prefix` and the response `xpath_spec` text.
"""
error_msg_parse = html.fromstring(response.text).xpath(xpath_spec)
error_message = error_msg_parse and error_msg_parse[0].text.strip()
if error_message:
if error_message.startswith("ERROR: "):
error_message = error_message[len("ERROR: "):]
log.error(error_prefix, error_message)
def check_length(self, filename, response):
"""Check the content-length reported by HEAD against the CRDS database's file size."""
archive_size = python23.long(response.headers["content-length"])
crds_size = python23.long(self.file_info[filename]["size"])
if archive_size != crds_size:
log.error("File", repr(filename),
"available but length bad. crds size:", crds_size,
"archive size:", archive_size)
self.bad_length_files.append(filename)
else:
log.verbose("File", repr(filename), "lengths agree:", crds_size)
def ignore_errors(self, i, affected):
"""Check each context switch for errors during bestrefs run. Fail or return False on errors."""
ignore = False
if affected.bestrefs_status != 0:
message = log.format("CRDS server-side errors for", i, affected.computation_dir)
if self.args.ignore_errant_history:
ignore = True
if self.args.fail_on_errant_history:
self.fatal_error(message)
else:
log.error(message)
return ignore
def handle_done(self, message):
"""Generic "done" handler issue info() message and stops monitoring / exits."""
status = message.data["status"]
if status == 0:
log.info(self.format_remote("COMPLETED:", message.data))
elif status == 1:
log.error(self.format_remote("FAILED:", message.data))
elif status == 2:
log.error(self.format_remote("CANCELLED:", message.data))
else:
log.info(self.format_remote("DONE:", message))
return message.data["result"]
def log_and_track_error(self, data, instrument, filekind, *params, **keys):
"""Issue an error message and record the first instance of each unique kind of error, where "unique"
is defined as (instrument, filekind, msg_text) and omits data id.
"""
msg = self.format_prefix(data, instrument, filekind, *params, **keys)
log.error(msg)
key = log.format(instrument, filekind, *params, **keys)
if key not in self.ue_mixin.messages:
self.ue_mixin.messages[key] = msg
self.ue_mixin.unique_data_names.add(data)
self.ue_mixin.count[key] += 1
self.ue_mixin.all_data_names.add(data)
return None # for log.exception_trap_logger --> don't reraise
def log_and_track_error(self, data, instrument, filekind, *params, **keys):
"""Issue an error message and record the first instance of each unique kind of error, where "unique"
is defined as (instrument, filekind, msg_text) and omits data id.
"""
msg = self.format_prefix(data, instrument, filekind, *params, **keys)
log.error(msg)
key = log.format(instrument, filekind, *params, **keys)
if key not in self.ue_mixin.messages:
self.ue_mixin.messages[key] = msg
self.ue_mixin.unique_data_names.add(data)
self.ue_mixin.count[key] += 1
self.ue_mixin.all_data_names.add(data)
return None # for log.exception_trap_logger --> don't reraise
def main(self):
"""Process command line parameters in to a context and list of
reference files. Print out the match tuples within the context
which contain the reference files.
"""
if self.matched_files:
self.dump_reference_matches()
elif self.args.datasets or self.args.instrument:
self.dump_dataset_headers()
else:
self.print_help()
log.error("Specify --files to dump reference match cases or --datasets to dump dataset matching parameters.")
return log.errors()
def verify_archive_file(self, filename):
"""Verify the likely presence of `filename` on the archive web server. Issue an ERROR if absent."""
url = self.archive_url(filename)
response = requests.head(url)
if response.status_code in [
200,
]:
log.verbose("File", repr(filename), "is available from", repr(url))
self.check_length(filename, response)
else:
log.error("File", repr(filename), "failed HTTP HEAD with code =",
response.status_code, "from", repr(url))
self.missing_files.append(filename)
def ignore_errors(self, i, affected):
"""Check each context switch for errors during bestrefs run. Fail or return False on errors."""
ignore = False
if affected.bestrefs_status != 0:
message = log.format("CRDS server-side errors for", i,
affected.computation_dir)
if self.args.ignore_errant_history:
ignore = True
if self.args.fail_on_errant_history:
self.fatal_error(message)
else:
log.error(message)
return ignore
def handle_done(self, message):
"""Generic "done" handler issue info() message and stops monitoring / exits."""
status = message.data["status"]
result = message.data.get("result", None)
if status == 0:
log.info(self.format_remote("COMPLETED:", result))
elif status == 1:
log.error(self.format_remote("FAILED:", result))
elif status == 2:
log.error(self.format_remote("CANCELLED:", result))
else:
log.info(self.format_remote("DONE:", result))
self.result = result
return result
def main(self):
"""Process command line parameters in to a context and list of
reference files. Print out the match tuples within the context
which contain the reference files.
"""
if self.matched_files:
self.dump_reference_matches()
elif self.args.datasets or self.args.instrument:
self.dump_dataset_headers()
else:
self.print_help()
log.error(
"Specify --files to dump reference match cases or --datasets to dump dataset matching parameters."
)
return log.errors()
def list_datasets(self):
"""List dataset header info for self.args.datasets with respect to self.args.context"""
for context in self.contexts:
with log.error_on_exception("Failed fetching dataset parameters with repect to", repr(context),
"for", repr(self.args.datasets)):
pars = api.get_dataset_headers_by_id(context, self.args.datasets)
pmap = rmap.get_cached_mapping(context)
for (dataset_id, header) in pars.items():
if isinstance(header, python23.string_types):
log.error("No header for", repr(dataset_id), ":", repr(header)) # header is reason
continue
header2 = pmap.minimize_header(header)
header2.pop("REFTYPE", None)
log.info("Dataset pars for", repr(dataset_id), "with respect to", repr(context) + ":\n",
log.PP(header2))
def main(self):
"""Synchronize files."""
if self.args.dry_run:
self.args.readonly_cache = True
if self.args.repair_files:
self.args.check_files = True
if self.args.organize: # do this before syncing anything under the current mode.
self.organize_references(self.args.organize)
self.require_server_connection()
if self.readonly_cache and self.args.verify_context_change:
log.error("--readonly-cache and --verify-context-change are incompatible, a readonly cache cannot change.")
if self.args.files:
self.sync_explicit_files()
verify_file_list = self.files
elif self.args.fetch_sqlite_db:
self.fetch_sqlite_db()
elif self.contexts:
active_mappings = self.get_context_mappings()
verify_file_list = active_mappings
if self.args.fetch_references or self.args.purge_references:
if self.args.dataset_files or self.args.dataset_ids:
active_references = self.sync_datasets()
else:
active_references = self.get_context_references()
active_references = sorted(set(active_references + self.get_conjugates(active_references)))
if self.args.fetch_references:
self.fetch_references(active_references)
verify_file_list += active_references
if self.args.purge_references:
self.purge_references(active_references)
if self.args.purge_mappings:
self.purge_mappings()
else:
log.error("Define --all, --contexts, --last, --range, --files, or --fetch-sqlite-db to sync.")
sys.exit(-1)
if self.args.check_files or self.args.check_sha1sum or self.args.repair_files:
self.verify_files(verify_file_list)
if self.args.verify_context_change:
old_context = heavy_client.load_server_info(self.observatory).operational_context
heavy_client.update_config_info(self.observatory)
if self.args.verify_context_change:
self.verify_context_change(old_context)
if self.args.push_context:
self.push_context()
self.report_stats()
log.standard_status()
return log.errors()
def verify_files(self, files):
"""Check `files` against the CRDS server database to ensure integrity and check reject status."""
basenames = [os.path.basename(file) for file in files]
try:
log.verbose("Downloading verification info for", len(basenames), "files.", verbosity=10)
infos = api.get_file_info_map(observatory=self.observatory, files=basenames,
fields=["size","rejected","blacklisted","state","sha1sum"])
except Exception as exc:
log.error("Failed getting file info. CACHE VERIFICATION FAILED. Exception: ", repr(str(exc)))
return
bytes_so_far = 0
total_bytes = api.get_total_bytes(infos)
for nth_file, file in enumerate(files):
bfile = os.path.basename(file)
if infos[bfile] == "NOT FOUND":
log.error("CRDS has no record of file", repr(bfile))
else:
self.verify_file(file, infos[bfile], bytes_so_far, total_bytes, nth_file, len(files))
bytes_so_far += int(infos[bfile]["size"])
def verify_files(self, files):
"""Check `files` against the CRDS server database to ensure integrity and check reject status."""
basenames = [os.path.basename(file) for file in files]
try:
log.verbose("Downloading verification info for", len(basenames), "files.", verbosity=10)
infos = api.get_file_info_map(observatory=self.observatory, files=basenames,
fields=["size","rejected","blacklisted","state","sha1sum"])
except Exception as exc:
log.error("Failed getting file info. CACHE VERIFICATION FAILED. Exception: ", repr(str(exc)))
return
bytes_so_far = 0
total_bytes = api.get_total_bytes(infos)
for nth_file, file in enumerate(files):
bfile = os.path.basename(file)
if infos[bfile] == "NOT FOUND":
log.error("CRDS has no record of file", repr(bfile))
else:
self.verify_file(file, infos[bfile], bytes_so_far, total_bytes, nth_file, len(files))
bytes_so_far += int(infos[bfile]["size"])
def print_new_files(self):
"""Print the references or mappings which are in the second (new) context and not
the firtst (old) context.
"""
if not rmap.is_mapping(self.old_file) or not rmap.is_mapping(self.new_file):
log.error("--print-new-files really only works for mapping differences.")
return -1
old = rmap.get_cached_mapping(self.old_file)
new = rmap.get_cached_mapping(self.new_file)
old_mappings = set(old.mapping_names())
new_mappings = set(new.mapping_names())
old_references = set(old.reference_names())
new_references = set(new.reference_names())
status = 0
for name in sorted(new_mappings - old_mappings):
print(name)
status = 1
for name in sorted(new_references - old_references):
print(name)
status = 1
return status
def validate_expansions(self, pmap):
for instrument in pmap.selections:
if instrument not in self:
log.verbose("Instrument", repr(instrument),
"has no substitutions.")
continue
imap = pmap.get_imap(instrument)
valid_values = imap.get_valid_values_map(condition=True,
remove_special=False)
for parameter, values in self[instrument].get_expansion_values(
).items():
for value in values:
if parameter not in valid_values or not valid_values[
parameter]:
log.verbose("For", repr(instrument), "parameter",
repr(parameter), "with value", repr(value),
"is unchecked.")
continue
if value not in valid_values[parameter]:
log.error("For", repr(instrument), "parameter",
repr(parameter), "value", repr(value),
"is not in", valid_values[parameter])
def download_files(self, downloads, localpaths):
"""Serial file-by-file download."""
self.info_map = get_file_info_map(
self.observatory, downloads, ["size", "rejected", "blacklisted", "state", "sha1sum", "instrument"])
if config.writable_cache_or_verbose("Readonly cache, skipping download of (first 5):", repr(downloads[:5]), verbosity=70):
bytes_so_far = 0
total_files = len(downloads)
total_bytes = get_total_bytes(self.info_map)
for nth_file, name in enumerate(downloads):
try:
if "NOT FOUND" in self.info_map[name]:
raise CrdsDownloadError("file is not known to CRDS server.")
bytes, path = self.catalog_file_size(name), localpaths[name]
log.info(file_progress("Fetching", name, path, bytes, bytes_so_far, total_bytes, nth_file, total_files))
self.download(name, path)
bytes_so_far += os.stat(path).st_size
except Exception as exc:
if self.raise_exceptions:
raise
else:
log.error("Failure downloading file", repr(name), ":", str(exc))
return bytes_so_far
return 0
def print_new_files(self):
"""Print the references or mappings which are in the second (new) context and not
the firtst (old) context.
"""
if not rmap.is_mapping(self.old_file) or not rmap.is_mapping(
self.new_file):
log.error(
"--print-new-files really only works for mapping differences.")
return -1
old = crds.get_pickled_mapping(self.old_file) # reviewed
new = crds.get_pickled_mapping(self.new_file) # reviewed
old_mappings = set(old.mapping_names())
new_mappings = set(new.mapping_names())
old_references = set(old.reference_names())
new_references = set(new.reference_names())
status = 0
for name in sorted(new_mappings - old_mappings):
print(name)
status = 1
for name in sorted(new_references - old_references):
print(name)
status = 1
return status
def handle_error(self, message):
"""Generic "error" handler issues an error message from remote process and
continues monitoring.
"""
log.error(self.format_remote(message.data))
return False
def main(self):
"""Synchronize files."""
if self.args.dry_run:
self.args.readonly_cache = True
if self.args.repair_files:
self.args.check_files = True
if self.args.output_dir:
os.environ["CRDS_MAPPATH_SINGLE"] = self.args.output_dir
os.environ["CRDS_REFPATH_SINGLE"] = self.args.output_dir
os.environ["CRDS_CFGPATH_SINGLE"] = self.args.output_dir
os.environ["CRDS_PICKLEPATH_SINGLE"] = self.args.output_dir
if self.args.clear_pickles or self.args.ignore_cache or self.args.repair_files:
self.clear_pickles(self.contexts)
if self.args.organize: # do this before syncing anything under the current mode.
self.organize_references(self.args.organize)
self.require_server_connection()
if self.readonly_cache and self.args.verify_context_change:
log.error("--readonly-cache and --verify-context-change are incompatible, a readonly cache cannot change.")
if self.args.files:
self.sync_explicit_files()
verify_file_list = self.files
elif self.args.fetch_sqlite_db:
self.fetch_sqlite_db()
elif self.contexts:
active_mappings = self.get_context_mappings()
verify_file_list = active_mappings
if self.args.fetch_references or self.args.purge_references:
if self.args.dataset_files or self.args.dataset_ids:
active_references = self.sync_datasets()
else:
active_references = self.get_context_references()
active_references = sorted(set(active_references + self.get_conjugates(active_references)))
if self.args.fetch_references:
self.fetch_references(active_references)
verify_file_list += active_references
if self.args.purge_references:
self.purge_references(active_references)
if self.args.purge_mappings:
self.purge_mappings()
else:
log.error("Define --all, --contexts, --last, --range, --files, or --fetch-sqlite-db to sync.")
sys.exit(-1)
if self.args.check_files or self.args.check_sha1sum or self.args.repair_files:
self.verify_files(verify_file_list)
if self.args.save_pickles:
self.pickle_contexts(self.contexts)
if self.args.verify_context_change:
old_context = heavy_client.load_server_info(self.observatory).operational_context
heavy_client.update_config_info(self.observatory)
if self.args.verify_context_change:
self.verify_context_change(old_context)
if self.args.push_context:
self.push_context()
self.report_stats()
log.standard_status()
return log.errors()
def handle_error(self, message):
"""Generic "error" handler issues an error message from remote process and
continues monitoring.
"""
log.error(self.format_remote(message.data))
return False
def handle_fail(self, message):
"""Generic "fail" handler reports on remote process fatal error / failure
and issues an error() message, then stops monitoring /exits.
"""
log.error(self.format_remote("Processing failed:", message.data))
return message.data["result"]
def check_naming_consistency(checked_instrument=None,
exhaustive_mapping_check=False):
"""Dev function to compare the properties returned by name decomposition
to the properties determined by file contents and make sure they're the same.
Also checks rmap membership.
>> from crds.tests import test_config
>> old_config = test_config.setup()
>> check_naming_consistency("acs")
>> check_naming_consistency("cos")
>> check_naming_consistency("nicmos")
>> check_naming_consistency("stis")
>> check_naming_consistency("wfc3")
>> check_naming_consistency("wfpc2")
>> test_config.cleanup(old_config)
"""
from crds import certify
for ref in rmap.list_references("*", observatory="hst", full_path=True):
with log.error_on_exception("Failed processing:", repr(ref)):
_path, _observ, instrument, filekind, _serial, _ext = ref_properties_from_cdbs_path(
ref)
if checked_instrument is not None and instrument != checked_instrument:
continue
if data_file.is_geis_data(ref):
if os.path.exists(data_file.get_conjugate(ref)):
continue
else:
log.warning("No GEIS header for", repr(ref))
log.verbose("Processing:", instrument, filekind, ref)
_path2, _observ2, instrument2, filekind2, _serial2, _ext2 = ref_properties_from_header(
ref)
if instrument != instrument2:
log.error("Inconsistent instruments", repr(instrument), "vs.",
repr(instrument2), "for", repr(ref))
if filekind != filekind2:
log.error("Inconsistent filekinds", repr(filekind), "vs.",
repr(filekind2), "for", repr(ref))
for pmap_name in reversed(
sorted(rmap.list_mappings("*.pmap", observatory="hst"))):
pmap = crds.get_cached_mapping(pmap_name)
r = certify.find_governing_rmap(pmap_name, ref)
if not r:
continue
if r.instrument != instrument:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with name derived instrument",
repr(instrument), "for", repr(ref), "in",
repr(pmap_name))
if r.filekind != filekind:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with name derived filekind",
repr(filekind), "for", repr(ref), "in",
repr(pmap_name))
if r.instrument != instrument2:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with content derived instrument",
repr(instrument2), "for", repr(ref), "in",
repr(pmap_name))
if r.filekind != filekind2:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with content derived filekind",
repr(filekind2), "for", repr(ref), "in",
repr(pmap_name))
if not exhaustive_mapping_check:
break
else:
log.error("Orphan reference", repr(ref),
"not found under any context.")
def check_naming_consistency(checked_instrument=None, exhaustive_mapping_check=False):
"""Dev function to compare the properties returned by name decomposition
to the properties determined by file contents and make sure they're the same.
Also checks rmap membership.
>> from crds.tests import test_config
>> old_config = test_config.setup()
>> check_naming_consistency("acs")
>> check_naming_consistency("cos")
>> check_naming_consistency("nicmos")
>> check_naming_consistency("stis")
>> check_naming_consistency("wfc3")
>> check_naming_consistency("wfpc2")
>> test_config.cleanup(old_config)
"""
from crds import certify
for ref in rmap.list_references("*", observatory="hst", full_path=True):
with log.error_on_exception("Failed processing:", repr(ref)):
_path, _observ, instrument, filekind, _serial, _ext = ref_properties_from_cdbs_path(ref)
if checked_instrument is not None and instrument != checked_instrument:
continue
if data_file.is_geis_data(ref):
if os.path.exists(data_file.get_conjugate(ref)):
continue
else:
log.warning("No GEIS header for", repr(ref))
log.verbose("Processing:", instrument, filekind, ref)
_path2, _observ2, instrument2, filekind2, _serial2, _ext2 = ref_properties_from_header(ref)
if instrument != instrument2:
log.error("Inconsistent instruments", repr(instrument), "vs.", repr(instrument2),
"for", repr(ref))
if filekind != filekind2:
log.error("Inconsistent filekinds", repr(filekind), "vs.", repr(filekind2),
"for", repr(ref))
for pmap_name in reversed(sorted(rmap.list_mappings("*.pmap", observatory="hst"))):
pmap = crds.get_cached_mapping(pmap_name)
r = certify.find_governing_rmap(pmap_name, ref)
if not r:
continue
if r.instrument != instrument:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with name derived instrument", repr(instrument), "for", repr(ref), "in", repr(pmap_name))
if r.filekind != filekind:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with name derived filekind", repr(filekind), "for", repr(ref), "in", repr(pmap_name))
if r.instrument != instrument2:
log.error("Rmap instrument", repr(r.instrument),
"inconsistent with content derived instrument", repr(instrument2), "for", repr(ref), "in", repr(pmap_name))
if r.filekind != filekind2:
log.error("Rmap filekind", repr(r.filekind),
"inconsistent with content derived filekind", repr(filekind2), "for", repr(ref), "in", repr(pmap_name))
if not exhaustive_mapping_check:
break
else:
log.error("Orphan reference", repr(ref), "not found under any context.")
def handle_fail(self, message):
"""Generic "fail" handler reports on remote process fatal error / failure
and issues an error() message, then stops monitoring /exits.
"""
log.error(self.format_remote("Processing failed:", message.data))
return message.data["result"]
