def _main(self):
"""_main() completes any complex generic setup, like determining contexts, and then
calls self.main() which does the real work of the script. _main() defines the full
call tree of code which is run inside the profiler or debugger.
"""
self.contexts = self.determine_contexts()
self._exit_status = self.main()
self.report_stats() # here if not called already
if self.print_status:
log.standard_status()
return self._exit_status
def log_all_ids(self, effects, ids):
"""PLUGIN: Summary output after all contexts processed."""
if self.args.quiet:
return
if not effects:
log.info("No new results are available.")
else:
print("#"*100, file=sys.stderr)
log.info("Contributing context switches =", len(effects))
log.info("Total products affected =", len(ids))
log.standard_status()
def log_all_ids(self, effects, ids):
"""PLUGIN: Summary output after all contexts processed."""
if self.args.quiet:
return
if not effects:
log.info("No new results are available.")
else:
print("#" * 100, file=sys.stderr)
log.info("Contributing context switches =", len(effects))
log.info("Total products affected =", len(ids))
log.standard_status()
def _main(self):
"""_main() completes any complex generic setup, like determining contexts, and then
calls self.main() which does the real work of the script. _main() defines the full
call tree of code which is run inside the profiler or debugger.
"""
self.contexts = self.determine_contexts()
self._exit_status = self.main()
self.report_stats() # here if not called already
if self.print_status:
log.standard_status()
return self._exit_status
def main(self):
"""Check files for availability from the archive."""
self.require_server_connection()
log.info("Mapping URL:", repr(self.mapping_url))
log.info("Reference URL:", repr(self.reference_url))
stats = utils.TimingStats()
self.file_info = api.get_file_info_map(self.observatory, self.files, fields=["size", "sha1sum"])
for filename in self.files:
self.verify_archive_file(filename)
stats.increment("files")
self.print_files()
stats.report_stat("files")
log.standard_status()
def main(self):
if self.args.best_effort:
os.environ["PASS_INVALID_VALUES"] = "1" # JWST SSB cal code data model
os.environ["CRDS_ALLOW_BAD_USEAFTER"] = "1" # Don't fail for bad USEAFTER values
os.environ["CRDS_ALLOW_SCHEMA_VIOLATIONS"] = "1" # Don't fail for data model bad value errors
os.environ["CRDS_ALLOW_BAD_PARKEY_VALUES"] = "1" # Don't fail for values which don't pass DM + .tpn checking
if self.args.rmaps: # clean up dead lines from file lists
self.args.rmaps = [ self.resolve_context(mapping) for mapping in self.args.rmaps if mapping.strip() ]
if self.args.references:
self.args.references = [self.locate_file(reference) for reference in self.args.references]
with log.error_on_exception("Refactoring operation FAILED"):
if self.args.command == "insert_reference":
if self.args.old_rmap:
old_rmap, new_rmap = self.resolve_context(self.args.old_rmap), self.resolve_context(self.args.new_rmap)
rmap_insert_references(old_rmap, new_rmap, self.args.references)
else:
self.insert_references() # figure it all out relative to --source-context
elif self.args.command == "delete_reference":
old_rmap, new_rmap = self.resolve_context(self.args.old_rmap), self.resolve_context(self.args.new_rmap)
rmap_delete_references(old_rmap, new_rmap, self.args.references)
elif self.args.command == "del_header":
self.del_header_key()
elif self.args.command == "set_header":
self.set_header_key()
elif self.args.command == "del_parameter":
self.del_parameter()
elif self.args.command == "set_parkey":
self.set_parkey()
elif self.args.command == "replace_text":
self.replace_text()
elif self.args.command == "set_substitution":
self.set_substitution()
elif self.args.command == "cat":
self.cat()
elif self.args.command == "add_useafter":
self.add_useafter()
elif self.args.command == "diff_rmaps":
self.diff_rmaps()
elif self.args.command == "certify_rmaps":
self.certify_rmaps()
else:
raise ValueError("Unknown refactoring command: " + repr(self.args.command))
log.standard_status()
return log.errors()
def main(self):
"""Check files for availability from the archive."""
self.require_server_connection()
log.info("Mapping URL:", repr(self.mapping_url))
log.info("Reference URL:", repr(self.reference_url))
stats = utils.TimingStats()
self.file_info = api.get_file_info_map(self.observatory,
self.files,
fields=["size", "sha1sum"])
for filename in self.files:
self.verify_archive_file(filename)
stats.increment("files")
self.print_files()
stats.report_stat("files")
log.standard_status()
def main(self):
"""Synchronize files."""
if self.args.dry_run:
self.args.readonly_cache = True
if self.args.repair_files:
self.args.check_files = True
if self.args.organize: # do this before syncing anything under the current mode.
self.organize_references(self.args.organize)
self.require_server_connection()
if self.readonly_cache and self.args.verify_context_change:
log.error("--readonly-cache and --verify-context-change are incompatible, a readonly cache cannot change.")
if self.args.files:
self.sync_explicit_files()
verify_file_list = self.files
elif self.args.fetch_sqlite_db:
self.fetch_sqlite_db()
elif self.contexts:
active_mappings = self.get_context_mappings()
verify_file_list = active_mappings
if self.args.fetch_references or self.args.purge_references:
if self.args.dataset_files or self.args.dataset_ids:
active_references = self.sync_datasets()
else:
active_references = self.get_context_references()
active_references = sorted(set(active_references + self.get_conjugates(active_references)))
if self.args.fetch_references:
self.fetch_references(active_references)
verify_file_list += active_references
if self.args.purge_references:
self.purge_references(active_references)
if self.args.purge_mappings:
self.purge_mappings()
else:
log.error("Define --all, --contexts, --last, --range, --files, or --fetch-sqlite-db to sync.")
sys.exit(-1)
if self.args.check_files or self.args.check_sha1sum or self.args.repair_files:
self.verify_files(verify_file_list)
if self.args.verify_context_change:
old_context = heavy_client.load_server_info(self.observatory).operational_context
heavy_client.update_config_info(self.observatory)
if self.args.verify_context_change:
self.verify_context_change(old_context)
if self.args.push_context:
self.push_context()
self.report_stats()
log.standard_status()
return log.errors()
def main(self):
if self.args.rmaps: # clean up dead lines from file lists
self.args.rmaps = [ self.resolve_context(mapping) for mapping in self.args.rmaps if mapping.strip() ]
if self.args.references:
self.args.references = [self.locate_file(reference) for reference in self.args.references]
with log.error_on_exception("Refactoring operation FAILED"):
if self.args.command == "insert_reference":
if self.args.old_rmap:
old_rmap, new_rmap = self.resolve_context(self.args.old_rmap), self.resolve_context(self.args.new_rmap)
rmap_insert_references(old_rmap, new_rmap, self.args.references)
else:
self.insert_references() # figure it all out relative to --source-context
elif self.args.command == "delete_reference":
old_rmap, new_rmap = self.resolve_context(self.args.old_rmap), self.resolve_context(self.args.new_rmap)
rmap_delete_references(old_rmap, new_rmap, self.args.references)
elif self.args.command == "del_header":
self.del_header_key()
elif self.args.command == "set_header":
self.set_header_key()
elif self.args.command == "del_parameter":
self.del_parameter()
elif self.args.command == "set_parkey":
self.set_parkey()
elif self.args.command == "replace_text":
self.replace_text()
elif self.args.command == "set_substitution":
self.set_substitution()
elif self.args.command == "cat":
self.cat()
elif self.args.command == "add_useafter":
self.add_useafter()
elif self.args.command == "diff_rmaps":
self.diff_rmaps()
elif self.args.command == "certify_rmaps":
self.certify_rmaps()
else:
raise ValueError("Unknown refactoring command: " + repr(self.args.command))
log.standard_status()
return log.errors()
def main(self):
"""Main control flow of submission directory and request manifest creation."""
log.divider("setting up", char="=")
self.require_server_connection()
self.finish_parameters()
if self.args.logout:
return self.logout()
self.submission = self.create_submission()
self.login()
if self.args.wipe_existing_files:
self.wipe_files()
self.jpoll_key = self.jpoll_open_channel()
if self.args.submission_kind == "batch":
submit_future = self.batch_submit_references()
elif self.args.submission_kind == "certify":
submit_future = self.certify_files()
elif self.args.submission_kind == "references":
submit_future = self.submit_references()
elif self.args.submission_kind == "mappings":
submit_future = self.submit_mappings()
if self.args.monitor_processing:
monitor_future = self.monitor()
if self.args.wait_for_completion:
self.submission_complete(submit_future)
if self.args.monitor_processing:
self.monitor_complete(monitor_future)
log.standard_status()
return log.errors()
def main(self):
"""Main control flow of submission directory and request manifest creation."""
log.divider("setting up", char="=")
self.require_server_connection()
self.finish_parameters()
if self.args.logout:
return self.logout()
self.submission = self.create_submission()
self.login()
if self.args.wipe_existing_files:
self.wipe_files()
self.jpoll_key = self.jpoll_open_channel()
if self.args.submission_kind == "batch":
submit_future = self.batch_submit_references()
elif self.args.submission_kind == "certify":
submit_future = self.certify_files()
elif self.args.submission_kind == "references":
submit_future = self.submit_references()
elif self.args.submission_kind == "mappings":
submit_future = self.submit_mappings()
if self.args.monitor_processing:
monitor_future = self.monitor()
if self.args.wait_for_completion:
self.submission_complete(submit_future)
if self.args.monitor_processing:
self.monitor_complete(monitor_future)
log.standard_status()
return log.errors()
"""
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
import sys
from crds import rmap, log
def update_checksum(file_):
"""Rewrite the checksum of a single mapping, in place."""
mapping = rmap.Mapping.from_file(file_, ignore_checksum=True)
mapping.write()
def update_checksums(files):
"""Rewrite the mapping checksums/hashes/sha1sums in all `files`."""
for file_ in files:
log.info("Updating checksum for", file_)
with log.error_on_exception("Failed updating checksum for",
repr(file_)):
update_checksum(file_)
if __name__ == "__main__":
import crds
crds.handle_version()
update_checksums(sys.argv[1:])
log.standard_status()
def main(self):
"""Synchronize files."""
if self.args.dry_run:
self.args.readonly_cache = True
if self.args.repair_files:
self.args.check_files = True
if self.args.output_dir:
os.environ["CRDS_MAPPATH_SINGLE"] = self.args.output_dir
os.environ["CRDS_REFPATH_SINGLE"] = self.args.output_dir
os.environ["CRDS_CFGPATH_SINGLE"] = self.args.output_dir
os.environ["CRDS_PICKLEPATH_SINGLE"] = self.args.output_dir
if self.args.clear_pickles or self.args.ignore_cache or self.args.repair_files:
self.clear_pickles(self.contexts)
if self.args.organize: # do this before syncing anything under the current mode.
self.organize_references(self.args.organize)
self.require_server_connection()
if self.readonly_cache and self.args.verify_context_change:
log.error("--readonly-cache and --verify-context-change are incompatible, a readonly cache cannot change.")
if self.args.files:
self.sync_explicit_files()
verify_file_list = self.files
elif self.args.fetch_sqlite_db:
self.fetch_sqlite_db()
elif self.contexts:
active_mappings = self.get_context_mappings()
verify_file_list = active_mappings
if self.args.fetch_references or self.args.purge_references:
if self.args.dataset_files or self.args.dataset_ids:
active_references = self.sync_datasets()
else:
active_references = self.get_context_references()
active_references = sorted(set(active_references + self.get_conjugates(active_references)))
if self.args.fetch_references:
self.fetch_references(active_references)
verify_file_list += active_references
if self.args.purge_references:
self.purge_references(active_references)
if self.args.purge_mappings:
self.purge_mappings()
else:
log.error("Define --all, --contexts, --last, --range, --files, or --fetch-sqlite-db to sync.")
sys.exit(-1)
if self.args.check_files or self.args.check_sha1sum or self.args.repair_files:
self.verify_files(verify_file_list)
if self.args.save_pickles:
self.pickle_contexts(self.contexts)
if self.args.verify_context_change:
old_context = heavy_client.load_server_info(self.observatory).operational_context
heavy_client.update_config_info(self.observatory)
if self.args.verify_context_change:
self.verify_context_change(old_context)
if self.args.push_context:
self.push_context()
self.report_stats()
log.standard_status()
return log.errors()
% python -m crds.checksum hst.pmap
will rewrite the sha1sum of hst.pmap.
"""
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
import sys
from crds import rmap, log
def update_checksum(file_):
"""Rewrite the checksum of a single mapping, in place."""
mapping = rmap.Mapping.from_file(file_, ignore_checksum=True)
mapping.write()
def update_checksums(files):
"""Rewrite the mapping checksums/hashes/sha1sums in all `files`."""
for file_ in files:
log.info("Updating checksum for", file_)
with log.error_on_exception("Failed updating checksum for", repr(file_)):
update_checksum(file_)
if __name__ == "__main__":
import crds
crds.handle_version()
update_checksums(sys.argv[1:])
log.standard_status()