def main(self):
"""Perform the high level sequence of tasks needed to download and
organize a version of pysynphot files under the specified directory.
"""
self.deferred_init()
# Blow away CRDS cache prior to syncing
if not self.args.keep_crds:
self.rmdir("crds")
self.crds_download()
syn_name_map = self.pysyn_cdbs_from_syn_tables()
# Blow away comp and mtab to remove old versions of links
if not self.args.keep_synphot:
self.rmdir("comp")
self.rmdir("mtab")
self.cross_link_cdbs_paths(syn_name_map)
# Blow away CRDS cache leaving only synphot organization.
# in the final product.
if not self.args.keep_crds:
self.rmdir("crds")
log.standard_status()
return log.errors()
def main(self):
"""Perform the high level sequence of tasks needed to download and
organize a version of pysynphot files under the specified directory.
"""
self.deferred_init()
# Blow away CRDS cache prior to syncing
if not self.args.keep_crds:
self.rmdir("crds")
self.crds_download()
syn_name_map = self.pysyn_cdbs_from_syn_tables()
# Blow away comp and mtab to remove old versions of links
if not self.args.keep_synphot:
self.rmdir("comp")
self.rmdir("mtab")
self.cross_link_cdbs_paths(syn_name_map)
# Blow away CRDS cache leaving only synphot organization.
# in the final product.
if not self.args.keep_crds:
self.rmdir("crds")
log.standard_status()
return log.errors()
def main(self):
"""Check files for availability from the archive."""
self.require_server_connection()
log.info("Mapping URL:", repr(self.mapping_url))
log.info("Reference URL:", repr(self.reference_url))
stats = utils.TimingStats()
self.init_files(self.files)
for filename in self.files:
self.verify_archive_file(filename)
stats.increment("files")
self.print_files()
stats.report_stat("files")
log.standard_status()
def log_all_ids(self, effects, ids):
"""PLUGIN: Summary output after all contexts processed."""
if self.args.quiet:
return
if not effects:
log.info("No new results are available.")
else:
if not ids:
log.info("No ids were affected.")
print("#" * 100, file=sys.stderr)
log.info("Contributing context switches =", len(effects))
log.info("Total products affected =", len(ids))
log.standard_status()
def log_all_ids(self, effects, ids):
"""PLUGIN: Summary output after all contexts processed."""
if self.args.quiet:
return
if not effects:
log.info("No new results are available.")
else:
if not ids:
log.info("No ids were affected.")
print("#"*100, file=sys.stderr)
log.info("Contributing context switches =", len(effects))
log.info("Total products affected =", len(ids))
log.standard_status()
def main(self):
"""Synchronize files."""
# clear any mutliprocessing locks associated with the CRDS cache.
if self.args.clear_locks:
crds_cache_locking.clear_cache_locks()
return log.errors()
self.handle_misc_switches() # simple side effects only
# if explicitly requested, or the cache is suspect or being ignored, clear
# cached context pickles.
if self.args.clear_pickles or self.args.ignore_cache or self.args.repair_files:
self.clear_pickles()
# utility to change cache structure, e.g. add instrument subdirs.
# do this before syncing anything under the current mode.
if self.args.organize:
self.organize_references(self.args.organize)
# fetching and verifying files both require a server connection.
self.require_server_connection()
# primary sync'ing occurs here, both mappings and references as well as odd-ball
# server sqlite3 database download.
verify_file_list = self.file_transfers()
# verification is relative to sync'ed files, and can include file replacement if
# defects are found.
if (self.args.check_files or self.args.check_sha1sum or self.args.repair_files or
self.args.purge_blacklisted or self.args.purge_rejected):
self.verify_files(verify_file_list)
# context pickles should only be (re)generated after mappings are fully sync'ed and verified
if self.args.save_pickles:
self.pickle_contexts(self.contexts)
# update CRDS cache config area, including stored version of operational context.
# implement pipeline support functions of context update verify and echo
# If --output-dir was specified, do not update cache config.
if self.args.output_dir:
log.verbose_warning("Used --output-dir, skipping cache server_config update including default context and bad files.")
if config.writable_cache_or_verbose("skipping removal of ref_cache_subdir_mode file."):
os.remove(config.get_crds_ref_subdir_file(self.observatory))
else:
self.update_context()
self.report_stats()
log.standard_status()
return log.errors()
def main(self):
if self.args.best_effort:
config.PASS_INVALID_VALUES.set(True) # JWST SSB cal code data model
config.ALLOW_BAD_USEAFTER.set(True) # Don't fail for bad USEAFTER values
config.ALLOW_SCHEMA_VIOLATIONS.set(True) # Don't fail for data model bad value errors
config.ALLOW_BAD_PARKEY_VALUES.set(True) # Don't fail for values which don't pass DM + .tpn checking
if self.args.rmaps: # clean up dead lines from file lists
self.args.rmaps = [ self.resolve_context(mapping) for mapping in self.args.rmaps if mapping.strip() ]
if self.args.references:
self.args.references = [self.locate_file(reference) for reference in self.args.references]
with log.error_on_exception("Refactoring operation FAILED"):
if self.args.command == "insert_reference":
if self.args.old_rmap:
old_rmap, new_rmap = self.resolve_context(self.args.old_rmap), self.resolve_context(self.args.new_rmap)
rmap_insert_references(old_rmap, new_rmap, self.args.references)
else:
self.insert_references() # figure it all out relative to --source-context
elif self.args.command == "delete_reference":
old_rmap, new_rmap = self.resolve_context(self.args.old_rmap), self.resolve_context(self.args.new_rmap)
rmap_delete_references(old_rmap, new_rmap, self.args.references)
elif self.args.command == "del_header":
self.del_header_key()
elif self.args.command == "set_header":
self.set_header_key()
elif self.args.command == "del_parameter":
self.del_parameter()
elif self.args.command == "set_parkey":
self.set_parkey()
elif self.args.command == "replace_text":
self.replace_text()
elif self.args.command == "set_substitution":
self.set_substitution()
elif self.args.command == "cat":
self.cat()
elif self.args.command == "add_useafter":
self.add_useafter()
elif self.args.command == "diff_rmaps":
self.diff_rmaps()
elif self.args.command == "certify_rmaps":
self.certify_rmaps()
else:
raise ValueError("Unknown refactoring command: " + repr(self.args.command))
log.standard_status()
return log.errors()
def main(self):
"""Main control flow of submission directory and request manifest creation."""
log.divider("setting up", char="=")
self.require_server_connection()
self.finish_parameters()
if self.args.certify_files:
self.certify_local_files()
if self.args.logout:
return self.logout()
self.login()
if self.args.wipe_existing_files:
self.wipe_files()
self.jpoll_key = self.jpoll_open_channel()
if self.args.submission_kind == "batch":
submit_future = self.batch_submit_references()
elif self.args.submission_kind == "certify":
submit_future = self.certify_files()
elif self.args.submission_kind == "references":
submit_future = self.submit_references()
elif self.args.submission_kind == "mappings":
submit_future = self.submit_mappings()
if self.args.monitor_processing:
monitor_future = self.monitor()
if self.args.wait_for_completion:
self.submission_complete(submit_future)
if self.args.monitor_processing:
monitor = self.monitor_complete(monitor_future)
if monitor.exit_status == 0:
self._ready_url = monitor.result
log.standard_status()
self._error_count = log.errors()
self._warning_count = log.warnings()
return log.errors()
def main(self):
"""Synchronize files."""
# clear any mutliprocessing locks associated with the CRDS cache.
if self.args.clear_locks:
crds_cache_locking.clear_cache_locks()
return log.errors()
self.handle_misc_switches() # simple side effects only
# if explicitly requested, or the cache is suspect or being ignored, clear
# cached context pickles.
if self.args.clear_pickles or self.args.ignore_cache or self.args.repair_files:
self.clear_pickles()
# utility to change cache structure, e.g. add instrument subdirs.
# do this before syncing anything under the current mode.
if self.args.organize:
self.organize_references(self.args.organize)
# fetching and verifying files both require a server connection.
self.require_server_connection()
# primary sync'ing occurs here, both mappings and references as well as odd-ball
# server sqlite3 database download.
verify_file_list = self.file_transfers()
# verification is relative to sync'ed files, and can include file replacement if
# defects are found.
if self.args.check_files or self.args.check_sha1sum or self.args.repair_files:
self.verify_files(verify_file_list)
# context pickles should only be (re)generated after mappings are fully sync'ed and verified
if self.args.save_pickles:
self.pickle_contexts(self.contexts)
# update CRDS cache config area, including stored version of operational context.
# implement pipeline support functions of context update verify and echo
# If explicit files were specified, do not update cache config.
if self.args.files and self.args.output_dir:
log.verbose_warning("Used explicit --files list and --output-dir, skipping cache server_config update including default context and bad files.")
else:
self.update_context()
self.report_stats()
log.standard_status()
return log.errors()
def main(self):
with log.error_on_exception("Refactoring operation FAILED"):
if self.args.command == "insert":
rmap_insert_references(self.old_rmap, self.new_rmap,
self.ref_paths)
elif self.args.command == "delete":
rmap_delete_references(self.old_rmap, self.new_rmap,
self.ref_paths)
elif self.args.command == "set_header":
field, setting = self.args.references[0], " ".join(
self.args.references[1:])
set_header_value(self.old_rmap, self.new_rmap, field, setting)
elif self.args.command == "del_header":
field = self.args.references[0]
del_header_value(self.old_rmap, self.new_rmap, field)
else:
raise ValueError("Unknown refactoring command: " +
repr(self.args.command))
log.standard_status()
return log.errors()
def main(self):
"""Main control flow of submission directory and request manifest creation."""
log.divider("setting up", char="=")
self.require_server_connection()
self.finish_parameters()
if self.args.logout:
return self.logout()
self.submission = self.create_submission()
self.login()
if self.args.wipe_existing_files:
self.wipe_files()
self.jpoll_key = self.jpoll_open_channel()
if self.args.submission_kind == "batch":
submit_future = self.batch_submit_references()
elif self.args.submission_kind == "certify":
submit_future = self.certify_files()
elif self.args.submission_kind == "references":
submit_future = self.submit_references()
elif self.args.submission_kind == "mappings":
submit_future = self.submit_mappings()
if self.args.monitor_processing:
monitor_future = self.monitor()
if self.args.wait_for_completion:
self.submission_complete(submit_future)
if self.args.monitor_processing:
self.monitor_complete(monitor_future)
log.standard_status()
return log.errors()
def main(self):
if self.args.best_effort:
config.PASS_INVALID_VALUES.set(
True) # JWST SSB cal code data model
config.ALLOW_BAD_USEAFTER.set(
True) # Don't fail for bad USEAFTER values
config.ALLOW_SCHEMA_VIOLATIONS.set(
True) # Don't fail for data model bad value errors
config.ALLOW_BAD_PARKEY_VALUES.set(
True
) # Don't fail for values which don't pass DM + .tpn checking
if self.args.rmaps: # clean up dead lines from file lists
self.args.rmaps = [
self.resolve_context(mapping) for mapping in self.args.rmaps
if mapping.strip()
]
if self.args.references:
self.args.references = [
self.locate_file(reference)
for reference in self.args.references
]
with log.error_on_exception("Refactoring operation FAILED"):
if self.args.command == "insert_reference":
if self.args.old_rmap:
old_rmap, new_rmap = self.resolve_context(
self.args.old_rmap), self.resolve_context(
self.args.new_rmap)
rmap_insert_references(old_rmap, new_rmap,
self.args.references)
else:
self.insert_references(
) # figure it all out relative to --source-context
elif self.args.command == "delete_reference":
old_rmap, new_rmap = self.resolve_context(
self.args.old_rmap), self.resolve_context(
self.args.new_rmap)
rmap_delete_references(old_rmap, new_rmap,
self.args.references)
elif self.args.command == "del_header":
self.del_header_key()
elif self.args.command == "set_header":
self.set_header_key()
elif self.args.command == "del_parameter":
self.del_parameter()
elif self.args.command == "set_parkey":
self.set_parkey()
elif self.args.command == "replace_text":
self.replace_text()
elif self.args.command == "set_substitution":
self.set_substitution()
elif self.args.command == "cat":
self.cat()
elif self.args.command == "add_useafter":
self.add_useafter()
elif self.args.command == "diff_rmaps":
self.diff_rmaps()
elif self.args.command == "certify_rmaps":
self.certify_rmaps()
else:
raise ValueError("Unknown refactoring command: " +
repr(self.args.command))
log.standard_status()
return log.errors()