def main(self): """Perform the high level sequence of tasks needed to download and organize a version of pysynphot files under the specified directory. """ self.deferred_init() # Blow away CRDS cache prior to syncing if not self.args.keep_crds: self.rmdir("crds") self.crds_download() syn_name_map = self.pysyn_cdbs_from_syn_tables() # Blow away comp and mtab to remove old versions of links if not self.args.keep_synphot: self.rmdir("comp") self.rmdir("mtab") self.cross_link_cdbs_paths(syn_name_map) # Blow away CRDS cache leaving only synphot organization. # in the final product. if not self.args.keep_crds: self.rmdir("crds") log.standard_status() return log.errors()
def main(self): """Check files for availability from the archive.""" self.require_server_connection() log.info("Mapping URL:", repr(self.mapping_url)) log.info("Reference URL:", repr(self.reference_url)) stats = utils.TimingStats() self.init_files(self.files) for filename in self.files: self.verify_archive_file(filename) stats.increment("files") self.print_files() stats.report_stat("files") log.standard_status()
def log_all_ids(self, effects, ids): """PLUGIN: Summary output after all contexts processed.""" if self.args.quiet: return if not effects: log.info("No new results are available.") else: if not ids: log.info("No ids were affected.") print("#" * 100, file=sys.stderr) log.info("Contributing context switches =", len(effects)) log.info("Total products affected =", len(ids)) log.standard_status()
def log_all_ids(self, effects, ids): """PLUGIN: Summary output after all contexts processed.""" if self.args.quiet: return if not effects: log.info("No new results are available.") else: if not ids: log.info("No ids were affected.") print("#"*100, file=sys.stderr) log.info("Contributing context switches =", len(effects)) log.info("Total products affected =", len(ids)) log.standard_status()
def main(self): """Synchronize files.""" # clear any mutliprocessing locks associated with the CRDS cache. if self.args.clear_locks: crds_cache_locking.clear_cache_locks() return log.errors() self.handle_misc_switches() # simple side effects only # if explicitly requested, or the cache is suspect or being ignored, clear # cached context pickles. if self.args.clear_pickles or self.args.ignore_cache or self.args.repair_files: self.clear_pickles() # utility to change cache structure, e.g. add instrument subdirs. # do this before syncing anything under the current mode. if self.args.organize: self.organize_references(self.args.organize) # fetching and verifying files both require a server connection. self.require_server_connection() # primary sync'ing occurs here, both mappings and references as well as odd-ball # server sqlite3 database download. verify_file_list = self.file_transfers() # verification is relative to sync'ed files, and can include file replacement if # defects are found. if (self.args.check_files or self.args.check_sha1sum or self.args.repair_files or self.args.purge_blacklisted or self.args.purge_rejected): self.verify_files(verify_file_list) # context pickles should only be (re)generated after mappings are fully sync'ed and verified if self.args.save_pickles: self.pickle_contexts(self.contexts) # update CRDS cache config area, including stored version of operational context. # implement pipeline support functions of context update verify and echo # If --output-dir was specified, do not update cache config. if self.args.output_dir: log.verbose_warning("Used --output-dir, skipping cache server_config update including default context and bad files.") if config.writable_cache_or_verbose("skipping removal of ref_cache_subdir_mode file."): os.remove(config.get_crds_ref_subdir_file(self.observatory)) else: self.update_context() self.report_stats() log.standard_status() return log.errors()
def main(self): if self.args.best_effort: config.PASS_INVALID_VALUES.set(True) # JWST SSB cal code data model config.ALLOW_BAD_USEAFTER.set(True) # Don't fail for bad USEAFTER values config.ALLOW_SCHEMA_VIOLATIONS.set(True) # Don't fail for data model bad value errors config.ALLOW_BAD_PARKEY_VALUES.set(True) # Don't fail for values which don't pass DM + .tpn checking if self.args.rmaps: # clean up dead lines from file lists self.args.rmaps = [ self.resolve_context(mapping) for mapping in self.args.rmaps if mapping.strip() ] if self.args.references: self.args.references = [self.locate_file(reference) for reference in self.args.references] with log.error_on_exception("Refactoring operation FAILED"): if self.args.command == "insert_reference": if self.args.old_rmap: old_rmap, new_rmap = self.resolve_context(self.args.old_rmap), self.resolve_context(self.args.new_rmap) rmap_insert_references(old_rmap, new_rmap, self.args.references) else: self.insert_references() # figure it all out relative to --source-context elif self.args.command == "delete_reference": old_rmap, new_rmap = self.resolve_context(self.args.old_rmap), self.resolve_context(self.args.new_rmap) rmap_delete_references(old_rmap, new_rmap, self.args.references) elif self.args.command == "del_header": self.del_header_key() elif self.args.command == "set_header": self.set_header_key() elif self.args.command == "del_parameter": self.del_parameter() elif self.args.command == "set_parkey": self.set_parkey() elif self.args.command == "replace_text": self.replace_text() elif self.args.command == "set_substitution": self.set_substitution() elif self.args.command == "cat": self.cat() elif self.args.command == "add_useafter": self.add_useafter() elif self.args.command == "diff_rmaps": self.diff_rmaps() elif self.args.command == "certify_rmaps": self.certify_rmaps() else: raise ValueError("Unknown refactoring command: " + repr(self.args.command)) log.standard_status() return log.errors()
def main(self): """Main control flow of submission directory and request manifest creation.""" log.divider("setting up", char="=") self.require_server_connection() self.finish_parameters() if self.args.certify_files: self.certify_local_files() if self.args.logout: return self.logout() self.login() if self.args.wipe_existing_files: self.wipe_files() self.jpoll_key = self.jpoll_open_channel() if self.args.submission_kind == "batch": submit_future = self.batch_submit_references() elif self.args.submission_kind == "certify": submit_future = self.certify_files() elif self.args.submission_kind == "references": submit_future = self.submit_references() elif self.args.submission_kind == "mappings": submit_future = self.submit_mappings() if self.args.monitor_processing: monitor_future = self.monitor() if self.args.wait_for_completion: self.submission_complete(submit_future) if self.args.monitor_processing: monitor = self.monitor_complete(monitor_future) if monitor.exit_status == 0: self._ready_url = monitor.result log.standard_status() self._error_count = log.errors() self._warning_count = log.warnings() return log.errors()
def main(self): """Synchronize files.""" # clear any mutliprocessing locks associated with the CRDS cache. if self.args.clear_locks: crds_cache_locking.clear_cache_locks() return log.errors() self.handle_misc_switches() # simple side effects only # if explicitly requested, or the cache is suspect or being ignored, clear # cached context pickles. if self.args.clear_pickles or self.args.ignore_cache or self.args.repair_files: self.clear_pickles() # utility to change cache structure, e.g. add instrument subdirs. # do this before syncing anything under the current mode. if self.args.organize: self.organize_references(self.args.organize) # fetching and verifying files both require a server connection. self.require_server_connection() # primary sync'ing occurs here, both mappings and references as well as odd-ball # server sqlite3 database download. verify_file_list = self.file_transfers() # verification is relative to sync'ed files, and can include file replacement if # defects are found. if self.args.check_files or self.args.check_sha1sum or self.args.repair_files: self.verify_files(verify_file_list) # context pickles should only be (re)generated after mappings are fully sync'ed and verified if self.args.save_pickles: self.pickle_contexts(self.contexts) # update CRDS cache config area, including stored version of operational context. # implement pipeline support functions of context update verify and echo # If explicit files were specified, do not update cache config. if self.args.files and self.args.output_dir: log.verbose_warning("Used explicit --files list and --output-dir, skipping cache server_config update including default context and bad files.") else: self.update_context() self.report_stats() log.standard_status() return log.errors()
def main(self): with log.error_on_exception("Refactoring operation FAILED"): if self.args.command == "insert": rmap_insert_references(self.old_rmap, self.new_rmap, self.ref_paths) elif self.args.command == "delete": rmap_delete_references(self.old_rmap, self.new_rmap, self.ref_paths) elif self.args.command == "set_header": field, setting = self.args.references[0], " ".join( self.args.references[1:]) set_header_value(self.old_rmap, self.new_rmap, field, setting) elif self.args.command == "del_header": field = self.args.references[0] del_header_value(self.old_rmap, self.new_rmap, field) else: raise ValueError("Unknown refactoring command: " + repr(self.args.command)) log.standard_status() return log.errors()
def main(self): """Main control flow of submission directory and request manifest creation.""" log.divider("setting up", char="=") self.require_server_connection() self.finish_parameters() if self.args.logout: return self.logout() self.submission = self.create_submission() self.login() if self.args.wipe_existing_files: self.wipe_files() self.jpoll_key = self.jpoll_open_channel() if self.args.submission_kind == "batch": submit_future = self.batch_submit_references() elif self.args.submission_kind == "certify": submit_future = self.certify_files() elif self.args.submission_kind == "references": submit_future = self.submit_references() elif self.args.submission_kind == "mappings": submit_future = self.submit_mappings() if self.args.monitor_processing: monitor_future = self.monitor() if self.args.wait_for_completion: self.submission_complete(submit_future) if self.args.monitor_processing: self.monitor_complete(monitor_future) log.standard_status() return log.errors()
def main(self): if self.args.best_effort: config.PASS_INVALID_VALUES.set( True) # JWST SSB cal code data model config.ALLOW_BAD_USEAFTER.set( True) # Don't fail for bad USEAFTER values config.ALLOW_SCHEMA_VIOLATIONS.set( True) # Don't fail for data model bad value errors config.ALLOW_BAD_PARKEY_VALUES.set( True ) # Don't fail for values which don't pass DM + .tpn checking if self.args.rmaps: # clean up dead lines from file lists self.args.rmaps = [ self.resolve_context(mapping) for mapping in self.args.rmaps if mapping.strip() ] if self.args.references: self.args.references = [ self.locate_file(reference) for reference in self.args.references ] with log.error_on_exception("Refactoring operation FAILED"): if self.args.command == "insert_reference": if self.args.old_rmap: old_rmap, new_rmap = self.resolve_context( self.args.old_rmap), self.resolve_context( self.args.new_rmap) rmap_insert_references(old_rmap, new_rmap, self.args.references) else: self.insert_references( ) # figure it all out relative to --source-context elif self.args.command == "delete_reference": old_rmap, new_rmap = self.resolve_context( self.args.old_rmap), self.resolve_context( self.args.new_rmap) rmap_delete_references(old_rmap, new_rmap, self.args.references) elif self.args.command == "del_header": self.del_header_key() elif self.args.command == "set_header": self.set_header_key() elif self.args.command == "del_parameter": self.del_parameter() elif self.args.command == "set_parkey": self.set_parkey() elif self.args.command == "replace_text": self.replace_text() elif self.args.command == "set_substitution": self.set_substitution() elif self.args.command == "cat": self.cat() elif self.args.command == "add_useafter": self.add_useafter() elif self.args.command == "diff_rmaps": self.diff_rmaps() elif self.args.command == "certify_rmaps": self.certify_rmaps() else: raise ValueError("Unknown refactoring command: " + repr(self.args.command)) log.standard_status() return log.errors()