def ingest_files(self):
"""Upload self.files to the user's ingest directory on the CRDS server."""
stats = self._start_stats()
total_size = utils.total_size(self.files)
ingest_info = self.get_ingested_files()
self.scan_for_nonsubmitted_ingests(ingest_info)
remaining_files = self.keep_existing_files(ingest_info, self.files) \
if self.args.keep_existing_files else self.files
for i, filename in enumerate(remaining_files):
file_size = utils.file_size(filename)
log.info("Upload started", repr(filename), "[", i + 1, "/",
len(self.files), " files ]", "[",
utils.human_format_number(file_size), "/",
utils.human_format_number(total_size), " bytes ]")
self.connection.upload_file("/upload/new/", filename)
stats.increment("bytes", file_size)
stats.increment("files", 1)
stats.log_status("files", "Upload complete", len(self.files))
stats.log_status("bytes", "Upload complete", total_size)
log.divider(func=log.verbose)
stats.report()
log.divider(char="=")
def ingest_files(self):
"""Copy self.files into the user's ingest directory on the CRDS server."""
stats = self._start_stats()
destination = self.submission_info.ingest_dir
host, path = destination.split(":")
self.ensure_ingest_exists(host, path)
total_size = utils.total_size(self.files)
ingest_info = self.get_ingested_files()
self.scan_for_nonsubmitted_ingests(ingest_info)
remaining_files = self.keep_existing_files(ingest_info, self.files) \
if self.args.keep_existing_files else self.files
for i, filename in enumerate(remaining_files):
file_size = utils.file_size(filename)
log.info("Copy started", repr(filename), "[", i + 1, "/",
len(self.files), " files ]", "[",
utils.human_format_number(file_size), "/",
utils.human_format_number(total_size), " bytes ]")
self.copy_file(filename, path, destination)
stats.increment("bytes", file_size)
stats.increment("files", 1)
stats.log_status("files", "Copy complete", len(self.files))
stats.log_status("bytes", "Copy complete", total_size)
log.divider(func=log.verbose)
stats.report()
log.divider(char="=")
def wipe_files(self):
"""Delete all files from the user's ingest directory on the CRDS server."""
log.divider(name="wipe files", char="=")
ingest_info = self.get_ingested_files()
for basename in ingest_info:
log.info("Wiping file", repr(basename))
self.connection.get(ingest_info[basename]["deleteUrl"])
def ingest_files(self):
"""Copy self.files into the user's ingest directory on the CRDS server."""
stats = self._start_stats()
destination = self.submission_info.ingest_dir
host, path = destination.split(":")
self.ensure_ingest_exists(host, path)
total_size = utils.total_size(self.files)
ingest_info = self.get_ingested_files()
self.scan_for_nonsubmitted_ingests(ingest_info)
remaining_files = self.keep_existing_files(ingest_info, self.files) \
if self.args.keep_existing_files else self.files
for i, filename in enumerate(remaining_files):
file_size = utils.file_size(filename)
log.info("Copy started", repr(filename), "[", i+1, "/", len(self.files), " files ]",
"[", utils.human_format_number(file_size),
"/", utils.human_format_number(total_size), " bytes ]")
self.copy_file(filename, path, destination)
stats.increment("bytes", file_size)
stats.increment("files", 1)
stats.log_status("files", "Copy complete", len(self.files))
stats.log_status("bytes", "Copy complete", total_size)
log.divider(func=log.verbose)
stats.report()
log.divider(char="=")
def log_section(section_name,
section_value,
verbosity=50,
log_function=log.verbose,
divider_name=None):
"""Issue log divider bar followed by a corresponding log message."""
log.divider(name=divider_name, verbosity=verbosity, func=log.verbose)
log_function(section_name, section_value, verbosity=verbosity + 5)
def _start_stats(self):
"""Helper method to initialize stats keeping for ingest."""
total_bytes = utils.total_size(self.files)
stats = utils.TimingStats(output=log.verbose)
stats.start()
log.divider(name="ingest files", char="=")
log.info("Uploading", len(self.files), "file(s) totalling", utils.human_format_number(total_bytes), "bytes")
log.divider(func=log.verbose)
return stats
def _start_stats(self):
"""Helper method to initialize stats keeping for ingest."""
total_bytes = utils.total_size(self.files)
stats = utils.TimingStats(output=log.verbose)
stats.start()
log.divider(name="ingest files", char="=")
log.info("Copying", len(self.files), "file(s) totalling", utils.human_format_number(total_bytes), "bytes")
log.divider(func=log.verbose)
return stats
def main(self):
"""Main control flow of submission directory and request manifest creation."""
log.divider("monitoring server on " + repr(self.args.key), char="=")
exit_flag = False
while not exit_flag:
for message in self._poll_status():
handler = getattr(self, "handle_" + message.type, self.handle_unknown)
exit_flag = handler(message)
time.sleep(self.args.poll_delay)
log.divider("monitoring server done", char="=")
return exit_flag
def dump_response(self, name, response):
"""Print out verbose output related to web `response` from activity `name`."""
log_section("headers:\n", response.headers, divider_name=name, verbosity=70)
log_section("status_code:", response.status_code, verbosity=50)
log_section("text:\n", response.text, verbosity=75)
try:
json_text = response.json()
log_section("json:\n", json_text)
except Exception:
pass
log.divider(func=log.verbose)
def main(self):
"""Main control flow of submission directory and request manifest creation."""
log.divider("monitoring server on " + repr(self.args.key), char="=")
exit_flag = False
while not exit_flag:
for message in self._poll_status():
handler = getattr(self, "handle_" + message.type, self.handle_unknown)
exit_flag = handler(message)
time.sleep(self.args.poll_delay)
log.divider("monitoring server done", char="=")
return exit_flag
def wipe_files(self):
"""Copy self.files into the user's ingest directory on the CRDS server."""
destination = self.submission_info.ingest_dir
log.divider(name="wipe files", char="=")
log.info("Wiping files at", repr(destination))
host, path = destination.split(":")
if destination.startswith(socket.gethostname()):
output = pysh.out_err("rm -vf ${path}/*")
else:
output = pysh.out_err("ssh ${host} rm -vf ${path}/*")
if output:
log.verbose(output)
def wipe_files(self):
"""Copy self.files into the user's ingest directory on the CRDS server."""
destination = self.submission_info.ingest_dir
log.divider(name="wipe files", char="=")
log.info("Wiping files at", repr(destination))
host, path = destination.split(":")
if destination.startswith(socket.gethostname()):
output = pysh.out_err("rm -vf ${path}/*")
else:
output = pysh.out_err("ssh ${host} rm -vf ${path}/*")
if output:
log.verbose(output)
def dump_response(self, name, response):
"""Print out verbose output related to web `response` from activity `name`."""
log_section("headers:\n",
response.headers,
divider_name=name,
verbosity=70)
log_section("status_code:", response.status_code, verbosity=50)
log_section("text:\n", response.text, verbosity=75)
try:
json_text = response.json()
log_section("json:\n", json_text)
except Exception:
pass
log.divider(func=log.verbose)
def main(self):
"""Main control flow of submission directory and request manifest creation."""
log.divider("setting up", char="=")
self.require_server_connection()
self.finish_parameters()
if self.args.certify_files:
self.certify_local_files()
if self.args.logout:
return self.logout()
self.login()
if self.args.wipe_existing_files:
self.wipe_files()
self.jpoll_key = self.jpoll_open_channel()
if self.args.submission_kind == "batch":
submit_future = self.batch_submit_references()
elif self.args.submission_kind == "certify":
submit_future = self.certify_files()
elif self.args.submission_kind == "references":
submit_future = self.submit_references()
elif self.args.submission_kind == "mappings":
submit_future = self.submit_mappings()
if self.args.monitor_processing:
monitor_future = self.monitor()
if self.args.wait_for_completion:
self.submission_complete(submit_future)
if self.args.monitor_processing:
monitor = self.monitor_complete(monitor_future)
if monitor.exit_status == 0:
self._ready_url = monitor.result
log.standard_status()
self._error_count = log.errors()
self._warning_count = log.warnings()
return log.errors()
def list_tpns(self):
"""Print out the .tpn information related to each of the files either
specified via the --tpns <files...> argument or implied by any of the
other standard --list mechanisms like --mappings or --references.
"""
constrained_files = self.get_words(self.args.tpns) + self.implied_files
for filename in constrained_files:
path = self.locate_file(filename)
if config.is_mapping(path):
tpn_text = reftypes.get_types_object(
self.observatory).reference_name_to_ld_tpn_text(path)
else:
tpn_text = reftypes.get_types_object(
self.observatory).reference_name_to_tpn_text(path)
log.divider(f"Certify constraints for '{path}'")
print(tpn_text)
def list_required_parkeys(self):
"""Print out the parkeys required for matching using the specified contexts."""
for name in self.contexts:
mapping = crds.get_cached_mapping(name)
if isinstance(mapping, rmap.PipelineContext):
log.divider(name="Parkeys required for " + repr(mapping.basename), func=log.write)
_print_dict("", mapping.get_required_parkeys())
elif isinstance(mapping, rmap.InstrumentContext):
for name in sorted(mapping.selections):
try:
rmapping = mapping.get_rmap(name)
except (crds.exceptions.IrrelevantReferenceTypeError,
crds.exceptions.OmitReferenceTypeError):
print(name +":", repr("N/A"))
else:
print(name + ":", rmapping.get_required_parkeys())
else:
print(name + ":", mapping.get_required_parkeys())
def list_required_parkeys(self):
"""Print out the parkeys required for matching using the specified contexts."""
for name in self.contexts:
mapping = crds.get_cached_mapping(name)
if isinstance(mapping, rmap.PipelineContext):
log.divider(name="Parkeys required for " +
repr(mapping.basename),
func=log.write)
_print_dict("", mapping.get_required_parkeys())
elif isinstance(mapping, rmap.InstrumentContext):
for name in sorted(mapping.selections):
try:
rmapping = mapping.get_rmap(name)
except (crds.exceptions.IrrelevantReferenceTypeError,
crds.exceptions.OmitReferenceTypeError):
print(name + ":", repr("N/A"))
else:
print(name + ":", rmapping.get_required_parkeys())
else:
print(name + ":", mapping.get_required_parkeys())
def main(self):
"""Main control flow of submission directory and request manifest creation."""
log.divider("setting up", char="=")
self.require_server_connection()
self.finish_parameters()
if self.args.logout:
return self.logout()
self.submission = self.create_submission()
self.login()
if self.args.wipe_existing_files:
self.wipe_files()
self.jpoll_key = self.jpoll_open_channel()
if self.args.submission_kind == "batch":
submit_future = self.batch_submit_references()
elif self.args.submission_kind == "certify":
submit_future = self.certify_files()
elif self.args.submission_kind == "references":
submit_future = self.submit_references()
elif self.args.submission_kind == "mappings":
submit_future = self.submit_mappings()
if self.args.monitor_processing:
monitor_future = self.monitor()
if self.args.wait_for_completion:
self.submission_complete(submit_future)
if self.args.monitor_processing:
self.monitor_complete(monitor_future)
log.standard_status()
return log.errors()
def main():
p = crds.get_cached_mapping("hst.pmap")
s = pickle.dumps(p)
q = pickle.loads(s)
p._trace_compare(q)
log.divider("p == q --> " + repr(p == q))
log.divider("__getstate__ --> " + repr(p.__getstate__() == q.__getstate__()))
log.divider("rmap __getstate__ --> " + repr(p.get_imap("acs").get_rmap("biasfile").__getstate__() == q.get_imap("acs").get_rmap("biasfile").__getstate__()))
def main():
p = crds.get_cached_mapping("hst.pmap")
s = pickle.dumps(p)
q = pickle.loads(s)
p._trace_compare(q)
log.divider("p == q --> " + repr(p == q))
log.divider("__getstate__ --> " + repr(p.__getstate__() == q.__getstate__()))
log.divider("rmap __getstate__ --> " + repr(p.get_imap("acs").get_rmap("biasfile").__getstate__() == q.get_imap("acs").get_rmap("biasfile").__getstate__()))
def log_section(section_name, section_value, verbosity=50, log_function=log.verbose,
divider_name=None):
"""Issue log divider bar followed by a corresponding log message."""
log.divider(name=divider_name, verbosity=verbosity, func=log.verbose)
log_function(section_name, section_value, verbosity=verbosity+5)
config.set_crds_state(new_state)
utils.clear_function_caches()
return old_state
def cleanup(old_state):
"""Strictly speaking test cleanup is more than restoring CRDS state."""
config.set_crds_state(old_state)
utils.clear_function_caches()
# ==============================================================================
def run_and_profile(name, case, globs={}, locs={}):
"""Using `name` for a banner and divider, execute code string `case` in the
global namespace, both evaled printing result and under the profiler.
"""
utils.clear_function_caches()
log.divider()
log.divider(name + " example")
log.divider()
print(eval(case, globs, locs))
utils.clear_function_caches()
log.divider()
log.divider(name + " profile")
log.divider()
cProfile.run(case, "profile.stats")
stats = pstats.Stats('profile.stats')
stats.strip_dirs()
stats.sort_stats('cumulative')
stats.print_stats(100)
os.remove('profile.stats')
