def list_dataset_headers(self):
"""List dataset header info for self.args.dataset_headers with respect to self.args.context"""
for context in self.contexts:
with log.error_on_exception("Failed fetching dataset parameters with repect to", repr(context),
"for", repr(self.args.dataset_headers)):
pars = api.get_dataset_headers_by_id(context, self.args.dataset_headers)
pmap = crds.get_cached_mapping(context)
for requested_id in self.args.dataset_headers:
for returned_id in sorted(pars.keys()):
if requested_id.upper() in returned_id.upper():
header = pars[returned_id]
if isinstance(header, python23.string_types):
log.error("No header for", repr(returned_id), ":", repr(header)) # header is reason
continue
if self.args.id_expansions_only:
print(returned_id, context if len(self.contexts) > 1 else "")
else:
if self.args.minimize_headers:
header2 = pmap.minimize_header(header)
else:
header2 = dict(header)
header2.pop("REFTYPE", None)
header2["dataset_id"] = returned_id
log.info("Dataset pars for", repr(returned_id), "with respect to", repr(context) + ":\n",
log.PP(header2))
if self.args.first_id_expansion_only:
break
def verify_context_change(self, old_context):
"""Verify that the starting and post-sync contexts are different, or issue an error."""
new_context = heavy_client.load_server_info(self.observatory).operational_context
if old_context == new_context:
log.error("Expected operational context switch but starting and post-sync contexts are both", repr(old_context))
else:
log.info("Operational context updated from", repr(old_context), "to", repr(new_context))
def update_checksums(files):
"""Rewrite the mapping checksums/hashes/sha1sums in all `files`."""
for file_ in files:
log.info("Updating checksum for", file_)
with log.error_on_exception("Failed updating checksum for",
repr(file_)):
update_checksum(file_)
def verify_context_change(self, old_context):
"""Verify that the starting and post-sync contexts are different, or issue an error."""
new_context = heavy_client.load_server_info(self.observatory).operational_context
if old_context == new_context:
log.error("Expected operational context switch but starting and post-sync contexts are both", repr(old_context))
else:
log.info("Operational context updated from", repr(old_context), "to", repr(new_context))
def ingest_files(self):
"""Copy self.files into the user's ingest directory on the CRDS server."""
stats = self._start_stats()
destination = self.submission_info.ingest_dir
host, path = destination.split(":")
total_size = utils.total_size(self.files)
ingest_info = self.get_ingested_files()
self.scan_for_nonsubmitted_ingests(ingest_info)
remaining_files = self.keep_existing_files(ingest_info, self.files) \
if self.args.keep_existing_files else self.files
for i, filename in enumerate(remaining_files):
file_size = utils.file_size(filename)
log.info("Copy started", repr(filename), "[", i+1, "/", len(self.files), " files ]",
"[", utils.human_format_number(file_size),
"/", utils.human_format_number(total_size), " bytes ]")
self.copy_file(filename, path, destination)
stats.increment("bytes", file_size)
stats.increment("files", 1)
stats.log_status("files", "Copy complete", len(self.files))
stats.log_status("bytes", "Copy complete", total_size)
log.divider(func=log.verbose)
stats.report()
log.divider(char="=")
def sync_datasets(self):
"""Sync mappings and references for datasets with respect to `self.contexts`."""
if not self.contexts:
log.error("Define --contexts under which references are fetched for --dataset-files or --dataset-ids.""")
sys.exit(-1)
active_references = []
for context in self.contexts:
if self.args.dataset_ids:
if len(self.args.dataset_ids) == 1 and self.args.dataset_ids[0].startswith("@"):
with open(self.args.dataset_ids[0][1:]) as pfile:
self.args.dataset_ids = pfile.read().splitlines()
with log.error_on_exception("Failed to get matching parameters for", self.args.dataset_ids):
id_headers = api.get_dataset_headers_by_id(context, self.args.dataset_ids)
for dataset in self.args.dataset_files or self.args.dataset_ids:
log.info("Syncing context '%s' dataset '%s'." % (context, dataset))
with log.error_on_exception("Failed to get matching parameters from", repr(dataset)):
if self.args.dataset_files:
headers = { dataset : data_file.get_conditioned_header(dataset, observatory=self.observatory) }
else:
headers = { dataset_id : header for (dataset_id, header) in id_headers.items() if
dataset.upper() in dataset_id }
for assc_dataset, header in headers.items():
with log.error_on_exception("Failed syncing references for dataset", repr(assc_dataset),
"under context", repr(context)):
bestrefs = crds.getrecommendations(header, context=context, observatory=self.observatory,
ignore_cache=self.args.ignore_cache)
log.verbose("Best references for", repr(assc_dataset), "are", bestrefs)
active_references.extend(bestrefs.values())
active_references = [ ref for ref in active_references if not ref.startswith("NOT FOUND") ]
log.verbose("Syncing references:", repr(active_references))
return list(set(active_references))
def ingest_files(self):
"""Copy self.files into the user's ingest directory on the CRDS server."""
stats = self._start_stats()
destination = self.submission_info.ingest_dir
host, path = destination.split(":")
total_size = utils.total_size(self.files)
ingest_info = self.get_ingested_files()
self.scan_for_nonsubmitted_ingests(ingest_info)
remaining_files = self.keep_existing_files(ingest_info, self.files) \
if self.args.keep_existing_files else self.files
for i, filename in enumerate(remaining_files):
file_size = utils.file_size(filename)
log.info("Copy started", repr(filename), "[", i + 1, "/",
len(self.files), " files ]", "[",
utils.human_format_number(file_size), "/",
utils.human_format_number(total_size), " bytes ]")
self.copy_file(filename, path, destination)
stats.increment("bytes", file_size)
stats.increment("files", 1)
stats.log_status("files", "Copy complete", len(self.files))
stats.log_status("bytes", "Copy complete", total_size)
log.divider(func=log.verbose)
stats.report()
log.divider(char="=")
def wfpc2_flatfile_filter(kmap):
log.info("Hacking WFPC2 Flatfile.")
# : ('MODE', 'FILTER1', 'FILTER2', 'IMAGETYP', 'FILTNAM1', 'FILTNAM2', 'LRFWAVE'), ('DATE-OBS', 'TIME-OBS')),
kmap[('*', '*', '*', 'EXT', 'FR*', '*', '# >3000 and <=4200 #')] = \
[rmap.Filemap(date='1990-01-01 00:00:00', file='m3c10045u.r4h', comment='')]
kmap[('*', '*', '*', 'EXT', '*', 'FR*', '# >3000 and <=4200 #')] = \
[rmap.Filemap(date='1990-01-01 00:00:00', file='m3c10045u.r4h', comment='')]
kmap[('*', '*', '*', 'EXT', 'FR*', '*', '# >4200 and <=5800 #')] = \
[rmap.Filemap(date='1990-01-01 00:00:00', file='m3c1004fu.r4h', comment='')]
kmap[('*', '*', '*', 'EXT', '*', 'FR*', '# >4200 and <=5800 #')] = \
[rmap.Filemap(date='1990-01-01 00:00:00', file='m3c1004fu.r4h', comment='')]
kmap[('*', '*', '*', 'EXT', 'FR*', '*', '# >5800 and <=7600 #')] = \
[rmap.Filemap(date='1990-01-01 00:00:00', file='m3c1004nu.r4h', comment='')]
kmap[('*', '*', '*', 'EXT', '*', 'FR*', '# >5800 and <=7600 #')] = \
[rmap.Filemap(date='1990-01-01 00:00:00', file='m3c1004nu.r4h', comment='')]
kmap[('*', '*', '*', 'EXT', 'FR*', '*', '# >7600 and <=10000 #')] = \
[rmap.Filemap(date='1990-01-01 00:00:00', file='m3c10052u.r4h', comment='')]
kmap[('*', '*', '*', 'EXT', '*', 'FR*', '# >7600 and <=10000 #')] = \
[rmap.Filemap(date='1990-01-01 00:00:00', file='m3c10052u.r4h', comment='')]
header_additions = [
("hooks", {
"fallback_header" : "fallback_header_wfpc2_flatfile_v1",
}),
]
return kmap, header_additions
def list_dataset_headers(self):
"""List dataset header info for self.args.dataset_headers with respect to self.args.context"""
for context in self.contexts:
with log.error_on_exception("Failed fetching dataset parameters with repect to", repr(context),
"for", repr(self.args.dataset_headers)):
pars = api.get_dataset_headers_by_id(context, self.args.dataset_headers)
pmap = crds.get_cached_mapping(context)
for requested_id in self.args.dataset_headers:
for returned_id in sorted(pars.keys()):
if requested_id.upper() in returned_id.upper():
header = pars[returned_id]
if isinstance(header, python23.string_types):
log.error("No header for", repr(returned_id), ":", repr(header)) # header is reason
continue
if self.args.id_expansions_only:
print(returned_id, context if len(self.contexts) > 1 else "")
else:
if self.args.minimize_headers:
header2 = pmap.minimize_header(header)
else:
header2 = dict(header)
header2.pop("REFTYPE", None)
header2["dataset_id"] = returned_id
log.info("Dataset pars for", repr(returned_id), "with respect to", repr(context) + ":\n",
log.PP(header2))
if self.args.first_id_expansion_only:
break
def rewrite(self, filename, uniqname):
"""Add a FITS checksum to `filename.`"""
with data_file.fits_open(filename, mode="readonly", checksum=self.args.verify_file, do_not_scale_image_data=True) as hdus:
verify_mode = "fix+warn" if not self.args.fits_errors else "fix+exception"
if self.args.verify_file:
hdus.verify(verify_mode)
basefile = os.path.basename(filename)
baseuniq = os.path.basename(uniqname)
if self.args.add_keywords:
now = datetime.datetime.utcnow()
hdus[0].header["FILENAME"] = baseuniq
hdus[0].header["ROOTNAME"] = os.path.splitext(baseuniq)[0].upper()
hdus[0].header["HISTORY"] = "{0} renamed to {1} on {2} {3} {4}".format(
basefile, baseuniq, MONTHS[now.month - 1], now.day, now.year)
if self.args.output_path:
uniqname = os.path.join(self.args.outpath, baseuniq)
try:
log.info("Rewriting", self.format_file(filename), "-->", self.format_file(uniqname))
hdus.writeto(uniqname, output_verify=verify_mode, checksum=self.args.add_checksum)
except Exception as exc:
if os.path.exists(uniqname):
os.remove(uniqname)
if "buffer is too small" in str(exc):
raise CrdsError("Failed to rename/rewrite", repr(basefile), "as", repr(baseuniq), ":",
"probable file truncation", ":", str(exc))
else:
raise CrdsError("Failed to rename/rewrite", repr(basefile), "as", repr(baseuniq), ":", str(exc))
def wfpc2_flatfile_filter(kmap):
log.info("Hacking WFPC2 Flatfile.")
# : ('MODE', 'FILTER1', 'FILTER2', 'IMAGETYP', 'FILTNAM1', 'FILTNAM2', 'LRFWAVE'), ('DATE-OBS', 'TIME-OBS')),
kmap[('*', '*', '*', 'EXT', 'FR*', '*', '# >3000 and <=4200 #')] = \
[rmap.Filemap(date='1990-01-01 00:00:00', file='m3c10045u.r4h', comment='')]
kmap[('*', '*', '*', 'EXT', '*', 'FR*', '# >3000 and <=4200 #')] = \
[rmap.Filemap(date='1990-01-01 00:00:00', file='m3c10045u.r4h', comment='')]
kmap[('*', '*', '*', 'EXT', 'FR*', '*', '# >4200 and <=5800 #')] = \
[rmap.Filemap(date='1990-01-01 00:00:00', file='m3c1004fu.r4h', comment='')]
kmap[('*', '*', '*', 'EXT', '*', 'FR*', '# >4200 and <=5800 #')] = \
[rmap.Filemap(date='1990-01-01 00:00:00', file='m3c1004fu.r4h', comment='')]
kmap[('*', '*', '*', 'EXT', 'FR*', '*', '# >5800 and <=7600 #')] = \
[rmap.Filemap(date='1990-01-01 00:00:00', file='m3c1004nu.r4h', comment='')]
kmap[('*', '*', '*', 'EXT', '*', 'FR*', '# >5800 and <=7600 #')] = \
[rmap.Filemap(date='1990-01-01 00:00:00', file='m3c1004nu.r4h', comment='')]
kmap[('*', '*', '*', 'EXT', 'FR*', '*', '# >7600 and <=10000 #')] = \
[rmap.Filemap(date='1990-01-01 00:00:00', file='m3c10052u.r4h', comment='')]
kmap[('*', '*', '*', 'EXT', '*', 'FR*', '# >7600 and <=10000 #')] = \
[rmap.Filemap(date='1990-01-01 00:00:00', file='m3c10052u.r4h', comment='')]
header_additions = [
("hooks", {
"fallback_header": "fallback_header_wfpc2_flatfile_v1",
}),
]
return kmap, header_additions
def sync_datasets(self):
"""Sync mappings and references for datasets with respect to `self.contexts`."""
if not self.contexts:
log.error("Define --contexts under which references are fetched for --dataset-files or --dataset-ids.""")
sys.exit(-1)
active_references = []
for context in self.contexts:
if self.args.dataset_ids:
if len(self.args.dataset_ids) == 1 and self.args.dataset_ids[0].startswith("@"):
with open(self.args.dataset_ids[0][1:]) as pfile:
self.args.dataset_ids = pfile.read().splitlines()
with log.error_on_exception("Failed to get matching parameters for", self.args.dataset_ids):
id_headers = api.get_dataset_headers_by_id(context, self.args.dataset_ids)
for dataset in self.args.dataset_files or self.args.dataset_ids:
log.info("Syncing context '%s' dataset '%s'." % (context, dataset))
with log.error_on_exception("Failed to get matching parameters from", repr(dataset)):
if self.args.dataset_files:
headers = { dataset : data_file.get_conditioned_header(dataset, observatory=self.observatory) }
else:
headers = { dataset_id : header for (dataset_id, header) in id_headers.items() if
dataset.upper() in dataset_id }
for assc_dataset, header in headers.items():
with log.error_on_exception("Failed syncing references for dataset", repr(assc_dataset),
"under context", repr(context)):
bestrefs = crds.getrecommendations(header, context=context, observatory=self.observatory,
ignore_cache=self.args.ignore_cache)
log.verbose("Best references for", repr(assc_dataset), "are", bestrefs)
active_references.extend(bestrefs.values())
active_references = [ ref for ref in active_references if not ref.startswith("NOT FOUND") ]
log.verbose("Syncing references:", repr(active_references))
return list(set(active_references))
def error_and_repair(self, file, *args, **keys):
"""Issue an error message and repair `file` if requested by command line args."""
log.error(*args, **keys)
if self.args.repair_files:
if config.writable_cache_or_info("Skipping remove and dump of", repr(file)):
log.info("Repairing file", repr(file))
utils.remove(file, observatory=self.observatory)
self.dump_files(self.default_context, [file])
def set_rmap_substitution(rmapping, new_filename, parameter_name, old_text, new_text, *args, **keys):
log.info("Adding substitution for", srepr(parameter_name),
"from", srepr(old_text), "to", srepr(new_text), "in", srepr(rmapping.basename))
new_mapping = rmapping.copy()
if "substitutions" not in new_mapping.header:
new_mapping.header["substitutions"] = {}
new_mapping.header["substitutions"][parameter_name] = { old_text : new_text }
new_mapping.write(new_filename)
def push_context(self):
"""Push the final context recorded in the local cache to the CRDS server so it can be displayed
as the operational state of a pipeline.
"""
info = heavy_client.load_server_info(self.observatory)
with log.error_on_exception("Failed pushing cached operational context name to CRDS server"):
api.push_remote_context(self.observatory, "operational", self.args.push_context, info.operational_context)
log.info("Pushed cached operatonal context name", repr(info.operational_context), "to CRDS server")
def error_and_repair(self, file, *args, **keys):
"""Issue an error message and repair `file` if requested by command line args."""
log.error(*args, **keys)
if self.args.repair_files:
if config.writable_cache_or_info("Skipping remove and dump of", repr(file)):
log.info("Repairing file", repr(file))
utils.remove(file, observatory=self.observatory)
self.dump_files(self.default_context, [file])
def set_rmap_substitution(rmapping, new_filename, parameter_name, old_text, new_text, *args, **keys):
log.info("Adding substitution for", srepr(parameter_name),
"from", srepr(old_text), "to", srepr(new_text), "in", srepr(rmapping.basename))
new_mapping = rmapping.copy()
if "substitutions" not in new_mapping.header:
new_mapping.header["substitutions"] = {}
new_mapping.header["substitutions"][parameter_name] = { old_text : new_text }
new_mapping.write(new_filename)
def push_context(self):
"""Push the final context recorded in the local cache to the CRDS server so it can be displayed
as the operational state of a pipeline.
"""
info = heavy_client.load_server_info(self.observatory)
with log.error_on_exception("Failed pushing cached operational context name to CRDS server"):
api.push_remote_context(self.observatory, "operational", self.args.push_context, info.operational_context)
log.info("Pushed cached operatonal context name", repr(info.operational_context), "to CRDS server")
def replace_rmap_text(rmapping, new_filename, old_text, new_text, *args, **keys):
"""Do simple text replacement from `old_text` to `new_text` in `rmapping`.
"""
log.info("Replacing", srepr(old_text), "with", srepr(new_text), "in",
srepr(rmapping.basename), "to", srepr(new_filename))
original_rmap = str(rmapping)
new_rmap = original_rmap.replace(old_text, new_text)
new_mapping = rmap.ReferenceMapping.from_string(new_rmap, ignore_checksum=True)
new_mapping.write(new_filename)
def _start_stats(self):
"""Helper method to initialize stats keeping for ingest."""
total_bytes = utils.total_size(self.files)
stats = utils.TimingStats(output=log.verbose)
stats.start()
log.divider(name="ingest files", char="=")
log.info("Copying", len(self.files), "file(s) totalling", utils.human_format_number(total_bytes), "bytes")
log.divider(func=log.verbose)
return stats
def _start_stats(self):
"""Helper method to initialize stats keeping for ingest."""
total_bytes = utils.total_size(self.files)
stats = utils.TimingStats(output=log.verbose)
stats.start()
log.divider(name="ingest files", char="=")
log.info("Copying", len(self.files), "file(s) totalling",
utils.human_format_number(total_bytes), "bytes")
log.divider(func=log.verbose)
return stats
def all_filekinds():
filekinds = set()
for context in rmap.list_mappings("*.pmap", "jwst"):
p = crds.get_cached_mapping(context)
for i in p.selections.values():
for r in i.selections.values():
if r.filekind not in filekinds:
log.info("Adding", repr(r.filekind), "from", repr(context))
filekinds.add(r.filekind)
return sorted(filekinds)
def update_header_names(name_map):
"""Update the .name and .derived_from fields in mapping new_path.header
to reflect derivation from old_path and name new_path.
"""
for old_path, new_path in sorted(name_map.items()):
old_base, new_base = os.path.basename(old_path), os.path.basename(new_path)
refactor.update_derivation(new_path, old_base)
log.info("Adjusting name", repr(new_base), "derived_from", repr(old_base),
"in", repr(new_path))
return name_map # no change
def update_header_names(name_map):
"""Update the .name and .derived_from fields in mapping new_path.header
to reflect derivation from old_path and name new_path.
"""
for old_path, new_path in sorted(name_map.items()):
old_base, new_base = os.path.basename(old_path), os.path.basename(
new_path)
refactor.update_derivation(new_path, old_base)
log.info("Adjusting name", repr(new_base), "derived_from",
repr(old_base), "in", repr(new_path))
return name_map # no change
def hack_in_new_maps(old, new, updated_maps):
"""Given mapping named `old`, create a modified copy named `new` which
installs each map of `updated_maps` in place of it's predecessor.
"""
copy_mapping(old, new)
for mapping in sorted(updated_maps):
key, replaced = insert_mapping(new, mapping)
if replaced:
log.info("Replaced", repr(replaced), "with", repr(mapping), "for", repr(key), "in", repr(old), "producing", repr(new))
else:
log.info("Added", repr(mapping), "for", repr(key), "in", repr(old), "producing", repr(new))
def del_rmap_parameter(rmapping, new_filename, parameter, *args, **keys):
"""Delete `parameter_name` from the parkey item of the `types` of the specified
`instruments` in `context`.
"""
log.info("Deleting parameter", repr(parameter), "from",repr(rmapping.basename))
parkey = rmapping.parkey
i, j = get_parameter_index(parkey, parameter)
del_parkey = parkey[:i] + ((parkey[i][:j] + parkey[i][j+1:]),) + parkey[i+1:]
log.verbose("Replacing", srepr(parkey), "with", srepr(del_parkey), "in", srepr(rmapping.basename))
rmapping.header["parkey"] = del_parkey
rmapping.selector.delete_match_param(parameter)
rmapping.write(new_filename)
def wipe_files(self):
"""Copy self.files into the user's ingest directory on the CRDS server."""
destination = self.submission_info.ingest_dir
log.divider(name="wipe files", char="=")
log.info("Wiping files at", repr(destination))
host, path = destination.split(":")
if destination.startswith(socket.gethostname()):
output = pysh.out_err("rm -vf ${path}/*")
else:
output = pysh.out_err("ssh ${host} rm -vf ${path}/*")
if output:
log.verbose(output)
def wipe_files(self):
"""Copy self.files into the user's ingest directory on the CRDS server."""
destination = self.submission_info.ingest_dir
log.divider(name="wipe files", char="=")
log.info("Wiping files at", repr(destination))
host, path = destination.split(":")
if destination.startswith(socket.gethostname()):
output = pysh.out_err("rm -vf ${path}/*")
else:
output = pysh.out_err("ssh ${host} rm -vf ${path}/*")
if output:
log.verbose(output)
def handle_done(self, message):
"""Generic "done" handler issue info() message and stops monitoring / exits."""
status = message.data["status"]
if status == 0:
log.info(self.format_remote("COMPLETED:", message.data))
elif status == 1:
log.error(self.format_remote("FAILED:", message.data))
elif status == 2:
log.error(self.format_remote("CANCELLED:", message.data))
else:
log.info(self.format_remote("DONE:", message))
return message.data["result"]
def del_rmap_parameter(rmapping, new_filename, parameter, *args, **keys):
"""Delete `parameter_name` from the parkey item of the `types` of the specified
`instruments` in `context`.
"""
log.info("Deleting parameter", repr(parameter), "from",repr(rmapping.basename))
parkey = rmapping.parkey
i, j = get_parameter_index(parkey, parameter)
del_parkey = parkey[:i] + ((parkey[i][:j] + parkey[i][j+1:]),) + parkey[i+1:]
log.verbose("Replacing", srepr(parkey), "with", srepr(del_parkey), "in", srepr(rmapping.basename))
rmapping.header["parkey"] = del_parkey
rmapping.selector.delete_match_param(parameter)
rmapping.write(new_filename)
def remove_dir(self, instrument):
"""Remove an instrument cache directory and any associated legacy link."""
if config.writable_cache_or_info("Skipping remove instrument", repr(instrument), "directory."):
crds_refpath = config.get_crds_refpath(self.observatory)
prefix = self.locator.get_env_prefix(instrument)
rootdir = os.path.join(crds_refpath, instrument)
refdir = os.path.join(crds_refpath, prefix[:-1])
if len(glob.glob(os.path.join(rootdir, "*"))):
log.info("Residual files in '{}'. Not removing.".format(rootdir))
return
if os.path.exists(refdir): # skip crds:// vs. oref
utils.remove(refdir, observatory=self.observatory)
utils.remove(rootdir, observatory=self.observatory)
def _setup_source_context(self):
"""Default the --source-context if necessary and then translate any symbolic name to a literal .pmap
name. e.g. jwst-edit --> jwst_0109.pmap. Then optionally sync the files to a local cache.
"""
if self.args.source_context is None:
self.source_context = self.observatory + "-edit"
log.info("Defaulting --source-context to", srepr(self.source_context))
else:
self.source_context = self.args.source_context
self.source_context = self.resolve_context(self.source_context)
if self.args.sync_files:
errs = sync.SyncScript("crds.sync --contexts {}".format(self.source_context))()
assert not errs, "Errors occurred while syncing all rules to CRDS cache."
def remove_dir(self, instrument):
"""Remove an instrument cache directory and any associated legacy link."""
if config.writable_cache_or_info("Skipping remove instrument", repr(instrument), "directory."):
crds_refpath = config.get_crds_refpath(self.observatory)
prefix = self.locator.get_env_prefix(instrument)
rootdir = os.path.join(crds_refpath, instrument)
refdir = os.path.join(crds_refpath, prefix[:-1])
if len(glob.glob(os.path.join(rootdir, "*"))):
log.info("Residual files in '{}'. Not removing.".format(rootdir))
return
if os.path.exists(refdir): # skip crds:// vs. oref
utils.remove(refdir, observatory=self.observatory)
utils.remove(rootdir, observatory=self.observatory)
def _setup_source_context(self):
"""Default the --source-context if necessary and then translate any symbolic name to a literal .pmap
name. e.g. jwst-edit --> jwst_0109.pmap. Then optionally sync the files to a local cache.
"""
if self.args.source_context is None:
self.source_context = self.observatory + "-edit"
log.info("Defaulting --source-context to", srepr(self.source_context))
else:
self.source_context = self.args.source_context
self.source_context = self.resolve_context(self.source_context)
if self.args.sync_files:
errs = sync.SyncScript("crds.sync --contexts {}".format(self.source_context))()
assert not errs, "Errors occurred while syncing all rules to CRDS cache."
def hack_in_new_maps(old, new, updated_maps):
"""Given mapping named `old`, create a modified copy named `new` which
installs each map of `updated_maps` in place of it's predecessor.
"""
copy_mapping(old, new)
for mapping in sorted(updated_maps):
key, replaced, replacement = insert_mapping(new, mapping)
if replaced:
log.info("Replaced", repr(replaced), "with", repr(replacement),
"for", repr(key), "in", repr(old), "producing", repr(new))
else:
log.info("Added", repr(replacement), "for", repr(key), "in",
repr(old), "producing", repr(new))
def main(self):
"""Check files for availability from the archive."""
self.require_server_connection()
log.info("Mapping URL:", repr(self.mapping_url))
log.info("Reference URL:", repr(self.reference_url))
stats = utils.TimingStats()
self.file_info = api.get_file_info_map(self.observatory, self.files, fields=["size", "sha1sum"])
for filename in self.files:
self.verify_archive_file(filename)
stats.increment("files")
self.print_files()
stats.report_stat("files")
log.standard_status()
def handle_done(self, message):
"""Generic "done" handler issue info() message and stops monitoring / exits."""
status = message.data["status"]
result = message.data.get("result", None)
if status == 0:
log.info(self.format_remote("COMPLETED:", result))
elif status == 1:
log.error(self.format_remote("FAILED:", result))
elif status == 2:
log.error(self.format_remote("CANCELLED:", result))
else:
log.info(self.format_remote("DONE:", result))
self.result = result
return result
def resolve_context(self, context):
"""Resolve context spec `context` into a .pmap, .imap, or .rmap filename, interpreting
date based specifications against the CRDS server operational context history.
"""
if isinstance(context, str) and context.lower() == "none":
return None
if config.is_date_based_mapping_spec(context):
if re.match(config.OBSERVATORY_RE_STR + r"-operational$", context):
final_context = self.server_info.operational_context
else:
_mode, final_context = heavy_client.get_processing_mode(self.observatory, context)
log.info("Symbolic context", repr(context), "resolves to", repr(final_context))
context = final_context
return context
def mapping_check_diffs(mapping, derived_from):
"""Issue warnings for *deletions* in self relative to parent derived_from
mapping. Issue warnings for *reversions*, defined as replacements which
where the replacement is older than the original, as defined by the names.
This is intended to check for missing modes and for inadvertent reversions
to earlier versions of files. For speed and simplicity, file time order
is currently determined by the names themselves, not file contents, file
system, or database info.
"""
mapping = rmap.asmapping(mapping, cached="readonly")
derived_from = rmap.asmapping(derived_from, cached="readonly")
log.info("Checking diffs from", repr(derived_from.basename), "to", repr(mapping.basename))
diffs = derived_from.difference(mapping)
mapping_check_diffs_core(diffs)
def resolve_context(self, context):
"""Resolve context spec `context` into a .pmap, .imap, or .rmap filename, interpreting
date based specifications against the CRDS server operational context history.
"""
if isinstance(context, str) and context.lower() == "none":
return None
if config.is_date_based_mapping_spec(context):
if re.match(config.OBSERVATORY_RE_STR + r"-operational$", context):
final_context = self.server_info.operational_context
else:
_mode, final_context = heavy_client.get_processing_mode(self.observatory, context)
if self.show_context_resolution:
log.info("Symbolic context", repr(context), "resolves to", repr(final_context))
context = final_context
return context
def list_datasets(self):
"""List dataset header info for self.args.datasets with respect to self.args.context"""
for context in self.contexts:
with log.error_on_exception("Failed fetching dataset parameters with repect to", repr(context),
"for", repr(self.args.datasets)):
pars = api.get_dataset_headers_by_id(context, self.args.datasets)
pmap = rmap.get_cached_mapping(context)
for (dataset_id, header) in pars.items():
if isinstance(header, python23.string_types):
log.error("No header for", repr(dataset_id), ":", repr(header)) # header is reason
continue
header2 = pmap.minimize_header(header)
header2.pop("REFTYPE", None)
log.info("Dataset pars for", repr(dataset_id), "with respect to", repr(context) + ":\n",
log.PP(header2))
def main(self):
"""Check files for availability from the archive."""
self.require_server_connection()
log.info("Mapping URL:", repr(self.mapping_url))
log.info("Reference URL:", repr(self.reference_url))
stats = utils.TimingStats()
self.file_info = api.get_file_info_map(self.observatory,
self.files,
fields=["size", "sha1sum"])
for filename in self.files:
self.verify_archive_file(filename)
stats.increment("files")
self.print_files()
stats.report_stat("files")
log.standard_status()
def mapping_check_diffs(mapping, derived_from):
"""Issue warnings for *deletions* in self relative to parent derived_from
mapping. Issue warnings for *reversions*, defined as replacements which
where the replacement is older than the original, as defined by the names.
This is intended to check for missing modes and for inadvertent reversions
to earlier versions of files. For speed and simplicity, file time order
is currently determined by the names themselves, not file contents, file
system, or database info.
"""
mapping = rmap.asmapping(mapping, cached="readonly")
derived_from = rmap.asmapping(derived_from, cached="readonly")
log.info("Checking diffs from", repr(derived_from.basename), "to",
repr(mapping.basename))
diffs = derived_from.difference(mapping)
mapping_check_diffs_core(diffs)
def _process_rmap(self, func, rmapping, *args, **keys):
"""Execute `func` on a single `rmapping` passing along *args and **keys"""
keywords = dict(keys)
rmapping_org = rmapping
new_filename = rmapping.filename if self.args.inplace else os.path.join(".", rmapping.basename)
if os.path.exists(new_filename):
log.info("Continuing refactoring from local copy", srepr(new_filename))
rmapping = rmap.load_mapping(new_filename)
keywords.update(locals())
fixers = self.args.fixers
if fixers:
rmapping = rmap.load_mapping(rmapping.filename)
keywords.update(locals())
apply_rmap_fixers(*args, **keywords)
func(*args, **keywords)
return new_filename
def dump_unique_errors(self):
"""Print out the first instance of errors recorded by log_and_track_error(). Write out error list files."""
if self.args.dump_unique_errors:
log.info("="*20, "unique error classes", "="*20)
if self.args.unique_threshold > 1:
log.info("Limiting error class reporting to cases with at least",
self.args.unique_threshold, "instances.")
for key in sorted(self.ue_mixin.messages):
if self.ue_mixin.count[key] >= self.args.unique_threshold:
log.info(self.ue_mixin.count[key], "total errors like::", self.ue_mixin.messages[key])
log.info("All unique error types:", len(self.ue_mixin.messages))
log.info("="*20, "="*len("unique error classes"), "="*20)
if self.args.all_errors_file:
self.dump_error_data(self.args.all_errors_file, self.ue_mixin.all_data_names)
if self.args.unique_errors_file:
self.dump_error_data(self.args.unique_errors_file, self.ue_mixin.unique_data_names)
def set_rmap_parkey(rmapping, new_filename, parkey, *args, **keys):
"""Set the parkey of `rmapping` to `parkey` and write out to `new_filename`.
"""
log.info("Setting parkey, removing all references from", srepr(rmapping.basename))
pktuple = eval(parkey)
required_keywords = tuple(utils.flatten(pktuple))
refnames = rmapping.reference_names()
references_headers = { refname : get_refactoring_header(rmapping.filename, refname, required_keywords)
for refname in refnames }
rmapping = rmap_delete_references(rmapping.filename, new_filename, refnames)
log.info("Setting parkey", srepr(parkey), "in", srepr(rmapping.basename))
rmapping.header["parkey"] = pktuple
rmapping.write(new_filename)
rmapping = rmap.load_mapping(new_filename)
rmapping = rmap_insert_references_by_matches(new_filename, new_filename, references_headers)
return rmapping
def get_affected(self, old_context, new_context):
"""Return the affected datasets Struct for the transition from old_context to new_context,
or None if the results aren't ready yet.
"""
try:
affected = api.get_affected_datasets(self.observatory, old_context, new_context)
except Exception as exc:
if "No precomputed affected datasets results exist" in str(exc):
if self.args.ignore_missing_results:
log.info("No results for", old_context, "-->", new_context, "ignoring and proceeding.")
affected = None
else:
self.fatal_error("Results for", old_context, "-->", new_context, "don't exist or are not yet complete.")
else:
self.fatal_error("get_affected_datasets failed: ", str(exc).replace("OtherError:",""))
return affected
def set_rmap_parkey(rmapping, new_filename, parkey, *args, **keys):
"""Set the parkey of `rmapping` to `parkey` and write out to `new_filename`.
"""
log.info("Setting parkey, removing all references from", srepr(rmapping.basename))
pktuple = eval(parkey)
required_keywords = tuple(utils.flatten(pktuple))
refnames = rmapping.reference_names()
references_headers = { refname : get_refactoring_header(rmapping.filename, refname, required_keywords)
for refname in refnames }
rmapping = rmap_delete_references(rmapping.filename, new_filename, refnames)
log.info("Setting parkey", srepr(parkey), "in", srepr(rmapping.basename))
rmapping.header["parkey"] = pktuple
rmapping.write(new_filename)
rmapping = rmap.load_mapping(new_filename)
rmapping = rmap_insert_references_by_matches(new_filename, new_filename, references_headers)
return rmapping
def dump_unique_errors(self):
"""Print out the first instance of errors recorded by log_and_track_error(). Write out error list files."""
if self.args.dump_unique_errors:
log.info("="*20, "unique error classes", "="*20)
if self.args.unique_threshold > 1:
log.info("Limiting error class reporting to cases with at least",
self.args.unique_threshold, "instances.")
for key in sorted(self.ue_mixin.messages):
if self.ue_mixin.count[key] >= self.args.unique_threshold:
log.info(self.ue_mixin.count[key], "total errors like::", self.ue_mixin.messages[key])
log.info("All unique error types:", len(self.ue_mixin.messages))
log.info("="*20, "="*len("unique error classes"), "="*20)
if self.args.all_errors_file:
self.dump_error_data(self.args.all_errors_file, self.ue_mixin.all_data_names)
if self.args.unique_errors_file:
self.dump_error_data(self.args.unique_errors_file, self.ue_mixin.unique_data_names)
def _process_rmap(self, func, rmapping, *args, **keys):
"""Execute `func` on a single `rmapping` passing along *args and **keys"""
keywords = dict(keys)
rmapping_org = rmapping
new_filename = rmapping.filename if self.args.inplace else os.path.join(".", rmapping.basename)
if os.path.exists(new_filename):
log.info("Continuing refactoring from local copy", srepr(new_filename))
rmapping = rmap.load_mapping(new_filename)
keywords.update(locals())
fixers = self.args.fixers
if fixers:
rmapping = rmap.load_mapping(rmapping.filename)
keywords.update(locals())
apply_rmap_fixers(*args, **keywords)
func(*args, **keywords)
return new_filename
def cat_rmap(rmapping, new_filename, header_key, *args, **keys):
"""Cat/print rmapping's source text or the value of `header_key` in the rmap header."""
if header_key is not None:
log.info("In", srepr(rmapping.basename), "parameter", srepr(header_key), "=", srepr(rmapping.header[header_key]))
else:
log.info("-"*80)
log.info("Rmap", srepr(rmapping.basename), "is:")
log.info("-"*80)
log.write(str(rmapping))
def cat_rmap(rmapping, new_filename, header_key, *args, **keys):
"""Cat/print rmapping's source text or the value of `header_key` in the rmap header."""
if header_key is not None:
log.info("In", srepr(rmapping.basename), "parameter", srepr(header_key), "=", srepr(rmapping.header[header_key]))
else:
log.info("-"*80)
log.info("Rmap", srepr(rmapping.basename), "is:")
log.info("-"*80)
log.write(str(rmapping))
def polled(self):
"""Output the latest affected datasets taken from the history starting item onward.
Since the history drives and ultimately precedes any affected datasets computation, there's
no guarantee that every history item is available.
"""
assert 0 <= self.history_start < len(self.history), "Invalid history interval with starting index " + repr(self.history_start)
assert 0 <= self.history_stop < len(self.history), "Invalid history interval with stopping index " + repr(self.history_stop)
assert self.history_start <= self.history_stop, "Invalid history interval, start >= stop."
effects = []
for i in range(self.history_start, self.history_stop):
log.info("Fetching effects for", (i,) + self.history[i+1])
old_context = self.history[i][1]
new_context = self.history[i+1][1]
affected = self.get_affected(old_context, new_context)
if affected:
effects.append((i, affected))
return effects
def insert_references(self):
"""Insert files specified by --references into the appropriate rmaps identified by --source-context."""
self._setup_source_context()
categorized = self.categorize_files(self.args.references)
pmap = crds.get_pickled_mapping(self.source_context) # reviewed
self.args.rmaps = []
for (instrument, filekind) in categorized:
try:
self.args.rmaps.append(pmap.get_imap(instrument).get_rmap(filekind).filename)
except crexc.CrdsError:
log.info("Existing rmap for", (instrument, filekind), "not found. Trying empty spec.")
spec_file = os.path.join(
os.path.dirname(self.obs_pkg.__file__), "specs", instrument + "_" + filekind + ".rmap")
rmapping = rmap.asmapping(spec_file)
log.info("Loaded spec file from", repr(spec_file))
self.args.rmaps.append(spec_file)
self.rmap_apply(insert_rmap_references, categorized=categorized)
def insert_references(self):
"""Insert files specified by --references into the appropriate rmaps identified by --source-context."""
self._setup_source_context()
categorized = self.categorize_files(self.args.references)
pmap = crds.get_cached_mapping(self.source_context)
self.args.rmaps = []
for (instrument, filekind) in categorized:
try:
self.args.rmaps.append(pmap.get_imap(instrument).get_rmap(filekind).filename)
except crexc.CrdsError:
log.info("Existing rmap for", (instrument, filekind), "not found. Trying empty spec.")
spec_file = os.path.join(
os.path.dirname(self.obs_pkg.__file__), "specs", instrument + "_" + filekind + ".rmap")
rmapping = rmap.asmapping(spec_file)
log.info("Loaded spec file from", repr(spec_file))
self.args.rmaps.append(spec_file)
self.rmap_apply(insert_rmap_references, categorized=categorized)
def fetch_references(self, references):
"""Gets all references required to support `only_contexts`. Removes
all references from the CRDS reference cache which are not required for
`only_contexts`.
"""
if not self.contexts:
return
if self.args.readonly_cache:
already_have = set(rmap.list_references("*", self.observatory))
fetched = [ x for x in sorted(set(references)-set(already_have)) if not x.startswith("NOT FOUND") ]
if fetched:
log.info("READONLY CACHE would fetch references:", repr(fetched))
with log.info_on_exception("Reference size information not available."):
info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"])
total_bytes = api.get_total_bytes(info_map)
log.info("READONLY CACHE would download", len(fetched), "references totaling",
utils.human_format_number(total_bytes).strip(), "bytes.")
else:
self.dump_files(self.contexts[0], references)
def gen_specs(context):
"""Generate spec files corresponding to all types in `context`."""
log.info("Generating specs from", repr(context))
p = crds.get_cached_mapping(context)
for mapping in p.mapping_names():
if mapping.endswith(".rmap"):
r = crds.get_cached_mapping(mapping)
specname = r.instrument + "_" + r.filekind + ".spec"
specpath = os.path.join(HERE, "specs", specname)
if os.path.exists(specpath):
continue
spec = dict(r.header)
spec["filetype"] = FILEKIND_TO_FILETYPE.get(
r.filekind.upper(), r.filekind.upper())
spec["file_ext"] = os.path.splitext(r.reference_names()[0])[-1]
spec["text_descr"] = TEXT_DESCR[r.filekind]
spec["suffix"] = r.filekind
log.write("Generating spec", repr(specpath))
reftypes.write_spec(specpath, spec)
def fetch_references(self, references):
"""Gets all references required to support `only_contexts`. Removes
all references from the CRDS reference cache which are not required for
`only_contexts`.
"""
if not self.contexts:
return
if self.args.readonly_cache:
already_have = set(rmap.list_references("*", self.observatory))
fetched = [ x for x in sorted(set(references)-set(already_have)) if not x.startswith("NOT FOUND") ]
if fetched:
log.info("READONLY CACHE would fetch references:", repr(fetched))
with log.info_on_exception("Reference size information not available."):
info_map = api.get_file_info_map(self.observatory, fetched, fields=["size"])
total_bytes = api.get_total_bytes(info_map)
log.info("READONLY CACHE would download", len(fetched), "references totaling",
utils.human_format_number(total_bytes).strip(), "bytes.")
else:
self.dump_files(self.contexts[0], references)
def _submission(self, relative_url):
"""Do a generic submission re-post to the specified relative_url."""
assert self.args.description is not None, "You must supply a --description for this function."
self.ingest_files()
log.info("Posting web request for", srepr(relative_url))
completion_args = self.connection.repost_start(
relative_url,
pmap_mode=self.pmap_mode,
pmap_name=self.pmap_name,
instrument=self.instrument,
change_level=self.args.change_level,
creator=self.args.creator,
description=self.args.description,
auto_rename=not self.args.dont_auto_rename,
compare_old_reference=not self.args.dont_compare_old_reference,
)
# give POST time to complete send, not response
time.sleep(10)
return completion_args
