def dump_dataset_headers(self): """Print out the matching parameters for the --datasets specified on the command line. """ multi_context_headers = defaultdict(list) for context in self.contexts: if self.args.datasets: headers = api.get_dataset_headers_by_id( context, self.args.datasets) elif self.args.instrument: headers = api.get_dataset_headers_by_instrument( context, self.args.instrument) for dataset_id, header in headers.items(): multi_context_headers[dataset_id].append((context, header)) for dataset_id, context_headers in multi_context_headers.items(): for (context, header) in context_headers: if self.args.condition_values: header = utils.condition_header(header) if self.args.minimize_headers: header = crds.get_cached_mapping(context).minimize_header( header) if len(self.contexts) == 1: print(dataset_id, ":", log.format_parameter_list(header)) else: print(dataset_id, ":", context, ":", log.format_parameter_list(header))
def list_dataset_headers(self): """List dataset header info for self.args.dataset_headers with respect to self.args.context""" for context in self.contexts: with log.error_on_exception("Failed fetching dataset parameters with repect to", repr(context), "for", repr(self.args.dataset_headers)): pars = api.get_dataset_headers_by_id(context, self.args.dataset_headers) pmap = crds.get_cached_mapping(context) for requested_id in self.args.dataset_headers: for returned_id in sorted(pars.keys()): if requested_id.upper() in returned_id.upper(): header = pars[returned_id] if isinstance(header, python23.string_types): log.error("No header for", repr(returned_id), ":", repr(header)) # header is reason continue if self.args.id_expansions_only: print(returned_id, context if len(self.contexts) > 1 else "") else: if self.args.minimize_headers: header2 = pmap.minimize_header(header) else: header2 = dict(header) header2.pop("REFTYPE", None) header2["dataset_id"] = returned_id log.info("Dataset pars for", repr(returned_id), "with respect to", repr(context) + ":\n", log.PP(header2)) if self.args.first_id_expansion_only: break
def sync_datasets(self): """Sync mappings and references for datasets with respect to `self.contexts`.""" if not self.contexts: log.error("Define --contexts under which references are fetched for --dataset-files or --dataset-ids.""") sys.exit(-1) active_references = [] for context in self.contexts: if self.args.dataset_ids: if len(self.args.dataset_ids) == 1 and self.args.dataset_ids[0].startswith("@"): with open(self.args.dataset_ids[0][1:]) as pfile: self.args.dataset_ids = pfile.read().splitlines() with log.error_on_exception("Failed to get matching parameters for", self.args.dataset_ids): id_headers = api.get_dataset_headers_by_id(context, self.args.dataset_ids) for dataset in self.args.dataset_files or self.args.dataset_ids: log.info("Syncing context '%s' dataset '%s'." % (context, dataset)) with log.error_on_exception("Failed to get matching parameters from", repr(dataset)): if self.args.dataset_files: headers = { dataset : data_file.get_conditioned_header(dataset, observatory=self.observatory) } else: headers = { dataset_id : header for (dataset_id, header) in id_headers.items() if dataset.upper() in dataset_id } for assc_dataset, header in headers.items(): with log.error_on_exception("Failed syncing references for dataset", repr(assc_dataset), "under context", repr(context)): bestrefs = crds.getrecommendations(header, context=context, observatory=self.observatory, ignore_cache=self.args.ignore_cache) log.verbose("Best references for", repr(assc_dataset), "are", bestrefs) active_references.extend(bestrefs.values()) active_references = [ ref for ref in active_references if not ref.startswith("NOT FOUND") ] log.verbose("Syncing references:", repr(active_references)) return list(set(active_references))
def sync_datasets(self): """Sync mappings and references for datasets with respect to `self.contexts`.""" if not self.contexts: log.error("Define --contexts under which references are fetched for --dataset-files or --dataset-ids.""") sys.exit(-1) active_references = [] for context in self.contexts: if self.args.dataset_ids: if len(self.args.dataset_ids) == 1 and self.args.dataset_ids[0].startswith("@"): with open(self.args.dataset_ids[0][1:]) as pfile: self.args.dataset_ids = pfile.read().splitlines() with log.error_on_exception("Failed to get matching parameters for", self.args.dataset_ids): id_headers = api.get_dataset_headers_by_id(context, self.args.dataset_ids) for dataset in self.args.dataset_files or self.args.dataset_ids: log.info("Syncing context '%s' dataset '%s'." % (context, dataset)) with log.error_on_exception("Failed to get matching parameters from", repr(dataset)): if self.args.dataset_files: headers = { dataset : data_file.get_conditioned_header(dataset, observatory=self.observatory) } else: headers = { dataset_id : header for (dataset_id, header) in id_headers.items() if dataset.upper() in dataset_id } for assc_dataset, header in headers.items(): with log.error_on_exception("Failed syncing references for dataset", repr(assc_dataset), "under context", repr(context)): bestrefs = crds.getrecommendations(header, context=context, observatory=self.observatory, ignore_cache=self.args.ignore_cache) log.verbose("Best references for", repr(assc_dataset), "are", bestrefs) active_references.extend(bestrefs.values()) active_references = [ ref for ref in active_references if not ref.startswith("NOT FOUND") ] log.verbose("Syncing references:", repr(active_references)) return list(set(active_references))
def list_dataset_headers(self): """List dataset header info for self.args.dataset_headers with respect to self.args.context""" for context in self.contexts: with log.error_on_exception("Failed fetching dataset parameters with repect to", repr(context), "for", repr(self.args.dataset_headers)): pars = api.get_dataset_headers_by_id(context, self.args.dataset_headers) pmap = crds.get_cached_mapping(context) for requested_id in self.args.dataset_headers: for returned_id in sorted(pars.keys()): if requested_id.upper() in returned_id.upper(): header = pars[returned_id] if isinstance(header, python23.string_types): log.error("No header for", repr(returned_id), ":", repr(header)) # header is reason continue if self.args.id_expansions_only: print(returned_id, context if len(self.contexts) > 1 else "") else: if self.args.minimize_headers: header2 = pmap.minimize_header(header) else: header2 = dict(header) header2.pop("REFTYPE", None) header2["dataset_id"] = returned_id log.info("Dataset pars for", repr(returned_id), "with respect to", repr(context) + ":\n", log.PP(header2)) if self.args.first_id_expansion_only: break
def __init__(self, context, datasets, datasets_since): """"Contact the CRDS server and get headers for the list of `datasets` ids with respect to `context`.""" super(DatasetHeaderGenerator, self).__init__(context, datasets, datasets_since) server = api.get_crds_server() log.info("Dumping dataset parameters from CRDS server at", repr(server), "for", repr(datasets)) self.headers = api.get_dataset_headers_by_id(context, datasets) log.info("Dumped", len(self.headers), "of", len(datasets), "datasets from CRDS server at", repr(server)) # every command line id should correspond to 1 or more headers for source in self.sources: if self.matching_two_part_id(source) not in self.headers.keys(): log.warning("Dataset", repr(source), "isn't represented by downloaded parameters.") # Process according to downloaded 2-part ids, not command line ids. self.sources = sorted(self.headers.keys())
def list_datasets(self): """List dataset header info for self.args.datasets with respect to self.args.context""" for context in self.contexts: with log.error_on_exception("Failed fetching dataset parameters with repect to", repr(context), "for", repr(self.args.datasets)): pars = api.get_dataset_headers_by_id(context, self.args.datasets) pmap = rmap.get_cached_mapping(context) for (dataset_id, header) in pars.items(): if isinstance(header, python23.string_types): log.error("No header for", repr(dataset_id), ":", repr(header)) # header is reason continue header2 = pmap.minimize_header(header) header2.pop("REFTYPE", None) log.info("Dataset pars for", repr(dataset_id), "with respect to", repr(context) + ":\n", log.PP(header2))
def fetch_source_segment(self, source): """Return the segment of dataset ids which surrounds id `source`.""" try: index = self.sources.index(source) // self.segment_size except ValueError as exc: raise CrdsError("Unknown dataset id " + repr(source)) from exc lower = index * self.segment_size upper = (index + 1) * self.segment_size segment_ids = self.sources[lower:upper] log.verbose("Dumping", len(segment_ids), "datasets from indices", lower, "to", lower + len(segment_ids), verbosity=20) dumped_headers = api.get_dataset_headers_by_id(self.context, segment_ids) log.verbose("Dumped", len(dumped_headers), "datasets", verbosity=20) if self.save_pickles: # keep all headers, causes memory problems with multiple instruments on ~8G ram. self.headers.update(dumped_headers) else: # conserve memory by keeping only the last N headers self.headers = dumped_headers
def dump_dataset_headers(self): """Print out the matching parameters for the --datasets specified on the command line. """ multi_context_headers = defaultdict(list) for context in self.contexts: if self.args.datasets: headers = api.get_dataset_headers_by_id(context, self.args.datasets) elif self.args.instrument: headers = api.get_dataset_headers_by_instrument(context, self.args.instrument) for dataset_id, header in headers.items(): multi_context_headers[dataset_id].append((context, header)) for dataset_id, context_headers in multi_context_headers.items(): for (context, header) in context_headers: if self.args.condition_values: header = utils.condition_header(header) if self.args.minimize_headers: header = rmap.get_cached_mapping(context).minimize_header(header) if len(self.contexts) == 1: print(dataset_id, ":", log.format_parameter_list(header)) else: print(dataset_id, ":", context, ":", log.format_parameter_list(header))