def test_metric_package_reload(self): # Create a Job without Metric definitions meas = Measurement('validate_drp.PA1', 15 * u.mmag) measurement_set = MeasurementSet([meas]) job = Job(measurements=measurement_set) job.reload_metrics_package('verify_metrics') # Should now have metrics and specs self.assertTrue(len(job.specs) > 0) self.assertTrue(len(job.metrics) > 0) self.assertIsInstance(job.measurements['validate_drp.PA1'].metric, Metric)
def ingest_data(filenames, metrics_package): """Load JSON files into a list of lsst.validate.base measurement Jobs. Parameters ---------- filenames : list of str Filenames of JSON files to load. Returns ------- job_list : list of lsst.validate.base.Job Each element is the Job representation of the JSON file. """ jobs = {} # Read in JSON output from metrics run for filename in filenames: with open(filename) as fh: data = json.load(fh) job = Job.deserialize(**data) filter_name = job.meta['filter_name'] metrics = MetricSet.load_metrics_package(metrics_package) job.metrics.update(metrics) specs = SpecificationSet.load_metrics_package(metrics_package) job.specs.update(specs) jobs[filter_name] = job return jobs
def run(self): jobs = {} for metric in self.metrics: data_ids = list(self.registry.queryDatasets((f'metricvalue_{metric.package}' f'_{metric.metric}'), collections=self.collection)) for did in data_ids: m = self.butler.get(did, collections=self.collection) # make the name the same as what SQuaSH Expects m.metric_name = metric # Grab the physical filter associated with the abstract filter # In general there may be more than one. Take the shortest assuming # it is the most generic. pfilts = [el.name for el in self.butler.registry.queryDimensionRecords('physical_filter', dataId=did.dataId)] pfilt = min(pfilts, key=len) tract = did.dataId['tract'] afilt = did.dataId['band'] key = f"{tract}_{afilt}" if key not in jobs.keys(): job_metadata = {'instrument': did.dataId['instrument'], 'filter': pfilt, 'band': afilt, 'tract': tract, 'butler_generation': 'Gen3', 'ci_dataset': self.dataset_name} # Get dataset_repo_url from repository somehow? jobs[key] = Job(meta=job_metadata, metrics=self.metrics) jobs[key].measurements.insert(m) return jobs
def runDataRefs(self, datarefs, customMetadata=None): """Call all registered metric tasks on each dataref. This method loads all datasets required to compute a particular metric, and persists the metrics as one or more `lsst.verify.Job` objects. Only metrics that successfully produce a `~lsst.verify.Measurement` will be included in a job. Parameters ---------- datarefs : `list` of `lsst.daf.persistence.ButlerDataRef` The data to measure. Datarefs may be complete or partial; each generates a measurement at the same granularity (e.g., a dataref with only ``"visit"`` specified generates visit-level measurements). customMetadata : `dict`, optional Any metadata that are needed for a specific pipeline, but that are not needed by the ``lsst.verify`` framework or by general-purpose measurement analysis code (these cases are handled by the `~MetricsControllerConfig.metadataAdder` subtask). If omitted, only generic metadata are added. Both keys and values must be valid inputs to `~lsst.verify.Metadata`. Returns ------- struct : `lsst.pipe.base.Struct` A `~lsst.pipe.base.Struct` containing the following component: - ``jobs`` : a list of collections of measurements (`list` of `lsst.verify.Job`). Each job in the list contains the measurement(s) for the corresponding dataref, and each job has at most one measurement for each element in `self.measurers`. A particular measurement is omitted if it could not be created. Notes ----- Some objects may be persisted, or incorrectly persisted, in the event of an exception. """ jobs = [] index = 0 for dataref in datarefs: job = Job.load_metrics_package() try: self.metadataAdder.run(job, dataref=dataref) if customMetadata: job.meta.update(customMetadata) for task in self.measurers: self._computeSingleMeasurement(job, task, dataref) finally: jobFile = self._getJobFilePath(index, dataref.dataId) self.log.info("Persisting metrics to %s...", jobFile) # This call order maximizes the chance that job gets # written, and to a unique file index += 1 job.write(jobFile) jobs.append(job) return Struct(jobs=jobs)
def __init__(self, butler=None, profile_jointcal=False, **kwargs): """ Instantiate a JointcalTask. Parameters ---------- butler : `lsst.daf.persistence.Butler` The butler is passed to the refObjLoader constructor in case it is needed. Ignored if the refObjLoader argument provides a loader directly. Used to initialize the astrometry and photometry refObjLoaders. profile_jointcal : `bool` Set to True to profile different stages of this jointcal run. """ pipeBase.CmdLineTask.__init__(self, **kwargs) self.profile_jointcal = profile_jointcal self.makeSubtask("sourceSelector") if self.config.doAstrometry: self.makeSubtask('astrometryRefObjLoader', butler=butler) self.makeSubtask("astrometryReferenceSelector") else: self.astrometryRefObjLoader = None if self.config.doPhotometry: self.makeSubtask('photometryRefObjLoader', butler=butler) self.makeSubtask("photometryReferenceSelector") else: self.photometryRefObjLoader = None # To hold various computed metrics for use by tests self.job = Job.load_metrics_package(subset='jointcal')
def unpersistJob(fileName): """Unpersist a Job object from the filename of its serialized form. Returns ------- The `lsst.verify.Job` object contained in `fileName`. """ with open(fileName) as handle: return Job.deserialize(**json.load(handle))
def test_metadataonly(self, mock_stdout): """Test that inspect_job can handle files with metadata but no metrics. """ # Job and its components were not designed to support deletion, so # create a new Job from scratch to ensure it's a valid object. job = Job() job.metrics.insert( Metric("foo.boringmetric", "", u.percent, tags=["redundant"])) job.metrics.insert( Metric("foo.fancymetric", "", u.meter, tags=["vital"])) job.meta["bar"] = "high" job.meta["shape"] = "rotund" job.specs.insert( ThresholdSpecification("utterly_ridiculous", 1e10 * u.meter, ">")) inspect_job(job) output = mock_stdout.getvalue() for key, value in [("bar", "high"), ("shape", "rotund")]: self._check_metadata(key, value, output)
def main(): """Present all Job files. """ args = build_argparser().parse_args() for filename in args.json_paths: if len(args.json_paths) > 1: print("\n%s:" % filename) with open(filename) as f: job = Job.deserialize(**json.load(f)) inspect_job(job)
def test_job_iadd(self): job_1 = Job(metrics=self.metric_set, specs=self.spec_set, measurements=self.measurement_set) job_2 = Job(metrics=self.metric_set_2, specs=self.spec_set_2, measurements=self.measurement_set_2) job_1 += job_2 self.assertIn(self.metric_photrms.name, job_1.metrics) self.assertIn(self.metric_test_2.name, job_1.metrics) self.assertIn('test.PhotRms.design', job_1.specs) self.assertIn('test2.SourceCount.design', job_1.specs) self.assertIn('test.PhotRms', job_1.measurements) self.assertIn('test2.SourceCount', job_1.measurements) self.assertIn('test.PhotRms', job_1.measurements['test.PhotRms'].blobs) self.assertIn('test2_blob', job_1.measurements['test2.SourceCount'].blobs)
def run(self): """Collate job information. Returns ------- jobs : `dict` [`str`, `lsst.verify.Job`] A mapping of `~lsst.verify.Job` objects, indexed by a string representation of their data ID. """ jobs = {} for metric in self.metrics: dataset = f'metricvalue_{metric.package}_{metric.metric}' datasetRefs = set( self.registry.queryDatasets(dataset, collections=self.collection, findFirst=True)) for ref in datasetRefs: # getDirect skips dataset resolution; ref is guaranteed to # be valid. m = self.butler.getDirect(ref) # make the name the same as what SQuaSH Expects m.metric_name = metric # queryDatasets guarantees ref.dataId.hasFull() dataId = ref.dataId.full.byName() key = make_key(ref) # For backward-compatibility with Gen 2 SQuaSH uploads pfilt = dataId.get('physical_filter') if not pfilt: # Grab the physical filter associated with the abstract # filter. In general there may be more than one. Take the # shortest assuming it is the most generic. pfilts = [ el.name for el in self.registry.queryDimensionRecords( 'physical_filter', dataId=ref.dataId) ] pfilt = min(pfilts, key=len) if key not in jobs.keys(): job_metadata = { 'filter': pfilt, 'butler_generation': 'Gen3', 'ci_dataset': self.dataset_name, } job_metadata.update(dataId) # Get dataset_repo_url from repository somehow? jobs[key] = Job(meta=job_metadata, metrics=self.metrics) jobs[key].measurements.insert(m) return jobs
def main(filenames): """Present all Job files. Parameters ---------- filenames : `list` of `str` The Job files to open. Must be in JSON format. """ for filename in filenames: if len(filenames) > 1: print("\n%s:" % filename) with open(filename) as f: job = Job.deserialize(**json.load(f)) inspect_job(job)
def setUp(self): self.job = Job() self.job.metrics.insert( Metric("foo.boringmetric", "", u.percent, tags=["redundant"])) self.job.metrics.insert( Metric("foo.fancymetric", "", u.meter, tags=["vital"])) self.job.measurements.insert( Measurement("foo.fancymetric", 2.0 * u.meter)) self.job.measurements.insert( Measurement("foo.fanciermetric", 3.5 * u.second)) self.job.measurements["foo.fanciermetric"].notes["fanciness"] \ = "moderate" self.job.measurements.insert( Measurement("foo.fanciestmetric", 3.1415927 * u.kilogram)) self.job.meta["bar"] = "high" self.job.meta["shape"] = "rotund" self.job.specs.insert( ThresholdSpecification("utterly_ridiculous", 1e10 * u.meter, ">"))
def test_query_metadata(self): job = Job(meta={'filter_name': 'r', 'camera': 'MegaCam'}) s1 = ThresholdSpecification(Name('validate_drp.AM1.design_r'), 5. * u.marcsec, '<', metadata_query={'filter_name': 'r'}) s2 = ThresholdSpecification(Name('validate_drp.AM1.design_i'), 5. * u.marcsec, '<', metadata_query={'filter_name': 'i'}) s3 = ThresholdSpecification(Name('validate_drp.AM1.design_HSC_r'), 5. * u.marcsec, '<', metadata_query={ 'filter_name': 'r', 'camera': 'HSC' }) self.assertTrue(s1.query_metadata(job.meta)) self.assertFalse(s2.query_metadata(job.meta)) self.assertFalse(s3.query_metadata(job.meta))
def test_metricsonly(self, mock_stdout): """Test that inspect_job can handle files with metrics but no metadata. """ # Job and its components were not designed to support deletion, so # create a new Job from scratch to ensure it's a valid object. job = Job() job.metrics.insert( Metric("foo.boringmetric", "", u.percent, tags=["redundant"])) job.metrics.insert( Metric("foo.fancymetric", "", u.meter, tags=["vital"])) job.measurements.insert(Measurement("foo.fancymetric", 2.0 * u.meter)) job.measurements.insert( Measurement("foo.fanciermetric", 3.5 * u.second)) job.measurements["foo.fanciermetric"].notes["fanciness"] = "moderate" job.measurements.insert( Measurement("foo.fanciestmetric", 3.1415927 * u.kilogram)) inspect_job(job) output = mock_stdout.getvalue() # MeasurementSet.values does not exist for _, measurement in job.measurements.items(): self._check_measurement(measurement, output)
def __init__(self, butler=None, profile_jointcal=False, **kwargs): """ Instantiate a JointcalTask. Parameters ---------- butler : lsst.daf.persistence.Butler The butler is passed to the refObjLoader constructor in case it is needed. Ignored if the refObjLoader argument provides a loader directly. Used to initialize the astrometry and photometry refObjLoaders. profile_jointcal : bool set to True to profile different stages of this jointcal run. """ pipeBase.CmdLineTask.__init__(self, **kwargs) self.profile_jointcal = profile_jointcal self.makeSubtask("sourceSelector") if self.config.doAstrometry: self.makeSubtask('astrometryRefObjLoader', butler=butler) if self.config.doPhotometry: self.makeSubtask('photometryRefObjLoader', butler=butler) # To hold various computed metrics for use by tests self.job = Job.load_metrics_package(subset='jointcal')
def load_json_output(filepath, metrics_package='verify_metrics'): """Read JSON from a file into a job object. Currently just does a trivial de-serialization with no checking to make sure that one results with a valid validate.base.job object. Parameters ---------- filepath : `str` Source file name for JSON output. Returns ------- job : A `validate.base.job` object. """ with open(filepath, 'r') as infile: json_data = json.load(infile) job = Job.deserialize(**json_data) metrics = MetricSet.load_metrics_package(metrics_package) job.metrics.update(metrics) specs = SpecificationSet.load_metrics_package(metrics_package) job.specs.update(specs) return job
def setUp(self): self.testbed = SquashMetadataTask() self.job = Job()
def main(): """Entrypoint for the ``dispatch_verify.py`` command line executable. """ log = lsst.log.Log.getLogger('verify.bin.dispatchverify.main') args = parse_args() config = Configuration(args) log.debug(str(config)) # Parse all Job JSON jobs = [] for json_path in config.json_paths: log.info('Loading {0}'.format(json_path)) with open(json_path) as fp: json_data = json.load(fp) job = Job.deserialize(**json_data) jobs.append(job) # Merge all Jobs into one job = jobs.pop(0) if len(jobs) > 0: log.info('Merging verification Job JSON.') for other_job in jobs: job += other_job # Ensure all measurements have a metric so that units are normalized log.info('Refreshing metric definitions from verify_metrics') job.reload_metrics_package('verify_metrics') # Insert package metadata from lsstsw if not config.ignore_lsstsw: log.info('Inserting lsstsw package metadata from ' '{0}.'.format(config.lsstsw)) job = insert_lsstsw_metadata(job, config) # Insert metadata from additional specified packages if config.extra_package_paths is not None: job = insert_extra_package_metadata(job, config) # Add environment variable metadata from the Jenkins CI environment if config.env_name == 'jenkins': log.info('Inserting Jenkins CI environment metadata.') jenkins_metadata = get_jenkins_env() job = insert_env_metadata(job, 'jenkins', jenkins_metadata) # Upload job if not config.test: log.info('Uploading Job JSON to {0}.'.format(config.api_url)) job.dispatch(api_user=config.api_user, api_password=config.api_password, api_url=config.api_url) if config.show_json: print( json.dumps(job.json, sort_keys=True, indent=4, separators=(',', ': '))) # Write a json file if config.output_filepath is not None: log.info('Writing Job JSON to {0}.'.format(config.output_filepath)) job.write(config.output_filepath)
def runOneFilter(repo, visitDataIds, brightSnrMin=None, brightSnrMax=None, makeJson=True, filterName=None, outputPrefix='', doApplyExternalPhotoCalib=False, externalPhotoCalibName=None, doApplyExternalSkyWcs=False, externalSkyWcsName=None, skipTEx=False, verbose=False, metrics_package='verify_metrics', instrument='Unknown', dataset_repo_url='./', skipNonSrd=False, **kwargs): r"""Main executable for the case where there is just one filter. Plot files and JSON files are generated in the local directory prefixed with the repository name (where '_' replace path separators), unless overriden by specifying `outputPrefix`. E.g., Analyzing a repository ``CFHT/output`` will result in filenames that start with ``CFHT_output_``. Parameters ---------- repo : string or Butler A Butler or a repository URL that can be used to construct one. dataIds : list of dict List of `butler` data IDs of Image catalogs to compare to reference. The `calexp` pixel image is needed for the photometric calibration unless doApplyExternalPhotoCalib is True such that the appropriate `photoCalib` dataset is used. Note that these have data IDs that include the tract number. brightSnrMin : float, optional Minimum median SNR for a source to be considered bright; passed to `lsst.validate.drp.matchreduce.build_matched_dataset`. brightSnrMax : float, optional Maximum median SNR for a source to be considered bright; passed to `lsst.validate.drp.matchreduce.build_matched_dataset`. makeJson : bool, optional Create JSON output file for metrics. Saved to current working directory. outputPrefix : str, optional Specify the beginning filename for output files. filterName : str, optional Name of the filter (bandpass). doApplyExternalPhotoCalib : bool, optional Apply external photoCalib to calibrate fluxes. externalPhotoCalibName : str, optional Type of external `PhotoCalib` to apply. Currently supported are jointcal, fgcm, and fgcm_tract. Must be set if doApplyExternalPhotoCalib is True. doApplyExternalSkyWcs : bool, optional Apply external wcs to calibrate positions. externalSkyWcsName : str, optional Type of external `wcs` to apply. Currently supported is jointcal. Must be set if "doApplyExternalSkyWcs" is True. skipTEx : bool, optional Skip TEx calculations (useful for older catalogs that don't have PsfShape measurements). verbose : bool, optional Output additional information on the analysis steps. skipNonSrd : bool, optional Skip any metrics not defined in the LSST SRD. Raises ------ RuntimeError: Raised if "doApplyExternalPhotoCalib" is True and "externalPhotoCalibName" is None, or if "doApplyExternalSkyWcs" is True and "externalSkyWcsName" is None. """ if kwargs: log.warn( f"Extra kwargs - {kwargs}, will be ignored. Did you add extra things to your config file?" ) if doApplyExternalPhotoCalib and externalPhotoCalibName is None: raise RuntimeError( "Must set externalPhotoCalibName if doApplyExternalPhotoCalib is True." ) if doApplyExternalSkyWcs and externalSkyWcsName is None: raise RuntimeError( "Must set externalSkyWcsName if doApplyExternalSkyWcs is True.") # collect just the common key, value pairs to omit the keys that are aggregated over job_metadata = dict( set.intersection(*[set(vid.items()) for vid in visitDataIds])) # update with metadata passed into the method job_metadata.update({ 'instrument': instrument, 'filter_name': filterName, 'dataset_repo_url': dataset_repo_url }) job = Job.load_metrics_package(meta=job_metadata, subset='validate_drp', package_name_or_path=metrics_package) matchedDataset = build_matched_dataset( repo, visitDataIds, doApplyExternalPhotoCalib=doApplyExternalPhotoCalib, externalPhotoCalibName=externalPhotoCalibName, doApplyExternalSkyWcs=doApplyExternalSkyWcs, externalSkyWcsName=externalSkyWcsName, skipTEx=skipTEx, skipNonSrd=skipNonSrd, brightSnrMin=brightSnrMin, brightSnrMax=brightSnrMax) snr = matchedDataset['snr'].quantity bright = (matchedDataset['brightSnrMin'].quantity < snr) & (snr < matchedDataset['brightSnrMax'].quantity) photomModel = build_photometric_error_model(matchedDataset, bright) astromModel = build_astrometric_error_model(matchedDataset, bright) linkedBlobs = [matchedDataset, photomModel, astromModel] metrics = job.metrics specs = job.specs def add_measurement(measurement): for blob in linkedBlobs: measurement.link_blob(blob) job.measurements.insert(measurement) for x, D in zip((1, 2, 3), (5., 20., 200.)): amxName = 'AM{0:d}'.format(x) afxName = 'AF{0:d}'.format(x) adxName = 'AD{0:d}'.format(x) amx = measureAMx(metrics['validate_drp.' + amxName], matchedDataset, D * u.arcmin, verbose=verbose) add_measurement(amx) afx_spec_set = specs.subset(required_meta={'instrument': 'HSC'}, spec_tags=[ afxName, ]) adx_spec_set = specs.subset(required_meta={'instrument': 'HSC'}, spec_tags=[ adxName, ]) for afx_spec_key, adx_spec_key in zip(afx_spec_set, adx_spec_set): afx_spec = afx_spec_set[afx_spec_key] adx_spec = adx_spec_set[adx_spec_key] adx = measureADx(metrics[adx_spec.metric_name], amx, afx_spec) add_measurement(adx) afx = measureAFx(metrics[afx_spec.metric_name], amx, adx, adx_spec) add_measurement(afx) pa1 = measurePA1(metrics['validate_drp.PA1'], filterName, matchedDataset.matchesBright, matchedDataset.magKey) add_measurement(pa1) pf1_spec_set = specs.subset(required_meta={ 'instrument': instrument, 'filter_name': filterName }, spec_tags=[ 'PF1', ]) pa2_spec_set = specs.subset(required_meta={ 'instrument': instrument, 'filter_name': filterName }, spec_tags=[ 'PA2', ]) # I worry these might not always be in the right order. Sorting... pf1_spec_keys = list(pf1_spec_set.keys()) pa2_spec_keys = list(pa2_spec_set.keys()) pf1_spec_keys.sort() pa2_spec_keys.sort() for pf1_spec_key, pa2_spec_key in zip(pf1_spec_keys, pa2_spec_keys): pf1_spec = pf1_spec_set[pf1_spec_key] pa2_spec = pa2_spec_set[pa2_spec_key] pa2 = measurePA2(metrics[pa2_spec.metric_name], pa1, pf1_spec.threshold) add_measurement(pa2) pf1 = measurePF1(metrics[pf1_spec.metric_name], pa1, pa2_spec) add_measurement(pf1) if not skipTEx: for x, D, bin_range_operator in zip((1, 2), (1.0, 5.0), ("<=", ">=")): texName = 'TE{0:d}'.format(x) tex = measureTEx(metrics['validate_drp.' + texName], matchedDataset, D * u.arcmin, bin_range_operator, verbose=verbose) add_measurement(tex) if not skipNonSrd: model_phot_reps = measure_model_phot_rep(metrics, filterName, matchedDataset) for measurement in model_phot_reps: add_measurement(measurement) if makeJson: job.write(outputPrefix + '.json') return job
def merge(jobs, lastJob): """Combine measurements from multiple chips or visits. Other job properties will be dictionary-merged (i.e., if multiple entries are assigned to the same key, only one will be preserved). Parameters ---------- jobs: iterable of `lsst.verify.Job` The jobs containing data to combine. lastJob: The job corresponding to the final run of ap_verify. Return ------ A single `lsst.verify.Job` object containing merged measurements from `jobs`. """ merged = Job.load_metrics_package() # Visible Job state: # job.measurements # job.meta # job.metrics (guaranteed by load_metrics_package) # job.specs (guaranteed by load_metrics_package) measurementsPerMetric = defaultdict(list) for job in jobs: for metricName in job.measurements: measurementsPerMetric[str(metricName)].append( job.measurements[metricName]) for metric in measurementsPerMetric: # Running times, object counts if metric.endswith("Time") or metric in { "ip_diffim.numSciSources", "association.numNewDiaObjects", "association.totalUnassociatedDiaObjects" }: addIfDefined(merged.measurements, sumMeasurements(measurementsPerMetric[metric])) # Fractions require special handling addIfDefined( merged.measurements, # Due to time constraints, no metric for total DIAObjects was implemented, # so we have to work around its absence mergeFractionsPartial( measurementsPerMetric["association.fracUpdatedDiaObjects"], measurementsPerMetric["association.numUnassociatedDiaObjects"])) addIfDefined( merged.measurements, mergeFractions( measurementsPerMetric["ip_diffim.fracDiaSourcesToSciSources"], measurementsPerMetric["ip_diffim.numSciSources"])) # L1 database metrics are cumulative, not per-CCD, so just copy them over for metric in ["association.totalUnassociatedDiaObjects"]: if metric in lastJob.measurements: addIfDefined(merged.measurements, lastJob.measurements[metric]) for job in jobs: merged.meta.update(job.meta) return merged
def test_job(self): """Create a Job from object sets.""" job = Job(metrics=self.metric_set, specs=self.spec_set, measurements=self.measurement_set) # Test object access via properties self.assertIn('test.PhotRms.design', job.specs) self.assertIn('test.PhotRms', job.metrics) self.assertIn('test.PhotRms', job.measurements) # Test metadata access self.assertIn('test.PhotRms.note', job.meta) self.assertEqual(job.meta['test.PhotRms.note'], 'value') # measurement metadata is always prefixed self.assertNotIn('note', job.meta) job.meta['job-level-key'] = 'yes' self.assertEqual(job.meta['job-level-key'], 'yes') self.assertIn('job-level-key', job.meta) self.assertEqual(len(job.meta), 2) job.meta.update({'test.PhotRms.note2': 'foo', 'dataset': 'ci_hsc'}) # note2 should be in measurement notes self.assertEqual(job.measurements['test.PhotRms'].notes['note2'], 'foo') self.assertEqual(job.meta['dataset'], 'ci_hsc') # Delete measurement and job-level metadata del job.meta['test.PhotRms.note2'] self.assertNotIn('test.PhotRms.note2', job.meta) self.assertNotIn('note2', job.measurements['test.PhotRms'].notes) del job.meta['dataset'] self.assertNotIn('dataset', job.meta) self.assertEqual(set(job.meta.keys()), set(['job-level-key', 'test.PhotRms.note'])) self.assertEqual(set([key for key in job.meta]), set(['job-level-key', 'test.PhotRms.note'])) keys = set() for key, value in job.meta.items(): keys.add(key) self.assertEqual(keys, set(['job-level-key', 'test.PhotRms.note'])) # Add a new measurement m = Measurement('test.PhotMedian', 28.5 * u.mag, notes={'aperture_corr': True}) job.measurements.insert(m) self.assertIn('test.PhotMedian', job.measurements) self.assertEqual(job.meta['test.PhotMedian.aperture_corr'], True) # Test serialization json_doc = job.json self.assertIn('measurements', json_doc) self.assertEqual(len(json_doc['measurements']), len(job.measurements)) self.assertIn('blobs', json_doc) self.assertIn('metrics', json_doc) self.assertEqual(len(json_doc['metrics']), len(job.metrics)) self.assertIn('specs', json_doc) self.assertEqual(len(json_doc['specs']), len(job.specs)) self.assertIn('meta', json_doc) self.assertEqual(len(json_doc['meta']), len(job.meta)) new_job = Job.deserialize(**json_doc) self.assertEqual(job, new_job) # check job-to-measurement metadata deserialization self.assertEqual(new_job.measurements['test.PhotRms'].notes['note'], 'value') self.assertEqual(new_job.meta['test.PhotRms.note'], 'value') self.assertEqual(new_job.meta['job-level-key'], 'yes')
def runOneFilter(repo, visitDataIds, metrics, brightSnr=100, makeJson=True, filterName=None, outputPrefix='', useJointCal=False, skipTEx=False, verbose=False, metrics_package='verify_metrics', **kwargs): """Main executable for the case where there is just one filter. Plot files and JSON files are generated in the local directory prefixed with the repository name (where '_' replace path separators), unless overriden by specifying `outputPrefix`. E.g., Analyzing a repository ``CFHT/output`` will result in filenames that start with ``CFHT_output_``. Parameters ---------- repo : string or Butler A Butler or a repository URL that can be used to construct one. dataIds : list of dict List of `butler` data IDs of Image catalogs to compare to reference. The `calexp` pixel image is needed for the photometric calibration unless useJointCal is True, in which the `photoCalib` and `wcs` datasets are used instead. Note that these have data IDs that include the tract number. metrics : `dict` or `collections.OrderedDict` Dictionary of `lsst.validate.base.Metric` instances. Typically this is data from ``validate_drp``\ 's ``metrics.yaml`` and loaded with `lsst.validate.base.load_metrics`. brightSnr : float, optional Minimum SNR for a star to be considered bright makeJson : bool, optional Create JSON output file for metrics. Saved to current working directory. outputPrefix : str, optional Specify the beginning filename for output files. filterName : str, optional Name of the filter (bandpass). useJointCal : bool, optional Use jointcal/meas_mosaic outputs to calibrate positions and fluxes. skipTEx : bool, optional Skip TEx calculations (useful for older catalogs that don't have PsfShape measurements). verbose : bool, optional Output additional information on the analysis steps. """ matchedDataset = build_matched_dataset(repo, visitDataIds, useJointCal=useJointCal, skipTEx=skipTEx) photomModel = build_photometric_error_model(matchedDataset) astromModel = build_astrometric_error_model(matchedDataset) linkedBlobs = [matchedDataset, photomModel, astromModel] try: instrument = kwargs['instrument'] dataset_repo_url = kwargs['dataset_repo_url'] except KeyError: raise ValueError( "Instrument name and input dataset URL must be set in config file") job = Job.load_metrics_package(meta={ 'instrument': instrument, 'filter_name': filterName, 'dataset_repo_url': dataset_repo_url }, subset='validate_drp', package_name_or_path=metrics_package) metrics = job.metrics specs = job.specs def add_measurement(measurement): for blob in linkedBlobs: measurement.link_blob(blob) job.measurements.insert(measurement) for x, D in zip((1, 2, 3), (5., 20., 200.)): amxName = 'AM{0:d}'.format(x) afxName = 'AF{0:d}'.format(x) adxName = 'AD{0:d}'.format(x) amx = measureAMx(metrics['validate_drp.' + amxName], matchedDataset, D * u.arcmin) add_measurement(amx) afx_spec_set = specs.subset(required_meta={'instrument': 'HSC'}, spec_tags=[ afxName, ]) adx_spec_set = specs.subset(required_meta={'instrument': 'HSC'}, spec_tags=[ adxName, ]) for afx_spec_key, adx_spec_key in zip(afx_spec_set, adx_spec_set): afx_spec = afx_spec_set[afx_spec_key] adx_spec = adx_spec_set[adx_spec_key] adx = measureADx(metrics[adx_spec.metric_name], amx, afx_spec) add_measurement(adx) afx = measureAFx(metrics[afx_spec.metric_name], amx, adx, adx_spec) add_measurement(afx) pa1 = measurePA1(metrics['validate_drp.PA1'], matchedDataset, filterName) add_measurement(pa1) pf1_spec_set = specs.subset(required_meta={ 'instrument': instrument, 'filter_name': filterName }, spec_tags=[ 'PF1', ]) pa2_spec_set = specs.subset(required_meta={ 'instrument': instrument, 'filter_name': filterName }, spec_tags=[ 'PA2', ]) # I worry these might not always be in the right order. Sorting... pf1_spec_keys = list(pf1_spec_set.keys()) pa2_spec_keys = list(pa2_spec_set.keys()) pf1_spec_keys.sort() pa2_spec_keys.sort() for pf1_spec_key, pa2_spec_key in zip(pf1_spec_keys, pa2_spec_keys): pf1_spec = pf1_spec_set[pf1_spec_key] pa2_spec = pa2_spec_set[pa2_spec_key] pa2 = measurePA2(metrics[pa2_spec.metric_name], pa1, pf1_spec.threshold) add_measurement(pa2) pf1 = measurePF1(metrics[pf1_spec.metric_name], pa1, pa2_spec) add_measurement(pf1) if not skipTEx: for x, D, bin_range_operator in zip((1, 2), (1.0, 5.0), ("<=", ">=")): texName = 'TE{0:d}'.format(x) tex = measureTEx(metrics['validate_drp.' + texName], matchedDataset, D * u.arcmin, bin_range_operator) add_measurement(tex) if makeJson: job.write(outputPrefix + '.json') return job
def main(): """Entrypoint for the ``dispatch_verify.py`` command line executable. """ logging.basicConfig(level=logging.INFO, stream=sys.stdout, format="{name} {levelname}: {message}", style="{") log = _LOG.getChild('main') parser = build_argparser() args = parser.parse_args() config = Configuration(args) log.debug(str(config)) # Parse all Job JSON jobs = [] for json_path in config.json_paths: log.info('Loading {0}'.format(json_path)) with open(json_path) as fp: json_data = json.load(fp) # Ignore blobs from the verification jobs if config.ignore_blobs: log.info('Ignoring blobs from Job JSON {0}'.format(json_path)) json_data = delete_blobs(json_data) job = Job.deserialize(**json_data) jobs.append(job) # Merge all Jobs into one job = jobs.pop(0) if len(jobs) > 0: log.info('Merging verification Job JSON.') for other_job in jobs: job += other_job # Ensure all measurements have a metric so that units are normalized log.info('Refreshing metric definitions from verify_metrics') job.reload_metrics_package('verify_metrics') # Insert package metadata from lsstsw if not config.ignore_lsstsw: log.info('Inserting lsstsw package metadata from ' '{0}.'.format(config.lsstsw)) job = insert_lsstsw_metadata(job, config) # Insert metadata from additional specified packages if config.extra_package_paths is not None: job = insert_extra_package_metadata(job, config) # Add environment variable metadata from the Jenkins CI environment if config.env_name == 'jenkins': log.info('Inserting Jenkins CI environment metadata.') jenkins_metadata = get_jenkins_env() job = insert_env_metadata(job, 'jenkins', jenkins_metadata, config.date_created) elif config.env_name == 'ldf': log.info('Inserting LSST Data Facility environment metadata.') ldf_metadata = get_ldf_env() job = insert_env_metadata(job, 'ldf', ldf_metadata, config.date_created) # Upload job if not config.test: log.info('Uploading Job JSON to {0}.'.format(config.api_url)) response = job.dispatch(api_user=config.api_user, api_password=config.api_password, api_url=config.api_url) log.info(response.json()['message']) if config.show_json: print( json.dumps(job.json, sort_keys=True, indent=4, separators=(',', ': '))) # Write a json file if config.output_filepath is not None: log.info('Writing Job JSON to {0}.'.format(config.output_filepath)) job.write(config.output_filepath)
def test_empty(self, mock_stdout): """Test that inspect_job can handle files with neither metrics nor metadata. """ inspect_job(Job())