def makeCompatibleRepo(self, repoDir, calibRepoDir): """Set up a directory as a repository compatible with this dataset. If the directory already exists, any files required by the dataset will be added if absent; otherwise the directory will remain unchanged. Parameters ---------- repoDir : `str` The directory where the output repository will be created. calibRepoDir : `str` The directory where the output calibration repository will be created. """ mapperArgs = {'mapperArgs': {'calibRoot': calibRepoDir}} if _isRepo(self.templateLocation): # Stub repo is not a parent because can't mix v1 and v2 repositories in parents list Butler(inputs=[{ "root": self.templateLocation, "mode": "r" }], outputs=[{ "root": repoDir, "mode": "rw", **mapperArgs }]) else: Butler(inputs=[{ "root": self._stubInputRepo, "mode": "r" }], outputs=[{ "root": repoDir, "mode": "rw", **mapperArgs }])
def inject_fakes_to_calexp(calexp_repo, calexp_id, magVar, coords, poisson=False): """This function injects fakes onto a calexp exposure at given positions. caelxp_repo: An empty calexp repo which is used to save the injected exposure. calexp_id: The data id of the calexp exposure magVar: The injected magnitude coords: A list of [x, y] pixel coordinates poisson: If true, add Poisson fluctuations to the fake model """ calexp_butler = Butler(calexp_repo) calexp_photoCalib = calexp_butler.get('calexp_photoCalib', calexp_id) calexp_exposure = calexp_butler.get('calexp', dataId=calexp_id) for coord in coords: x, y = coord[0], coord[1] inject_star(calexp_exposure, calexp_photoCalib, x, y, magVar, poisson=poisson) calexp_butler.put(calexp_exposure, 'calexp', dataId=calexp_id)
def setUp(self): refCatDir = os.path.join(os.path.dirname(__file__), "data", "sdssrefcat") self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(3001, 3001)) self.ctrPix = afwGeom.Point2I(1500, 1500) metadata = dafBase.PropertySet() metadata.set("RADECSYS", "FK5") metadata.set("EQUINOX", 2000.0) metadata.set("CTYPE1", "RA---TAN") metadata.set("CTYPE2", "DEC--TAN") metadata.set("CUNIT1", "deg") metadata.set("CUNIT2", "deg") metadata.set("CRVAL1", 215.5) metadata.set("CRVAL2", 53.0) metadata.set("CRPIX1", self.ctrPix[0] + 1) metadata.set("CRPIX2", self.ctrPix[1] + 1) metadata.set("CD1_1", 5.1e-05) metadata.set("CD1_2", 0.0) metadata.set("CD2_2", -5.1e-05) metadata.set("CD2_1", 0.0) self.tanWcs = afwImage.makeWcs(metadata) self.exposure = afwImage.ExposureF(self.bbox) self.exposure.setWcs(self.tanWcs) self.exposure.setFilter(afwImage.Filter("r", True)) butler = Butler(refCatDir) self.refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler)
def load_tract(repo, tract, patches=None, **kwargs): """Merge catalogs from forced-photometry coadds across available filters. Parameters -- tract: int Tract of sky region to load repo: str File location of Butler repository+rerun to load. patches: list of str List of patches. If not specified, will default to '0,0'--'7,7'. Returns -- AstroPy Table of merged catalog """ butler = Butler(repo) if patches is None: patches = ['%d,%d' % (i, j) for i in range(8) for j in range(8)] merged_patch_cats = [] for patch in patches: try: this_patch_merged_cat = load_patch(butler, tract, patch, **kwargs) except NoResults as e: print(e) continue merged_patch_cats.append(this_patch_merged_cat) merged_tract_cat = vstack(merged_patch_cats) return merged_tract_cat
def load_tract(repo, tract, patches=None, **kwargs): """Merge catalogs from forced-photometry coadds across available filters. Parameters -- tract: int Tract of sky region to load repo: str File location of Butler repository+rerun to load. patches: list of str List of patches. If not specified, will default to '0,0'--'7,7'. Returns -- AstroPy Table of merged catalog """ butler = Butler(repo) if patches is None: # Extract the patches for this tract from the skymap skymap = butler.get(datasetType='deepCoadd_skyMap') patches = ['%d,%d' % patch.getIndex() for patch in skymap[tract]] merged_patch_cats = [] for patch in patches: this_patch_merged_cat = load_patch(butler, tract, patch, **kwargs) # Event if this_patch_merged_cat is an empty Table, it's still fine to append to the list here. # They will get vstacked away below. merged_patch_cats.append(this_patch_merged_cat) merged_tract_cat = vstack(merged_patch_cats) return merged_tract_cat
def get_butler_by_repo(repo, **kwargs): """Construct and return a Bulter for the requested repository Parameters ---------- repo : `str` Name of the repo, e.g., 'TS8' | 'BOT' kwargs Passed to the Bulter constructor Returns ------- butler : `Butler` the requested Bulter Raises ------ KeyError : If repo does not match any known repository """ try: repo_path = BUTLER_REPO_DICT[repo] except KeyError: raise KeyError("Unknown Bulter repo key %s" % repo) butler = Butler(repo_path, **kwargs) return butler
def load_tract(repo, tract, patches=None, **kwargs): """Merge catalogs from forced-photometry coadds across available filters. Parameters -- tract: int Tract of sky region to load repo: str File location of Butler repository+rerun to load. patches: list of str List of patches. If not specified, will default to '0,0'--'7,7'. Returns -- Pandas DataFrame of merged catalog """ butler = Butler(repo) if patches is None: # Extract the patches for this tract from the skymap skymap = butler.get(datasetType='deepCoadd_skyMap') patches = ['%d,%d' % patch.getIndex() for patch in skymap[tract]] merged_tract_cat = pd.DataFrame() for patch in patches: this_patch_merged_cat = load_patch(butler, tract, patch, **kwargs) merged_tract_cat.append(this_patch_merged_cat) return merged_tract_cat
def setUp(self): # Load sample input from disk testDir = os.path.dirname(__file__) self.srcCat = afwTable.SourceCatalog.readFits( os.path.join(testDir, "data", "v695833-e0-c000.xy.fits")) self.srcCat["slot_ApFlux_fluxSigma"] = 1 self.srcCat["slot_PsfFlux_fluxSigma"] = 1 # The .xy.fits file has sources in the range ~ [0,2000],[0,4500] # which is bigger than the exposure self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(2048, 4612)) smallExposure = afwImage.ExposureF( os.path.join(testDir, "data", "v695833-e0-c000-a00.sci.fits")) self.exposure = afwImage.ExposureF(self.bbox) self.exposure.setWcs(smallExposure.getWcs()) self.exposure.setFilter(smallExposure.getFilter()) self.exposure.setCalib(smallExposure.getCalib()) # Make a reference loader butler = Butler(RefCatDir) self.refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler) logLevel = Log.TRACE self.log = Log.getLogger('testPhotoCal') self.log.setLevel(logLevel) self.config = PhotoCalConfig() # The test and associated data have been prepared on the basis that we # use the PsfFlux to perform photometry. self.config.fluxField = "base_PsfFlux_flux"
def setUp(self): self.butler = Butler( os.path.join(weeklyRerun, "pipeline", self.configuration, "pipeline")) self.visits = dict(brn=getBrnVisits, bmn=getBmnVisits)[self.configuration]() self.design = PfsDesign.read(1, weeklyRaw)
def __init__( self, butlerpath, destination=None, dataset=None, engine="pyarrow", sample_frac=None, num_buckets=8, ): self._butler = Butler(butlerpath) if dataset is None: dataset = self._default_dataset self.dataset = dataset if destination is None: destination = f"{butlerpath}/ktk" self.destination = destination self.sample_frac = sample_frac self.num_buckets = num_buckets self.stats_path = f"{self.destination}/{self.dataset}_stats.parq" self._store = None self.engine = engine self.metadata = self.butler.get("qaDashboard_info") self.dataIds = [ dataId for dataId in self.iter_dataId() if self.butler.datasetExists(self.dataset, dataId) ] self.filters = [filt for filt in self.metadata["visits"].keys()] self.dataIds_by_filter = { filt: [d for d in self.dataIds if d["filter"] == filt] for filt in self.filters } self._filenames = None self._filenames_by_filter = None
def setUp(self): # Load sample input from disk testDir = os.path.dirname(__file__) self.srcSet = SourceCatalog.readFits(os.path.join(testDir, "v695833-e0-c000.xy.fits")) self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(2048, 4612)) # approximate # create an exposure with the right metadata; the closest thing we have is # apparently v695833-e0-c000-a00.sci.fits, which is much too small smallExposure = ExposureF(os.path.join(testDir, "v695833-e0-c000-a00.sci.fits")) self.exposure = ExposureF(self.bbox) self.exposure.setWcs(smallExposure.getWcs()) self.exposure.setFilter(smallExposure.getFilter()) # copy the pixels we can, in case the user wants a debug display mi = self.exposure.getMaskedImage() mi.assign(smallExposure.getMaskedImage(), smallExposure.getBBox()) logLevel = Log.INFO refCatDir = os.path.join(testDir, "data", "sdssrefcat") butler = Butler(refCatDir) refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler) astrometryConfig = AstrometryTask.ConfigClass() self.astrom = AstrometryTask(config=astrometryConfig, refObjLoader=refObjLoader) self.astrom.log.setLevel(logLevel) # Since our sourceSelector is a registry object we have to wait for it to be created # before setting default values. self.astrom.matcher.sourceSelector.config.minSnr = 0
def loadAllTables(filters, newpath, oldpath, dataPath, patch, tract): logger.info("Loading new catalog") newCats = loadCatalogs(filters, newpath, "template") logger.info("Loading new flux conserved catalog") newCats2 = loadCatalogs(filters, newpath) logger.info("Loading old catalog") oldCats = loadCatalogs(filters, oldpath) logger.info("Building astropy tables") newFlux, newSed = getAllFlux(newCats, filters) newFlux2, newSed2 = getAllFlux(newCats2, filters) newTable = buildAllTables(newCats, newFlux, newSed, filters) newTable2 = buildAllTables(newCats2, newFlux2, newSed2, filters) newTable = newTable[(newTable["parent"] != 0) & ~np.isnan(newTable["x"])] newTable2 = newTable2[(newTable2["parent"] != 0) & ~np.isnan(newTable2["x"])] oldTables = OrderedDict([(f, buildTable(oldCats[f])) for f in filters]) logger.info("matching results") oldTable, matches = matchAllCatalogs(newTable, oldTables, filters) matchedNew = newTable[matches] matchedNew2 = newTable2[matches] logger.info("loading calexps") butler = Butler(inputs=dataPath) calexp = OrderedDict() for f in filters: calexp[f] = butler.get('deepCoadd_calexp', patch=patch, filter="HSC-" + f, tract=tract) return oldTable, matchedNew, matchedNew2, calexp, newCats
def setUp(self): inputDir = os.path.join(ROOT, "data", "input") self.testDir = tempfile.mkdtemp(dir=os.path.join(ROOT, 'tests'), prefix=type(self).__name__ + '-') self.butler = Butler(inputs=inputDir, outputs={ "root": self.testDir, "mode": 'rw' })
def get_diff_calexp_coadd_butler(diff_repo, calexp_repo, coadd_repo): """ Returns: butler_dict = { 'diff': diff_butler, 'calexp_butler': calexp_butler, 'coadd_butler': coadd_butler } """ diff_butler = Butler(diff_repo) calexp_butler = Butler(calexp_repo) coadd_butler = Butler(coadd_repo) butler_dict = { 'diff': diff_butler, 'calexp': calexp_butler, 'coadd': coadd_butler } return butler_dict
def setUp(self): np.random.seed(12345) self.butler = Butler(RefCatDir) refObjLoader = LoadIndexedReferenceObjectsTask(butler=self.butler) center = lsst.afw.geom.SpherePoint(215.5, 53.0, lsst.afw.geom.degrees) radius = 0.5 * lsst.afw.geom.degrees self.filter = "r" self.references = refObjLoader.loadSkyCircle(center, radius, self.filter).refCat
def makeLinearizerDecam(fromFile, force=False, verbose=False): """Convert the specified DECam linearity FITS table to standard LSST format Details: - Input format is one table per CCD, HDU is amplifier number, the table has 3 columns: ADU, ADU_LINEAR_A, ADU_LINEAR_B. The values of ADU contiguous (0, 1, 2...) but check and error out if not. The values of the latter two are replacements (0+delta0, 1+delta1, 2+delta2...) and this is converted to offsets for the LSST linearization tables (delta0, delta1, delta2...) - Output is a set of LinearizeLookupTable instances, one per CCD, saved as dataset type "linearizer" - The row indices for the linearization lookup table are (row index=amp name): 0=A, 1=B @param[in] fromFile path to DECam linearity table (a FITS file with one HDU per amplifier) """ print("Making DECam linearizers from %r" % (fromFile, )) butler = Butler(mapper=DecamMapper) linearizerDir = DecamMapper.getLinearizerDir() if os.path.exists(linearizerDir): if not force: print("Output directory %r exists; use --force to replace" % (linearizerDir, )) sys.exit(1) print("Replacing data in linearizer directory %r" % (linearizerDir, )) else: print("Creating linearizer directory %r" % (linearizerDir, )) os.makedirs(linearizerDir) camera = DecamMapper().camera fromHDUs = fits.open(fromFile)[1:] # HDU 0 has no data assert len(fromHDUs) == len(camera) for ccdind, (detector, hdu) in enumerate(zip(camera, fromHDUs)): ccdnum = ccdind + 1 if verbose: print("ccdnum=%s; detector=%s" % (ccdnum, detector.getName())) fromData = hdu.data assert len(fromData.dtype) == 3 lsstTable = np.zeros((2, len(fromData)), dtype=np.float32) uncorr = fromData["ADU"] if not np.allclose(uncorr, np.arange(len(fromData))): raise RuntimeError( "ADU data not a range of integers starting at 0") for i, ampName in enumerate("AB"): # convert DECam replacement table to LSST offset table if verbose: print("DECam table for %s=%s..." % ( ampName, fromData["ADU_LINEAR_" + ampName][0:5], )) lsstTable[i, :] = fromData["ADU_LINEAR_" + ampName] - uncorr if verbose: print("LSST table for %s=%s..." % ( ampName, lsstTable[i, 0:5], )) linearizer = LinearizeLookupTable(table=lsstTable, detector=detector) butler.put(linearizer, "linearizer", dataId=dict(ccdnum=ccdnum)) print("Wrote %s linearizers" % (ccdind + 1, ))
def ingest(cls, root, camera, visit, filenames, sensors, metadata): """Add all images from an external visit (a full-focal-plane exposure) to a data repository. This both symlinks the external data files to the appropriate location in the directory structure and adds the necessary rows to the SQLite registry tables. Parameters ---------- root : str Directory of the data repository to add data to. Must have an existing "registry.sqlite3" file present directly in the root and a _mapper file pointing to HscAndExtMapper. camera : str Name of the camera used to produced the external observation. Must have an entry in ExternalImage.CAMERA_INFO. visit : int Original integer visit ID for the observation, *before* adding CAMERA_INFO[camera]["ID"]*CAMERA_ID_MULTIPLIER. filenames : list A list of file names containing the external data files, either relative to the current directory or absolute. sensors : list A list of integer sensor IDs corresponding to the filenames list. metadata : VisitMetadata An object containing additional metadata for this visit to be added to the registry. See VisitMetadata for a description of what attributes are required. """ db = sqlite3.connect(os.path.join(root, "registry.sqlite3")) butler = Butler(inputs=[root]) visit += cls.CAMERA_INFO[camera]["id"] * cls.CAMERA_ID_MULTIPLIER ccdCols = [ "filter", "dateObs", "taiObs", "field", "expId", "pointing", "dataType", "pa" ] ccdSql = "INSERT INTO raw (visit, ccd, {}) VALUES (?, ?, {})".format( ", ".join(ccdCols), ", ".join(["?"] * len(ccdCols))) ccdValues = tuple(getattr(metadata, col) for col in ccdCols) visitCols = ["filter", "dateObs", "taiObs", "field"] visitSql = "INSERT INTO raw_visit (visit, {}) VALUES (?, {})".format( ", ".join(visitCols), ", ".join(["?"] * len(visitCols))) visitValues = tuple(getattr(metadata, col) for col in visitCols) for filename, sensor in zip(filenames, sensors): outputFileName = butler.get("external_filename", visit=visit, ccd=sensor)[0] os.symlink(filename, outputFileName) db.execute(ccdSql, ( visit, sensor, ) + ccdValues) db.execute(visitSql, (visit, ) + visitValues) db.commit() return visit
def butler(self): if not self._butler: if self.gen3: GEN3_REPO_ROOT = os.path.join(getPackageDir("ci_hsc_gen2"), "DATAgen3") self._butler = lsst.daf.butler.Butler( GEN3_REPO_ROOT, collections=self.collection) else: self._butler = Butler(self.root) return self._butler
def testDetectors(self): """Test that the detector returned by the gen 2 butler is the same as the expected one. """ for root, did, expected in zip(self.roots, self.ids, self.expecteds): butler = Butler(root) raw = butler.get("raw", dataId=did) for amp1, amp2 in zip(expected['detector'], raw.getDetector()): with self.subTest(amp=amp1.getName()): self.assertEqual(amp1.getName(), amp2.getName()) self.assertAmpRawBBoxesEqual(amp1, amp2)
def create(self): """Create a Butler. @returns a new Butler. """ if hasattr(self.mapper, 'root'): root = self.mapper.root else: root = None return Butler(root=root, mapper=self.mapper)
def installExternalData(): """Command-line interface for installing external data""" parser = argparse.ArgumentParser(description="Install extenral data") parser.add_argument("source", help="Source of external data") parser.add_argument("root", help="Butler data root") parser.add_argument("--tract", type=int, default=0, help="Tract identifier") parser.add_argument("--visitCcd", nargs=2, type=int, default=[], action="append", help="Visit and CCD of jointcal data to ingest (multiple OK)") args = parser.parse_args() butler = Butler(args.root) installJointcal(args.source, butler, args.tract, args.visitCcd)
def testAssemble(self): """Test the assembly of E2V and ITL sensors """ task = AssembleCcdTask() # exclude LATISS for this test since we don't have an expected output for root, did, expected in zip(self.roots, self.ids, self.expecteds): butler = Butler(root) raw = butler.get("raw", dataId=did) assembled = task.assembleCcd(raw) count = numpy.sum(expected['expected'].read().array - assembled.getImage().array) self.assertEqual(count, 0)
def load_and_save_tract(repo, tract, filename, key_prefix='coadd', patches=None, overwrite=True, verbose=False, **kwargs): """Save catalogs to HDF5 from forced-photometry coadds across available filters. Iterates through patches, saving each in append mode to the save HDF5 file. Parameters -- repo: str File location of Butler repository+rerun to load. tract: int Tract of sky region to load filename: str Filename for HDF file. key_prefix: str Base for the key in the HDF file. Keys will be of the form "%s_%d_%s" % (keybase, tract, patch) With the addition that the comma will be removed from the patch name to provide a valid Python identifier: e.g., 'coadd_4849_11' overwrite: bool Overwrite an existing HDF file. """ butler = Butler(repo) if patches is None: # Extract the patches for this tract from the skymap skymap = butler.get(datasetType='deepCoadd_skyMap') patches = ['%d,%d' % patch.getIndex() for patch in skymap[tract]] for patch in patches: if verbose: print("Processing tract %d, patch %s" % (tract, patch)) patch_merged_cat = load_patch(butler, tract, patch, verbose=verbose, **kwargs) if len(patch_merged_cat) == 0: if verbose: print(" No good entries for tract %d, patch %s" % (tract, patch)) continue key = '%s_%d_%s' % (key_prefix, tract, patch) key = valid_identifier_name(key) patch_merged_cat.to_pandas().to_hdf(filename, key, format='fixed')
def setUp(self): refCatDir = os.path.join(os.path.dirname(__file__), "data", "sdssrefcat") self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(3001, 3001)) crpix = afwGeom.Box2D(self.bbox).getCenter() self.tanWcs = afwGeom.makeSkyWcs(crpix=crpix, crval=afwGeom.SpherePoint(215.5, 53.0, afwGeom.degrees), cdMatrix=afwGeom.makeCdMatrix(scale=5.1e-5*afwGeom.degrees)) self.exposure = afwImage.ExposureF(self.bbox) self.exposure.setWcs(self.tanWcs) self.exposure.setFilter(afwImage.Filter("r", True)) butler = Butler(refCatDir) self.refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler)
def connect(self): # search for metadata.yaml file # 1. Look in path directory i.e. '/project/tmorton/tickets/DM-20015/RC2_w18/metadata.yaml' # 2. Look for datafolder in current directory i.e. './RC2_w18/metadata.yaml' # 3. Look for datafolder in dir specified in LSST_META env variable i.e. /user/name/lsst_meta/RC2_w18/metadata.yaml' # when LSST_META='/user/name/lsst_meta' print('-- read metadata file --') # if Butler is available use it to connect. If not available we are reading from disk if Butler: try: print('-- connect to butler --') self.conn = Butler(str(self.path)) self.metadata = self.conn.get('qaDashboard_metadata') self.failures = self.metadata.get('failures', {}) if not self.filters: self.filters = list(self.metadata['visits'].keys()) if not self.tracts: all_tracts = [list(self.metadata['visits'][filt].keys()) for filt in self.filters] self.tracts = list(set([int(y) for x in all_tracts for y in x])) except: print(f'{self.path} is not available in Butler attempting to read parquet files instead') else: if self.path.joinpath(METADATA_FILENAME).exists(): self.metadata_path = self.path.joinpath(METADATA_FILENAME) else: self.metadata_path = Path(os.environ.get('LSST_META', os.curdir)).joinpath(self.path.name, METADATA_FILENAME) with self.metadata_path.open('r') as f: self.metadata = yaml.load(f, Loader=yaml.SafeLoader) self.failures = self.metadata.get('failures', {}) if self.tracts is None: self.tracts = list(set(x for v in self.metadata['visits'].values() for x in v.keys())) print('-- read coadd table --') self.fetch_coadd_table() # currently ignoring forced/unforced # update metadata based on coadd table fields print('-- generate other metadata fields --') df = self.coadd['qaDashboardCoaddTable'] self.flags = df.columns[df.dtypes == bool].to_list() if not Butler: self.filters = list(self.metadata['visits'].keys()) self.metrics = set(df.columns.to_list()) - set(self.flags) - set(['patch', 'dec', 'label', 'psfMag', 'ra', 'filter', 'dataset', 'dir0', 'tract']) print('-- read visit data --') self.fetch_visits_by_metric() print('-- done with reads --')
def __init__(self, repo, vb=False, path_to_tracts=''): self.repo = repo # Instantiate a butler, or report failure: from lsst.daf.persistence import Butler try: self.butler = Butler(repo) except: self.butler = None print("Warning: failed to instantiate a butler to get data from repo '"+repo+"'") return None # Set up some internal variables: self.vb = vb self.exists = {} self.existence = False self.counts = {} self.tracts = [] self.path_to_tracts = path_to_tracts if path_to_tracts != '': try: self.skymap_butler = Butler(repo + path_to_tracts) except: self.skymap_butler = None print("Warning: failed to find a skyMap for the path " + repo + path_to_tracts) return
def testBasics(self): """Test detection and measurement on simple synthesized data """ bbox = Box2I(Point2I(256, 100), Extent2I(128, 127)) minCounts = 5000 maxCounts = 50000 starSigma = 1.5 numX = 5 numY = 5 coordList = self.makeCoordList( bbox=bbox, numX=numX, numY=numY, minCounts=minCounts, maxCounts=maxCounts, sigma=starSigma, ) kwid = 11 # kernel width sky = 2000 # create an exposure without a Wcs; add the Wcs later exposure = plantSources(bbox=bbox, kwid=kwid, sky=sky, coordList=coordList, addPoissonNoise=True) schema = SourceTable.makeMinimalSchema() config = DetectAndMeasureTask.ConfigClass() task = DetectAndMeasureTask(config=config, schema=schema) butler = Butler(root=InputDir) dataRef = butler.dataRef("calexp", dataId=dict(visit=1)) wcs = dataRef.get("raw").getWcs() exposure.setWcs(wcs) exposureIdInfo = dataRef.get("expIdInfo") taskRes = task.run(exposure=exposure, exposureIdInfo=exposureIdInfo) self.assertEqual(len(taskRes.sourceCat), numX * numY) schema = taskRes.sourceCat.schema centroidFlagKey = schema.find("slot_Centroid_flag").getKey() parentKey = schema.find("parent").getKey() psfFluxFlagKey = schema.find("slot_PsfFlux_flag").getKey() psfFluxKey = schema.find("slot_PsfFlux_flux").getKey() for src in taskRes.sourceCat: self.assertFalse(src.get(centroidFlagKey)) # centroid found self.assertEqual(src.get(parentKey), 0) # not debelended self.assertFalse(src.get(psfFluxFlagKey)) # flux measured self.assertGreater(src.get(psfFluxKey), 4000) # flux sane
def __main__(args): repo_directory = "/lsstdata/offline/teststand/BOT/gen2repo" butler = Butler(repo_directory) runs = ['12672', '12673', '12844', '12845', '12853', '12855'] run = runs[0] sensors = set( butler.queryMetadata('raw', ['raftName', 'detectorName'], dataId={ 'run': run, 'imageType': 'BIAS' })) run_dark_regression_analysis_on_all_sensors(butler, runs, sensors, show_plots=False)
def getTargetList(parsedCmd, **kwargs): parentDir = parsedCmd.input while os.path.exists(os.path.join(parentDir, "_parent")): parentDir = os.path.realpath(os.path.join(parentDir, "_parent")) butler2 = Butler(root=os.path.join(parentDir, "rerun", parsedCmd.prior_rerun), calibRoot=parsedCmd.calib) idParser = parsedCmd.prior.__class__(parsedCmd.prior.level) idParser.idList = parsedCmd.prior.idList idParser.datasetType = parsedCmd.prior.datasetType butler = parsedCmd.butler parsedCmd.butler = butler2 idParser.makeDataRefList(parsedCmd) parsedCmd.butler = butler return [(parsedCmd.id.refList, dict(priorRefList=idParser.refList, **kwargs))]
def setUp(self): # Load sample input from disk testDir = os.path.dirname(__file__) self.srcCat = afwTable.SourceCatalog.readFits( os.path.join(testDir, "data", "v695833-e0-c000.xy.fits")) self.srcCat["slot_ApFlux_fluxSigma"] = 1 self.srcCat["slot_PsfFlux_fluxSigma"] = 1 # The .xy.fits file has sources in the range ~ [0,2000],[0,4500] # which is bigger than the exposure self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(2048, 4612)) smallExposure = afwImage.ExposureF( os.path.join(testDir, "data", "v695833-e0-c000-a00.sci.fits")) self.exposure = afwImage.ExposureF(self.bbox) self.exposure.setWcs(smallExposure.getWcs()) self.exposure.setFilter(smallExposure.getFilter()) self.exposure.setCalib(smallExposure.getCalib()) coordKey = self.srcCat.getCoordKey() centroidKey = self.srcCat.getCentroidKey() wcs = self.exposure.getWcs() for src in self.srcCat: src.set(coordKey, wcs.pixelToSky(src.get(centroidKey))) # Make a reference loader butler = Butler(RefCatDir) self.refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler) logLevel = Log.TRACE self.log = Log.getLogger('testPhotoCal') self.log.setLevel(logLevel) self.config = PhotoCalConfig() self.config.match.matchRadius = 0.5 self.config.match.referenceSelection.doMagLimit = True self.config.match.referenceSelection.magLimit.maximum = 22.0 self.config.match.referenceSelection.magLimit.fluxField = "i_flux" self.config.match.referenceSelection.doFlags = True self.config.match.referenceSelection.flags.good = ['photometric'] self.config.match.referenceSelection.flags.bad = ['resolved'] self.config.match.sourceSelection.doUnresolved = False # Don't have star/galaxy in the srcCat # The test and associated data have been prepared on the basis that we # use the PsfFlux to perform photometry. self.config.fluxField = "base_PsfFlux_flux"