def inject_fakes_to_calexp(calexp_repo, calexp_id, magVar, coords, poisson=False): """This function injects fakes onto a calexp exposure at given positions. caelxp_repo: An empty calexp repo which is used to save the injected exposure. calexp_id: The data id of the calexp exposure magVar: The injected magnitude coords: A list of [x, y] pixel coordinates poisson: If true, add Poisson fluctuations to the fake model """ calexp_butler = Butler(calexp_repo) calexp_photoCalib = calexp_butler.get('calexp_photoCalib', calexp_id) calexp_exposure = calexp_butler.get('calexp', dataId=calexp_id) for coord in coords: x, y = coord[0], coord[1] inject_star(calexp_exposure, calexp_photoCalib, x, y, magVar, poisson=poisson) calexp_butler.put(calexp_exposure, 'calexp', dataId=calexp_id)
def load_tract(repo, tract, patches=None, **kwargs): """Merge catalogs from forced-photometry coadds across available filters. Parameters -- tract: int Tract of sky region to load repo: str File location of Butler repository+rerun to load. patches: list of str List of patches. If not specified, will default to '0,0'--'7,7'. Returns -- AstroPy Table of merged catalog """ butler = Butler(repo) if patches is None: # Extract the patches for this tract from the skymap skymap = butler.get(datasetType='deepCoadd_skyMap') patches = ['%d,%d' % patch.getIndex() for patch in skymap[tract]] merged_patch_cats = [] for patch in patches: this_patch_merged_cat = load_patch(butler, tract, patch, **kwargs) # Event if this_patch_merged_cat is an empty Table, it's still fine to append to the list here. # They will get vstacked away below. merged_patch_cats.append(this_patch_merged_cat) merged_tract_cat = vstack(merged_patch_cats) return merged_tract_cat
def loadAllTables(filters, newpath, oldpath, dataPath, patch, tract): logger.info("Loading new catalog") newCats = loadCatalogs(filters, newpath, "template") logger.info("Loading new flux conserved catalog") newCats2 = loadCatalogs(filters, newpath) logger.info("Loading old catalog") oldCats = loadCatalogs(filters, oldpath) logger.info("Building astropy tables") newFlux, newSed = getAllFlux(newCats, filters) newFlux2, newSed2 = getAllFlux(newCats2, filters) newTable = buildAllTables(newCats, newFlux, newSed, filters) newTable2 = buildAllTables(newCats2, newFlux2, newSed2, filters) newTable = newTable[(newTable["parent"] != 0) & ~np.isnan(newTable["x"])] newTable2 = newTable2[(newTable2["parent"] != 0) & ~np.isnan(newTable2["x"])] oldTables = OrderedDict([(f, buildTable(oldCats[f])) for f in filters]) logger.info("matching results") oldTable, matches = matchAllCatalogs(newTable, oldTables, filters) matchedNew = newTable[matches] matchedNew2 = newTable2[matches] logger.info("loading calexps") butler = Butler(inputs=dataPath) calexp = OrderedDict() for f in filters: calexp[f] = butler.get('deepCoadd_calexp', patch=patch, filter="HSC-" + f, tract=tract) return oldTable, matchedNew, matchedNew2, calexp, newCats
def load_tract(repo, tract, patches=None, **kwargs): """Merge catalogs from forced-photometry coadds across available filters. Parameters -- tract: int Tract of sky region to load repo: str File location of Butler repository+rerun to load. patches: list of str List of patches. If not specified, will default to '0,0'--'7,7'. Returns -- Pandas DataFrame of merged catalog """ butler = Butler(repo) if patches is None: # Extract the patches for this tract from the skymap skymap = butler.get(datasetType='deepCoadd_skyMap') patches = ['%d,%d' % patch.getIndex() for patch in skymap[tract]] merged_tract_cat = pd.DataFrame() for patch in patches: this_patch_merged_cat = load_patch(butler, tract, patch, **kwargs) merged_tract_cat.append(this_patch_merged_cat) return merged_tract_cat
def testCameraFromButler(self): """Test that the butler can return the camera """ butler = Butler(self.input) camera = butler.get("camera", immediate=True) self.assertEqual(camera.getName(), self.mapper.camera.getName()) self.assertEqual(len(camera), len(self.mapper.camera)) self.assertEqual(len(camera[0]), len(self.mapper.camera[0]))
def main(visit, detector, diarepo, outputdir='.'): diabutler = Butler(diarepo) diff_img_path = os.path.join(outputdir, f'diff_exposure_v{visit}_d{detector}.fits') science_img_path = os.path.join( outputdir, f'science_exposure_v{visit}_d{detector}.fits') imgD = diabutler.get('deepDiff_differenceExp', visit=visit, detector=detector) imgD.writeFits(diff_img_path) imgS = diabutler.get('calexp', visit=visit, detector=detector) imgS.writeFits(science_img_path) print(f'files copied to {outputdir}') return
class FitsStorageTestCase(lsst.utils.tests.TestCase): def setUp(self): inputDir = os.path.join(ROOT, "data", "input") self.testDir = tempfile.mkdtemp(dir=os.path.join(ROOT, 'tests'), prefix=type(self).__name__+'-') self.butler = Butler(inputs=inputDir, outputs={"root": self.testDir, "mode": 'rw'}) def tearDown(self): del self.butler if os.path.exists(self.testDir): shutil.rmtree(self.testDir) def testReadPropertyList(self): """Test that reading a FitsStorage dataset into a PropertyList is the same as using the special _md Exposure hook. """ md1 = self.butler.get("raw_md", visit=1, filter="g") md2 = self.butler.get("rawMetadataDirect", visit=1, filter="g") self.assertEqual(md1, md2)
def ingest(cls, root, camera, visit, filenames, sensors, metadata): """Add all images from an external visit (a full-focal-plane exposure) to a data repository. This both symlinks the external data files to the appropriate location in the directory structure and adds the necessary rows to the SQLite registry tables. Parameters ---------- root : str Directory of the data repository to add data to. Must have an existing "registry.sqlite3" file present directly in the root and a _mapper file pointing to HscAndExtMapper. camera : str Name of the camera used to produced the external observation. Must have an entry in ExternalImage.CAMERA_INFO. visit : int Original integer visit ID for the observation, *before* adding CAMERA_INFO[camera]["ID"]*CAMERA_ID_MULTIPLIER. filenames : list A list of file names containing the external data files, either relative to the current directory or absolute. sensors : list A list of integer sensor IDs corresponding to the filenames list. metadata : VisitMetadata An object containing additional metadata for this visit to be added to the registry. See VisitMetadata for a description of what attributes are required. """ db = sqlite3.connect(os.path.join(root, "registry.sqlite3")) butler = Butler(inputs=[root]) visit += cls.CAMERA_INFO[camera]["id"] * cls.CAMERA_ID_MULTIPLIER ccdCols = [ "filter", "dateObs", "taiObs", "field", "expId", "pointing", "dataType", "pa" ] ccdSql = "INSERT INTO raw (visit, ccd, {}) VALUES (?, ?, {})".format( ", ".join(ccdCols), ", ".join(["?"] * len(ccdCols))) ccdValues = tuple(getattr(metadata, col) for col in ccdCols) visitCols = ["filter", "dateObs", "taiObs", "field"] visitSql = "INSERT INTO raw_visit (visit, {}) VALUES (?, {})".format( ", ".join(visitCols), ", ".join(["?"] * len(visitCols))) visitValues = tuple(getattr(metadata, col) for col in visitCols) for filename, sensor in zip(filenames, sensors): outputFileName = butler.get("external_filename", visit=visit, ccd=sensor)[0] os.symlink(filename, outputFileName) db.execute(ccdSql, ( visit, sensor, ) + ccdValues) db.execute(visitSql, (visit, ) + visitValues) db.commit() return visit
def testDetectors(self): """Test that the detector returned by the gen 2 butler is the same as the expected one. """ for root, did, expected in zip(self.roots, self.ids, self.expecteds): butler = Butler(root) raw = butler.get("raw", dataId=did) for amp1, amp2 in zip(expected['detector'], raw.getDetector()): with self.subTest(amp=amp1.getName()): self.assertEqual(amp1.getName(), amp2.getName()) self.assertAmpRawBBoxesEqual(amp1, amp2)
class SimpleButlerInterface(BaseGenericCatalog): """ A simple butler interface. Args: ----- repo: str path to repository containing the DM-processed data. dataId: dict """ def _subclass_init(self, repo, datasetType, dataId=None, **kwargs): if not _HAS_LSST_STACK: raise RuntimeError('LSST Stack not available') self._butler = Butler(repo) self._datasetType = datasetType self._dataId_cache = self._butler.subset(self._datasetType, dataId=dataId).cache self._columns = None for dataId in self._dataId_cache: data = self._get_data(dataId) if data is not None: self._columns = data.schema.getNames() break if not self._columns: raise RuntimeError('No datasets or columns found!') def _get_data(self, dataId, datasetType=None): try: data = self._butler.get(datasetType or self._datasetType, dataId=dataId) except NoResults: return None return data def _generate_quantity_getter(self, dataId): data = self._get_data(dataId) if data is None: return return data.get def _iter_native_dataset(self, native_filters=None): for dataId in self._dataId_cache: if native_filters is None or native_filters.check_scalar(dataId): quantity_getter = self._generate_quantity_getter(dataId) if quantity_getter is not None: yield quantity_getter def _generate_native_quantity_list(self): return self._columns
def testAssemble(self): """Test the assembly of E2V and ITL sensors """ task = AssembleCcdTask() # exclude LATISS for this test since we don't have an expected output for root, did, expected in zip(self.roots, self.ids, self.expecteds): butler = Butler(root) raw = butler.get("raw", dataId=did) assembled = task.assembleCcd(raw) count = numpy.sum(expected['expected'].read().array - assembled.getImage().array) self.assertEqual(count, 0)
def load_and_save_tract(repo, tract, filename, key_prefix='coadd', patches=None, overwrite=True, verbose=False, **kwargs): """Save catalogs to HDF5 from forced-photometry coadds across available filters. Iterates through patches, saving each in append mode to the save HDF5 file. Parameters -- repo: str File location of Butler repository+rerun to load. tract: int Tract of sky region to load filename: str Filename for HDF file. key_prefix: str Base for the key in the HDF file. Keys will be of the form "%s_%d_%s" % (keybase, tract, patch) With the addition that the comma will be removed from the patch name to provide a valid Python identifier: e.g., 'coadd_4849_11' overwrite: bool Overwrite an existing HDF file. """ butler = Butler(repo) if patches is None: # Extract the patches for this tract from the skymap skymap = butler.get(datasetType='deepCoadd_skyMap') patches = ['%d,%d' % patch.getIndex() for patch in skymap[tract]] for patch in patches: if verbose: print("Processing tract %d, patch %s" % (tract, patch)) patch_merged_cat = load_patch(butler, tract, patch, verbose=verbose, **kwargs) if len(patch_merged_cat) == 0: if verbose: print(" No good entries for tract %d, patch %s" % (tract, patch)) continue key = '%s_%d_%s' % (key_prefix, tract, patch) key = valid_identifier_name(key) patch_merged_cat.to_pandas().to_hdf(filename, key, format='fixed')
class ArcTestCase(lsst.utils.tests.TestCase): def setUp(self): self.butler = Butler( os.path.join(weeklyRerun, "calib", self.arms, f"arc_{self.arms}", "detectorMap")) def tearDown(self): del self.butler def testResiduals(self): """Test that wavelength fit residuals are reasonable""" tolerance = { "b": 0.05, "r": 0.02, "m": 0.01, "n": 0.05, } # tolerance for each arm, nm minSigNoise = 10 # Minimum (flux) signal-to-noise for arc lines to consider for arm in self.arms: atol = tolerance[arm] detMap = self.butler.get("detectorMap", visit=self.visit, arm=arm) lines = self.butler.get("arcLines", visit=self.visit, arm=arm) fitWavelength = detMap.findWavelength(lines.fiberId, lines.y) good = ~lines.flag & (lines.status == 0) sigNoise = lines.intensity / lines.intensityErr for fiberId in set(lines.fiberId): with self.subTest(arm=arm, fiberId=fiberId): select = (lines.fiberId == fiberId) & good & (sigNoise > minSigNoise) num = select.sum() self.assertGreater(num, 10) residual = lines.wavelength[select] - fitWavelength[select] lq, median, uq = np.percentile(residual, (25.0, 50.0, 75.0)) robustRms = 0.741 * (uq - lq) self.assertFloatsAlmostEqual(median, 0.0, atol=atol) self.assertFloatsAlmostEqual(robustRms, 0.0, atol=atol)
class FitsStorageTestCase(lsst.utils.tests.TestCase): def setUp(self): inputDir = os.path.join(ROOT, "data", "input") self.testDir = tempfile.mkdtemp(dir=os.path.join(ROOT, 'tests'), prefix=type(self).__name__ + '-') self.butler = Butler(inputs=inputDir, outputs={ "root": self.testDir, "mode": 'rw' }) def tearDown(self): del self.butler if os.path.exists(self.testDir): shutil.rmtree(self.testDir) def testReadPropertyList(self): """Test that reading a FitsStorage dataset into a PropertyList is the same as using the special _md Exposure hook. """ md1 = self.butler.get("raw_md", visit=1, filter="g") md2 = self.butler.get("rawMetadataDirect", visit=1, filter="g") self.assertEqual(md1, md2)
def testExposureIdInfo(self): butler = Butler(self.input) expIdBits = self.mapper.bypass_ccdExposureId_bits( # args are ignored datasetType = None, pythonType = int, location = None, dataId = dict(), ) for visit in (1, 2, 3): dataId = dict(visit=visit) expIdInfo = butler.get("expIdInfo", dataId=dataId) self.assertEqual(expIdInfo.expId, visit) self.assertEqual(expIdInfo.expBits, expIdBits) self.assertEqual(expIdInfo.maxBits, 64) self.assertEqual(expIdInfo.unusedBits, expIdInfo.maxBits-expIdBits)
def testBasics(self): """Test construction of a discrete sky map """ butler = Butler(inputs=self.inPath, outputs={ 'root': self.outPath, 'mode': 'rw' }) coordList = [] # list of sky coords of all corners of all calexp for dataId in ( dict(visit=1, filter="g"), dict(visit=2, filter="g"), dict(visit=3, filter="r"), ): # TODO: pybind11 remove `immediate=True` once DM-9112 is resolved rawImage = butler.get("raw", dataId, immediate=True) # fake calexp by simply copying raw data; the task just cares about its bounding box # (which is slightly larger for raw, but that doesn't matter for this test) calexp = rawImage butler.put(calexp, "calexp", dataId) calexpWcs = calexp.getWcs() calexpBoxD = Box2D(calexp.getBBox()) coordList += [ calexpWcs.pixelToSky(corner) for corner in calexpBoxD.getCorners() ] # use the calexp to make a sky map retVal = MakeDiscreteSkyMapTask.parseAndRun( args=[self.inPath, "--output", self.outPath, "--id", "filter=g^r"], config=self.config, doReturnResults=True, ) self.assertEqual(len(retVal.resultList), 1) skyMap = retVal.resultList[0].result.skyMap self.assertEqual(type(skyMap), DiscreteSkyMap) self.assertEqual(len(skyMap), 1) tractInfo = skyMap[0] self.assertEqual(tractInfo.getId(), 0) self.assertEqual(tractInfo.getNumPatches(), Extent2I(3, 3)) tractWcs = tractInfo.getWcs() tractBoxD = Box2D(tractInfo.getBBox()) for skyPoint in coordList: self.assertTrue(tractBoxD.contains(tractWcs.skyToPixel(skyPoint)))
def testBasics(self): """Test construction of a discrete sky map """ butler = Butler(inputs=self.inPath, outputs={'root': self.outPath, 'mode': 'rw'}) coordList = [] # list of sky coords of all corners of all calexp for dataId in ( dict(visit=1, filter="g"), dict(visit=2, filter="g"), dict(visit=3, filter="r"), ): rawImage = butler.get("raw", dataId) # fake calexp by simply copying raw data; the task just cares about its bounding box # (which is slightly larger for raw, but that doesn't matter for this test) calexp = rawImage butler.put(calexp, "calexp", dataId) calexpWcs = calexp.getWcs() calexpBoxD = Box2D(calexp.getBBox()) coordList += [calexpWcs.pixelToSky(corner) for corner in calexpBoxD.getCorners()] # use the calexp to make a sky map retVal = MakeDiscreteSkyMapTask.parseAndRun( args=[self.inPath, "--output", self.outPath, "--id", "filter=g^r"], config=self.config, doReturnResults=True, ) self.assertEqual(len(retVal.resultList), 1) skyMap = retVal.resultList[0].result.skyMap self.assertEqual(type(skyMap), DiscreteSkyMap) self.assertEqual(len(skyMap), 1) tractInfo = skyMap[0] self.assertEqual(tractInfo.getId(), 0) self.assertEqual(tractInfo.getNumPatches(), Extent2I(3, 3)) tractWcs = tractInfo.getWcs() tractBoxD = Box2D(tractInfo.getBBox()) for skyPoint in coordList: self.assertTrue(tractBoxD.contains(tractWcs.skyToPixel(skyPoint)))
import lsst.afw.image as afwImage import lsst.afw.display as afwDisplay import lsst.afw.cameraGeom as cameraGeom from lsst.daf.persistence import Butler import create_coaddComands as ccoadd import create_multiBandCommands as multib import create_diaCommands as cdia import create_assocCommands as cassoc import create_forcedPhotCommands as cfPhot repo = '/global/cscratch1/sd/desc/DC2/data/Run2.1i/rerun/calexp-v1' b = Butler(repo) skymap = b.get('deepCoadd_skyMap') # creating a rectangle of 2 sq. degree for tract/patch search radec_NE = afwGeom.SpherePoint(58, -31, afwGeom.degrees) radec_SE = afwGeom.SpherePoint(58, -32, afwGeom.degrees) radec_SW = afwGeom.SpherePoint(56, -32, afwGeom.degrees) radec_NW = afwGeom.SpherePoint(56, -31, afwGeom.degrees) rect = [radec_NE, radec_NW, radec_SW, radec_SE] tpatches = skymap.findTractPatchList(rect) # to figure the visits on this patches we need the DB # this will help us dtermine seeing and time cuts for coadds database = repo + '/tracts_mapping.sqlite3' query_tmpl = "select DISTINCT(visit), filter from overlaps WHERE tract={} and patch={} order by visit" conn = sqlite3.connect(database)
filt = filt[-1] if filt in ('u', 'y') or n_visit not in list(visitab.obsHistID): print('continue') continue raft_list = glob(avisit+'/R*') for araft in raft_list: raft_n = int(os.path.basename(araft)[1:]) #print(raft_n) # have visit number and raft number detlist = glob(araft+'/diaSrc_*.fits') for adet in detlist: det_n = int(os.path.basename(adet)[-8:-5]) print(n_visit, filt, raft_n, det_n) im = b.get('calexp', visit=n_visit, detector=det_n) #im.getBBox().getCorners() image_wcs = im.getWcs() corners = [geom.Point2D(c) for c in im.getBBox().getCorners()] sky_corners = image_wcs.pixelToSky(corners) #print('we have wcs, break') vinfo = visitab.query(f'obsHistID=={n_visit}').values[0] bsight = geom.SpherePoint(vinfo[4]*geom.degrees, vinfo[5]*geom.degrees) orient = vinfo[10]*geom.degrees minionwcs = makeSkyWcs(trans[det_n], orient, flipX=True, boresight=bsight, projection='TAN') minion_skycorners = minionwcs.pixelToSky(corners) print(minion_skycorners, sky_corners)
def prep(self): self.prior = bfd.MomentPrior() priorFiles = [] priorButler = Butler(self.config.priorRerun) prior_skyMap = priorButler.get('deepCoadd_skyMap') for tract in self.config.priorTracts: for patchInfo in prior_skyMap[tract]: patch = '%d,%d' % patchInfo.getIndex() if self.config.priorPatches: if patch not in self.config.priorPatches: continue if priorButler.datasetExists('deepCoadd_momentPrior', tract=tract, patch=patch, filter=self.config.priorFilter, label=self.config.priorLabel): priorFiles.append( priorButler.get('deepCoadd_momentPrior_filename', tract=tract, patch=patch, filter=self.config.priorFilter, label=self.config.priorLabel)[0]) max_file = len(priorFiles) if self.config.maxPriorFiles > 0: max_file = self.config.maxPriorFiles first = True for file in priorFiles[:max_file]: if file.find('_parent') > 0: self.log.info("Skipping %s, from parent" % file) continue self.log.info("Adding prior %s" % file) try: cat = lsst.afw.table.BaseCatalog.readFits(file) self.prior.addCatalog(cat, self.config.invariantCovariance, self.config.sampleFraction, self.config.sampleSeed) # Should be same for all prior catalogs if first: self.cov = numpy.array( cat.getTable().getMetadata().getArrayDouble( 'COV')).reshape(6, 6) first = False except Exception as e: print('Failed to read', e) continue self.prior.prepare() self.fluxMin = self.prior.getFluxMin() self.fluxMax = self.prior.getFluxMax() self.varMin = self.prior.getVarMin() self.varMax = self.prior.getVarMax() selectionPqr = self.prior.selectionProbability( self.cov.astype(numpy.float32)) deselect = selectionPqr.copy() deselect[0] = 1 - selectionPqr[0] for i in range(1, 6): deselect[i] *= -1. self.noSelectPqr = deselect
def load_patch(butler_or_repo, tract, patch, fields_to_join=('id', ), filters={ 'u': 'u', 'g': 'g', 'r': 'r', 'i': 'i', 'z': 'z', 'y': 'y' }, trim_colnames_for_fits=False, verbose=False): """Load patch catalogs. Return merged catalog across filters. butler_or_repo: Butler object or str Either a Butler object or a filename to the repo tract: int Tract in skymap patch: str Patch in the tract in the skymap fields_to_join: iterable of str Join the catalogs for each filter on these fields filters: iterable of str Filter names to load trim_colnames_for_fits: bool Trim column names to satisfy the FITS standard character limit of <68. Returns -- AstroPy Table of patch catalog merged across filters. """ if isinstance(butler_or_repo, str): butler = Butler(butler_or_repo) else: butler = butler_or_repo # Define the filters and order in which to sort them.: tract_patch_data_id = {'tract': tract, 'patch': patch} try: ref_table = butler.get(datasetType='deepCoadd_ref', dataId=tract_patch_data_id).asAstropy() except NoResults as e: if verbose: print(" ", e) return Table() isPrimary = ref_table['detect_isPrimary'] ref_table = ref_table[isPrimary] if len(ref_table) == 0: if verbose: print(" No good isPrimary entries for tract %d, patch %s" % (tract, patch)) return ref_table merge_filter_cats = {} for filt in filters: this_data = tract_patch_data_id.copy() this_data['filter'] = filters[filt] try: cat = butler.get(datasetType='deepCoadd_forced_src', dataId=this_data).asAstropy() except NoResults as e: if verbose: print(" ", e) continue CoaddCalib = butler.get('deepCoadd_calexp_calib', this_data) CoaddCalib.setThrowOnNegativeFlux(False) mag, mag_err = CoaddCalib.getMagnitude(cat['base_PsfFlux_flux'], cat['base_PsfFlux_fluxSigma']) cat['mag'] = mag cat['mag_err'] = mag_err cat['SNR'] = np.abs( cat['base_PsfFlux_flux']) / cat['base_PsfFlux_fluxSigma'] cat = cat[isPrimary] merge_filter_cats[filt] = cat merged_patch_cat = ref_table for filt in filters: if filt not in merge_filter_cats: continue cat = merge_filter_cats[filt] if len(cat) < 1: continue # Rename duplicate columns with prefix of filter prefix_columns(cat, filt, fields_to_skip=fields_to_join) # Merge metadata with concatenation with enable_merge_strategies(MergeNumbersAsList, MergeListNumbersAsList): merged_patch_cat = join(merged_patch_cat, cat, keys=fields_to_join) if trim_colnames_for_fits: # FITS column names can't be longer that 68 characters # Trim here to ensure consistency across any format we write this out to trim_long_colnames(merged_patch_cat) return merged_patch_cat
class Taster(object): """ Worker for tasting the datasets in a Butler's repo (based mostly off of querying metadata). Instantiate with a repo. """ def __init__(self, repo, vb=False, path_to_tracts=''): self.repo = repo # Instantiate a butler, or report failure: from lsst.daf.persistence import Butler try: self.butler = Butler(repo) except: self.butler = None print("Warning: failed to instantiate a butler to get data from repo '"+repo+"'") return None # Set up some internal variables: self.vb = vb self.exists = {} self.existence = False self.counts = {} self.tracts = [] self.path_to_tracts = path_to_tracts if path_to_tracts != '': try: self.skymap_butler = Butler(repo + path_to_tracts) except: self.skymap_butler = None print("Warning: failed to find a skyMap for the path " + repo + path_to_tracts) return def what_exists(self, all=False): """ Check for the existence of various useful things. Parameters ========== all: boolean If true, the method will check all possible dataset types Returns ======= exists: dict Checklist of what exists (True) and what does not (False) """ # Get mappers for all tested repos from lsst.obs.hsc import HscMapper from lsst.obs.comCam import ComCamMapper #from lsst.obs.lsst import LsstCamMapper from lsst.obs.ctio0m9 import Ctio0m9Mapper #select proper mapper if self.repo.find('hsc') != -1: mapper = HscMapper(root=self.repo) elif self.repo.find('comCam') != -1: mapper = ComCamMapper(root=self.repo) #elif self.repo.find('DC2') != -1: mapper = LsstCamMapper(root=self.repo) elif self.repo.find('ctio0m9') != -1: mapper = Ctio0m9Mapper(root=self.repo) else: print("Unable to locate Mapper file in specified repo. Check that you selected a valid repo.") if all: #collect a list of all possible dataset types mapper = HscMapper(root=self.repo) all_dataset_types = mapper.getDatasetTypes() remove = ['_config', '_filename', '_md', '_sub', '_len', '_schema', '_metadata'] interesting = [] for dataset_type in all_dataset_types: keep = True for word in remove: if word in dataset_type: keep = False if keep: interesting.append(dataset_type) else: interesting = ['raw', 'calexp', 'src', 'deepCoadd_calexp', 'deepCoadd_meas'] self.look_for_datasets_of_type(interesting) self.look_for_skymap() self.existence = True return def look_for_datasets_of_type(self, datasettypes): """ Check whether dataset of given type is in the metadata. Parameters ========== datasettype: list of strings Types of dataset to check for, eg 'calexp', 'raw', 'wcs' etc. """ datasets_that_exist = [] datasets_that_do_not_exist = [] for datasettype in datasettypes: try: datasetkeys = self.butler.getKeys(datasettype) onekey = list(datasetkeys.keys())[0] metadata = self.butler.queryMetadata(datasettype, [onekey]) #if self.vb: print("{} dataset exists.".format(datasettype)) datasets_that_exist.append(datasettype) self.exists[datasettype] = True except: #if self.vb: print("{} dataset doesn't exist.".format(datasettype)) datasets_that_do_not_exist.append(datasettype) self.exists[datasettype] = False #Organize output if self.vb: print("Datasets that exist\n-------------------") print(datasets_that_exist) print("\nDatasets that do not exist\n--------------------------") print(datasets_that_do_not_exist) return def look_for_skymap(self): """ Check for the existence of a skymap. """ try: self.skyMap = self.skymap_butler.get('deepCoadd_skyMap') self.exists['deepCoadd_skyMap'] = True if self.vb: print("\nSkymap\n-------------------\ndeepCoadd_skyMap exists.") except: self.skyMap = None self.exists['deepCoadd_skyMap'] = False if self.vb: print("\nSkymap\n-------------------\ndeepCoadd_skyMap doesn't exist.") return def estimate_sky_area(self): """ Use available skymap to estimate sky area covered by tracts and patches. Returns ======= area: float Sky area in square degrees """ if self.skyMap is None: return None area_label = 'Total Sky Area (deg$^2$)' if area_label in self.counts.keys(): return self.counts[area_label] # Collect tracts from files import os, glob tracts = sorted([int(os.path.basename(x)) for x in glob.glob(os.path.join(self.repo + self.path_to_tracts, 'deepCoadd-results', 'merged', '*'))]) self.tracts = tracts self.counts['Number of Tracts'] = len(tracts) # Note: We'd like to do this with the butler, but it appears 'tracts' have to be # specified in the dataId to be queried, so the queryMetadata method fails # Calculate area from all tracts total_area = 0.0 #deg^2 plotting_vertices = [] for test_tract in tracts: # Get inner vertices for tract tractInfo = self.skyMap[test_tract] vertices = tractInfo._vertexCoordList plotting_vertices.append(vertices) # Calculate area of box av_dec = 0.5 * (vertices[2][1] + vertices[0][1]) av_dec = av_dec.asRadians() delta_ra_raw = vertices[0][0] - vertices[1][0] delta_ra = delta_ra_raw.asDegrees() * np.cos(av_dec) delta_dec= vertices[2][1] - vertices[0][1] area = delta_ra * delta_dec.asDegrees() # Combine areas total_area += area if self.vb: print(area_label, ": ", total_area) # Round of the total area for table purposes self.counts[area_label] = round(total_area, 2) return self.counts[area_label] def count_things(self): """ Count the available number of calexp visits, sensors, fields etc. """ # Collect numbers of images of various kinds: if self.exists['calexp']: self.counts['Number of Visits'] = \ len(self.butler.queryMetadata('calexp', ['visit'])) self.counts['Number of Pointings'] = \ len(self.butler.queryMetadata('calexp', ['pointing'])) self.counts['Number of Sensor Visits'] = \ len(self.butler.queryMetadata('calexp', ['ccd'])) self.counts['Number of Fields'] = \ len(self.butler.queryMetadata('calexp', ['field'])) self.counts['Number of Filters'] = \ len(self.butler.queryMetadata('calexp', ['filter'])) # Collect number of objects from Source Catalog if self.exists['src']: self.counts['Number of Sources'] = \ len(self.butler.queryMetadata('src', ['id'])) return def plot_sky_coverage(self): import matplotlib.pyplot as plt fig = plt.figure() for tract in self.tracts: tractInfo = self.skyMap[tract] corners = [(x[0].asDegrees(), x[1].asDegrees()) for x in tractInfo.getVertexList()] x = [k[0] for k in corners] + [corners[0][0]] y = [k[1] for k in corners] + [corners[0][1]] plt.plot(x,y, color='b') plt.xlabel('RA (deg)') plt.ylabel('Dec (deg)') plt.title('2D Projection of Sky Coverage') plt.show() return def report(self): """ Print a nice report of the data available in this repo. """ # First check what's there: if not self.existence: self.what_exists() # Then, get the numbers: self.count_things() self.estimate_sky_area() # A nice bold section heading: display(Markdown('### Main Repo: %s' % self.repo)) if self.path_to_tracts != '': display(Markdown('### Specified Tract Directory: %s' %self.path_to_tracts)) # Make a table of the collected metadata output_table = "| Metadata Characteristics | | \n | :---: | --- | \n " for key in self.counts.keys(): output_table += "| %s | %s | \n" %(key, self.counts[key]) # Display it: display(Markdown(output_table)) # Plot sky coverage self.plot_sky_coverage() return
from astropy.coordinates import SkyCoord from astropy.table import vstack from lsst.afw.geom import makeSkyWcs from lsst.daf.persistence import Butler from lsst.obs.lsst.imsim import ImsimMapper template_repo = '/global/cscratch1/sd/bos0109/templates_rect' diarepo = template_repo + '/rerun/diff_rect' assocrepo = diarepo + '/rerun/assoc_secrun' forcerepo = assocrepo + '/rerun/forcedPhot' tmprepo = template_repo + '/rerun/multiband' diabutler = Butler(forcerepo) skymap = diabutler.get('deepCoadd_skyMap') truth_lightc = pd.read_csv('./lightcurves/lightcurves_cat_rect_58.0_56.0_-31.0_-32.0.csv') sntab = pd.read_csv('./catalogs+tables/supernovae_cat_rect_58.0_56.0_-31.0_-32.0.csv') #truth_lightc = pd.read_csv('./lightcurves/lightcurves_cat_rect_58_56_-31_-32.csv') #sntab = pd.read_csv('./catalogs+tables/supernovae_cat_rect_58_56_-31_-32.csv') #diaSrc_store = pd.HDFStore('/global/cscratch1/sd/bos0109/diaSrc_forced_fulltables_v4.h5') diaSrc_store = pd.HDFStore('/global/homes/b/bos0109/run2_diaproc/results/diaSrc_secrun_fulltables_v4.h5') diaSrc_store.open() metacols = ['id', 'visit', 'filter', 'raftName', 'detectorName', 'detector'] #region ------------------------------------------------------------------------ ## =================== Convenient functions ================================= ## def get_truth_LC(truth_tab, snid): sffx = ('_observable', '_observed', '_flux', '_fluxErr', '_mag', '_magErr')
class ButlerWrapper(object): def __init__(self, inputs, outputs=None): """Initialize the butler wrapper class. Parameters ---------- inputs : RepositoryArgs, dict, or str Can be a single item or a list. Provides arguments to load an existing repository (or repositories). String is assumed to be a URI and is used as the cfgRoot (URI to the location of the cfg file). (Local file system URI does not have to start with 'file://' and in this way can be a relative path). The 'RepositoryArgs' class can be used to provide more parameters with which to initialize a repository (such as 'mapper', 'mapperArgs', 'tags', etc. See the 'RepositoryArgs' documentation for more details). A dict may be used as shorthand for a 'RepositoryArgs' class instance. The dict keys must match parameters to the 'RepositoryArgs.__init__' function. outputs : RepositoryArgs, dict, or str, optional Provides arguments to load one or more existing repositories or create new ones. The different types are handled the same as for 'inputs'. (the default is None.) """ self._butler = Butler(inputs=inputs, outputs=outputs) def setInputsAndOutputs(self, inputs, outputs=None): """Set the inputs and outputs of butler. Parameters ---------- inputs : RepositoryArgs, dict, or str Can be a single item or a list. Provides arguments to load an existing repository (or repositories). String is assumed to be a URI and is used as the cfgRoot (URI to the location of the cfg file). (Local file system URI does not have to start with 'file://' and in this way can be a relative path). The 'RepositoryArgs' class can be used to provide more parameters with which to initialize a repository (such as 'mapper', 'mapperArgs', 'tags', etc. See the 'RepositoryArgs' documentation for more details). A dict may be used as shorthand for a 'RepositoryArgs' class instance. The dict keys must match parameters to the 'RepositoryArgs.__init__' function. outputs : RepositoryArgs, dict, or str, optional Provides arguments to load one or more existing repositories or create new ones. The different types are handled the same as for 'inputs'. (the default is None.) """ self._butler = Butler(inputs=inputs, outputs=outputs) def getRawExp(self, visit, raft, sensor, snap=None): """Get the raw exposure. Parameters ---------- visit : int Visit Id. raft : str Abbreviated raft name (e.g. "R22"). sensor : str Abbreviated sensor name (e.g. "S11"). snap : int, optional Snap time (0 or 1) means first/ second exposure. (the default is None.) Returns ------- lsst.afw.image.exposure.exposure.ExposureF Raw exposure object. """ dataId = self._getDefaultDataId(visit, raft, sensor) if (snap is not None) and isinstance(snap, (int, float)): dataId["snap"] = int(snap) return self._butler.get("raw", dataId=dataId) def _getDefaultDataId(self, visit, raft, sensor): """Get the default data Id. Parameters ---------- visit : int Visit Id. raft : str Abbreviated raft name (e.g. "R22"). sensor : str Abbreviated sensor name (e.g. "S11"). Returns ------- dict Default data Id. """ dataId = dict(visit=int(visit), raftName=raft, detectorName=sensor) return dataId def getPostIsrCcd(self, visit, raft, sensor, afilter=None): """Get the post-ISR CCD exposure. ISR: Instrument signature removal. CCD: Charge-coupled device. Parameters ---------- visit : int Visit Id. raft : str Abbreviated raft name (e.g. "R22"). sensor : str Abbreviated sensor name (e.g. "S11"). afilter : str, optional Active filter ("u", "g", "r", "i", "z", "y") (the default is None.) Returns ------- lsst.afw.image.exposure.exposure.ExposureF Post-ISR CCD object. """ dataId = self._getDefaultDataId(visit, raft, sensor) if (afilter is not None) and isinstance(afilter, str): dataId["filter"] = afilter return self._butler.get("postISRCCD", dataId=dataId) @staticmethod def getImageData(exposure): """Get the image data. Parameters ---------- exposure : lsst.afw.image.exposure.exposure.ExposureF Exposure object. Returns ------- numpy.ndarray Image data. """ # Get the numpy array data based on the input object type if isinstance(exposure, np.ndarray): data = exposure elif hasattr(exposure, "getMaskedImage"): data = exposure.getMaskedImage().getImage().getArray() elif hasattr(exposure, "getImage"): data = exposure.getImage().getArray() else: data = exposure.getArray() # Return the data in numpy array return data
from lsst.daf.persistence import Butler import lsst.afw.detection as afwDetection import lsst.afw.geom as afwGeom import lsst.geom as geom import bfd import galsim import numpy as np b = Butler('/datasets/hsc/repo/rerun/RC/w_2018_24/DM-14688/') #exposure =b.get('deepCoadd_calexp',tract=9813,patch='4,4',filter='HSC-I') #meas = b.get('deepCoadd_meas',tract=9813,patch='4,4',filter='HSC-I') exposure = b.get('calexp', visit=1228, ccd=49) meas = b.get('src', visit=1228, ccd=49) ds = [] for ii in range(1000): sns = [] sigmas = np.arange(0.2, 2, 0.1) for i in sigmas: weight = bfd.KSigmaWeight(4, i) measRecord = meas[ii] center = measRecord.getCentroid() orig_box = measRecord.getFootprint().getBBox() w, h = orig_box.getWidth(), orig_box.getHeight() box_size = max(w, h) if box_size % 2 == 1: box_size += 1
def prep(self): if self.initialized: return self.prior = None priorFiles = [] priorButler = Butler(self.config.priorRerun) prior_skyMap = priorButler.get('deepCoadd_skyMap') for tract in self.config.priorTracts: for patchInfo in prior_skyMap[tract]: patch = '%d,%d' % patchInfo.getIndex() if self.config.priorPatches: if patch not in self.config.priorPatches: continue if priorButler.datasetExists('deepCoadd_prior', tract=tract, patch=patch, filter=self.config.priorFilter, label=self.config.priorLabel): priorFiles.append(priorButler.getUri('deepCoadd_prior', tract=tract, patch=patch, filter=self.config.priorFilter, label=self.config.priorLabel)) max_file = len(priorFiles) if self.config.maxPriorFiles > 0: max_file = self.config.maxPriorFiles self.zBin = None for file in priorFiles[:max_file]: if file.find('_parent') > 0: self.log.info("Skipping %s, from parent" % file) continue self.log.info("Adding prior %s" % file) try: cat = afwTable.BaseCatalog.readFits(file) md = cat.getTable().getMetadata().toDict() if self.prior is None: self.fluxMin = md['FLUXMIN'] self.fluxMax = md['FLUXMAX'] self.varMin = md['VARMIN'] self.varMax = md['VARMAX'] cov_even = np.array(md['COV_EVEN']) cov_odd = np.array(md['COV_ODD']) self.zMax = md['ZMAXCUT'] self.zMin = md['ZMINCUT'] self.noiseFactor = md['noiseFactor'] self.priorSigmaCutoff = md['priorSigmaCutoff'] self.priorSigmaStep = md['priorSigmaStep'] self.priorSigmaBuffer = md['priorSigmaBuffer'] self.nSample = md['NSAMPLE'] self.selectionOnly = md['selectionOnly'] self.invariantCovariance = md['invariantCovariance'] covMat = self.bfd.MomentCov(cov_even.reshape(self.n_even, self.n_even), cov_odd.reshape(self.n_odd, self.n_odd)) self.prior = self.bfd.KDTreePrior(self.fluxMin, self.fluxMax, covMat, self.ud, self.nSample, self.selectionOnly, self.noiseFactor, self.priorSigmaStep, self.priorSigmaCutoff, self.priorSigmaBuffer, self.invariantCovariance) else: fluxMin = md['FLUXMIN'] fluxMax = md['FLUXMAX'] varMin = md['VARMIN'] varMax = md['VARMAX'] cov_even = np.array(md['COV_EVEN']) cov_odd = np.array(md['COV_ODD']) zMax = md['ZMAXCUT'] zMin = md['ZMINCUT'] noiseFactor = md['noiseFactor'] priorSigmaCutoff = md['priorSigmaCutoff'] priorSigmaStep = md['priorSigmaStep'] priorSigmaBuffer = md['priorSigmaBuffer'] nSample = md['NSAMPLE'] selectionOnly = md['selectionOnly'] invariantCovariance = md['invariantCovariance'] mismatch = False if fluxMin != self.fluxMin: self.log.info('does not match fluxMin') mismatch = True if fluxMax != self.fluxMax: self.log.info('does not match fluxMax') mismatch = True if varMin != self.varMin: self.log.info('does not match varMin') mismatch = True if varMax != self.varMax: self.log.info('does not match varMax') mismatch = True if zMin != self.zMin: self.log.info('does not match zMin') mismatch = True if zMax != self.zMax: self.log.info('does not match zMax') mismatch = True if noiseFactor != self.noiseFactor: self.log.info('does not match fluxMin') mismatch = True if priorSigmaBuffer != self.priorSigmaBuffer: self.log.info('does not match priorSigmaBuffer') mismatch = True if priorSigmaStep != self.priorSigmaStep: self.log.info('does not match priorSigmaStep') mismatch = True if priorSigmaCutoff != self.priorSigmaCutoff: self.log.info('does not match priorSigmaCutoff') mismatch = True if nSample != self.nSample: self.log.info('does not match nSample') mismatch = True if selectionOnly != self.selectionOnly: self.log.info('does not match selectionOnly') mismatch = True if invariantCovariance != self.invariantCovariance: self.log.info('does not match invariantCovariance') mismatch = True if mismatch: self.log.info('Skipping %s' % file) continue for s in cat: ti = self.bfd.TemplateInfo() ti.m = s.get('m') ti.dm = s.get('dm').reshape(self.bfd.BFDConfig.MSIZE, self.bfd.BFDConfig.DSIZE) ti.dxy = s.get('dxy').reshape(self.bfd.BFDConfig.XYSIZE, self.bfd.BFDConfig.DSIZE) ti.nda = s.get('nda') ti.id = s.get('bfd_id') self.prior.addTemplateInfo(ti) except Exception as e: print('Failed to read', e) continue self.prior.prepare() self.initialized = True
from lsst.daf.persistence import Butler from importlib import reload import lsst.afw.geom as afwGeom import lsst.afw.cameraGeom import lsst.geom as geom from lsst.afw.geom import makeSkyWcs from lsst.obs.lsst.imsim import ImsimMapper from collections import OrderedDict as Odict import dm_utilities as dmu calexprepo = '/global/cscratch1/sd/desc/DC2/data/Run2.1i/rerun/calexp-v1' b = Butler(calexprepo) skymap = b.get('deepCoadd_skyMap') template_repo = '/global/cscratch1/sd/bos0109/templates_rect' tmprepo = template_repo + '/rerun/multiband' diarepo = template_repo + '/rerun/diff_rect' assocrepo = diarepo + '/rerun/assoc_thirrun' forcerepo = assocrepo + '/rerun/forcedPhot' diabutler = Butler(forcerepo) truth_lightc = pd.read_csv( './lightcurves/lightcurves_cat_rect_58.0_56.0_-31.0_-32.0.csv') #sntab = pd.read_csv('./catalogs+tables/supernovae_cat_rect_58_56_-31_-32.csv') sntab = pd.read_csv('./results/sntab_matched.csv') snlcs = pd.read_csv('lightcurves/sn_matched_lcs.csv')
import numpy as np from matplotlib import pyplot as plt plt.ion() from lsst.daf.persistence import Butler butler = Butler('/datasets/hsc/repo/rerun/RC/w_2018_10/DM-13647/') # Get a particular calexp # In order to find this file, I had to look through the directory structure # $(pointing)/$(filter)/corr/CORR-$(visit)-$(ccd).fits #subset = butler.subset('calexp', **{'ccd':95, 'pointing':1111, 'filter':'HSC-Z', 'visit':17962}) subset = butler.subset('calexp', **{'ccd':95, 'filter':'HSC-Z', 'visit':17962}) calexp = butler.get('calexp', **subset.cache[0]) src = butler.get('src', **subset.cache[0]) # Image plt.figure(figsize=(5, 8)) plt.imshow(calexp.getImage().array, vmin=np.percentile(calexp.getImage().array, 10), vmax=np.percentile(calexp.getImage().array, 90)) plt.colorbar() plt.scatter(src.getX(), src.getY(), c='red', s=2) # Variance plt.figure(figsize=(5, 8)) plt.imshow(calexp.getVariance().array, vmin=np.percentile(calexp.getVariance().array, 10), vmax=np.percentile(calexp.getVariance().array, 90))
import matplotlib.pyplot as plt import pandas as pd from astropy import time from astropy import units as u from astropy.coordinates import SkyCoord from astropy.table import vstack from lsst.afw.geom import makeSkyWcs from lsst.daf.persistence import Butler from lsst.obs.lsst.imsim import ImsimMapper calexprepo = '/global/cscratch1/sd/desc/DC2/data/Run2.1i/rerun/calexp-v1' b = Butler(calexprepo) skymap = b.get('deepCoadd_skyMap') template_repo = '/global/cscratch1/sd/bos0109/templates_rect' diarepo = template_repo + '/rerun/diff_rect' assocrepo = diarepo + '/rerun/assoc_thirrun' forcerepo = assocrepo + '/rerun/forcedPhot' tmprepo = template_repo + '/rerun/multiband' diabutler = Butler(forcerepo) truth_lightc = pd.read_csv('./lightcurves/lightcurves_cat_rect_58.0_56.0_-31.0_-32.0.csv') sntab = pd.read_csv('./catalogs+tables/supernovae_cat_rect_58.0_56.0_-31.0_-32.0.csv') #region --------- we are going to clean the tables using the visits in forced repo files = glob(diarepo+'/deepDiff/*') visitn = []
import lsst.afw.table as afwTable from lsst.meas.base import measurementInvestigationLib as mil repoPath = '/datasets/hsc/repo/rerun/private/nate/shapeTest2' tract = 9813 filt = "HSC-I" patches = ['{},{}'.format(x, y) for x in range(4, 7) for y in range(4, 7)] repo = Butler(repoPath) images = [] catalogs = [] for patch in patches: dataId = {'tract': tract, 'patch': patch, 'filter': filt} tmpIm = repo.get('deepCoadd_calexp', dataId) tmpCat = repo.get('deepCoadd_meas', dataId) images.append(tmpIm) catalogs.append(tmpCat) count = 0 for cat in catalogs: count += len(cat) totalCatalog = afwTable.SourceCatalog(catalogs[0].schema) totalCatalog.reserve(count) imageId = [] for n in range(len(catalogs)): for record in catalogs[n]: tmpRecord = totalCatalog.addNew() tmpRecord.assign(record)
class ButlerWrapper(object): def __init__(self, inputs, outputs=None): """Initialize the butler wrapper class. Parameters ---------- inputs : RepositoryArgs, dict, or str Can be a single item or a list. Provides arguments to load an existing repository (or repositories). String is assumed to be a URI and is used as the cfgRoot (URI to the location of the cfg file). (Local file system URI does not have to start with 'file://' and in this way can be a relative path). The 'RepositoryArgs' class can be used to provide more parameters with which to initialize a repository (such as 'mapper', 'mapperArgs', 'tags', etc. See the 'RepositoryArgs' documentation for more details). A dict may be used as shorthand for a 'RepositoryArgs' class instance. The dict keys must match parameters to the 'RepositoryArgs.__init__' function. outputs : RepositoryArgs, dict, or str, optional Provides arguments to load one or more existing repositories or create new ones. The different types are handled the same as for 'inputs'. (the default is None.) """ self._butler = Butler(inputs=inputs, outputs=outputs) def setInputsAndOutputs(self, inputs, outputs=None): """Set the inputs and outputs of butler. Parameters ---------- inputs : RepositoryArgs, dict, or str Can be a single item or a list. Provides arguments to load an existing repository (or repositories). String is assumed to be a URI and is used as the cfgRoot (URI to the location of the cfg file). (Local file system URI does not have to start with 'file://' and in this way can be a relative path). The 'RepositoryArgs' class can be used to provide more parameters with which to initialize a repository (such as 'mapper', 'mapperArgs', 'tags', etc. See the 'RepositoryArgs' documentation for more details). A dict may be used as shorthand for a 'RepositoryArgs' class instance. The dict keys must match parameters to the 'RepositoryArgs.__init__' function. outputs : RepositoryArgs, dict, or str, optional Provides arguments to load one or more existing repositories or create new ones. The different types are handled the same as for 'inputs'. (the default is None.) """ self._butler = Butler(inputs=inputs, outputs=outputs) def getRawExp(self, visit, raft, sensor, snap=None): """Get the raw exposure. Parameters ---------- visit : int Visit Id. raft : str Abbreviated raft name (e.g. "R22"). sensor : str Abbreviated sensor name (e.g. "S11"). snap : int, optional Snap time (0 or 1) means first/ second exposure. (the default is None.) Returns ------- lsst.afw.image.exposure.exposure.ExposureF Raw exposure object. """ dataId = self._getDefaultDataId(visit, raft, sensor) return self._butler.get("raw", dataId=dataId) def _getDefaultDataId(self, visit, raft, sensor): """Get the default data Id. Parameters ---------- visit : int Visit Id. raft : str Abbreviated raft name (e.g. "R22"). sensor : str Abbreviated sensor name (e.g. "S11"). Returns ------- dict Default data Id. """ dataId = dict(expId=int(visit), raftName=raft, detectorName=sensor) return dataId def getPostIsrCcd(self, visit, raft, sensor, aFilter=None): """Get the post-ISR CCD exposure. ISR: Instrument signature removal. CCD: Charge-coupled device. Parameters ---------- visit : int Visit Id. raft : str Abbreviated raft name (e.g. "R22"). sensor : str Abbreviated sensor name (e.g. "S11"). aFilter : str, optional Active filter ("u", "g", "r", "i", "z", "y") (the default is None.) Returns ------- lsst.afw.image.exposure.exposure.ExposureF Post-ISR CCD object. """ dataId = self._getDefaultDataId(visit, raft, sensor) self._extendDataId(dataId, aFilter=aFilter) return self._butler.get("postISRCCD", dataId=dataId) def _extendDataId(self, dataId, snap=None, aFilter=None): """Extend the data Id. Parameters ---------- dataId : dict Data Id. snap : int, optional Snap time (0 or 1) means first/ second exposure. (the default is None.) aFilter : str, optional Active filter ("u", "g", "r", "i", "z", "y") (the default is None.) """ if (snap is not None) and isinstance(snap, (int, float)): dataId["snap"] = int(snap) if (aFilter is not None) and isinstance(aFilter, str): dataId["filter"] = aFilter def getEimage(self, visit, raft, sensor, snap=None): """Get the PhoSim eimage exposure. Parameters ---------- visit : int Visit Id. raft : str Abbreviated raft name (e.g. "R22"). sensor : str Abbreviated sensor name (e.g. "S11"). snap : int, optional Snap time (0 or 1) means first/ second exposure. (the default is None.) Returns ------- lsst.afw.image.exposure.exposure.ExposureF Eimage exposure object. """ dataId = self._getDefaultDataId(visit, raft, sensor) self._extendDataId(dataId, snap=snap) return self._butler.get("eimage", dataId=dataId) @staticmethod def getImageData(exposure): """Get the image data. Parameters ---------- exposure : lsst.afw.image.exposure.exposure.ExposureF Exposure object. Returns ------- numpy.ndarray Image data. """ # Get the numpy array data based on the input object type if isinstance(exposure, np.ndarray): data = exposure elif hasattr(exposure, "getMaskedImage"): data = exposure.getMaskedImage().getImage().getArray() elif hasattr(exposure, "getImage"): data = exposure.getImage().getArray() else: data = exposure.getArray() # Return the data in numpy array return data
field, tract = 'PSNJ07250042+2347030', 21 J_cat, H_cat, ref_table = read_cats(field, tract=tract, repo=rerun) snr_threshold = 5 good_color = (J_cat['J_SNR'] > snr_threshold) & (H_cat['H_SNR'] > snr_threshold) J_cat = J_cat[good_color] H_cat = H_cat[good_color] ref_table = ref_table[good_color] butler = Butler(rerun) dId = {'field': field, 'filter': 'H', 'tract': tract, 'patch': '0,0'} calexp = butler.get('deepCoadd', dataId=dId) # Find PSNJ07250042+2347030 based on RA, Dec: 07:25:00.408 +23:47:03.15 (J2000) sn_RA, sn_Dec = 111.251700, 23.784208 sn_coord = afwGeom.SpherePoint(sn_RA, sn_Dec, afwGeom.degrees) distList = [] for s in ref_table: this_coord = afwGeom.SpherePoint(s['coord_ra'], s['coord_dec'], afwGeom.radians) angSep = sn_coord.separation(this_coord) distList.append(angSep) distance = np.array(distList)
import matplotlib.pyplot as plt import pandas as pd from astropy import time from astropy import units as u from astropy.coordinates import SkyCoord from astropy.table import vstack from lsst.afw.geom import makeSkyWcs from lsst.daf.persistence import Butler from lsst.obs.lsst.imsim import ImsimMapper calexprepo = '/global/cscratch1/sd/desc/DC2/data/Run2.1i/rerun/calexp-v1' b = Butler(calexprepo) skymap = b.get('deepCoadd_skyMap') template_repo = '/global/cscratch1/sd/bos0109/templates_rect' tmprepo = template_repo + '/rerun/multiband' diarepo = template_repo + '/rerun/diff_rect' assocrepo = diarepo + '/rerun/assoc_thirrun' forcerepo = assocrepo + '/rerun/forcedPhot' diabutler = Butler(forcerepo) #truth_lightc = pd.read_csv('./lightcurves/lightcurves_cat_rect_58.0_56.0_-31.0_-32.0.csv') #sntab = pd.read_csv('./catalogs+tables/supernovae_cat_rect_58.0_56.0_-31.0_-32.0.csv') store = f'{SCRATCH}/results/diaSrc_thirrun_fulltables_v1.h5' diaSrc_store = pd.HDFStore(store) diaSrc_store.open()
def runIsr(self): """ this method takes the amplifier images from phosim, runs isr, and writes the new post-isr e-images with '_isr' appended to the filename. """ outputDir = os.path.join(self.imageDir, 'iter{}'.format(str(self.iIter))) flatsDir = os.path.join(self.aosSrcDir, '..', 'data', 'flats') repackagedDir = os.path.join(outputDir, 'repackaged') butlerDir = os.path.join(outputDir, 'butler') postISRDir = os.path.join(butlerDir, 'rerun', 'run1') if not os.path.exists(flatsDir): cwd = os.getcwd() os.mkdir(flatsDir) os.chdir(flatsDir) runProgram( 'makeGainImages.py --detector_list R00_S22 R40_S02 R04_S20 R44_S00', verbose=True) os.chdir(cwd) if os.path.exists(repackagedDir): shutil.rmtree(repackagedDir) os.mkdir(repackagedDir) runProgram('phosim_repackager.py {} --out_dir {}'.format( outputDir, repackagedDir), verbose=True) if os.path.exists(butlerDir): shutil.rmtree(butlerDir) os.mkdir(butlerDir) runProgram( 'echo lsst.obs.lsst.phosim.PhosimMapper > {}/_mapper'.format( butlerDir), verbose=True) runProgram( 'ingestCalibs.py {} {}/* --validity 9999 --output {} --mode copy'. format(butlerDir, flatsDir, butlerDir), verbose=True) runProgram('ingestImages.py {} {}/*.fits --clobber-config'.format( butlerDir, repackagedDir), verbose=True) runProgram('runIsr.py {} --id --rerun run1 --config isr.doBias=False isr.doDark=False '\ 'isr.doFlat=True isr.doFringe=False --clobber-config'.format(butlerDir), verbose=True) # We don't want to import LSST stack dependencies unless we have to. # Once we have imported for the first time then it is fast. from lsst.daf.persistence import Butler butler = Butler(postISRDir) pattern = re.compile( 'lsst_e_(\d+)_f\d_(R\d{2})_(S\d{2})_(C\d)_E(\d{3}).fits') for fname in os.listdir(outputDir): match = pattern.match(fname) if match: visit, raft, sensor, chip, snap = match.groups() dataId = { 'raftName': raft, 'visit': int(visit), 'detectorName': sensor, 'snap': int(snap) } data = butler.get('postISRCCD', dataId) img = data.getImage().getArray().transpose() if 'C0' in fname: img = img[:, :2000] else: img = img[:, 2000:] fitsIn = os.path.join(outputDir, fname) fitsOut = os.path.join(outputDir, '{}_isr.fits'.format(fname[:-5])) fitsPrimary = fits.open(fitsIn)[0] fitsPrimary.data = img fitsPrimary.writeto(fitsOut, overwrite=True) # butlerDir is ~ 5GB shutil.rmtree(butlerDir) shutil.rmtree(repackagedDir)
def load_patch(butler_or_repo, tract, patch, fields_to_join=('id', ), filters={ 'u': 'u', 'g': 'g', 'r': 'r', 'i': 'i', 'z': 'z', 'y': 'y' }, trim_colnames_for_fits=False, verbose=False, debug=False): """Load patch catalogs. Return merged catalog across filters. butler_or_repo: Butler object or str Either a Butler object or a filename to the repo tract: int Tract in skymap patch: str Patch in the tract in the skymap fields_to_join: iterable of str Join the catalogs for each filter on these fields filters: iterable of str Filter names to load trim_colnames_for_fits: bool Trim column names to satisfy the FITS standard character limit of <68. Returns -- Pandas DataFrame of patch catalog merged across filters. """ if isinstance(butler_or_repo, str): butler = Butler(butler_or_repo) else: butler = butler_or_repo # Define the filters and order in which to sort them.: tract_patch_data_id = {'tract': tract, 'patch': patch} try: ref_table = butler.get(datasetType='deepCoadd_ref', dataId=tract_patch_data_id) ref_table = ref_table.asAstropy().to_pandas() except NoResults as e: if verbose: print(" ", e) return pd.DataFrame() isPrimary = ref_table['detect_isPrimary'] ref_table = ref_table[isPrimary] if len(ref_table) == 0: if verbose: print(" No good isPrimary entries for tract %d, patch %s" % (tract, patch)) return ref_table flux_field_names_per_schema_version = { 1: { 'psf_flux': 'base_PsfFlux_flux', 'psf_flux_err': 'base_PsfFlux_fluxSigma', 'modelfit_flux': 'modelfit_CModel_flux', 'modelfit_flux_err': 'modelfit_CModel_fluxSigma' }, 2: { 'psf_flux': 'base_PsfFlux_flux', 'psf_flux_err': 'base_PsfFlux_fluxErr', 'modelfit_flux': 'modelfit_CModel_flux', 'modelfit_flux_err': 'modelfit_CModel_fluxErr' }, 3: { 'psf_flux': 'base_PsfFlux_instFlux', 'psf_flux_err': 'base_PsfFlux_instFluxErr', 'modelfit_flux': 'modelfit_CModel_instFlux', 'modelfit_flux_err': 'modelfit_CModel_instFluxErr' }, } merge_filter_cats = {} for filt in filters: this_data = tract_patch_data_id.copy() this_data['filter'] = filters[filt] try: cat = butler.get(datasetType='deepCoadd_forced_src', dataId=this_data) except NoResults as e: if verbose: print(" ", e) continue if debug: print("AFW photometry catalog schema version: {}".format( cat.schema.VERSION)) flux_names = flux_field_names_per_schema_version[cat.schema.VERSION] # Convert the AFW table to an AstroPy table # because it's much easier to add column to an AstroPy table # than it is to set up a new schema for an AFW table. # cat = cat.asAstropy() # Try instead out converting the AFW->AstroPy->Pandas per cat # hoping to avoid memory copy # Then join in memory space. cat = cat.asAstropy().to_pandas() calib = butler.get('deepCoadd_calexp_calib', this_data) calib.setThrowOnNegativeFlux(False) mag, mag_err = calib.getMagnitude( cat[flux_names['psf_flux']].values, cat[flux_names['psf_flux_err']].values) cat['mag'] = mag cat['mag_err'] = mag_err cat['SNR'] = np.abs(cat[flux_names['psf_flux']] / cat[flux_names['psf_flux_err']]) modelfit_mag, modelfit_mag_err = calib.getMagnitude( cat[flux_names['modelfit_flux']].values, cat[flux_names['modelfit_flux_err']].values) cat['modelfit_mag'] = modelfit_mag cat['modelfit_mag_err'] = modelfit_mag_err cat['modelfit_SNR'] = np.abs(cat[flux_names['modelfit_flux']] / cat[flux_names['modelfit_flux_err']]) cat = cat[isPrimary] merge_filter_cats[filt] = cat merged_patch_cat = ref_table for filt in filters: if filt not in merge_filter_cats: continue cat = merge_filter_cats[filt] if len(cat) < 1: continue # Rename duplicate columns with prefix of filter prefix_columns(cat, filt, fields_to_skip=fields_to_join) # Merge metadata with concatenation merged_patch_cat = pd.merge(merged_patch_cat, cat, on=fields_to_join, sort=False) if trim_colnames_for_fits: # FITS column names can't be longer that 68 characters # Trim here to ensure consistency across any format we write this out to trim_long_colnames(merged_patch_cat) return merged_patch_cat