def __init__(self, detectorFileName, ampFileNameList, inAmpCoords=True, plateScale=1., radialCoeffs=(0., 1.), clobberMetadata=False, doRaise=True): ''' @param[in] detectorFileName FITS file containing the detector description. May use [] notation to specify an extension in an MEF. @param[in] ampFileNameList List of FITS file names to use in building the amps. May contain duplicate entries if the raw data are assembled. @param[in] inAmpCoords Boolean, True if raw data are in amp coordinates, False if raw data are assembled into pseudo detector pixel arrays @param[in] plateScale Nominal platescale (arcsec/mm) @param[in] radialCoeffs Radial distortion coefficients for a radial polynomial in normalized units. @param[in] clobberMetadata Clobber metadata from input files if overridden in the _sanitizeMetadata method @param[in] doRaise Raise exception if not all non-defaulted keywords are defined? Default is True. ''' self.inAmpCoords = inAmpCoords self.defaultAmpMap = self._makeDefaultAmpMap() self.defaultDetectorMap = self._makeDefaultDetectorMap() self.detectorMetadata = afwImage.readMetadata(detectorFileName) self._sanitizeHeaderMetadata(self.detectorMetadata, clobber=clobberMetadata) self.ampMetadataList = [] self.detector = None self.doRaise = doRaise for fileName in ampFileNameList: self.ampMetadataList.append(afwImage.readMetadata(fileName)) self._sanitizeHeaderMetadata(self.ampMetadataList[-1], clobber=clobberMetadata) self.plateScale = plateScale self.focalPlaneToPupil = self._makeRadialTransform(radialCoeffs)
def _get_catalog(self, dataset, **kwargs): """Load the catalogs from the butler.""" filenames = (self.butler.get(dataset + "_filename", dataId, immediate=True)[0] for dataId in self.dataIds[dataset]) try: # In recent stack version, metadata are in HDU 1 headers = (afwimage.readMetadata(fn, 1) for fn in filenames) size = sum(md.get("NAXIS2") for md in headers) except: # Older stack version headers = (afwimage.readMetadata(fn, 2) for fn in filenames) size = sum(md.get("NAXIS2") for md in headers) cat = self.butler.get(dataset, self.dataIds[dataset][0], flags=afwtable.SOURCE_IO_NO_FOOTPRINTS, immediate=True) self.from_butler['schema'] = cat.schema catadic = {k: [] for k in sorted(self.dataIds[dataset][0].keys())} catalog = afwtable.SourceCatalog(self.from_butler['schema']) catalog.reserve(size) # pbar = cutils.progressbar(len(self.dataIds[dataset])) print("INFO: Looping over the dataids") for i, dataid in enumerate(self.dataIds[dataset]): cat = self.butler.get(dataset, dataid, flags=afwtable.SOURCE_IO_NO_FOOTPRINTS) catalog.extend(cat, deep=True) for newkey in catadic: catadic[newkey].extend([dataid[newkey]] * len(cat)) # pbar.update(i + 1) # pbar.finish() print("INFO: Merging the dictionnaries") catadic.update(catalog.getColumnView().extract(*self.keys[dataset], copy=True, ordered=True)) return catadic
def bypass_raw_visitInfo(self, datasetType, pythonType, location, dataId): """Work around for reading metadata from multi-HDU files. afwImage.readMetadata() doesn't honour [hdu] suffixes in filenames. Parameters ---------- datasetType : anything, unused Unused pythonType : anything, unused Unused location : `lsst.daf.persistence.ButlerLocation` Butler location dataId : anything, unused Unused Returns ------- visitInfo : `lsst.afw.image.visitInfo` The visitInfo """ import re import lsst.afw.image as afwImage fileName = location.getLocationsWithRoot()[0] mat = re.search(r"\[(\d+)\]$", fileName) if mat: hdu = int(mat.group(1)) md = afwImage.readMetadata(fileName, hdu=hdu) else: md = afwImage.readMetadata(fileName) # or hdu = INT_MIN; -(1 << 31) return afwImage.VisitInfo(md)
def _get_catalog(self, dataset, **kwargs): """Load the catalogs from the butler.""" filenames = (self.butler.get(dataset + "_filename", dataId, immediate=True)[0] for dataId in self.dataids[dataset]) try: # In recent stack version, metadata are in HDU 1 headers = (afwimage.readMetadata(fn, 1) for fn in filenames) size = sum(md.get("NAXIS2") for md in headers) except: # Older stack version headers = (afwimage.readMetadata(fn, 2) for fn in filenames) size = sum(md.get("NAXIS2") for md in headers) cat = self.butler.get(dataset, self.dataids[dataset][0], flags=afwtable.SOURCE_IO_NO_FOOTPRINTS, immediate=True) self.from_butler['schema'] = cat.schema catadic = {k: [] for k in sorted(self.dataids[dataset][0].keys())} catalog = afwtable.SourceCatalog(self.from_butler['schema']) catalog.reserve(size) pbar = cutils.progressbar(len(self.dataids[dataset])) print("INFO: Looping over the dataids") for i, dataid in enumerate(self.dataids[dataset]): cat = self.butler.get(dataset, dataid, flags=afwtable.SOURCE_IO_NO_FOOTPRINTS) catalog.extend(cat, deep=True) for newkey in catadic: catadic[newkey].extend([dataid[newkey]] * len(cat)) pbar.update(i + 1) pbar.finish() print("INFO: Merging the dictionnaries") catadic.update(catalog.getColumnView().extract(*self.keys[dataset], copy=True, ordered=True)) # Clean memory before going further # gc.collect() return catadic
def getInfo(self, filename): """Get information about the image from the filename and its contents Here, we open the image and parse the header, but one could also look at the filename itself and derive information from that, or set values from the configuration. @param filename Name of file to inspect @return File properties; list of file properties for each extension """ md = afwImage.readMetadata(filename, self.config.hdu) phuInfo = self.getInfoFromMetadata(md) if len(self.config.extnames) == 0: # No extensions to worry about return phuInfo, [phuInfo] # Look in the provided extensions extnames = set(self.config.extnames) extnum = 1 infoList = [] while len(extnames) > 0: extnum += 1 try: md = afwImage.readMetadata(filename, extnum) except: self.log.warn("Error reading %s extensions %s" % (filename, extnames)) break ext = self.getExtensionName(md) if ext in extnames: infoList.append(self.getInfoFromMetadata(md, info=phuInfo.copy())) extnames.discard(ext) return phuInfo, infoList
def getInfo(self, filename): """Get information about the image from the filename and its contents Here, we open the image and parse the header, but one could also look at the filename itself and derive information from that, or set values from the configuration. @param filename Name of file to inspect @return File properties; list of file properties for each extension """ md = afwImage.readMetadata(filename, self.config.hdu) phuInfo = self.getInfoFromMetadata(md) if len(self.config.extnames) == 0: # No extensions to worry about return phuInfo, [phuInfo] # Look in the provided extensions extnames = set(self.config.extnames) extnum = 0 infoList = [] while len(extnames) > 0: extnum += 1 try: md = afwImage.readMetadata(filename, extnum) except: self.log.warn("Error reading %s extensions %s" % (filename, extnames)) break ext = self.getExtensionName(md) if ext in extnames: hduInfo = self.getInfoFromMetadata(md, info=phuInfo.copy()) # We need the HDU number when registering MEF files. hduInfo["hdu"] = extnum infoList.append(hduInfo) extnames.discard(ext) return phuInfo, infoList
def getInfo(self, filename, filetype): """ The science pixels, mask, and weight (inverse variance) are stored in separate files each with a unique name but with a common unique identifier EXPNUM in the FITS header. We have to aggregate the 3 filenames for a given EXPNUM and return this information along with that returned by the base class. We expect a directory structure that looks like the following: dqmask/ instcal/ wtmap/ The user creates the registry by running ingestImagesDecam.py outputRepository --mode=link instcal/*fits """ if filetype == "instcal": if self.expnumMapper is None: self.buildExpnumMapper(os.path.dirname(os.path.abspath(filename))) # Note that phuInfo will have # 'side': 'X', 'ccd': 0 phuInfo, infoList = super(DecamParseTask, self).getInfo(filename) expnum = phuInfo["visit"] phuInfo[self.instcalPrefix] = self.expnumMapper[expnum][self.instcalPrefix] phuInfo[self.dqmaskPrefix] = self.expnumMapper[expnum][self.dqmaskPrefix] phuInfo[self.wtmapPrefix] = self.expnumMapper[expnum][self.wtmapPrefix] for idx, info in enumerate(infoList): expnum = info["visit"] info[self.instcalPrefix] = self.expnumMapper[expnum][self.instcalPrefix] info[self.dqmaskPrefix] = self.expnumMapper[expnum][self.dqmaskPrefix] info[self.wtmapPrefix] = self.expnumMapper[expnum][self.wtmapPrefix] elif filetype == "raw": md = afwImage.readMetadata(filename, self.config.hdu) phuInfo = self.getInfoFromMetadata(md) extnames = set(self.config.extnames) extnum = 1 infoList = [] while len(extnames) > 0: extnum += 1 try: md = afwImage.readMetadata(filename, extnum) except: self.log.warn("Error reading %s extensions %s" % (filename, extnames)) break ext = self.getExtensionName(md) if ext in extnames: info = self.getInfoFromMetadata(md, info=phuInfo.copy()) info['hdu'] = extnum - 1 info[self.instcalPrefix] = "" info[self.dqmaskPrefix] = "" info[self.wtmapPrefix] = "" infoList.append(info) extnames.discard(ext) return phuInfo, infoList
def makeitLsst(prefs, context, saveWcs=False, plot=dict()): """This is the python wrapper that reads lsst tables""" # Create an array of PSFs (one PSF for each extension) if prefs.getVerboseType() != prefs.QUIET: print("----- %d input catalogues:" % prefs.getNcat()) if saveWcs: # only needed for making plots wcssList = [] fields = psfexLib.vectorField() for cat in prefs.getCatalogs(): field = psfexLib.Field(cat) wcss = [] wcssList.append(wcss) with pyfits.open(cat): # Hack: I want the WCS so I'll guess where the calexp is to be found calexpFile = guessCalexp(cat) md = afwImage.readMetadata(calexpFile) wcs = afwImage.makeWcs(md) if not wcs: crval = afwCoord.makeCoord(afwCoord.ICRS, 0.0 * afwGeom.degrees, 0.0 * afwGeom.degrees) wcs = afwImage.makeWcs(crval, afwGeom.PointD(0, 0), 1.0, 0, 0, 1.0) naxis1, naxis2 = md.get("NAXIS1"), md.get("NAXIS2") # Find how many rows there are in the catalogue md = afwImage.readMetadata(cat) field.addExt(wcs, naxis1, naxis2, md.get("NAXIS2")) if saveWcs: wcss.append((wcs, naxis1, naxis2)) field.finalize() fields.append(field) fields[0].getNext() # number of extensions prefs.getPsfStep() sets = psfexLib.vectorSet() for set in load_samplesLsst(prefs, context, plot=plot): sets.append(set) psfexLib.makeit(fields, sets) ret = [[f.getPsfs() for f in fields], sets] if saveWcs: ret.append(wcssList) return ret
def makeitLsst(prefs, context, saveWcs=False, plot=dict()): """This is the python wrapper that reads lsst tables""" # Create an array of PSFs (one PSF for each extension) if prefs.getVerboseType() != prefs.QUIET: print("----- %d input catalogues:" % prefs.getNcat()) if saveWcs: # only needed for making plots wcssList = [] fields = psfexLib.vectorField() for cat in prefs.getCatalogs(): field = psfexLib.Field(cat) wcss = [] wcssList.append(wcss) with pyfits.open(cat): # Hack: I want the WCS so I'll guess where the calexp is to be found calexpFile = guessCalexp(cat) md = afwImage.readMetadata(calexpFile) wcs = afwImage.makeWcs(md) if not wcs: crval = afwCoord.makeCoord(afwCoord.ICRS, 0.0*afwGeom.degrees, 0.0*afwGeom.degrees) wcs = afwImage.makeWcs(crval, afwGeom.PointD(0, 0), 1.0, 0, 0, 1.0) naxis1, naxis2 = md.get("NAXIS1"), md.get("NAXIS2") # Find how many rows there are in the catalogue md = afwImage.readMetadata(cat) field.addExt(wcs, naxis1, naxis2, md.get("NAXIS2")) if saveWcs: wcss.append((wcs, naxis1, naxis2)) field.finalize() fields.append(field) fields[0].getNext() # number of extensions prefs.getPsfStep() sets = psfexLib.vectorSet() for set in load_samplesLsst(prefs, context, plot=plot): sets.append(set) psfexLib.makeit(fields, sets) ret = [[f.getPsfs() for f in fields], sets] if saveWcs: ret.append(wcssList) return ret
def linearity(directory, infilebase, outfile, amps, x0, y0, boxsize): #get list of files files = glob.glob(directory+infilebase) #write output file header f=open(directory+outfile, 'w+') f.write('amp\tx0\ty0\tboxsize\tmedian\texptime\n') for filename in files: #get exposure time from header hdr = afwImage.readMetadata(filename, 1) exptime = hdr.get('EXPTIME') for amp in amps: #define selected region box = afwGeom.Box2I(afwGeom.Point2I(x0, y0), afwGeom.Extent2I(boxsize, boxsize)) #read in selected region of file im = afwImage.ExposureF(filename, amp+1, box) #get median of region of image box_median = afwMath.makeStatistics(im.getMaskedImage(), afwMath.MEDIAN).getValue() #write amp, box parameters, region median, and exptime to file f.write(str(amp) + '\t' + str(x0) + '\t' + str(y0) + '\t' + str(boxsize) + '\t' + str(box_median) + '\t' + str(exptime) + '\n') f.close()
def makeCcd(fileName, fromHeader=False, trim=True): ccd = cameraGeom.Ccd(cameraGeom.Id(0)) a = 0 # one-less than the next HDU while True: # while there are valid HDUs a += 1 if fromHeader: try: hdu = 1 + a md, channelNo = afwImage.readMetadata(fileName, hdu), None except lsst.pex.exceptions.LsstCppException: if hdu == 1: # an empty PDU continue break if "TTYPE1" in md.names(): # not an image break else: md, channelNo = None, a # # Add amp to the Ccd # try: ccd.addAmp(makeAmp(md, channelNo, trim)) except StopIteration: break return ccd
def testFitsHeader(self): """Test that XY0 and crpix are written to the header as expected""" #getPixelOrigin() returns origin in lsst coordinates, so need to add 1 to #compare to values stored in fits headers parentCrpix = self.parent.getWcs().getPixelOrigin() #Make a sub-image x0, y0 = 20, 30 llc = afwGeom.Point2I(x0, y0) bbox = afwGeom.Box2I(llc, afwGeom.Extent2I(60, 50)) deep = False subImg = afwImage.ExposureF(self.parent, bbox, afwImage.LOCAL, deep) with utilsTests.getTempFilePath(".fits") as outFile: subImg.writeFits(outFile) hdr = afwImage.readMetadata(outFile) def checkLtvHeader(hdr, name, value): # Per DM-4133, LTVn headers are required to be floating point self.assertTrue(hdr.exists(name), name + " not saved to FITS header") self.assertIsInstance(hdr.get(name), numbers.Real, name + " is not numeric") self.assertNotIsInstance(hdr.get(name), numbers.Integral, name + " is an int") self.assertEqual(hdr.get(name), value, name + " has wrong value") checkLtvHeader(hdr, "LTV1", -1*x0) checkLtvHeader(hdr, "LTV2", -1*y0) self.assertTrue( hdr.exists("CRPIX1"), "CRPIX1 not saved to fits header") self.assertTrue( hdr.exists("CRPIX2"), "CRPIX2 not saved to fits header") fitsCrpix = [hdr.get("CRPIX1"), hdr.get("CRPIX2")] self.assertAlmostEqual(fitsCrpix[0] - hdr.get("LTV1"), parentCrpix[0]+1, 6, "CRPIX1 saved wrong") self.assertAlmostEqual(fitsCrpix[1] - hdr.get("LTV2"), parentCrpix[1]+1, 6, "CRPIX2 saved wrong")
def bypass_instcal(self, datasetType, pythonType, butlerLocation, dataId): # Workaround until I can access the butler instcalMap = self.map_instcal(dataId) dqmaskMap = self.map_dqmask(dataId) wtmapMap = self.map_wtmap(dataId) rawMap = self.map_raw(dataId) instcalType = getattr(afwImage, instcalMap.getPythonType().split(".")[-1]) rawType = getattr(afwImage, rawMap.getPythonType().split(".")[-1]) dqmaskType = getattr(afwImage, dqmaskMap.getPythonType().split(".")[-1]) wtmapType = getattr(afwImage, wtmapMap.getPythonType().split(".")[-1]) instcal = instcalType(instcalMap.getLocations()[0]) dqmask = dqmaskType(dqmaskMap.getLocations()[0]) wtmap = wtmapType(wtmapMap.getLocations()[0]) raw = rawType(rawMap.getLocations()[0]) mask = self.translate_dqmask(dqmask) variance = self.translate_wtmap(wtmap) mi = afwImage.MaskedImageF(afwImage.ImageF(instcal.getImage()), mask, variance) md = raw.getMetadata() wcs = afwImage.makeWcs(md) exp = afwImage.ExposureF(mi, wcs) # Set the calib by hand; need to grab the zeroth extension header = re.sub(r'[\[](\d+)[\]]$', "[0]", rawMap.getLocations()[0]) md0 = afwImage.readMetadata(header) calib = afwImage.Calib() calib.setExptime(md0.get("EXPTIME")) calib.setFluxMag0(10**(0.4 * md0.get("MAGZERO"))) exp.setCalib(calib) exp.setMetadata(md) # Do we need to remove WCS/calib info? return exp
def readRawFile(fileName, dataId={}, detector=None): """Read a raw file from fileName, assembling it nicely. Parameters ---------- filename : `str` The fully-qualified filename. dataId : `lsst.daf.persistence.DataId` If provided, used to look up e.g. the filter. detector : `lsst.afw.cameraGeom.Detector` If provided, add this detector to the returned Exposure Returns ------- exposure : `lsst.afw.image.Exposure` The assembled exposure from the supplied filename. """ class Info(): def __init__(self, obj): self.obj = obj amps = [] for hdu in range(1, 16+1): exp = afwImage.makeExposure(afwImage.makeMaskedImage(afwImage.ImageF(fileName, hdu=hdu))) exp.setDetector(detector) amps.append(exp) component_info = {} component_info["raw_hdu"] = Info(afwImage.readMetadata(fileName, hdu=0)) component_info["raw_amp"] = Info(amps) exp = assemble_raw(dataId, component_info, None) return exp
def _standardizeMasterCal(self, datasetType, item, dataId, setFilter=False): """Standardize a MasterCal image obtained from NOAO archive into Exposure These MasterCal images are MEF files with one HDU for each detector. Some WCS header, eg CTYPE1, exists only in the zeroth extensionr, so info in the zeroth header need to be copied over to metadata. @param datasetType: Dataset type ("bias" or "flat") @param item: The image read by the butler @param dataId: Data identifier @param setFilter: Whether to set the filter in the Exposure @return (lsst.afw.image.Exposure) the standardized Exposure """ mi = afwImage.makeMaskedImage(item.getImage()) md = item.getMetadata() masterCalMap = getattr(self, "map_" + datasetType) masterCalPath = masterCalMap(dataId).getLocations()[0] headerPath = re.sub(r'[\[](\d+)[\]]$', "[0]", masterCalPath) md0 = afwImage.readMetadata(headerPath) for kw in ('CTYPE1', 'CTYPE2', 'CRVAL1', 'CRVAL2', 'CUNIT1', 'CUNIT2', 'CD1_1', 'CD1_2', 'CD2_1', 'CD2_2'): if kw in md0.paramNames() and kw not in md.paramNames(): md.add(kw, md0.get(kw)) wcs = afwImage.makeWcs(md, True) exp = afwImage.makeExposure(mi, wcs) exp.setMetadata(md) return self._standardizeExposure(self.calibrations[datasetType], exp, dataId, filter=setFilter)
def testFitsHeader(self): """Test that XY0 and crpix are written to the header as expected""" #getPixelOrigin() returns origin in lsst coordinates, so need to add 1 to #compare to values stored in fits headers parentCrpix = self.parent.getWcs().getPixelOrigin() #Make a sub-image x0, y0 = 20, 30 llc = afwGeom.Point2I(x0, y0) bbox = afwGeom.Box2I(llc, afwGeom.Extent2I(60, 50)) deep = False subImg = afwImage.ExposureF(self.parent, bbox, afwImage.LOCAL, deep) outFile = "tmp.fits" subImg.writeFits(outFile) hdr = afwImage.readMetadata(outFile) os.remove(outFile) self.assertTrue(hdr.exists("LTV1"), "LTV1 not saved to fits header") self.assertTrue(hdr.exists("LTV2"), "LTV2 not saved to fits header") self.assertEqual(hdr.get("LTV1"), -1 * x0, "LTV1 has wrong value") self.assertEqual(hdr.get("LTV2"), -1 * y0, "LTV1 has wrong value") self.assertTrue(hdr.exists("CRPIX1"), "CRPIX1 not saved to fits header") self.assertTrue(hdr.exists("CRPIX2"), "CRPIX2 not saved to fits header") fitsCrpix = [hdr.get("CRPIX1"), hdr.get("CRPIX2")] self.assertAlmostEqual(fitsCrpix[0] - hdr.get("LTV1"), parentCrpix[0] + 1, 6, "CRPIX1 saved wrong") self.assertAlmostEqual(fitsCrpix[1] - hdr.get("LTV2"), parentCrpix[1] + 1, 6, "CRPIX2 saved wrong")
def assembleImage(self, gains=None): imDict = {} hdus = {} for i in range(self._namps): hdu = i + 1 filename = '%s[%i]' % (self.infile, hdu) md = afwImage.readMetadata(filename) imDict[md.get('EXTNAME')] = afwImage.ImageF(filename) hdus[md.get('EXTNAME')] = hdu det = self.buildDetector() assembleInput = {} for amp in det: im = imDict[amp.getName()] oscanim = im.Factory(im, amp.getRawHorizontalOverscanBBox()) oscan = afwMath.makeStatistics(oscanim, afwMath.MEDIAN).getValue() im -= oscan if gains is not None: im *= gains[hdus[amp.getName()]] assembleInput[amp.getName()] = self.makeExposure(im) assembleConfig = ipIsr.AssembleCcdTask.ConfigClass() assembleConfig.doTrim = True assembler = ipIsr.AssembleCcdTask(config=assembleConfig) resultExp = assembler.assembleCcd(assembleInput) return resultExp.getMaskedImage().getImage()
def testFitsHeader(self): """Test that XY0 and crpix are written to the header as expected""" # getPixelOrigin() returns origin in lsst coordinates, so need to add 1 to # compare to values stored in fits headers parentCrpix = self.parent.getWcs().getPixelOrigin() # Make a sub-image x0, y0 = 20, 30 llc = afwGeom.Point2I(x0, y0) bbox = afwGeom.Box2I(llc, afwGeom.Extent2I(60, 50)) deep = False subImg = afwImage.ExposureF(self.parent, bbox, afwImage.LOCAL, deep) with utilsTests.getTempFilePath(".fits") as outFile: subImg.writeFits(outFile) hdr = afwImage.readMetadata(outFile) def checkLtvHeader(hdr, name, value): # Per DM-4133, LTVn headers are required to be floating point self.assertTrue(hdr.exists(name), name + " not saved to FITS header") self.assertIsInstance(hdr.get(name), numbers.Real, name + " is not numeric") self.assertNotIsInstance(hdr.get(name), numbers.Integral, name + " is an int") self.assertEqual(hdr.get(name), value, name + " has wrong value") checkLtvHeader(hdr, "LTV1", -1 * x0) checkLtvHeader(hdr, "LTV2", -1 * y0) self.assertTrue(hdr.exists("CRPIX1"), "CRPIX1 not saved to fits header") self.assertTrue(hdr.exists("CRPIX2"), "CRPIX2 not saved to fits header") fitsCrpix = [hdr.get("CRPIX1"), hdr.get("CRPIX2")] self.assertAlmostEqual(fitsCrpix[0] - hdr.get("LTV1"), parentCrpix[0] + 1, 6, "CRPIX1 saved wrong") self.assertAlmostEqual(fitsCrpix[1] - hdr.get("LTV2"), parentCrpix[1] + 1, 6, "CRPIX2 saved wrong")
def getInfo(self, filename): """Get information about the image from the filename and its contents @param filename Name of file to inspect @return File properties; list of file properties for each extension """ #matches = re.search("^PFSA(\d{6})(\d)(\d).fits", filename) matches = re.search("PF([JLXIASPF])([ABCDS])(\d{6})(\d)(\d).fits", filename) if not matches: matches = re.search("PF([JLXIASPF])([ABCDS])-(\d{6})(\d)(\d).fits", filename) if not matches: raise RuntimeError("Unable to interpret filename: %s" % filename) site, category, visit, filterInt, spectrograph = matches.groups() if int(spectrograph) > 4: spectrograph = '4' ccd = int(spectrograph)-1 filter = '' if filterInt == '0': filter = 'b' elif filterInt == '1': filter = 'r' ccd += 4 elif filterInt == '2': filter = 'n' ccd += 8 else: filter = 'm' ccd += 4 arm = filter header = afwImage.readMetadata(filename) info = dict(site=site, category=category, visit=int(visit), filter=filter, arm=arm, spectrograph=int(spectrograph), ccd=int(ccd)) info = self.getInfoFromMetadata(header, info=info) return info, [info]
def _get_catalog(self, dataset, **kwargs): """Load the catalogs from the butler.""" filenames = (self.butler.get(dataset + "_filename", dataId, immediate=True)[0] for dataId in self.dataids[dataset]) headers = (afwimage.readMetadata(fn, 2) for fn in filenames) size = sum(md.get("NAXIS2") for md in headers) cat = self.butler.get(dataset, self.dataids[dataset][0], flags=afwtable.SOURCE_IO_NO_FOOTPRINTS, immediate=True) self.from_butler['schema'] = cat.schema catadic = {k: [] for k in sorted(self.dataids[dataset][0].keys())} catalog = afwtable.SourceCatalog(self.from_butler['schema']) catalog.reserve(size) pbar = progressbar(len(self.dataids[dataset])) print "INFO: Looping over the dataids" for i, dataid in enumerate(self.dataids[dataset]): cat = self.butler.get(dataset, dataid, flags=afwtable.SOURCE_IO_NO_FOOTPRINTS) catalog.extend(cat, deep=True) for newkey, idk in zip(catadic, sorted(self.dataids[dataset][0].keys())): catadic[newkey].extend([dataid[idk]] * len(cat)) pbar.update(i + 1) pbar.finish() print "INFO: Merging the dictionnaries" catadic.update(catalog.getColumnView().extract(*self.keys[dataset], copy=True, ordered=True)) # Clean memory before going further gc.collect() return catadic
def testFitsHeader(self): """Test that XY0 and crpix are written to the header as expected""" #getPixelOrigin() returns origin in lsst coordinates, so need to add 1 to #compare to values stored in fits headers parentCrpix = self.parent.getWcs().getPixelOrigin() #Make a sub-image x0, y0 = 20, 30 llc = afwGeom.Point2I(x0, y0) bbox = afwGeom.Box2I(llc, afwGeom.Extent2I(60, 50)) deep = False subImg = afwImage.ExposureF(self.parent, bbox, afwImage.LOCAL, deep) outFile = "tmp.fits" subImg.writeFits(outFile) hdr = afwImage.readMetadata(outFile) os.remove(outFile) self.assertTrue( hdr.exists("LTV1"), "LTV1 not saved to fits header") self.assertTrue( hdr.exists("LTV2"), "LTV2 not saved to fits header") self.assertEqual(hdr.get("LTV1"), -1*x0, "LTV1 has wrong value") self.assertEqual(hdr.get("LTV2"), -1*y0, "LTV1 has wrong value") self.assertTrue( hdr.exists("CRPIX1"), "CRPIX1 not saved to fits header") self.assertTrue( hdr.exists("CRPIX2"), "CRPIX2 not saved to fits header") fitsCrpix = [hdr.get("CRPIX1"), hdr.get("CRPIX2")] self.assertAlmostEqual(fitsCrpix[0] - hdr.get("LTV1"), parentCrpix[0]+1, 6, "CRPIX1 saved wrong") self.assertAlmostEqual(fitsCrpix[1] - hdr.get("LTV2"), parentCrpix[1]+1, 6, "CRPIX2 saved wrong")
def setUp(self): maskedImage = afwImage.MaskedImageF(inFilePathSmall) maskedImageMD = afwImage.readMetadata(inFilePathSmall) self.smallExposure = afwImage.ExposureF(inFilePathSmall) self.width = maskedImage.getWidth() self.height = maskedImage.getHeight() self.wcs = afwImage.makeWcs(maskedImageMD) self.psf = DummyPsf(2.0) self.detector = DetectorWrapper().detector self.exposureBlank = afwImage.ExposureF() self.exposureMiOnly = afwImage.makeExposure(maskedImage) self.exposureMiWcs = afwImage.makeExposure(maskedImage, self.wcs) self.exposureCrWcs = afwImage.ExposureF( 100, 100, self.wcs) # n.b. the (100, 100, ...) form self.exposureCrOnly = afwImage.ExposureF(afwGeom.ExtentI( 100, 100)) # test with ExtentI(100, 100) too afwImage.Filter.reset() afwImage.FilterProperty.reset() filterPolicy = pexPolicy.Policy() filterPolicy.add("lambdaEff", 470.0) afwImage.Filter.define(afwImage.FilterProperty("g", filterPolicy))
def test1(self): # This fails due to #1386 #wcsfn = os.path.join(self.datadir, 'imsim-v85518312-fu-R43-S12.wcs') wcsfn = os.path.join(self.datadir, 'imsim-v85518312-fu-R43-S12.wcs2') hdr = afwImage.readMetadata(wcsfn) wcs1 = afwImage.makeWcs(hdr) crval = wcs1.getSkyOrigin() cd = wcs1.getCDMatrix() print cd crval_p = afwGeom.Point2D(crval.getLongitude().asDegrees(), crval.getLatitude().asDegrees()) origin = wcs1.getPixelOrigin() print crval_p print origin wcs2 = afwImage.Wcs(crval_p, origin, cd) for wcs in [wcs1,wcs2]: print wcs print 'x, y, RA, Dec, pixscale("/pix), pixscale2' for x,y in [(0,0),(300,0),(350,0),(360,0),(370,0),(380,0),(400,0)]: radec = wcs.pixelToSky(x,y) ra = radec.getLongitude().asDegrees() dec = radec.getLatitude ().asDegrees() pixscale = 3600. * sqrt(wcs.pixArea(afwGeom.Point2D(x,y))) ps2 = wcs.pixelScale().asArcseconds() print x,y,ra,dec,pixscale,ps2 self.assertTrue(abs(pixscale - 0.2) < 1e-3) self.assertTrue(abs(ps2 - 0.2) < 1e-3)
def testReadMetadata(self): im = afwImage.DecoratedImageF(self.fileForMetadata) meta = afwImage.readMetadata(self.fileForMetadata) self.assertIn("NAXIS1", meta.names()) self.assertEqual(im.getWidth(), meta.get("NAXIS1")) self.assertEqual(im.getHeight(), meta.get("NAXIS2"))
def processRaft(raftDir, conn, done, qsp): nProcessed = 0 nSkipped = 0 nUnrecognized = 0 for fits in glob.glob( os.path.join( raftDir, "S[0-2][0-2]", "imsim_*_R[0-4][0-4]_S[0-2][0-2]_C[01][0-7]_E00[01].fits*")): m = re.search( r'v(\d+)-f(\w)/E00(\d)/R(\d)(\d)/S(\d)(\d)/' + r'imsim_\1_R\4\5_S\6\7_C(\d)(\d)_E00\3\.fits', fits) if not m: print >> sys.stderr, "Warning: Unrecognized file:", fits nUnrecognized += 1 continue (visit, filter, snap, raft1, raft2, sensor1, sensor2, channel1, channel2) = m.groups() key = "%s_F%s_E%s_R%s,%s_S%s,%s_C%s,%s" % (visit, filter, snap, raft1, raft2, sensor1, sensor2, channel1, channel2) if key in done: nSkipped += 1 continue md = afwImage.readMetadata(fits) expTime = md.get("EXPTIME") mjdObs = md.get("MJD-OBS") taiObs = dafBase.DateTime(mjdObs, dafBase.DateTime.MJD, dafBase.DateTime.TAI).toString( dafBase.DateTime.UTC)[:-1] conn.execute( """INSERT INTO raw VALUES (NULL, ?, ?, ?, ?, ?, ?, ?, ?)""", (visit, filter, snap, "%s,%s" % (raft1, raft2), "%s,%s" % (sensor1, sensor2), "%s,%s" % (channel1, channel2), taiObs, expTime)) for row in conn.execute("SELECT last_insert_rowid()"): id = row[0] break wcs = afwImage.makeWcs(md) poly = skypix.imageToPolygon(wcs, md.get("NAXIS1"), md.get("NAXIS2"), padRad=0.000075) # about 15 arcsec pix = qsp.intersect(poly) for skyTileId in pix: conn.execute("INSERT INTO raw_skyTile VALUES(?, ?)", (id, skyTileId)) conn.commit() nProcessed += 1 print >>sys.stderr, raftDir, \ "... %d processed, %d skipped, %d unrecognized" % \ (nProcessed, nSkipped, nUnrecognized)
def EventFromInputfile(inputfile, datatypePolicy, metadataPolicy, topicName='triggerImageprocEvent', hostName='lsst4.ncsa.uiuc.edu'): """generate a new file event for a given FITS file @param inputfile the name of the FITS file @param datatyepPolicy the policy describing the metadata transformation @param metadataPolicy the policy describing the event metadata types @param topicName the name of the topic to send event as @param hostName the event broker hostname """ global exposureCount exposureCount += 1 # For DC3a, inputfile is a .fits file on disk metadata = afwImage.readMetadata(inputfile) # First, transform the input metdata transformMetadata(metadata, datatypePolicy, metadataPolicy, 'Keyword') # To be consistent... if not validateMetadata(metadata, metadataPolicy): logger.log(logger.FATAL, 'Unable to create event from %s' % inputfile) # Create event policy, using defaults from input metadata event = dafBase.PropertySet() event.copy('visitId', metadata, 'visitId') event.copy('ccdId', metadata, 'ccdId') event.copy('ampId', metadata, 'ampId') event.copy('exposureId', metadata, 'exposureId') event.copy('datasetId', metadata, 'datasetId') event.copy('filter', metadata, 'filter') event.copy('expTime', metadata, 'expTime') event.copy('ra', metadata, 'ra') event.copy('decl', metadata, 'decl') event.copy('equinox', metadata, 'EQUINOX') event.copy('airmass', metadata, 'airmass') event.copy('dateObs', metadata, 'dateObs') if event.getInt('exposureId') == 0: eventTransmitter = ctrlEvents.EventTransmitter(hostName, topicName + '0') elif event.getInt('exposureId') == 1: eventTransmitter = ctrlEvents.EventTransmitter(hostName, topicName + '1') logger.log(logger.INFO, 'Sending event for %s' % os.path.basename(inputfile)) if logger.sends(logger.DEBUG): logger.log(logger.DEBUG, "Data Event data:\n%s" % event.toString()) elif logger.sends(VERB3): logger.log( VERB3, "Event data: datasetId=%s; ra=%f, dec=%f" % (event.get("datasetId"), event.get("ra"), event.get("decl"))) eventTransmitter.publish(event) return True
def fromDir(cls, root, visit, **kwds): ffp = {} wcs = {} fcrPattern = os.path.join(root, "fcr-%07d-*.fits" % visit) # meas_mosaic coords wcsPattern = os.path.join(root, "wcs-%07d-*.fits" % visit) # LSST coords start = fcrPattern.index("*") for filename in glob.glob(fcrPattern): ccd = int(filename[start:start + 3]) md = afwImage.readMetadata(filename) ffp[ccd] = FluxFitParams(md) for filename in glob.glob(wcsPattern): ccd = int(filename[start:start + 3]) md = afwImage.readMetadata(filename) wcs[ccd] = afwImage.makeWcs(md) return CorrectionImageSource(ffp, wcs, **kwds)
def std_dark(self, item, dataId): exp = afwImage.makeExposure(afwImage.makeMaskedImage(item)) rawPath = self.map_raw(dataId).getLocations()[0] headerPath = re.sub(r'[\[](\d+)[\]]$', "[0]", rawPath) md0 = afwImage.readMetadata(headerPath) visitInfo = self.makeRawVisitInfo(md0) exp.getInfo().setVisitInfo(visitInfo) return self._standardizeExposure(self.calibrations["dark"], exp, dataId, filter=False)
def __init__(self, infile, hdu): self.header = None try: self.md = afwImage.readMetadata(infile, hdu) except: # This exception occurs when DM stack encounters a "." in # a FITS header keyword. self.header = fits.open(infile)[hdu-1].header
def checkExtName(self, name, value, extNum): filename = DATA + "[%s]" % name header = afwImage.readMetadata(filename) self.assertEqual(header.get("EXT_NUM"), extNum) self.assertEqual(header.get("EXTNAME").strip(), name) image = afwImage.ImageI(filename) self.assertEqual(image.get(0,0), value)
def testReadWcs(self): """Test reading a Wcs directly from a fits file""" meta = afwImage.readMetadata(InputImagePath + "_img.fits") wcs = afwImage.makeWcs(meta) sky0 = wcs.pixelToSky(0.0, 0.0).getPosition() sky1 = self.wcs.pixelToSky(0.0, 0.0).getPosition() self.assertEqual(sky0, sky1)
def __init__(self, imfile, mask_files=(), ccdtemp_par=-100): self.ccd = MaskedCCD(imfile, mask_files=mask_files) self.md = afwImage.readMetadata(imfile, 0) try: ccdtemp = self.md.get('CCDTEMP') except: ccdtemp = ccdtemp_par self.fe55_yield = Fe55Yield(ccdtemp).alpha() self._footprint_signal = self._footprint_signal_spans
def testReadWcs(self): """Test reading a Wcs directly from a fits file""" meta = afwImage.readMetadata(InputImagePath + ".fits") wcs = afwImage.makeWcs(meta) sky0 = wcs.pixelToSky(0.0, 0.0).getPosition() sky1 = self.wcs.pixelToSky(0.0, 0.0).getPosition() self.assertEqual(sky0, sky1)
def getInfo(self, filename): """Get information about the image from the filename and its contents The information we want isn't in the headers, but in the filname. Parameters ---------- filename : `str` Name of file to inspect Returns ------- info : `dict` File properties from the PHU. infoList : `list` of `dict` File properties from the extensions. """ self.log.debug('interpreting filename <%s>' % filename) path, name = os.path.split(filename) matches = re.search(r"^PF(%s)(%s)-?(\d{6})(\d)(\d)\.fits$" % (self.sites, self.categories), name) if not matches: raise RuntimeError("Unable to interpret filename: %s" % filename) site, category, expId, spectrograph, armNum = matches.groups() armNum = int(armNum) spectrograph = int(spectrograph) expId = int(expId, base=10) # spectrograph 2 was called 9 at JHU, at least in the early days, so if we find # a spectrograph 9 we re-assign its number to 2 if spectrograph == 9: spectrograph = 2 self.log.info("Visit %06d has spectrograph == 9" % expId) if spectrograph < 1 or spectrograph > self.nSpectrograph + 1: raise RuntimeError('spectrograph (=%d) out of bounds 1..%d' % (spectrograph, self.nSpectrograph)) try: arm = self.arms[armNum - 1] # 1-indexed except IndexError as exc: raise IndexError('armNum=%d out of bounds 1..%d' % (armNum, max(self.arms.keys()))) from exc ccd = PfsMapper.computeDetectorId(spectrograph, arm) self.log.debug('site = <%s>, category = <%s>, expId = <%s>, spectrograph = <%d>, armNum = <%d>, ' 'arm = <%s>, ccd = <%d>' % (site, category, expId, spectrograph, armNum, arm, ccd)) info = dict(site=site, category=category, expId=expId, visit=expId, filter=arm, arm=arm, spectrograph=spectrograph, ccd=ccd) if os.path.exists(filename): header = afwImage.readMetadata(filename) info = self.getInfoFromMetadata(header, info=info) return info, [info]
def _listdir(self, path, prefix): for file in os.listdir(path): fileName = os.path.join(path, file) md = afwImage.readMetadata(fileName) if "EXPNUM" not in md.names(): return expnum = md.get("EXPNUM") if expnum not in self.expnumMapper.keys(): self.expnumMapper[expnum] = {self.instcalPrefix: None, self.wtmapPrefix: None, self.dqmaskPrefix: None} self.expnumMapper[expnum][prefix] = fileName
def testMakeWcs(self): path = eups.productDir("afw") path = os.path.join(path, "tests", "data", "parent.fits") fitsHdr = image.readMetadata(path) wcs = image.makeWcs(fitsHdr) c = wcs.pixelToSky(0, 0) print type(c) c.getPosition()
def testMakeWcs(self): path= eups.productDir("afw") path = os.path.join(path, "tests", "data", "parent.fits") fitsHdr = image.readMetadata(path) wcs = image.makeWcs(fitsHdr) c = wcs.pixelToSky(0,0) print type(c) c.getPosition()
def testMakeWcs(self): afwdataDir = lsst.utils.getPackageDir("afw") path = os.path.join(afwdataDir, "tests", "data", "parent.fits") fitsHdr = image.readMetadata(path) wcs = image.makeWcs(fitsHdr) c = wcs.pixelToSky(0, 0) print(type(c)) c.getPosition()
def testXY0(self): """Test that XY0 values are handled correctly when building an exposure and also when reading the WCS header directly. #2205""" bbox = afwGeom.Box2I(afwGeom.Point2I(1000, 1000), afwGeom.Extent2I(10, 10)) def makeWcs(crPixPos, crValDeg, projection): ps = dafBase.PropertySet() ctypes = [ ("%-5s%3s" % (("RA", "DEC")[i], projection)).replace(" ", "-") for i in range(2) ] for i in range(2): ip1 = i + 1 ps.add("CTYPE%1d" % (ip1, ), ctypes[i]) ps.add("CRPIX%1d" % (ip1, ), crPixPos[i]) ps.add("CRVAL%1d" % (ip1, ), crValDeg[i]) ps.add("RADECSYS", "ICRS") ps.add("EQUINOX", 2000) ps.add("CD1_1", -0.001) ps.add("CD2_1", 0.0) ps.add("CD1_2", 0.0) ps.add("CD2_2", 0.001) return afwImage.makeWcs(ps) wcs = makeWcs( crPixPos=(100.0, 100.0), crValDeg=(10.0, 35.0), projection="STG", # also fails for TAN ) exposure = afwImage.ExposureF(bbox, wcs) pixPos = afwGeom.Box2D(bbox).getMax() if verbose: print "XY0=", exposure.getXY0() print "pixPos=", pixPos skyPos = wcs.pixelToSky(pixPos) with utilsTests.getTempFilePath(".fits") as tmpFile: exposure.writeFits(tmpFile) for useExposure in (False, True): if useExposure: unpExp = afwImage.ExposureF(tmpFile) unpWcs = unpExp.getWcs() else: md = afwImage.readMetadata(tmpFile) unpWcs = afwImage.makeWcs(md, False) unpPixPos = unpWcs.skyToPixel(skyPos) if verbose: print "useExposure=%s; unp pixPos=%s" % (useExposure, unpPixPos) for i in range(2): self.assertAlmostEqual(unpPixPos[i], 1009.5)
def testDM882(self): """Test that we can write a dotted header unit to a FITS file. See DM-882.""" self.dimage1.getMetadata().add("A.B.C.D", 12345) tempdir = tempfile.mkdtemp() testfile = os.path.join(tempdir, "test.fits") try: self.dimage1.writeFits(testfile) meta = afwImage.readMetadata(testfile) self.assertEqual(meta.get("A.B.C.D"), 12345) finally: shutil.rmtree(tempdir)
def checkEquinoxHeader(coordSysName, writeEquinox): coordSys = getattr(afwCoord, coordSysName) # We should get the same behaviour with both Wcs and TanWcs: check them both. for dummyWcs in (makeWcs(coordSys=coordSys), afwImage.TanWcs.cast(makeWcs(coordSys=coordSys))): dummyExposure = afwImage.ExposureF() dummyExposure.setWcs(dummyWcs) with utilsTests.getTempFilePath(".fits") as tmpFile: dummyExposure.writeFits(tmpFile) metadata = afwImage.readMetadata(tmpFile) self.assertEqual(metadata.get("RADESYS"), coordSysName) self.assertTrue(("EQUINOX" in metadata.names()) == writeEquinox)
def testReadMetadata(self): if self.fileForMetadata: im = afwImage.DecoratedImageF(self.fileForMetadata) else: print >> sys.stderr, "Warning: afwdata is not set up; not running the FITS metadata I/O tests" return meta = afwImage.readMetadata(self.fileForMetadata) self.assertTrue("NAXIS1" in meta.names()) self.assertEqual(im.getWidth(), meta.get("NAXIS1")) self.assertEqual(im.getHeight(), meta.get("NAXIS2"))
def makeCcdMosaic(dir, basename, e, c, aList, imageFactory=afwImage.MaskedImageF, verbose=0): """Return an image of all the specified amplifiers, aList, for the given CCD E.g. sl = makeCcdMosaic("/lsst/DC3root/rlp1173", "v704897", 0, 3, range(8)) """ try: aList[0] except TypeError: aList = [aList] for what in ("header", "data"): if what == "header": bbox = afwGeom.Box2I() ampBBox = {} wcs = {} else: ccdImage = imageFactory(bbox.getWidth(), bbox.getHeight()) ccdImage.set(0) ccdImage.setXY0(bbox.getLLC()) for a in aList: filename = os.path.join(dir, "IPSD", "output", "sci", "%s-e%d" % (basename, e), "%s-e%d-c%03d-a%02d.sci" % (basename, e, c, a)) if verbose and what == "header": print filename if what == "header": md = afwImage.readMetadata(filename + "_img.fits") xy0 = afwGeom.Point2I(md.get("CRVAL1A"), md.get("CRVAL2A")) xy1 = xy0 + afwGeom.Extent2I(md.get("NAXIS1") - 1, md.get("NAXIS2") - 1) bbox.grow(xy0) bbox.grow(xy1) ampBBox[a] = afwGeom.Box2I(xy0, xy1) wcs[a] = afwImage.Wcs(md) else: try: data = imageFactory(filename + "_img.fits") except: data = imageFactory(filename) ampImage = ccdImage.Factory(ccdImage, ampBBox[a]) ampImage[:] = data del ampImage try: ccdImage.getMask() if wcs.has_key(0): ccdImage = afwImage.ExposureF(ccdImage, wcs[0]) except AttributeError: pass return ccdImage
def readout_time(infile): """ Return readout time as an astropy.time.Time object. It is computed from the PHDU keyword values as MJD-OBS + EXPTIME """ md = afwImage.readMetadata(infile, 1) mjd_obs = astropy.time.Time(md.get('MJD-OBS'), format='mjd', scale='utc') t_readout = astropy.time.Time( mjd_obs.datetime + datetime.timedelta(seconds=md.get('EXPTIME')), scale='utc') return t_readout
def testXY0(self): """Test that XY0 values are handled correctly when building an exposure and also when reading the WCS header directly. #2205""" bbox = afwGeom.Box2I(afwGeom.Point2I(1000, 1000), afwGeom.Extent2I(10, 10)) def makeWcs(crPixPos, crValDeg, projection): ps = dafBase.PropertySet() ctypes = [("%-5s%3s" % (("RA", "DEC")[i], projection)).replace(" ", "-") for i in range(2)] for i in range(2): ip1 = i + 1 ps.add("CTYPE%1d" % (ip1,), ctypes[i]) ps.add("CRPIX%1d" % (ip1,), crPixPos[i]) ps.add("CRVAL%1d" % (ip1,), crValDeg[i]) ps.add("RADECSYS", "ICRS") ps.add("EQUINOX", 2000) ps.add("CD1_1", -0.001) ps.add("CD2_1", 0.0) ps.add("CD1_2", 0.0) ps.add("CD2_2", 0.001) return afwImage.makeWcs(ps) wcs = makeWcs( crPixPos = (100.0, 100.0), crValDeg = (10.0, 35.0), projection = "STG", # also fails for TAN ) exposure = afwImage.ExposureF(bbox, wcs) pixPos = afwGeom.Box2D(bbox).getMax() if verbose: print "XY0=", exposure.getXY0() print "pixPos=", pixPos skyPos = wcs.pixelToSky(pixPos) tmpFile = "temp.fits" try: exposure.writeFits(tmpFile) for useExposure in (False, True): if useExposure: unpExp = afwImage.ExposureF(tmpFile) unpWcs = unpExp.getWcs() else: md = afwImage.readMetadata("temp.fits") unpWcs = afwImage.makeWcs(md, False) unpPixPos = unpWcs.skyToPixel(skyPos) if verbose: print "useExposure=%s; unp pixPos=%s" % (useExposure, unpPixPos) for i in range(2): self.assertAlmostEqual(unpPixPos[i], 1009.5) finally: os.remove(tmpFile)
def processRaft(raftDir, conn, done, qsp): nProcessed = 0 nSkipped = 0 nUnrecognized = 0 for fits in glob.glob(os.path.join(raftDir, "S[0-2][0-2]", "imsim_*_R[0-4][0-4]_S[0-2][0-2]_C[01][0-7]_E00[01].fits*")): import pdb; pdb.set_trace() m = re.search(r'v(\d+)-f(\w)/E00(\d)/R(\d)(\d)/S(\d)(\d)/' + r'imsim_\1_R\4\5_S\6\7_C(\d)(\d)_E00\3\.fits', fits) if not m: print >>sys.stderr, "Warning: Unrecognized file:", fits nUnrecognized += 1 continue (visit, filter, snap, raft1, raft2, sensor1, sensor2, channel1, channel2) = m.groups() key = "%s_F%s_E%s_R%s,%s_S%s,%s_C%s,%s" % (visit, filter, snap, raft1, raft2, sensor1, sensor2, channel1, channel2) if done.has_key(key): nSkipped += 1 continue md = afwImage.readMetadata(fits) expTime = md.get("EXPTIME") mjdObs = md.get("MJD-OBS") taiObs = dafBase.DateTime(mjdObs, dafBase.DateTime.MJD, dafBase.DateTime.TAI).toString()[:-1] conn.execute("""INSERT INTO raw VALUES (NULL, ?, ?, ?, ?, ?, ?, ?, ?)""", (visit, filter, snap, "%s,%s" % (raft1, raft2), "%s,%s" % (sensor1, sensor2), "%s,%s" % (channel1, channel2), taiObs, expTime)) for row in conn.execute("SELECT last_insert_rowid()"): id = row[0] break wcs = afwImage.makeWcs(md) poly = skypix.imageToPolygon(wcs, md.get("NAXIS1"), md.get("NAXIS2"), padRad=0.000075) # about 15 arcsec pix = qsp.intersect(poly) for skyTileId in pix: conn.execute("INSERT INTO raw_skyTile VALUES(?, ?)", (id, skyTileId)) conn.commit() nProcessed += 1 print >>sys.stderr, raftDir, \ "... %d processed, %d skipped, %d unrecognized" % \ (nProcessed, nSkipped, nUnrecognized)
def EventFromInputfile(inputfile, datatypePolicy, metadataPolicy, topicName='triggerImageprocEvent', hostName='newfield.as.arizona.edu'): """generate a new file event for a given FITS file @param inputfile the name of the FITS file @param datatyepPolicy the policy describing the metadata transformation @param metadataPolicy the policy describing the event metadata types @param topicName the name of the topic to send event as @param hostName the event broker hostname """ global exposureCount exposureCount += 1 # For mosphot, inputfile is a .fits file on disk metadata = afwImage.readMetadata(inputfile) # logger.log(logger.INFO,"Original metadata:\n" + metadata.toString()) # First, transform the input metdata transformMetadata(metadata, datatypePolicy, metadataPolicy, 'Keyword') # To be consistent... if not validateMetadata(metadata, metadataPolicy): logger.log(logger.FATAL, 'Unable to create event from %s' % inputfile) # Create event policy, using defaults from input metadata event = dafBase.PropertySet() event.copy('exposureId', metadata, 'exposureId') event.copy('datasetId', metadata, 'datasetId') event.copy('filter', metadata, 'filter') event.copy('expTime', metadata, 'expTime') event.copy('ra', metadata, 'ra') event.copy('decl', metadata, 'decl') event.copy('equinox', metadata, 'equinox') event.copy('airmass', metadata, 'airmass') event.copy('dateObs', metadata, 'dateObs') eventTransmitter = ctrlEvents.EventTransmitter(hostName, topicName) logger.log(logger.INFO, 'Sending event for %s' % os.path.basename(inputfile)) if logger.sends(logger.DEBUG): logger.log(logger.DEBUG, "Data Event data:\n%s" % event.toString()) elif logger.sends(VERB3): logger.log(VERB3, "Event data: datasetId=%s; ra=%f, dec=%f" % (event.get("datasetId"), event.get("ra"), event.get("decl"))) eventTransmitter.publish(event) return True
def fileNames(fileLoop): """This function searches for LSST FITS files with associated trimfiles. obss is a list of the FITS subdirectories in the /astro/net/pogo3/.../obs/ directory. They start with a v, then 9 numbers, then -fu, -fg, -fr, -fi, -fz, or -fy. Each subdirectory has an E000 path and an E001 path; they simulate consecutive exposures of the same area of sky. Each of those has a subdirectory, R22 (the central raft in the camera) containing one zipped eimage (compressed FITS file). There are 5287 v...-f* directories. Load each directory in the obs directory that starts with a 'v', name it and add '/E000/R22/' to it and call it the eDir. For this run I only want images from the r filters. Not all of the files have trim files associated with them, so I also skip the ones that don't.""" #ePath = '/astro/net/pogo3/rgibson/testSim/testPM/obs/v858247512-fr/E000/R22/eimage_858247512_R22_S11_E000.fits.gz' ePath = '/astro/net/pogo3/rgibson/testSim/testPM/obs/v858903212-fr/E000/R22/eimage_858903212_R22_S11_E000.fits.gz' #eFile = 'eimage_858247512_R22_S11_E000.fits.gz' eFile = 'eimage_858903212_R22_S11_E000.fits.gz' trimPath = None; metaPath = None; eEPath = None trimDir = '/astro/net/pogo3/rgibson/testSim/testPM/trim/obsid' t0 = eFile.split('_') t1 = t0[1][0:-1] metaPath = '%s%s/metadata_%s.dat' % (trimDir, t1, t1) trimPath = '%s%s/pops/trim_%s_MSSTARS.dat.gz' % (trimDir, t1, t1) eEPath = '%s%s/pops/trim_%s_EASTEREGGS.dat.gz' % (trimDir, t1, t1) outDir = '/astro/net/pogo3/rgibson/testSim/testPM/out%s/' % eFile # [0:-3] removes the ".gz" newEFile = 'eImageTANSIPa' + eFile[0:-3] newEPath = outDir + newEFile fileLoop = 1 # Get the FITS image and find the positive footprints therein. eIm = afwImage.ImageF(newEPath) metaData = afwImage.readMetadata(newEPath) wcs = afwImage.makeWcs(metaData) eHDUList = pyfits.open(newEPath) x0 = wcs.pixelToSky(0, 0).getPosition()[0] y0 = wcs.pixelToSky(0, 0).getPosition()[1] x1 = wcs.pixelToSky(0, 4096).getPosition()[0] y1 = wcs.pixelToSky(0, 4096).getPosition()[1] x2 = wcs.pixelToSky(4096, 4096).getPosition()[0] y2 = wcs.pixelToSky(4096, 4096).getPosition()[1] x3 = wcs.pixelToSky(4096, 0).getPosition()[0] y3 = wcs.pixelToSky(4096, 0).getPosition()[1] # You just want the minimum radius so you don't go off the edge of the image boundary. deltax = min(abs(x2-x1), abs(x2-x0), abs(x3-x1), abs(x3-x0)) deltay = min(abs(y1-y0), abs(y2-y0), abs(y1-y3), abs(y2-y3)) radius = min(deltax/2., deltay/2.) ra_center = wcs.pixelToSky(2048, 2048).getPosition()[0] dec_center = wcs.pixelToSky(2048, 2048).getPosition()[1] return ra_center, dec_center, radius, eIm, eHDUList, wcs, ePath, fileLoop
def writeDefects(self, maskFile, ccd, format=None): """Given a Mask, find the defects for each amp in the specified CCD (0-indexed). If format is provided, it's expected to be used as "format % (ccd, amp)" to generate a .paf filename to write the defects too. If it's "-", write to stdout""" # Metadata should have validity range keywords, but it doesn't. if False: metaData = afwImage.readMetadata(maskFile, 0) hdu = ccd + 2 im = afwImage.ImageF(maskFile, hdu) im -= 1 im *= -1 mask = afwImage.MaskU(im.getBBox(afwImage.PARENT)); mask.set(0x0) mask = afwImage.makeMaskedImage(im, mask) for amp in self.ampList(): subMask = mask.Factory(mask, self.getTrimSecBBox(amp), afwImage.LOCAL) ds = afwDetection.makeFootprintSet(subMask, afwDetection.Threshold(0.5), "INTRP") if False: ds9.setDefaultFrame(amp) ds9.mtv(subMask) ds9.dot("Amp %d" % amp, 0, 0) if not format: return if format == "-": fd = sys.stdout else: fd = open(format % (ccd, amp), "w") print >> fd, """\ #<?cfg paf policy ?> # # Defects for CCD%03d amp %02d generated from %s, HDU %d # # Coordinates are trimmed and per-amp not per-CCD. If you change this, the ISR will have to change # # Generated by $HeadURL$ # """ % (ccd, amp, maskFile, hdu), for foot in ds.getFootprints(): afwDetectionUtils.writeFootprintAsDefects(fd, foot)
def testReadMetadata(self): with utilsTests.getTempFilePath(".fits") as tmpFile: self.exposureCrWcs.getMetadata().set("FRAZZLE", True) # This will write the main metadata (inc. FRAZZLE) to the primary HDU, and the # WCS to subsequent HDUs, along with INHERIT=T. self.exposureCrWcs.writeFits(tmpFile) # This should read the first non-empty HDU (i.e. it skips the primary), but # goes back and reads it if it finds INHERIT=T. That should let us read # frazzle and the Wcs from the PropertySet returned by readMetadata. md = afwImage.readMetadata(tmpFile) wcs = afwImage.makeWcs(md, True) self.assertEqual(wcs.getPixelOrigin(), self.wcs.getPixelOrigin()) self.assertEqual(wcs.getSkyOrigin(), self.wcs.getSkyOrigin()) self.assert_(numpy.all(wcs.getCDMatrix() == self.wcs.getCDMatrix())) frazzle = md.get("FRAZZLE") self.assert_(frazzle is True)
def processBand(filterDir, conn, done, qsp): nProcessed = 0 nSkipped = 0 nUnrecognized = 0 print >>sys.stderr, filterDir, "... started" for fits in glob.iglob( os.path.join(filterDir, "fpC*_ts_coaddNorm_NN.fit.gz")): m = re.search(r'/([ugriz])/fpC-(\d{6})-\1(\d)-(\d{4})_ts_coaddNorm_NN.fit.gz', fits) if not m: print >>sys.stderr, "Warning: Unrecognized file:", fits nUnrecognized += 1 continue (filter, run, camcol, field) = m.groups() camcol = int(camcol) run = int(run) field = int(field) key = "%d_B%s_C%d_F%d" % (run, filter, camcol, field) if done.has_key(key): nSkipped += 1 continue md = afwImage.readMetadata(fits) conn.execute("""INSERT INTO raw VALUES (NULL, ?, ?, ?, ?)""", (run, filter, camcol, field)) for row in conn.execute("SELECT last_insert_rowid()"): id = row[0] break wcs = afwImage.makeWcs(md) poly = skypix.imageToPolygon(wcs, md.get("NAXIS1"), md.get("NAXIS2"), padRad=0.000075) # about 15 arcsec pix = qsp.intersect(poly) for skyTileId in pix: conn.execute("INSERT INTO raw_skyTile VALUES(?, ?)", (id, skyTileId)) nProcessed += 1 if nProcessed % 100 == 0: conn.commit() conn.commit() print >>sys.stderr, filterDir, \ "... %d processed, %d skipped, %d unrecognized" % \ (nProcessed, nSkipped, nUnrecognized)
def AssembleImage(input_file, doGainCorrection = False, trim = True): import os imDict = {} afilelist = [] dfilename = '%s[%s]' % (input_file, 0) for i in range(16): filename = '%s[%i]' % (input_file, i+1) md = afwImage.readMetadata(filename) afilelist.append(filename) imDict[md.get('EXTNAME')] = afwImage.ImageF(filename) db = TestCamDetectorBuilder(dfilename, afilelist, inAmpCoords=True, clobberMetadata=True) det = db.buildDetector() assembleInput = {} for amp in det: im = imDict[amp.getName()] oscanim = im.Factory(im, amp.getRawHorizontalOverscanBBox()) oscan = numpy.median(oscanim.getArray()) imArr = im.getArray() imArr -= oscan #Calculate and correct for gain if doGainCorrection: # Buffer so edge rolloff doesn't interfere buffImArr = imArr[30:-30][30:-30] medCounts = numpy.median(buffImArr) stdCounts = numpy.std(buffImArr) gain = medCounts/stdCounts**2 imArr *= gain assembleInput[amp.getName()] = db.makeExposure(im) assembleConfig = AssembleCcdTask.ConfigClass() if trim: assembleConfig.doTrim = True assembler = AssembleCcdTask(config=assembleConfig) resultExp = assembler.assembleCcd(assembleInput) # camGeomUtils.showCcd(resultExp.getDetector(), imageSource(resultExp), frame=0) return resultExp else: assembleConfig.doTrim = False assembler = AssembleCcdTask(config=assembleConfig) resultExp = assembler.assembleCcd(assembleInput) # camGeomUtils.showCcd(resultExp.getDetector(), imageSource(resultExp), frame=1) return resultExp