def testRedir(self): """ Test redirection to stream. """ with TestRedir.StdoutCapture(self.outputFilename): log.configure() dest = io.StringIO() log_utils.enable_notebook_logging(dest) log.log(log.getDefaultLogger().getName(), log.INFO, "This is INFO") log.info(u"This is unicode INFO") log.trace("This is TRACE") log.debug("This is DEBUG") log.warn("This is WARN") log.error("This is ERROR") log.fatal("This is FATAL") log_utils.disable_notebook_logging() log.warn("Format %d %g %s", 3, 2.71828, "foo") self.assertEqual( dest.getvalue(), """root INFO: This is INFO root INFO: This is unicode INFO root WARN: This is WARN root ERROR: This is ERROR root FATAL: This is FATAL """, ) self.check( """ root WARN: Format 3 2.71828 foo """ )
def test_readInFits(self): credFile = os.path.expanduser('~/.mysqlAuthLSST') if not os.path.isfile(credFile): log.warn("Required file with credentials '%s' not found.", credFile) return testFile = ("./tests/testData/imsim_886258731_R33_S21_C12_E000.fits.gz") self.assertTrue(isFitsExt('stuf.fits')) self.assertFalse(isFitsExt('thing.txt')) self.assertFalse(isFitsExt('item.tx.gz')) self.assertTrue(isFitsExt(testFile)) self.assertTrue(isFits(testFile)) # Destroy existing tables and re-create them dbDestroyCreate(credFile, "DELETE") # Open a connection to the database. metadataFits = MetadataFitsDb(credFile) # test a specific file self.assertFalse(metadataFits.isFileInDb(testFile)) metadataFits.insertFile(testFile) log.info(metadataFits.showColumnsInTables()) self.assertTrue(metadataFits.isFileInDb(testFile)) # test crawler rootDir = '~/test_md' rootDir = os.path.expanduser(rootDir) if not os.path.exists(rootDir): log.error("Data directory {} is required".format(rootDir)) return directoryCrawl(rootDir, metadataFits)
def getCcdImage(self, ccd, imageFactory=afwImage.ImageF, binSize=1, asMaskedImage=False): """Return an image of the specified ccd, and also the (possibly updated) ccd""" log = lsst.log.Log.getLogger("afw.cameraGeom.utils.ButlerImage") if self.isTrimmed: bbox = ccd.getBBox() else: bbox = calcRawCcdBBox(ccd) im = None if self.butler is not None: err = None for dataId in [dict(detector=ccd.getId()), dict(ccd=ccd.getId()), dict(ccd=ccd.getName())]: try: im = self.butler.get(self.type, dataId, **self.kwargs) except FitsError as e: # no point trying another dataId err = IOError(e.args[0].split('\n')[0]) # It's a very chatty error break except Exception as e: # try a different dataId if err is None: err = e continue else: ccd = im.getDetector() # possibly modified by assembleCcdTask break if im: if asMaskedImage: im = im.getMaskedImage() else: im = im.getMaskedImage().getImage() else: if self.verbose: # Lost by jupyterlab. print(f"Reading {ccd.getId()}: {err}") log.warn(f"Reading {ccd.getId()}: {err}") if im is None: return self._prepareImage(ccd, imageFactory(*bbox.getDimensions()), binSize), ccd if self.type == "raw": if hasattr(im, 'convertF'): im = im.convertF() if False and self.callback is None: # we need to trim the raw image self.callback = rawCallback allowRotate = True if self.callback: try: im = self.callback(im, ccd, imageSource=self) except Exception as e: if self.verbose: log.error(f"callback failed: {e}") im = imageFactory(*bbox.getDimensions()) else: allowRotate = False # the callback was responsible for any rotations return self._prepareImage(ccd, im, binSize, allowRotate=allowRotate), ccd
def testBasic(self): """ Test basic log output with default configuration. Since the default threshold is INFO, the DEBUG or TRACE message is not emitted. """ with TestLog.StdoutCapture(self.outputFilename): log.configure() log.log(log.getDefaultLogger(), log.INFO, "This is INFO") log.info(u"This is unicode INFO") log.trace("This is TRACE") log.debug("This is DEBUG") log.warn("This is WARN") log.error("This is ERROR") log.fatal("This is FATAL") log.critical("This is CRITICAL") log.warning("Format %d %g %s", 3, 2.71828, "foo") self.check(""" root INFO: This is INFO root INFO: This is unicode INFO root WARN: This is WARN root ERROR: This is ERROR root FATAL: This is FATAL root FATAL: This is CRITICAL root WARN: Format 3 2.71828 foo """)
def test_readInFits(self): credFile = os.path.expanduser('~/.mysqlAuthLSST') if not os.path.isfile(credFile): log.warn("Required file with credentials '%s' not found.", credFile) return testFile = ( "./tests/testData/imsim_886258731_R33_S21_C12_E000.fits.gz") self.assertTrue(isFitsExt('stuf.fits')) self.assertFalse(isFitsExt('thing.txt')) self.assertFalse(isFitsExt('item.tx.gz')) self.assertTrue(isFitsExt(testFile)) self.assertTrue(isFits(testFile)) # Destroy existing tables and re-create them dbDestroyCreate(credFile, "DELETE") # Open a connection to the database. metadataFits = MetadataFitsDb(credFile) # test a specific file self.assertFalse(metadataFits.isFileInDb(testFile)) metadataFits.insertFile(testFile) log.info(metadataFits.showColumnsInTables()) self.assertTrue(metadataFits.isFileInDb(testFile)) # test crawler rootDir = '~/test_md' rootDir = os.path.expanduser(rootDir) if not os.path.exists(rootDir): log.error("Data directory {} is required".format(rootDir)) return directoryCrawl(rootDir, metadataFits)
def getCcdImage(self, ccd, imageFactory=afwImage.ImageF, binSize=1, asMaskedImage=False): """Return an image of the specified ccd, and also the (possibly updated) ccd""" log = lsst.log.Log.getLogger("afw.cameraGeom.utils.ButlerImage") if self.isTrimmed: bbox = ccd.getBBox() else: bbox = calcRawCcdBBox(ccd) im = None if self.butler is not None: err = None for dataId in [dict(detector=ccd.getId()), dict(ccd=ccd.getId()), dict(ccd=ccd.getName())]: try: im = self.butler.get(self.type, dataId, **self.kwargs) except FitsError as e: # no point trying another dataId err = IOError(e.args[0].split('\n')[0]) # It's a very chatty error break except Exception as e: # try a different dataId if err is None: err = e continue else: ccd = im.getDetector() # possibly modified by assembleCcdTask break if im: if asMaskedImage: im = im.getMaskedImage() else: im = im.getMaskedImage().getImage() else: if self.verbose: print("Reading %s: %s" % (ccd.getId(), err)) # lost by jupyterLab log.warn("Reading %s: %s", ccd.getId(), err) if im is None: return self._prepareImage(ccd, imageFactory(*bbox.getDimensions()), binSize), ccd if self.type == "raw": if hasattr(im, 'convertF'): im = im.convertF() if False and self.callback is None: # we need to trim the raw image self.callback = rawCallback allowRotate = True if self.callback: try: im = self.callback(im, ccd, imageSource=self) except Exception as e: if self.verbose: log.error("callback failed: %s" % e) im = imageFactory(*bbox.getDimensions()) else: allowRotate = False # the callback was responsible for any rotations return self._prepareImage(ccd, im, binSize, allowRotate=allowRotate), ccd
def getCcdImage(self, ccd, imageFactory=afwImage.ImageF, binSize=1): """Return an image of the specified ccd, and also the (possibly updated) ccd""" log = lsst.log.Log.getLogger("afw.cameraGeom.utils.ButlerImage") if self.isTrimmed: bbox = ccd.getBBox() else: bbox = calcRawCcdBBox(ccd) im = None if self.butler is not None: e = None if self.type == "calexp": # reading the exposure can die if the PSF's unknown try: # need to switch to cid as below. RHL has no calexp to test with fileName = self.butler.get(self.type + "_filename", ccd=ccd.getId(), **self.kwargs)[0] im = imageFactory(fileName) except Exception as e: pass else: im = None for cid in [ccd.getId(), ccd.getName()]: try: im = self.butler.get(self.type, ccd=cid, **self.kwargs) ccd = im.getDetector() # possibly modified by assembleCcdTask e = None break except Exception as e: continue if im: im = im.getMaskedImage().getImage() else: raise e if e: if self.verbose: log.info("Reading %s: %s" % (ccd.getId(), e)) if im is None: return self._prepareImage(ccd, imageFactory(*bbox.getDimensions()), binSize), ccd if self.type == "raw": allowRotate = True # all other images were rotated by the ISR if hasattr(im, 'convertF'): im = im.convertF() if False and self.callback is None: # we need to trim the raw image self.callback = rawCallback if self.callback: try: im = self.callback(im, ccd, imageSource=self) except Exception as e: if self.verbose: log.error("callback failed: %s" % e) im = imageFactory(*bbox.getDimensions()) return self._prepareImage(ccd, im, binSize, allowRotate=allowRotate), ccd
def testBasic(self): """ Test basic log output. Since the default threshold is INFO, the TRACE message is not emitted. """ with TestLog.StdoutCapture(self.outputFilename): log.configure() log.trace("This is TRACE") log.info("This is INFO") log.debug("This is DEBUG") log.warn("This is WARN") log.error("This is ERROR") log.fatal("This is FATAL") log.info("Format %d %g %s", 3, 2.71828, "foo") self.check(""" INFO root null - This is INFO DEBUG root null - This is DEBUG WARN root null - This is WARN ERROR root null - This is ERROR FATAL root null - This is FATAL INFO root null - Format 3 2.71828 foo """)
def testBasic(self): """ Test basic log output with default configuration. Since the default threshold is INFO, the DEBUG or TRACE message is not emitted. """ with TestLog.StdoutCapture(self.outputFilename): log.configure() log.log(log.getDefaultLoggerName(), log.INFO, "This is INFO") log.info(u"This is unicode INFO") log.trace("This is TRACE") log.debug("This is DEBUG") log.warn("This is WARN") log.error("This is ERROR") log.fatal("This is FATAL") log.warn("Format %d %g %s", 3, 2.71828, "foo") self.check(""" root INFO: This is INFO root INFO: This is unicode INFO root WARN: This is WARN root ERROR: This is ERROR root FATAL: This is FATAL root WARN: Format 3 2.71828 foo """)
def makeImageFromCamera(camera, detectorNameList=None, background=numpy.nan, bufferSize=10, imageSource=FakeImageDataSource(), imageFactory=afwImage.ImageU, binSize=1): """Make an Image of a Camera. Put each detector's image in the correct location and orientation on the focal plane. The input images can be binned to an integer fraction of their original bboxes. Parameters ---------- camera : `lsst.afw.cameraGeom.Camera` Camera object to use to make the image. detectorNameList : `list` of `str` List of detector names from `camera` to use in building the image. Use all Detectors if None. background : `float` Value to use where there is no Detector. bufferSize : `int` Size of border in binned pixels to make around the camera image. imageSource : `FakeImageDataSource` or None Source to get ccd images. Must have a ``getCcdImage()`` method. imageFactory : callable like `lsst.afw.image.Image` Type of image to build. binSize : `int` Bin the image by this factor in both dimensions. Returns ------- image : `lsst.afw.image.Image` Image of the entire camera. """ log = lsst.log.Log.getLogger("afw.cameraGeom.utils.makeImageFromCamera") if detectorNameList is None: ccdList = camera else: ccdList = [camera[name] for name in detectorNameList] if detectorNameList is None: camBbox = camera.getFpBBox() else: camBbox = lsst.geom.Box2D() for detName in detectorNameList: for corner in camera[detName].getCorners(FOCAL_PLANE): camBbox.include(corner) pixelSize_o = camera[next(camera.getNameIter())].getPixelSize() camBbox = getCameraImageBBox(camBbox, pixelSize_o, bufferSize * binSize) origin = camBbox.getMin() camIm = imageFactory( int(math.ceil(camBbox.getDimensions().getX() / binSize)), int(math.ceil(camBbox.getDimensions().getY() / binSize))) camIm[:] = imageSource.background assert imageSource.isTrimmed, "isTrimmed is False isn't supported by getCcdInCamBBoxList" boxList = getCcdInCamBBoxList(ccdList, binSize, pixelSize_o, origin) for det, bbox in zip(ccdList, boxList): im = imageSource.getCcdImage(det, imageFactory, binSize)[0] if im is None: continue nQuarter = det.getOrientation().getNQuarter() im = afwMath.rotateImageBy90(im, nQuarter) imView = camIm.Factory(camIm, bbox, afwImage.LOCAL) try: imView[:] = im except pexExceptions.LengthError as e: log.error( "Unable to fit image for detector \"%s\" into image of camera: %s" % (det.getName(), e)) return camIm
def get(api_url, api_endpoint=None, api_user=None, api_password=None, timeout=None, version=None): """GET request to the SQUASH API. Parameters ---------- api_url : `str` Root URL of the SQUASH API. For example, ``'https://squash.lsst.codes/api'``. api_endpoint : `str`, optional Name of the API endpoint to post to. The ``api_url`` is requested if unset. api_user : `str`, optional API username. api_password : `str`, optional API password. timeout : `float`, optional Request timeout. The value of `get_default_timeout` is used by default. version : `str`, optional API version. The value of `get_default_api_version` is used by default. Raises ------ requests.exceptions.RequestException Raised if the HTTP request fails. Returns ------- response : `requests.Response` Response object. Obtain JSON content with ``response.json()``. """ log = lsst.log.Log.getLogger('verify.squash.get') if api_user is not None and api_password is not None: auth = (api_user, api_password) else: auth = None if api_endpoint is not None: api_endpoint_url = get_endpoint_url(api_url, api_endpoint) else: api_endpoint_url = api_url headers = {'Accept': make_accept_header(version)} try: r = requests.get(api_endpoint_url, auth=auth, headers=headers, timeout=timeout or get_default_timeout()) log.info('GET {0} status: {1}'.format(api_endpoint_url, r.status_code)) r.raise_for_status() except requests.exceptions.RequestException as e: log.error(str(e)) raise e return r
def post(api_url, api_endpoint, json_doc=None, timeout=None, version=None, access_token=None): """POST a JSON document to SQUASH. Parameters ---------- api_url : `str` Root URL of the SQUASH API. For example, ``'https://squash.lsst.codes/api'``. api_endpoint : `str` Name of the API endpoint to post to. json_doc : `dict` A JSON-serializable object. timeout : `float`, optional Request timeout. The value of `get_default_timeout` is used by default. version : `str`, optional API version. The value of `get_default_api_version` is used by default. access_token : `str`, optional Access token (see `get_access_token`). Not required when a POST is done to the API authorization endpoint. Raises ------ requests.exceptions.RequestException Raised if the HTTP request fails. Returns ------- response : `requests.Response` Response object. Obtain JSON content with ``response.json()``. """ log = lsst.log.Log.getLogger('verify.squash.post') api_endpoint_url = get_endpoint_url(api_url, api_endpoint) headers = {'Accept': make_accept_header(version)} if access_token: headers['Authorization'] = make_authorization_header(access_token) try: # Disable redirect following for POST as requests will turn a POST into # a GET when following a redirect. http://ls.st/pbx r = requests.post(api_endpoint_url, json=json_doc, allow_redirects=False, headers=headers, timeout=timeout or get_default_timeout()) log.info('POST {0} status: {1}'.format(api_endpoint_url, r.status_code)) r.raise_for_status() # be pedantic about return status. requests#status_code will not error # on 3xx codes if r.status_code != 200 and r.status_code != 201 \ and r.status_code != 202: message = 'Expected status = 200(OK), 201(Created) or 202' \ '(Accepted). Got status={0}. {1}'.format(r.status_code, r.reason) raise requests.exceptions.RequestException(message) except requests.exceptions.RequestException as e: log.error(str(e)) raise e return r
def makeImageFromCamera(camera, detectorNameList=None, background=numpy.nan, bufferSize=10, imageSource=FakeImageDataSource(), imageFactory=afwImage.ImageU, binSize=1): """Make an Image of a Camera. Put each detector's image in the correct location and orientation on the focal plane. The input images can be binned to an integer fraction of their original bboxes. Parameters ---------- camera : `lsst.afw.cameraGeom.Camera` Camera object to use to make the image. detectorNameList : `list` of `str` List of detector names from ``camera`` to use in building the image. Use all Detectors if `None`. background : `float` Value to use where there is no Detector. bufferSize : `int` Size of border in binned pixels to make around the camera image. imageSource : `FakeImageDataSource` or `None` Source to get ccd images. Must have a ``getCcdImage()`` method. imageFactory : callable like `lsst.afw.image.Image` Type of image to build. binSize : `int` Bin the image by this factor in both dimensions. Returns ------- image : `lsst.afw.image.Image` Image of the entire camera. """ log = lsst.log.Log.getLogger("afw.cameraGeom.utils.makeImageFromCamera") if detectorNameList is None: ccdList = camera else: ccdList = [camera[name] for name in detectorNameList] if detectorNameList is None: camBbox = camera.getFpBBox() else: camBbox = lsst.geom.Box2D() for detName in detectorNameList: for corner in camera[detName].getCorners(FOCAL_PLANE): camBbox.include(corner) pixelSize_o = camera[next(camera.getNameIter())].getPixelSize() camBbox = getCameraImageBBox(camBbox, pixelSize_o, bufferSize*binSize) origin = camBbox.getMin() camIm = imageFactory(int(math.ceil(camBbox.getDimensions().getX()/binSize)), int(math.ceil(camBbox.getDimensions().getY()/binSize))) camIm[:] = imageSource.background assert imageSource.isTrimmed, "isTrimmed is False isn't supported by getCcdInCamBBoxList" boxList = getCcdInCamBBoxList(ccdList, binSize, pixelSize_o, origin) for det, bbox in zip(ccdList, boxList): im = imageSource.getCcdImage(det, imageFactory, binSize)[0] if im is None: continue imView = camIm.Factory(camIm, bbox, afwImage.LOCAL) try: imView[:] = im except pexExceptions.LengthError as e: log.error("Unable to fit image for detector \"%s\" into image of camera: %s" % (det.getName(), e)) return camIm