def testRedir(self):
"""
Test redirection to stream.
"""
with TestRedir.StdoutCapture(self.outputFilename):
log.configure()
dest = io.StringIO()
log_utils.enable_notebook_logging(dest)
log.log(log.getDefaultLogger().getName(), log.INFO, "This is INFO")
log.info(u"This is unicode INFO")
log.trace("This is TRACE")
log.debug("This is DEBUG")
log.warn("This is WARN")
log.error("This is ERROR")
log.fatal("This is FATAL")
log_utils.disable_notebook_logging()
log.warn("Format %d %g %s", 3, 2.71828, "foo")
self.assertEqual(
dest.getvalue(),
"""root INFO: This is INFO
root INFO: This is unicode INFO
root WARN: This is WARN
root ERROR: This is ERROR
root FATAL: This is FATAL
""",
)
self.check(
"""
root WARN: Format 3 2.71828 foo
"""
)
def test_readInFits(self):
credFile = os.path.expanduser('~/.mysqlAuthLSST')
if not os.path.isfile(credFile):
log.warn("Required file with credentials '%s' not found.", credFile)
return
testFile = ("./tests/testData/imsim_886258731_R33_S21_C12_E000.fits.gz")
self.assertTrue(isFitsExt('stuf.fits'))
self.assertFalse(isFitsExt('thing.txt'))
self.assertFalse(isFitsExt('item.tx.gz'))
self.assertTrue(isFitsExt(testFile))
self.assertTrue(isFits(testFile))
# Destroy existing tables and re-create them
dbDestroyCreate(credFile, "DELETE")
# Open a connection to the database.
metadataFits = MetadataFitsDb(credFile)
# test a specific file
self.assertFalse(metadataFits.isFileInDb(testFile))
metadataFits.insertFile(testFile)
log.info(metadataFits.showColumnsInTables())
self.assertTrue(metadataFits.isFileInDb(testFile))
# test crawler
rootDir = '~/test_md'
rootDir = os.path.expanduser(rootDir)
if not os.path.exists(rootDir):
log.error("Data directory {} is required".format(rootDir))
return
directoryCrawl(rootDir, metadataFits)
def getCcdImage(self, ccd, imageFactory=afwImage.ImageF, binSize=1, asMaskedImage=False):
"""Return an image of the specified ccd, and also the (possibly updated) ccd"""
log = lsst.log.Log.getLogger("afw.cameraGeom.utils.ButlerImage")
if self.isTrimmed:
bbox = ccd.getBBox()
else:
bbox = calcRawCcdBBox(ccd)
im = None
if self.butler is not None:
err = None
for dataId in [dict(detector=ccd.getId()), dict(ccd=ccd.getId()), dict(ccd=ccd.getName())]:
try:
im = self.butler.get(self.type, dataId, **self.kwargs)
except FitsError as e: # no point trying another dataId
err = IOError(e.args[0].split('\n')[0]) # It's a very chatty error
break
except Exception as e: # try a different dataId
if err is None:
err = e
continue
else:
ccd = im.getDetector() # possibly modified by assembleCcdTask
break
if im:
if asMaskedImage:
im = im.getMaskedImage()
else:
im = im.getMaskedImage().getImage()
else:
if self.verbose:
# Lost by jupyterlab.
print(f"Reading {ccd.getId()}: {err}")
log.warn(f"Reading {ccd.getId()}: {err}")
if im is None:
return self._prepareImage(ccd, imageFactory(*bbox.getDimensions()), binSize), ccd
if self.type == "raw":
if hasattr(im, 'convertF'):
im = im.convertF()
if False and self.callback is None: # we need to trim the raw image
self.callback = rawCallback
allowRotate = True
if self.callback:
try:
im = self.callback(im, ccd, imageSource=self)
except Exception as e:
if self.verbose:
log.error(f"callback failed: {e}")
im = imageFactory(*bbox.getDimensions())
else:
allowRotate = False # the callback was responsible for any rotations
return self._prepareImage(ccd, im, binSize, allowRotate=allowRotate), ccd
def testBasic(self):
"""
Test basic log output with default configuration.
Since the default threshold is INFO, the DEBUG or TRACE
message is not emitted.
"""
with TestLog.StdoutCapture(self.outputFilename):
log.configure()
log.log(log.getDefaultLogger(), log.INFO, "This is INFO")
log.info(u"This is unicode INFO")
log.trace("This is TRACE")
log.debug("This is DEBUG")
log.warn("This is WARN")
log.error("This is ERROR")
log.fatal("This is FATAL")
log.critical("This is CRITICAL")
log.warning("Format %d %g %s", 3, 2.71828, "foo")
self.check("""
root INFO: This is INFO
root INFO: This is unicode INFO
root WARN: This is WARN
root ERROR: This is ERROR
root FATAL: This is FATAL
root FATAL: This is CRITICAL
root WARN: Format 3 2.71828 foo
""")
def test_readInFits(self):
credFile = os.path.expanduser('~/.mysqlAuthLSST')
if not os.path.isfile(credFile):
log.warn("Required file with credentials '%s' not found.",
credFile)
return
testFile = (
"./tests/testData/imsim_886258731_R33_S21_C12_E000.fits.gz")
self.assertTrue(isFitsExt('stuf.fits'))
self.assertFalse(isFitsExt('thing.txt'))
self.assertFalse(isFitsExt('item.tx.gz'))
self.assertTrue(isFitsExt(testFile))
self.assertTrue(isFits(testFile))
# Destroy existing tables and re-create them
dbDestroyCreate(credFile, "DELETE")
# Open a connection to the database.
metadataFits = MetadataFitsDb(credFile)
# test a specific file
self.assertFalse(metadataFits.isFileInDb(testFile))
metadataFits.insertFile(testFile)
log.info(metadataFits.showColumnsInTables())
self.assertTrue(metadataFits.isFileInDb(testFile))
# test crawler
rootDir = '~/test_md'
rootDir = os.path.expanduser(rootDir)
if not os.path.exists(rootDir):
log.error("Data directory {} is required".format(rootDir))
return
directoryCrawl(rootDir, metadataFits)
def getCcdImage(self, ccd, imageFactory=afwImage.ImageF, binSize=1, asMaskedImage=False):
"""Return an image of the specified ccd, and also the (possibly updated) ccd"""
log = lsst.log.Log.getLogger("afw.cameraGeom.utils.ButlerImage")
if self.isTrimmed:
bbox = ccd.getBBox()
else:
bbox = calcRawCcdBBox(ccd)
im = None
if self.butler is not None:
err = None
for dataId in [dict(detector=ccd.getId()), dict(ccd=ccd.getId()), dict(ccd=ccd.getName())]:
try:
im = self.butler.get(self.type, dataId, **self.kwargs)
except FitsError as e: # no point trying another dataId
err = IOError(e.args[0].split('\n')[0]) # It's a very chatty error
break
except Exception as e: # try a different dataId
if err is None:
err = e
continue
else:
ccd = im.getDetector() # possibly modified by assembleCcdTask
break
if im:
if asMaskedImage:
im = im.getMaskedImage()
else:
im = im.getMaskedImage().getImage()
else:
if self.verbose:
print("Reading %s: %s" % (ccd.getId(), err)) # lost by jupyterLab
log.warn("Reading %s: %s", ccd.getId(), err)
if im is None:
return self._prepareImage(ccd, imageFactory(*bbox.getDimensions()), binSize), ccd
if self.type == "raw":
if hasattr(im, 'convertF'):
im = im.convertF()
if False and self.callback is None: # we need to trim the raw image
self.callback = rawCallback
allowRotate = True
if self.callback:
try:
im = self.callback(im, ccd, imageSource=self)
except Exception as e:
if self.verbose:
log.error("callback failed: %s" % e)
im = imageFactory(*bbox.getDimensions())
else:
allowRotate = False # the callback was responsible for any rotations
return self._prepareImage(ccd, im, binSize, allowRotate=allowRotate), ccd
def getCcdImage(self, ccd, imageFactory=afwImage.ImageF, binSize=1):
"""Return an image of the specified ccd, and also the (possibly updated) ccd"""
log = lsst.log.Log.getLogger("afw.cameraGeom.utils.ButlerImage")
if self.isTrimmed:
bbox = ccd.getBBox()
else:
bbox = calcRawCcdBBox(ccd)
im = None
if self.butler is not None:
e = None
if self.type == "calexp": # reading the exposure can die if the PSF's unknown
try: # need to switch to cid as below. RHL has no calexp to test with
fileName = self.butler.get(self.type + "_filename", ccd=ccd.getId(),
**self.kwargs)[0]
im = imageFactory(fileName)
except Exception as e:
pass
else:
im = None
for cid in [ccd.getId(), ccd.getName()]:
try:
im = self.butler.get(self.type, ccd=cid, **self.kwargs)
ccd = im.getDetector() # possibly modified by assembleCcdTask
e = None
break
except Exception as e:
continue
if im:
im = im.getMaskedImage().getImage()
else:
raise e
if e:
if self.verbose:
log.info("Reading %s: %s" % (ccd.getId(), e))
if im is None:
return self._prepareImage(ccd, imageFactory(*bbox.getDimensions()), binSize), ccd
if self.type == "raw":
allowRotate = True # all other images were rotated by the ISR
if hasattr(im, 'convertF'):
im = im.convertF()
if False and self.callback is None: # we need to trim the raw image
self.callback = rawCallback
if self.callback:
try:
im = self.callback(im, ccd, imageSource=self)
except Exception as e:
if self.verbose:
log.error("callback failed: %s" % e)
im = imageFactory(*bbox.getDimensions())
return self._prepareImage(ccd, im, binSize, allowRotate=allowRotate), ccd
def testBasic(self):
"""
Test basic log output. Since the default threshold is INFO, the
TRACE message is not emitted.
"""
with TestLog.StdoutCapture(self.outputFilename):
log.configure()
log.trace("This is TRACE")
log.info("This is INFO")
log.debug("This is DEBUG")
log.warn("This is WARN")
log.error("This is ERROR")
log.fatal("This is FATAL")
log.info("Format %d %g %s", 3, 2.71828, "foo")
self.check("""
INFO root null - This is INFO
DEBUG root null - This is DEBUG
WARN root null - This is WARN
ERROR root null - This is ERROR
FATAL root null - This is FATAL
INFO root null - Format 3 2.71828 foo
""")
def testBasic(self):
"""
Test basic log output with default configuration.
Since the default threshold is INFO, the DEBUG or TRACE
message is not emitted.
"""
with TestLog.StdoutCapture(self.outputFilename):
log.configure()
log.log(log.getDefaultLoggerName(), log.INFO, "This is INFO")
log.info(u"This is unicode INFO")
log.trace("This is TRACE")
log.debug("This is DEBUG")
log.warn("This is WARN")
log.error("This is ERROR")
log.fatal("This is FATAL")
log.warn("Format %d %g %s", 3, 2.71828, "foo")
self.check("""
root INFO: This is INFO
root INFO: This is unicode INFO
root WARN: This is WARN
root ERROR: This is ERROR
root FATAL: This is FATAL
root WARN: Format 3 2.71828 foo
""")
def makeImageFromCamera(camera,
detectorNameList=None,
background=numpy.nan,
bufferSize=10,
imageSource=FakeImageDataSource(),
imageFactory=afwImage.ImageU,
binSize=1):
"""Make an Image of a Camera.
Put each detector's image in the correct location and orientation on the
focal plane. The input images can be binned to an integer fraction of their
original bboxes.
Parameters
----------
camera : `lsst.afw.cameraGeom.Camera`
Camera object to use to make the image.
detectorNameList : `list` of `str`
List of detector names from `camera` to use in building the image.
Use all Detectors if None.
background : `float`
Value to use where there is no Detector.
bufferSize : `int`
Size of border in binned pixels to make around the camera image.
imageSource : `FakeImageDataSource` or None
Source to get ccd images. Must have a ``getCcdImage()`` method.
imageFactory : callable like `lsst.afw.image.Image`
Type of image to build.
binSize : `int`
Bin the image by this factor in both dimensions.
Returns
-------
image : `lsst.afw.image.Image`
Image of the entire camera.
"""
log = lsst.log.Log.getLogger("afw.cameraGeom.utils.makeImageFromCamera")
if detectorNameList is None:
ccdList = camera
else:
ccdList = [camera[name] for name in detectorNameList]
if detectorNameList is None:
camBbox = camera.getFpBBox()
else:
camBbox = lsst.geom.Box2D()
for detName in detectorNameList:
for corner in camera[detName].getCorners(FOCAL_PLANE):
camBbox.include(corner)
pixelSize_o = camera[next(camera.getNameIter())].getPixelSize()
camBbox = getCameraImageBBox(camBbox, pixelSize_o, bufferSize * binSize)
origin = camBbox.getMin()
camIm = imageFactory(
int(math.ceil(camBbox.getDimensions().getX() / binSize)),
int(math.ceil(camBbox.getDimensions().getY() / binSize)))
camIm[:] = imageSource.background
assert imageSource.isTrimmed, "isTrimmed is False isn't supported by getCcdInCamBBoxList"
boxList = getCcdInCamBBoxList(ccdList, binSize, pixelSize_o, origin)
for det, bbox in zip(ccdList, boxList):
im = imageSource.getCcdImage(det, imageFactory, binSize)[0]
if im is None:
continue
nQuarter = det.getOrientation().getNQuarter()
im = afwMath.rotateImageBy90(im, nQuarter)
imView = camIm.Factory(camIm, bbox, afwImage.LOCAL)
try:
imView[:] = im
except pexExceptions.LengthError as e:
log.error(
"Unable to fit image for detector \"%s\" into image of camera: %s"
% (det.getName(), e))
return camIm
def get(api_url,
api_endpoint=None,
api_user=None,
api_password=None,
timeout=None,
version=None):
"""GET request to the SQUASH API.
Parameters
----------
api_url : `str`
Root URL of the SQUASH API. For example,
``'https://squash.lsst.codes/api'``.
api_endpoint : `str`, optional
Name of the API endpoint to post to. The ``api_url`` is requested if
unset.
api_user : `str`, optional
API username.
api_password : `str`, optional
API password.
timeout : `float`, optional
Request timeout. The value of `get_default_timeout` is used by default.
version : `str`, optional
API version. The value of `get_default_api_version` is used by default.
Raises
------
requests.exceptions.RequestException
Raised if the HTTP request fails.
Returns
-------
response : `requests.Response`
Response object. Obtain JSON content with ``response.json()``.
"""
log = lsst.log.Log.getLogger('verify.squash.get')
if api_user is not None and api_password is not None:
auth = (api_user, api_password)
else:
auth = None
if api_endpoint is not None:
api_endpoint_url = get_endpoint_url(api_url, api_endpoint)
else:
api_endpoint_url = api_url
headers = {'Accept': make_accept_header(version)}
try:
r = requests.get(api_endpoint_url,
auth=auth,
headers=headers,
timeout=timeout or get_default_timeout())
log.info('GET {0} status: {1}'.format(api_endpoint_url, r.status_code))
r.raise_for_status()
except requests.exceptions.RequestException as e:
log.error(str(e))
raise e
return r
def post(api_url,
api_endpoint,
json_doc=None,
timeout=None,
version=None,
access_token=None):
"""POST a JSON document to SQUASH.
Parameters
----------
api_url : `str`
Root URL of the SQUASH API. For example,
``'https://squash.lsst.codes/api'``.
api_endpoint : `str`
Name of the API endpoint to post to.
json_doc : `dict`
A JSON-serializable object.
timeout : `float`, optional
Request timeout. The value of `get_default_timeout` is used by default.
version : `str`, optional
API version. The value of `get_default_api_version` is used by default.
access_token : `str`, optional
Access token (see `get_access_token`). Not required when a POST is done
to the API authorization endpoint.
Raises
------
requests.exceptions.RequestException
Raised if the HTTP request fails.
Returns
-------
response : `requests.Response`
Response object. Obtain JSON content with ``response.json()``.
"""
log = lsst.log.Log.getLogger('verify.squash.post')
api_endpoint_url = get_endpoint_url(api_url, api_endpoint)
headers = {'Accept': make_accept_header(version)}
if access_token:
headers['Authorization'] = make_authorization_header(access_token)
try:
# Disable redirect following for POST as requests will turn a POST into
# a GET when following a redirect. http://ls.st/pbx
r = requests.post(api_endpoint_url,
json=json_doc,
allow_redirects=False,
headers=headers,
timeout=timeout or get_default_timeout())
log.info('POST {0} status: {1}'.format(api_endpoint_url,
r.status_code))
r.raise_for_status()
# be pedantic about return status. requests#status_code will not error
# on 3xx codes
if r.status_code != 200 and r.status_code != 201 \
and r.status_code != 202:
message = 'Expected status = 200(OK), 201(Created) or 202' \
'(Accepted). Got status={0}. {1}'.format(r.status_code,
r.reason)
raise requests.exceptions.RequestException(message)
except requests.exceptions.RequestException as e:
log.error(str(e))
raise e
return r
def makeImageFromCamera(camera, detectorNameList=None, background=numpy.nan, bufferSize=10,
imageSource=FakeImageDataSource(), imageFactory=afwImage.ImageU, binSize=1):
"""Make an Image of a Camera.
Put each detector's image in the correct location and orientation on the
focal plane. The input images can be binned to an integer fraction of their
original bboxes.
Parameters
----------
camera : `lsst.afw.cameraGeom.Camera`
Camera object to use to make the image.
detectorNameList : `list` of `str`
List of detector names from ``camera`` to use in building the image.
Use all Detectors if `None`.
background : `float`
Value to use where there is no Detector.
bufferSize : `int`
Size of border in binned pixels to make around the camera image.
imageSource : `FakeImageDataSource` or `None`
Source to get ccd images. Must have a ``getCcdImage()`` method.
imageFactory : callable like `lsst.afw.image.Image`
Type of image to build.
binSize : `int`
Bin the image by this factor in both dimensions.
Returns
-------
image : `lsst.afw.image.Image`
Image of the entire camera.
"""
log = lsst.log.Log.getLogger("afw.cameraGeom.utils.makeImageFromCamera")
if detectorNameList is None:
ccdList = camera
else:
ccdList = [camera[name] for name in detectorNameList]
if detectorNameList is None:
camBbox = camera.getFpBBox()
else:
camBbox = lsst.geom.Box2D()
for detName in detectorNameList:
for corner in camera[detName].getCorners(FOCAL_PLANE):
camBbox.include(corner)
pixelSize_o = camera[next(camera.getNameIter())].getPixelSize()
camBbox = getCameraImageBBox(camBbox, pixelSize_o, bufferSize*binSize)
origin = camBbox.getMin()
camIm = imageFactory(int(math.ceil(camBbox.getDimensions().getX()/binSize)),
int(math.ceil(camBbox.getDimensions().getY()/binSize)))
camIm[:] = imageSource.background
assert imageSource.isTrimmed, "isTrimmed is False isn't supported by getCcdInCamBBoxList"
boxList = getCcdInCamBBoxList(ccdList, binSize, pixelSize_o, origin)
for det, bbox in zip(ccdList, boxList):
im = imageSource.getCcdImage(det, imageFactory, binSize)[0]
if im is None:
continue
imView = camIm.Factory(camIm, bbox, afwImage.LOCAL)
try:
imView[:] = im
except pexExceptions.LengthError as e:
log.error("Unable to fit image for detector \"%s\" into image of camera: %s" % (det.getName(), e))
return camIm
