def process_one(filename, write=False, quiet=False):
"""Convert one file in-place from Jy (or no units) to nJy fluxes.
Parameters
----------
filename : `str`
The file to convert.
write : `bool`, optional
Write the converted catalog out, overwriting the read in catalog?
quiet : `bool`, optional
Do not print messages about files read/written or fields found?
"""
log = lsst.log.Log()
if quiet:
log.setLevel(lsst.log.WARN)
log.info(f"Reading: {filename}")
catalog = lsst.afw.table.SimpleCatalog.readFits(filename)
output = convertToNanojansky(catalog, log, doConvert=write)
if write:
addRefCatMetadata(output)
output.writeFits(filename)
log.info(f"Wrote: {filename}")
def lsst_camera():
"""
Return a copy of the LSST Camera model as stored in obs_lsstSim.
"""
if not hasattr(lsst_camera, '_lsst_camera'):
lsstLog.setLevel('CameraMapper', lsstLog.WARN)
lsst_camera._lsst_camera = LsstSimMapper().camera
return lsst_camera._lsst_camera
def lsst_camera():
"""
Return a copy of the LSST Camera model as stored in obs_lsstSim.
"""
if not hasattr(lsst_camera, '_lsst_camera'):
lsstLog.setLevel('CameraMapper', lsstLog.WARN)
lsst_camera._lsst_camera = LsstSimMapper().camera
return lsst_camera._lsst_camera
def _configure_logging(level):
lsst.log.configure_prop("""
log4j.rootLogger=INFO, A1
log4j.appender.A1=ConsoleAppender
log4j.appender.A1.Target=System.out
log4j.appender.A1.layout=PatternLayout
log4j.appender.A1.layout.ConversionPattern=%-5p %d{yyyy-MM-ddTHH:mm:ss.SSSZ} %c - %m%n
""")
log = lsst.log.Log.getLogger("convertRepo")
log.setLevel(level)
def get_obs_lsstSim_camera(log_level=lsstLog.WARN):
"""
Get the obs_lsstSim CameraMapper object, setting the default
log-level at WARN in order to silence the INFO message about
"Loading Posix exposure registry from .". Note that this only
affects the 'CameraMapper' logging level. The logging level set
by any calling code (e.g., imsim.py) will still apply to other log
messages made by imSim code.
"""
lsstLog.setLevel('CameraMapper', log_level)
return lsst_camera()
def _main(args=None):
parser = argparse.ArgumentParser(
description="Patch a DESC DC2 gen2 butler repo for conversion to gen3")
parser.add_argument("option_filename",
type=str,
help="file from which to load options")
parser.add_argument(
"-v",
"--verbose",
action="store_const",
dest="verbose",
default=lsst.log.Log.INFO,
const=lsst.log.Log.DEBUG,
help="Set the log level to DEBUG.",
)
commandline_options = parser.parse_args(args)
lsst.log.configure_prop("""
log4j.rootLogger=INFO, A1
log4j.appender.A1=ConsoleAppender
log4j.appender.A1.Target=System.out
log4j.appender.A1.layout=PatternLayout
log4j.appender.A1.layout.ConversionPattern=%-5p %d{yyyy-MM-ddTHH:mm:ss.SSSZ} %c - %m%n
""")
log = lsst.log.Log.getLogger("convertRepo")
log.setLevel(commandline_options.verbose)
with open(commandline_options.option_filename, "r") as options_file:
options = yaml.safe_load(options_file)
origin_root = options["origin_root"]
gen2_root = options["gen2_root"]
rerun_paths = [
os.path.join(gen2_root, r["path"]) for r in options["reruns"]
]
calib_paths = [
os.path.join(gen2_root, c["path"]) for c in options["calibs"]
]
repository_paths = rerun_paths + calib_paths
for repository_path in repository_paths:
filename = os.path.join(repository_path, "repositoryCfg.yaml")
replace_root(filename, origin_root, gen2_root)
calib_root = os.path.join(gen2_root, "CALIB")
for calib_root in calib_paths:
move_files(calib_root)
update_calib_registry(calib_root)
update_registry(gen2_root)
return 0
def testContext(self):
"""Test the log context/component stack."""
with TestLog.StdoutCapture(self.outputFilename):
log.configure()
log.setLevel('', log.DEBUG)
log.trace("This is TRACE")
log.info("This is INFO")
log.debug("This is DEBUG")
with log.LogContext("component"):
log.trace("This is TRACE")
log.info("This is INFO")
log.debug("This is DEBUG")
log.trace("This is TRACE 2")
log.info("This is INFO 2")
log.debug("This is DEBUG 2")
with log.LogContext("comp") as ctx:
log.trace("This is TRACE 3")
log.info("This is INFO 3")
log.debug("This is DEBUG 3")
ctx.setLevel(log.INFO)
self.assertEqual(ctx.getLevel(), log.INFO)
self.assertTrue(ctx.isEnabledFor(log.INFO))
log.trace("This is TRACE 3a")
log.info("This is INFO 3a")
log.debug("This is DEBUG 3a")
with log.LogContext("subcomp", log.TRACE) as ctx2:
self.assertEqual(ctx2.getLevel(), log.TRACE)
log.trace("This is TRACE 4")
log.info("This is INFO 4")
log.debug("This is DEBUG 4")
log.trace("This is TRACE 5")
log.info("This is INFO 5")
log.debug("This is DEBUG 5")
self.check("""
root INFO: This is INFO
root DEBUG: This is DEBUG
component INFO: This is INFO
component DEBUG: This is DEBUG
root INFO: This is INFO 2
root DEBUG: This is DEBUG 2
comp INFO: This is INFO 3
comp DEBUG: This is DEBUG 3
comp INFO: This is INFO 3a
comp.subcomp TRACE: This is TRACE 4
comp.subcomp INFO: This is INFO 4
comp.subcomp DEBUG: This is DEBUG 4
comp INFO: This is INFO 5
""")
def testContext(self):
"""Test the log context/component stack."""
with TestLog.StdoutCapture(self.outputFilename):
log.configure()
log.setLevel('', log.DEBUG)
log.trace("This is TRACE")
log.info("This is INFO")
log.debug("This is DEBUG")
with log.LogContext("component"):
log.trace("This is TRACE")
log.info("This is INFO")
log.debug("This is DEBUG")
log.trace("This is TRACE 2")
log.info("This is INFO 2")
log.debug("This is DEBUG 2")
with log.LogContext("comp") as ctx:
log.trace("This is TRACE 3")
log.info("This is INFO 3")
log.debug("This is DEBUG 3")
ctx.setLevel(log.INFO)
self.assertEqual(ctx.getLevel(), log.INFO)
self.assert_(ctx.isEnabledFor(log.INFO))
log.trace("This is TRACE 3a")
log.info("This is INFO 3a")
log.debug("This is DEBUG 3a")
with log.LogContext("subcomp", log.TRACE) as ctx2:
self.assertEqual(ctx2.getLevel(), log.TRACE)
log.trace("This is TRACE 4")
log.info("This is INFO 4")
log.debug("This is DEBUG 4")
log.trace("This is TRACE 5")
log.info("This is INFO 5")
log.debug("This is DEBUG 5")
self.check("""
root INFO: This is INFO
root DEBUG: This is DEBUG
component INFO: This is INFO
component DEBUG: This is DEBUG
root INFO: This is INFO 2
root DEBUG: This is DEBUG 2
comp INFO: This is INFO 3
comp DEBUG: This is DEBUG 3
comp INFO: This is INFO 3a
comp.subcomp TRACE: This is TRACE 4
comp.subcomp INFO: This is INFO 4
comp.subcomp DEBUG: This is DEBUG 4
comp INFO: This is INFO 5
""")
def resetLog(cls):
"""Uninitialize the butler CLI Log handler and reset component log
levels.
If the lsst.log handler was added to the python root logger's handlers
in `initLog`, it will be removed here.
For each loger level that was set by this class, sets that logger's
level to the value it was before this class set it. For lsst.log, if a
component level was uninitialzed, it will be set to
`Log.defaultLsstLogLevel` because there is no log4cxx api to set a
component back to an uninitialized state.
"""
if cls._lsstLogHandler is not None:
logging.getLogger().removeHandler(cls._lsstLogHandler)
for componentSetting in reversed(cls._componentSettings):
if lsstLog is not None and componentSetting.lsstLogLevel is not None:
lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel)
logger = logging.getLogger(componentSetting.component)
logger.setLevel(componentSetting.pythonLogLevel)
cls._setLogLevel(None, "INFO")
cls._initialized = False
def get_logger(log_level):
"""
Set up standard logging module and set lsst.log to the same log
level.
Parameters
----------
log_level : str
This is converted to logging.<log_level> and set in the logging
config.
"""
# Setup logging output.
logging.basicConfig(format="%(message)s", stream=sys.stdout)
logger = logging.getLogger()
logger.setLevel(eval('logging.' + log_level))
# Set similar logging level for Stack code.
if log_level == "CRITICAL":
log_level = "FATAL"
lsstLog.setLevel(lsstLog.getDefaultLoggerName(),
eval('lsstLog.%s' % log_level))
return logger
matplotlib.rcParams['axes.prop_cycle'] = cycler("color", color_cycle)
from astropy.table import Table as ApTable
import lsst.afw.table as afwTable
import lsst.afw.image as afwImage
from lsst.afw.image import PARENT
from lsst.utils import getPackageDir
import lsst.log as log
import lsst.meas.deblender
from lsst.meas.deblender import proximal, display, sim, baseline
import lsst.meas.deblender.utils as debUtils
from lsst.daf.persistence import Butler
logger = logging.getLogger("lsst.meas.deblender")
logger.setLevel(logging.INFO)
log.setLevel("", log.INFO)
plogger = logging.getLogger("proxmin")
plogger.setLevel(logging.INFO)
dlogger = logging.getLogger("deblender")
dlogger.setLevel(logging.INFO)
def loadCatalogs(filters, path, filename="catalog"):
catalogs = OrderedDict()
for f in filters:
catalogs[f] = afwTable.SourceCatalog.readFits(
os.path.join(path, "{0}_{1}.fits".format(filename, f)))
return catalogs
def getAllFlux(catalogs, filters):
self.assertTrue(isFits(testFile))
# Destroy existing tables and re-create them
dbDestroyCreate(credFile, "DELETE")
# Open a connection to the database.
metadataFits = MetadataFitsDb(credFile)
# test a specific file
self.assertFalse(metadataFits.isFileInDb(testFile))
metadataFits.insertFile(testFile)
log.info(metadataFits.showColumnsInTables())
self.assertTrue(metadataFits.isFileInDb(testFile))
# test crawler
rootDir = '~/test_md'
rootDir = os.path.expanduser(rootDir)
if not os.path.exists(rootDir):
log.error("Data directory {} is required".format(rootDir))
return
directoryCrawl(rootDir, metadataFits)
def main():
global _options
unittest.main()
if __name__ == "__main__":
log.setLevel("", log.INFO)
main()
# Destroy existing tables and re-create them
dbDestroyCreate(credFile, "DELETE")
# Open a connection to the database.
metadataFits = MetadataFitsDb(credFile)
#root = '/lsst3/DC3/data/obs/ImSim/pt1_2/eimage/v886946741-fi/E000'
log.debug(rootDir)
rootDir = os.path.expanduser(rootDir)
log.debug(rootDir)
directoryCrawl(rootDir, metadataFits)
def deleteTestDb():
credFile = "~/.lsst/dbAuth-dbServ.ini"
# Destroy existing tables and re-create them
dbDestroyCreate(credFile, "DELETE")
if __name__ == "__main__":
log.setLevel("", log.DEBUG)
if len(sys.argv) > 1:
if (sys.argv[1] == "-DELETE"):
deleteTestDb()
else:
test(sys.argv[1])
else:
test()
# Destroy existing tables and re-create them
dbDestroyCreate(credFile, "DELETE")
# Open a connection to the database.
metadataFits = MetadataFitsDb(credFile)
#root = '/lsst3/DC3/data/obs/ImSim/pt1_2/eimage/v886946741-fi/E000'
log.debug(rootDir)
rootDir = os.path.expanduser(rootDir)
log.debug(rootDir)
directoryCrawl(rootDir, metadataFits)
def deleteTestDb():
credFile = "~/.lsst/dbAuth-dbServ.ini"
# Destroy existing tables and re-create them
dbDestroyCreate(credFile, "DELETE")
if __name__ == "__main__":
log.setLevel("", log.DEBUG)
if len(sys.argv) > 1:
if (sys.argv[1] == "-DELETE"):
deleteTestDb()
else:
test(sys.argv[1])
else:
test()
parser.add_argument(
"filename",
help=
"Path to YAML file describing external files (usually resources/external.yaml)."
)
parser.add_argument("-v",
"--verbose",
action="store_const",
dest="logLevel",
default=lsst.log.Log.INFO,
const=lsst.log.Log.DEBUG,
help="Set the log level to DEBUG.")
args = parser.parse_args()
log = lsst.log.Log.getLogger("lsst.daf.butler")
log.setLevel(args.logLevel)
# Forward python logging to lsst logger
lgr = logging.getLogger("lsst.daf.butler")
lgr.setLevel(logging.INFO if args.logLevel ==
lsst.log.Log.INFO else logging.DEBUG)
lgr.addHandler(lsst.log.LogHandler())
butler = Butler(args.root, collections=["HSC/calib"])
def rewrite(dataset: FileDataset) -> FileDataset:
# Join the datastore root to the exported path. This should yield
# absolute paths that start with $CI_HSC_GEN2_DIR.
dataset.path = os.path.join(butler.datastore.root.ospath, dataset.path)
# Remove symlinks in the path; this should result in absolute paths
# that start with $TESTDATA_CI_HSC_DIR, because ci_hsc_gen2 always
import os
import lsst.log
from lsst.daf.butler import Butler, DatasetType, FileDataset
from lsst.obs.base.gen2to3 import ConvertRepoTask, RootRepoConverter, \
CalibRepo
from lsst.obs.lsst import LsstCamImSim
log = lsst.log.Log.getLogger('lsst.daf.butler')
log.setLevel(lsst.log.Log.DEBUG)
def makeRawCalibConvertTask(butler: Butler, fresh_start: bool=True):
instrument = LsstCamImSim()
config = ConvertRepoTask.ConfigClass()
instrument.applyConfigOverrides(ConvertRepoTask._DefaultName, config)
config.relatedOnly = True
config.transfer = "symlink"
# config.datasetIncludePatterns = ["flat", "bias", "dark", "fringe", "SKY",
# "raw"]
config.datasetIncludePatterns = ["flat", "bias", "dark", "fringe", "sky"]
config.datasetIgnorePatterns.append("*_camera")
config.fileIgnorePatterns.extend(["*.log", "*.png", "rerun*"])
config.doRegisterInstrument = fresh_start
return ConvertRepoTask(config=config, butler3=butler, instrument=instrument)
def makeRefCatConvertTask(butler: Butler):
instrument = LsstCamImSim()
config = ConvertRepoTask.ConfigClass()
instrument.applyConfigOverrides(ConvertRepoTask._DefaultName, config)
config.refCats = ["cal_ref_cat"]
config.relatedOnly = True
DC2 cutout service
"""
import sys
import io
import base64
import argparse
import json
import numpy as np
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
import lsst.log as lsstLog
lsstLog.setLevel('CameraMapper', lsstLog.FATAL)
import lsst.daf.persistence as dafPersist
import lsst.afw.geom as afwGeom
from astropy.visualization import ZScaleInterval, make_lupton_rgb
zscale = ZScaleInterval()
_DEFAULT_REPO = '/global/projecta/projectdirs/lsst/global/in2p3/Run1.1/output'
_POSSIBLE_FILTERS = 'ugrizy'
class CutoutMaker():
""" CutoutMaker class
"""
def __init__(self, repo=_DEFAULT_REPO, **kwargs):
self._repo = repo
def main():
log.setLevel("", log.DEBUG)
dataCatCfg = DataCatCfg()
c = Crawler(dataCatCfg)
c.start()
