def process_one(filename, write=False, quiet=False): """Convert one file in-place from Jy (or no units) to nJy fluxes. Parameters ---------- filename : `str` The file to convert. write : `bool`, optional Write the converted catalog out, overwriting the read in catalog? quiet : `bool`, optional Do not print messages about files read/written or fields found? """ log = lsst.log.Log() if quiet: log.setLevel(lsst.log.WARN) log.info(f"Reading: {filename}") catalog = lsst.afw.table.SimpleCatalog.readFits(filename) output = convertToNanojansky(catalog, log, doConvert=write) if write: addRefCatMetadata(output) output.writeFits(filename) log.info(f"Wrote: {filename}")
def lsst_camera(): """ Return a copy of the LSST Camera model as stored in obs_lsstSim. """ if not hasattr(lsst_camera, '_lsst_camera'): lsstLog.setLevel('CameraMapper', lsstLog.WARN) lsst_camera._lsst_camera = LsstSimMapper().camera return lsst_camera._lsst_camera
def _configure_logging(level): lsst.log.configure_prop(""" log4j.rootLogger=INFO, A1 log4j.appender.A1=ConsoleAppender log4j.appender.A1.Target=System.out log4j.appender.A1.layout=PatternLayout log4j.appender.A1.layout.ConversionPattern=%-5p %d{yyyy-MM-ddTHH:mm:ss.SSSZ} %c - %m%n """) log = lsst.log.Log.getLogger("convertRepo") log.setLevel(level)
def get_obs_lsstSim_camera(log_level=lsstLog.WARN): """ Get the obs_lsstSim CameraMapper object, setting the default log-level at WARN in order to silence the INFO message about "Loading Posix exposure registry from .". Note that this only affects the 'CameraMapper' logging level. The logging level set by any calling code (e.g., imsim.py) will still apply to other log messages made by imSim code. """ lsstLog.setLevel('CameraMapper', log_level) return lsst_camera()
def _main(args=None): parser = argparse.ArgumentParser( description="Patch a DESC DC2 gen2 butler repo for conversion to gen3") parser.add_argument("option_filename", type=str, help="file from which to load options") parser.add_argument( "-v", "--verbose", action="store_const", dest="verbose", default=lsst.log.Log.INFO, const=lsst.log.Log.DEBUG, help="Set the log level to DEBUG.", ) commandline_options = parser.parse_args(args) lsst.log.configure_prop(""" log4j.rootLogger=INFO, A1 log4j.appender.A1=ConsoleAppender log4j.appender.A1.Target=System.out log4j.appender.A1.layout=PatternLayout log4j.appender.A1.layout.ConversionPattern=%-5p %d{yyyy-MM-ddTHH:mm:ss.SSSZ} %c - %m%n """) log = lsst.log.Log.getLogger("convertRepo") log.setLevel(commandline_options.verbose) with open(commandline_options.option_filename, "r") as options_file: options = yaml.safe_load(options_file) origin_root = options["origin_root"] gen2_root = options["gen2_root"] rerun_paths = [ os.path.join(gen2_root, r["path"]) for r in options["reruns"] ] calib_paths = [ os.path.join(gen2_root, c["path"]) for c in options["calibs"] ] repository_paths = rerun_paths + calib_paths for repository_path in repository_paths: filename = os.path.join(repository_path, "repositoryCfg.yaml") replace_root(filename, origin_root, gen2_root) calib_root = os.path.join(gen2_root, "CALIB") for calib_root in calib_paths: move_files(calib_root) update_calib_registry(calib_root) update_registry(gen2_root) return 0
def testContext(self): """Test the log context/component stack.""" with TestLog.StdoutCapture(self.outputFilename): log.configure() log.setLevel('', log.DEBUG) log.trace("This is TRACE") log.info("This is INFO") log.debug("This is DEBUG") with log.LogContext("component"): log.trace("This is TRACE") log.info("This is INFO") log.debug("This is DEBUG") log.trace("This is TRACE 2") log.info("This is INFO 2") log.debug("This is DEBUG 2") with log.LogContext("comp") as ctx: log.trace("This is TRACE 3") log.info("This is INFO 3") log.debug("This is DEBUG 3") ctx.setLevel(log.INFO) self.assertEqual(ctx.getLevel(), log.INFO) self.assertTrue(ctx.isEnabledFor(log.INFO)) log.trace("This is TRACE 3a") log.info("This is INFO 3a") log.debug("This is DEBUG 3a") with log.LogContext("subcomp", log.TRACE) as ctx2: self.assertEqual(ctx2.getLevel(), log.TRACE) log.trace("This is TRACE 4") log.info("This is INFO 4") log.debug("This is DEBUG 4") log.trace("This is TRACE 5") log.info("This is INFO 5") log.debug("This is DEBUG 5") self.check(""" root INFO: This is INFO root DEBUG: This is DEBUG component INFO: This is INFO component DEBUG: This is DEBUG root INFO: This is INFO 2 root DEBUG: This is DEBUG 2 comp INFO: This is INFO 3 comp DEBUG: This is DEBUG 3 comp INFO: This is INFO 3a comp.subcomp TRACE: This is TRACE 4 comp.subcomp INFO: This is INFO 4 comp.subcomp DEBUG: This is DEBUG 4 comp INFO: This is INFO 5 """)
def testContext(self): """Test the log context/component stack.""" with TestLog.StdoutCapture(self.outputFilename): log.configure() log.setLevel('', log.DEBUG) log.trace("This is TRACE") log.info("This is INFO") log.debug("This is DEBUG") with log.LogContext("component"): log.trace("This is TRACE") log.info("This is INFO") log.debug("This is DEBUG") log.trace("This is TRACE 2") log.info("This is INFO 2") log.debug("This is DEBUG 2") with log.LogContext("comp") as ctx: log.trace("This is TRACE 3") log.info("This is INFO 3") log.debug("This is DEBUG 3") ctx.setLevel(log.INFO) self.assertEqual(ctx.getLevel(), log.INFO) self.assert_(ctx.isEnabledFor(log.INFO)) log.trace("This is TRACE 3a") log.info("This is INFO 3a") log.debug("This is DEBUG 3a") with log.LogContext("subcomp", log.TRACE) as ctx2: self.assertEqual(ctx2.getLevel(), log.TRACE) log.trace("This is TRACE 4") log.info("This is INFO 4") log.debug("This is DEBUG 4") log.trace("This is TRACE 5") log.info("This is INFO 5") log.debug("This is DEBUG 5") self.check(""" root INFO: This is INFO root DEBUG: This is DEBUG component INFO: This is INFO component DEBUG: This is DEBUG root INFO: This is INFO 2 root DEBUG: This is DEBUG 2 comp INFO: This is INFO 3 comp DEBUG: This is DEBUG 3 comp INFO: This is INFO 3a comp.subcomp TRACE: This is TRACE 4 comp.subcomp INFO: This is INFO 4 comp.subcomp DEBUG: This is DEBUG 4 comp INFO: This is INFO 5 """)
def resetLog(cls): """Uninitialize the butler CLI Log handler and reset component log levels. If the lsst.log handler was added to the python root logger's handlers in `initLog`, it will be removed here. For each loger level that was set by this class, sets that logger's level to the value it was before this class set it. For lsst.log, if a component level was uninitialzed, it will be set to `Log.defaultLsstLogLevel` because there is no log4cxx api to set a component back to an uninitialized state. """ if cls._lsstLogHandler is not None: logging.getLogger().removeHandler(cls._lsstLogHandler) for componentSetting in reversed(cls._componentSettings): if lsstLog is not None and componentSetting.lsstLogLevel is not None: lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel) logger = logging.getLogger(componentSetting.component) logger.setLevel(componentSetting.pythonLogLevel) cls._setLogLevel(None, "INFO") cls._initialized = False
def get_logger(log_level): """ Set up standard logging module and set lsst.log to the same log level. Parameters ---------- log_level : str This is converted to logging.<log_level> and set in the logging config. """ # Setup logging output. logging.basicConfig(format="%(message)s", stream=sys.stdout) logger = logging.getLogger() logger.setLevel(eval('logging.' + log_level)) # Set similar logging level for Stack code. if log_level == "CRITICAL": log_level = "FATAL" lsstLog.setLevel(lsstLog.getDefaultLoggerName(), eval('lsstLog.%s' % log_level)) return logger
matplotlib.rcParams['axes.prop_cycle'] = cycler("color", color_cycle) from astropy.table import Table as ApTable import lsst.afw.table as afwTable import lsst.afw.image as afwImage from lsst.afw.image import PARENT from lsst.utils import getPackageDir import lsst.log as log import lsst.meas.deblender from lsst.meas.deblender import proximal, display, sim, baseline import lsst.meas.deblender.utils as debUtils from lsst.daf.persistence import Butler logger = logging.getLogger("lsst.meas.deblender") logger.setLevel(logging.INFO) log.setLevel("", log.INFO) plogger = logging.getLogger("proxmin") plogger.setLevel(logging.INFO) dlogger = logging.getLogger("deblender") dlogger.setLevel(logging.INFO) def loadCatalogs(filters, path, filename="catalog"): catalogs = OrderedDict() for f in filters: catalogs[f] = afwTable.SourceCatalog.readFits( os.path.join(path, "{0}_{1}.fits".format(filename, f))) return catalogs def getAllFlux(catalogs, filters):
self.assertTrue(isFits(testFile)) # Destroy existing tables and re-create them dbDestroyCreate(credFile, "DELETE") # Open a connection to the database. metadataFits = MetadataFitsDb(credFile) # test a specific file self.assertFalse(metadataFits.isFileInDb(testFile)) metadataFits.insertFile(testFile) log.info(metadataFits.showColumnsInTables()) self.assertTrue(metadataFits.isFileInDb(testFile)) # test crawler rootDir = '~/test_md' rootDir = os.path.expanduser(rootDir) if not os.path.exists(rootDir): log.error("Data directory {} is required".format(rootDir)) return directoryCrawl(rootDir, metadataFits) def main(): global _options unittest.main() if __name__ == "__main__": log.setLevel("", log.INFO) main()
# Destroy existing tables and re-create them dbDestroyCreate(credFile, "DELETE") # Open a connection to the database. metadataFits = MetadataFitsDb(credFile) #root = '/lsst3/DC3/data/obs/ImSim/pt1_2/eimage/v886946741-fi/E000' log.debug(rootDir) rootDir = os.path.expanduser(rootDir) log.debug(rootDir) directoryCrawl(rootDir, metadataFits) def deleteTestDb(): credFile = "~/.lsst/dbAuth-dbServ.ini" # Destroy existing tables and re-create them dbDestroyCreate(credFile, "DELETE") if __name__ == "__main__": log.setLevel("", log.DEBUG) if len(sys.argv) > 1: if (sys.argv[1] == "-DELETE"): deleteTestDb() else: test(sys.argv[1]) else: test()
parser.add_argument( "filename", help= "Path to YAML file describing external files (usually resources/external.yaml)." ) parser.add_argument("-v", "--verbose", action="store_const", dest="logLevel", default=lsst.log.Log.INFO, const=lsst.log.Log.DEBUG, help="Set the log level to DEBUG.") args = parser.parse_args() log = lsst.log.Log.getLogger("lsst.daf.butler") log.setLevel(args.logLevel) # Forward python logging to lsst logger lgr = logging.getLogger("lsst.daf.butler") lgr.setLevel(logging.INFO if args.logLevel == lsst.log.Log.INFO else logging.DEBUG) lgr.addHandler(lsst.log.LogHandler()) butler = Butler(args.root, collections=["HSC/calib"]) def rewrite(dataset: FileDataset) -> FileDataset: # Join the datastore root to the exported path. This should yield # absolute paths that start with $CI_HSC_GEN2_DIR. dataset.path = os.path.join(butler.datastore.root.ospath, dataset.path) # Remove symlinks in the path; this should result in absolute paths # that start with $TESTDATA_CI_HSC_DIR, because ci_hsc_gen2 always
import os import lsst.log from lsst.daf.butler import Butler, DatasetType, FileDataset from lsst.obs.base.gen2to3 import ConvertRepoTask, RootRepoConverter, \ CalibRepo from lsst.obs.lsst import LsstCamImSim log = lsst.log.Log.getLogger('lsst.daf.butler') log.setLevel(lsst.log.Log.DEBUG) def makeRawCalibConvertTask(butler: Butler, fresh_start: bool=True): instrument = LsstCamImSim() config = ConvertRepoTask.ConfigClass() instrument.applyConfigOverrides(ConvertRepoTask._DefaultName, config) config.relatedOnly = True config.transfer = "symlink" # config.datasetIncludePatterns = ["flat", "bias", "dark", "fringe", "SKY", # "raw"] config.datasetIncludePatterns = ["flat", "bias", "dark", "fringe", "sky"] config.datasetIgnorePatterns.append("*_camera") config.fileIgnorePatterns.extend(["*.log", "*.png", "rerun*"]) config.doRegisterInstrument = fresh_start return ConvertRepoTask(config=config, butler3=butler, instrument=instrument) def makeRefCatConvertTask(butler: Butler): instrument = LsstCamImSim() config = ConvertRepoTask.ConfigClass() instrument.applyConfigOverrides(ConvertRepoTask._DefaultName, config) config.refCats = ["cal_ref_cat"] config.relatedOnly = True
DC2 cutout service """ import sys import io import base64 import argparse import json import numpy as np import matplotlib matplotlib.use('agg') import matplotlib.pyplot as plt import lsst.log as lsstLog lsstLog.setLevel('CameraMapper', lsstLog.FATAL) import lsst.daf.persistence as dafPersist import lsst.afw.geom as afwGeom from astropy.visualization import ZScaleInterval, make_lupton_rgb zscale = ZScaleInterval() _DEFAULT_REPO = '/global/projecta/projectdirs/lsst/global/in2p3/Run1.1/output' _POSSIBLE_FILTERS = 'ugrizy' class CutoutMaker(): """ CutoutMaker class """ def __init__(self, repo=_DEFAULT_REPO, **kwargs): self._repo = repo
def main(): log.setLevel("", log.DEBUG) dataCatCfg = DataCatCfg() c = Crawler(dataCatCfg) c.start()