def make_sensor_flat(det_name, wcs, counts_per_iter, niter, rng,
overwrite=True):
"""Pickleable function to use with multiprocessing module."""
global camera_wrapper, phot_params, obs_md
logger = desc.imsim.get_logger('INFO', name=det_name)
visit = obs_md.OpsimMetaData['obshistID']
ccd_id = "R{}_S{}".format(det_name[2:5:2], det_name[8:11:2])
outfile = 'lsst_a_{}_{}_{}.fits'.format(visit, ccd_id, obs_md.bandpass)
if not overwrite and os.path.isfile(outfile):
logger.info("%s exists. Skipping.", outfile)
return
logger.info("running %s", det_name)
gs_det = make_galsim_detector(camera_wrapper, det_name, phot_params, obs_md)
desc.imsim.add_treering_info([gs_det])
wcs.set_keywords(outfile, det_name, obs_md, phot_params)
my_flat = desc.imsim.make_flat(gs_det, counts_per_iter, niter, rng,
logger=logger, wcs=wcs)
exptime = wcs.header['EXPTIME']
with fits.open(wcs.eimage_file) as eimage:
eimage[0].header.update(wcs.header)
eimage[0].data = my_flat.array
raw_image = desc.imsim.ImageSource(eimage[0].data, exptime, ccd_id,
visit=visit)
raw_image.eimage = eimage
raw_image.eimage_data = eimage[0].data
raw_image._read_pointing_info(None)
raw_image.write_fits_file(outfile, image_type='FLAT')
def test_read_tree_rings(self):
"""Check reading of tree_ring_parameters file"""
camera_wrapper = LSSTCameraWrapper()
desc.imsim.read_config()
needed_stuff = desc.imsim.parsePhoSimInstanceFile(self.instcat_file, ())
obs_md = needed_stuff.obs_metadata
phot_params = needed_stuff.phot_params
detector_list = []
for sensor in self.sensors:
detector_list.append(make_galsim_detector(camera_wrapper, sensor, phot_params, obs_md))
gs_interpreter = GalSimInterpreter(detectors=detector_list)
tr_filename = os.path.join(lsstUtils.getPackageDir('imsim'),
'data', 'tree_ring_data',
'tree_ring_parameters_19mar18.txt')
desc.imsim.add_treering_info(gs_interpreter.detectors,
tr_filename=tr_filename)
for i, detector in enumerate(gs_interpreter.detectors):
center = detector.tree_rings.center
shifted_center = (center.x - detector._xCenterPix, center.y - detector._yCenterPix)
self.assertAlmostEqual(shifted_center, self.centers[i], 1)
r_value_test = detector.tree_rings.func(self.rtest)
self.assertAlmostEqual(r_value_test, self.rvalues[i], 6)
def render_checkpoint(instcat, checkpoint_file, create_eimage=False):
"""
Render a checkpoint file as a raw file.
"""
config = desc.imsim.read_config()
RS_digits = [x for x in checkpoint_file.split('-')[-1] if x.isdigit()]
det_name = 'R:{},{} S:{},{}'.format(*RS_digits)
obs_md, phot_params, _ \
= desc.imsim.parsePhoSimInstanceFile(instcat, [det_name], numRows=50,
log_level='DEBUG')
camera_wrapper = LSSTCameraWrapper()
det = make_galsim_detector(camera_wrapper, det_name, phot_params, obs_md)
gs_interpreter = make_gs_interpreter(obs_md, [det], None, None)
gs_interpreter.checkpoint_file = checkpoint_file
gs_interpreter.restore_checkpoint(camera_wrapper, phot_params, obs_md)
desc.imsim.add_cosmic_rays(gs_interpreter, phot_params)
full_well = int(config['ccd']['full_well'])
desc.imsim.apply_channel_bleeding(gs_interpreter, full_well)
visit = obs_md.OpsimMetaData['obshistID']
name_root = f'lsst_a_{visit}'
for name, gs_image in gs_interpreter.detectorImages.items():
raw = desc.imsim.ImageSource.create_from_galsim_image(gs_image)
outfile = '_'.join((name_root, name))
raw.write_fits_file(outfile, compress=True)
if create_eimage:
name_root = f'lsst_e_{visit}'
gs_interpreter.writeImages(nameRoot=name_root)
def _make_gs_interpreters(self, seed, sensor_list, file_id):
"""
Create a separate GalSimInterpreter for each sensor so that
they can be run in parallel and maintain separate checkpoint
files.
Also extract GsObjectLists from gs_obj_dict for only the
sensors in sensor_list so that the memory in the underlying
InstCatTrimmer object in gs_obj_dict can be recovered.
"""
bp_dict = BandpassDict.loadTotalBandpassesFromFiles(bandpassNames=self.obs_md.bandpass)
disable_sky_model = self.config['sky_model']['disable_sky_model']
noise_and_background \
= make_sky_model(self.obs_md, self.phot_params, seed=seed,
apply_sensor_model=self.apply_sensor_model,
disable_sky_model=disable_sky_model)
self.gs_interpreters = dict()
for det in self.camera_wrapper.camera:
det_type = det.getType()
det_name = det.getName()
if sensor_list is not None and det_name not in sensor_list:
continue
if det_type in (DetectorType.WAVEFRONT, DetectorType.GUIDER):
continue
gs_det = make_galsim_detector(self.camera_wrapper, det_name,
self.phot_params, self.obs_md)
self.gs_interpreters[det_name] \
= make_gs_interpreter(self.obs_md, [gs_det], bp_dict,
noise_and_background,
epoch=2000.0, seed=seed,
apply_sensor_model=self.apply_sensor_model,
bf_strength=self.config['ccd']['bf_strength'])
self.gs_interpreters[det_name].sky_bg_per_pixel \
= noise_and_background.sky_counts(det_name)
self.gs_interpreters[det_name].setPSF(PSF=self.psf)
if self.apply_sensor_model:
add_treering_info(self.gs_interpreters[det_name].detectors)
if file_id is not None:
self.gs_interpreters[det_name].checkpoint_file \
= self.checkpoint_file(file_id, det_name)
self.gs_interpreters[det_name].nobj_checkpoint \
= self.config['checkpointing']['nobj']
self.gs_interpreters[det_name]\
.restore_checkpoint(self.camera_wrapper,
self.phot_params,
self.obs_md)
if self.create_centroid_file:
self.gs_interpreters[det_name].centroid_base_name = \
os.path.join(self.outdir,
self.config['persistence']['centroid_prefix'])
def restore_checkpoint(self,
camera_wrapper,
phot_params,
obs_metadata,
epoch=2000.0):
"""
Restore self.detectorImages, self._rng, and self.drawn_objects states
from the checkpoint file.
Parameters
----------
camera_wrapper: lsst.sims.GalSimInterface.GalSimCameraWrapper
An object representing the camera being simulated
phot_params: lsst.sims.photUtils.PhotometricParameters
An object containing the physical parameters representing
the photometric properties of the system
obs_metadata: lsst.sims.utils.ObservationMetaData
Characterizing the pointing of the telescope
epoch: float
Representing the Julian epoch against which RA, Dec are
reckoned (default = 2000)
"""
if (self.checkpoint_file is None
or not os.path.isfile(self.checkpoint_file)):
return
with open(self.checkpoint_file, 'rb') as input_:
image_state = pickle.load(input_)
images = image_state['images']
for key in images:
# Unmangle the detector name.
detname = "R:{},{} S:{},{}".format(*tuple(key[1:3] + key[5:7]))
# Create the galsim.Image from scratch as a blank image and
# set the pixel data from the persisted image data array.
detector = make_galsim_detector(camera_wrapper,
detname,
phot_params,
obs_metadata,
epoch=epoch)
self.detectorImages[key] = self.blankImage(detector=detector)
self.detectorImages[key] += image_state['images'][key]
self._rng = image_state['rng']
self.drawn_objects = image_state['drawn_objects']
def compare_to_imsim(phosim_commands):
bandpass = bandpass_all[int(phosim_commands['filter'].values[0])]
obs_md = ObservationMetaData(
pointingRA=float(phosim_commands['rightascension'].values[0]),
pointingDec=float(phosim_commands['declination'].values[0]),
mjd=float(phosim_commands['mjd'].values[0]),
rotSkyPos=float(phosim_commands['rotskypos'].values[0]),
bandpassName=bandpass,
m5=LSSTdefaults().m5(bandpass),
seeing=float(phosim_commands['seeing'].values[0]))
noise_and_background = ESOSkyModel(obs_md,
addNoise=True,
addBackground=True)
phot_params = PhotometricParameters(
exptime=float(phosim_commands['vistime'].values[0]),
nexp=int(phosim_commands['nsnap'].values[0]),
gain=1,
readnoise=0,
darkcurrent=0,
bandpass=bandpass)
# We are going to check one sensor only
detector_list = [
make_galsim_detector(camera_wrapper, "R:2,2 S:1,1", phot_params,
obs_md)
]
bp_dict = BandpassDict.loadTotalBandpassesFromFiles(
bandpassNames=obs_md.bandpass)
gs_interpreter = GalSimInterpreter(obs_metadata=obs_md,
epoch=2000.0,
detectors=detector_list,
bandpassDict=bp_dict,
noiseWrapper=noise_and_background,
seed=1234)
image = gs_interpreter.blankImage(detector=detector_list[0])
image_2 = noise_and_background.addNoiseAndBackground(
image,
bandpass=obs_md.bandpass,
m5=obs_md.m5,
FWHMeff=obs_md.seeing,
photParams=phot_params,
chipName=detector_list[0].name)
return compute_bkg(image_2.array)
def _initializeGalSimInterpreter(self):
"""
This method creates the GalSimInterpreter (if it is None)
This method reads in all of the data about the camera and pass it into
the GalSimInterpreter.
This method calls _getBandpasses to construct the paths to
the files containing the bandpass data.
"""
if not isinstance(self.camera_wrapper, GalSimCameraWrapper):
raise RuntimeError(
"GalSimCatalog.camera_wrapper must be an instantiation of "
"GalSimCameraWrapper or one of its daughter classes\n"
"It is actually of type %s" % str(type(self.camera_wrapper)))
if self.galSimInterpreter is None:
# This list will contain instantiations of the GalSimDetector class
# (see galSimInterpreter.py), which stores detector information in a way
# that the GalSimInterpreter will understand
detectors = []
for dd in self.camera_wrapper.camera:
if dd.getType() == WAVEFRONT or dd.getType() == GUIDER:
# This package does not yet handle the 90-degree rotation
# in WCS that occurs for wavefront or guide sensors
continue
if self.allowed_chips is None or dd.getName(
) in self.allowed_chips:
detectors.append(
make_galsim_detector(self.camera_wrapper,
dd.getName(),
self.photParams,
self.obs_metadata,
epoch=self.db_obj.epoch))
if not hasattr(self, 'bandpassDict'):
if self.noise_and_background is not None:
if self.obs_metadata.m5 is None:
raise RuntimeError(
'WARNING in GalSimCatalog; you did not specify m5 in your '
'obs_metadata. m5 is required in order to '
'add noise to your images')
for name in self.bandpassNames:
if name not in self.obs_metadata.m5:
raise RuntimeError(
'WARNING in GalSimCatalog; your obs_metadata does not have '
+ 'm5 values for all of your bandpasses \n' +
'bandpass has: %s \n' %
self.bandpassNames.__repr__() + 'm5 has: %s ' %
list(self.obs_metadata.m5.keys()).__repr__())
if self.obs_metadata.seeing is None:
raise RuntimeError(
'WARNING in GalSimCatalog; you did not specify seeing in your '
'obs_metadata. seeing is required in order to add '
'noise to your images')
for name in self.bandpassNames:
if name not in self.obs_metadata.seeing:
raise RuntimeError(
'WARNING in GalSimCatalog; your obs_metadata does not have '
+
'seeing values for all of your bandpasses \n' +
'bandpass has: %s \n' %
self.bandpassNames.__repr__() +
'seeing has: %s ' %
list(self.obs_metadata.seeing.keys()).__repr__(
))
(self.bandpassDict,
hardwareDict) = BandpassDict.loadBandpassesFromFiles(
bandpassNames=self.bandpassNames,
filedir=self.bandpassDir,
bandpassRoot=self.bandpassRoot,
componentList=self.componentList,
atmoTransmission=os.path.join(self.bandpassDir,
self.atmoTransmissionName))
self.galSimInterpreter = GalSimInterpreter(
obs_metadata=self.obs_metadata,
epoch=self.db_obj.epoch,
detectors=detectors,
bandpassDict=self.bandpassDict,
noiseWrapper=self.noise_and_background,
seed=self.seed)
self.galSimInterpreter.setPSF(PSF=self.PSF)
def main():
"""
Drive GalSim to simulate the LSST.
"""
# Setup a parser to take command line arguments
parser = argparse.ArgumentParser()
parser.add_argument('file', help="The instance catalog")
parser.add_argument('-n',
'--numrows',
default=None,
type=int,
help="Read the first numrows of the file.")
parser.add_argument('--outdir',
type=str,
default='fits',
help='Output directory for eimage file')
parser.add_argument(
'--sensor',
type=str,
default=None,
help='Sensor to simulate, e.g., "R:2,2 S:1,1".' +
'If None, then simulate all sensors with sources on them')
parser.add_argument(
'--config_file',
type=str,
default=None,
help="Config file. If None, the default config will be used.")
parser.add_argument('--log_level',
type=str,
choices=['DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL'],
default='INFO',
help='Logging level. Default: "INFO"')
parser.add_argument(
'--psf',
type=str,
default='Kolmogorov',
choices=['DoubleGaussian', 'Kolmogorov'],
help="PSF model to use; either the double Gaussian "
"from LSE=40 (equation 30), or the Kolmogorov convolved "
"with a Gaussian proposed by David Kirkby at the "
"23 March 2017 SSims telecon")
parser.add_argument('--checkpoint_file',
type=str,
default=None,
help='Checkpoint file name.')
parser.add_argument('--nobj_checkpoint',
type=int,
default=1000,
help='# objects to process between checkpoints')
parser.add_argument('--seed',
type=int,
default=267,
help='integer used to seed random number generator')
arguments = parser.parse_args()
config = desc.imsim.read_config(arguments.config_file)
logger = desc.imsim.get_logger(arguments.log_level)
# Get the number of rows to read from the instance file. Use
# default if not specified.
numRows = arguments.numrows
if numRows is not None:
logger.info("Reading %i rows from the instance catalog %s.", numRows,
arguments.file)
else:
logger.info("Reading all rows from the instance catalog %s.",
arguments.file)
camera_wrapper = LSSTCameraWrapper()
catalog_contents = desc.imsim.parsePhoSimInstanceFile(arguments.file,
numRows=numRows)
obs_md = catalog_contents.obs_metadata
phot_params = catalog_contents.phot_params
sources = catalog_contents.sources
gs_object_arr = sources[0]
gs_object_dict = sources[1]
# Sub-divide the source dataframe into stars and galaxies.
if arguments.sensor is not None:
detector_list = [
make_galsim_detector(camera_wrapper, arguments.sensor, phot_params,
obs_md)
]
else:
detector_list = []
for det in camera_wrapper.camera:
det_type = det.getType()
if det_type != WAVEFRONT and det_type != GUIDER:
detector_list.append(
make_galsim_detector(camera_wrapper, det.getName(),
phot_params, obs_md))
# Add noise and sky background
# The simple code using the default lsst-GalSim interface would be:
#
# PhoSimStarCatalog.noise_and_background = ExampleCCDNoise(addNoise=True,
# addBackground=True)
#
# But, we need a more realistic sky model and we need to pass more than
# this basic info to use Peter Y's ESO sky model.
# We must pass obs_metadata, chip information etc...
noise_and_background \
= ESOSkyModel(obs_md, addNoise=True, addBackground=True)
bp_dict = BandpassDict.loadTotalBandpassesFromFiles(
bandpassNames=obs_md.bandpass)
gs_interpreter = GalSimInterpreter(obs_metadata=obs_md,
epoch=2000.0,
detectors=detector_list,
bandpassDict=bp_dict,
noiseWrapper=noise_and_background,
seed=arguments.seed)
gs_interpreter.checkpoint_file = arguments.checkpoint_file
gs_interpreter.nobj_checkpoint = arguments.nobj_checkpoint
gs_interpreter.restore_checkpoint(camera_wrapper, phot_params, obs_md)
# Add a PSF.
if arguments.psf.lower() == "doublegaussian":
# This one is taken from equation 30 of
# www.astro.washington.edu/users/ivezic/Astr511/LSST_SNRdoc.pdf .
#
# Set seeing from self.obs_metadata.
local_PSF = \
SNRdocumentPSF(obs_md.OpsimMetaData['FWHMgeom'])
elif arguments.psf.lower() == "kolmogorov":
# This PSF was presented by David Kirkby at the 23 March 2017
# Survey Simulations Working Group telecon
#
# https://confluence.slac.stanford.edu/pages/viewpage.action?spaceKey=LSSTDESC&title=SSim+2017-03-23
# equation 3 of Krisciunas and Schaefer 1991
airmass = 1.0 / np.sqrt(
1.0 - 0.96 *
(np.sin(0.5 * np.pi - obs_md.OpsimMetaData['altitude']))**2)
local_PSF = \
Kolmogorov_and_Gaussian_PSF(airmass=airmass,
rawSeeing=obs_md.OpsimMetaData['rawSeeing'],
band=obs_md.bandpass)
else:
raise RuntimeError("Do not know what to do with psf model: "
"%s" % arguments.psf)
gs_interpreter.setPSF(PSF=local_PSF)
if arguments.sensor is not None:
gs_objects_to_draw = gs_object_dict[arguments.sensor]
else:
gs_objects_to_draw = gs_object_arr
for gs_obj in gs_objects_to_draw:
if gs_obj.uniqueId in gs_interpreter.drawn_objects:
continue
gs_interpreter.drawObject(gs_obj)
desc.imsim.add_cosmic_rays(gs_interpreter, phot_params)
# Write out the fits files
outdir = arguments.outdir
if not os.path.isdir(outdir):
os.makedirs(outdir)
prefix = config['persistence']['eimage_prefix']
gs_interpreter.writeImages(nameRoot=os.path.join(outdir, prefix) +
str(obs_md.OpsimMetaData['obshistID']))
rng = galsim.UniformDeviate(args.seed)
niter = int(args.counts_total / args.counts_per_iter + 0.5)
counts_per_iter = args.counts_total / niter
logger = desc.imsim.get_logger(args.log_level, name='make_flats')
visit = obs_md.OpsimMetaData['obshistID']
camera_wrapper = LSSTCameraWrapper()
wcs = galsim.FitsWCS(args.wcs_file) if args.wcs_file is not None else None
for det in camera_wrapper.camera:
det_name = det.getName()
if (det.getType() != cameraGeom.SCIENCE
or (args.sensors is not None and det_name not in sensor_list)):
continue
logger.info("processing %s", det_name)
gs_det = make_galsim_detector(camera_wrapper, det_name, phot_params,
obs_md)
desc.imsim.add_treering_info([gs_det])
my_flat = desc.imsim.make_flat(gs_det,
counts_per_iter,
niter,
rng,
logger=logger,
wcs=wcs)
ccd_id = "R{}_S{}".format(det_name[2:5:2], det_name[8:11:2])
prefix = config['persistence']['eimage_prefix']
my_flat.write('{}_{}_{}_{}.fits'.format(prefix, visit, ccd_id,
obs_md.bandpass))
def test_checkpointing(self):
"Test checkpointing of .detectorImages data."
camera = camTestUtils.CameraWrapper().camera
camera_wrapper = GalSimCameraWrapper(camera)
phot_params = PhotometricParameters()
obs_md = ObservationMetaData(pointingRA=23.0,
pointingDec=12.0,
rotSkyPos=13.2,
mjd=59580.0,
bandpassName='r')
detectors = [
make_galsim_detector(camera_wrapper, dd.getName(), phot_params,
obs_md) for dd in camera_wrapper.camera
]
# Create a GalSimInterpreter object and set the checkpoint
# attributes.
gs_interpreter = GalSimInterpreter(detectors=detectors)
gs_interpreter.checkpoint_file = self.cp_file
nobj = 10
gs_interpreter.nobj_checkpoint = nobj
# Set the image data by hand.
key = "R00_S00_r.fits"
detname = "R:0,0 S:0,0"
detector = make_galsim_detector(camera_wrapper, detname, phot_params,
obs_md)
image = gs_interpreter.blankImage(detector=detector)
image += 17
gs_interpreter.detectorImages[key] = image
# Add some drawn objects and check that the checkpoint file is
# written at the right cadence.
for uniqueId in range(1, nobj + 1):
gs_interpreter.drawn_objects.add(uniqueId)
gs_interpreter.write_checkpoint()
if uniqueId < nobj:
self.assertFalse(os.path.isfile(self.cp_file))
else:
self.assertTrue(os.path.isfile(self.cp_file))
# Verify that the checkpointed data has the expected content.
with open(self.cp_file, 'rb') as input_:
cp_data = pickle.load(input_)
self.assertTrue(np.array_equal(cp_data['images'][key], image.array))
# Check the restore_checkpoint function.
new_interpreter = GalSimInterpreter(detectors=detectors)
new_interpreter.checkpoint_file = self.cp_file
new_interpreter.restore_checkpoint(camera_wrapper, phot_params, obs_md)
self.assertEqual(new_interpreter.drawn_objects,
gs_interpreter.drawn_objects)
self.assertEqual(set(new_interpreter.detectorImages.keys()),
set(gs_interpreter.detectorImages.keys()))
for det_name in new_interpreter.detectorImages.keys():
new_img = new_interpreter.detectorImages[det_name]
gs_img = gs_interpreter.detectorImages[det_name]
np.testing.assert_array_equal(new_img.array, gs_img.array)
self.assertEqual(new_img.bounds, gs_img.bounds)
self.assertEqual(new_img.wcs.crpix1, gs_img.wcs.crpix1)
self.assertEqual(new_img.wcs.crpix2, gs_img.wcs.crpix2)
self.assertEqual(new_img.wcs.crval1, gs_img.wcs.crval1)
self.assertEqual(new_img.wcs.crval2, gs_img.wcs.crval2)
self.assertEqual(new_img.wcs.detectorName, gs_img.wcs.detectorName)
for name in new_img.wcs.fitsHeader.names():
self.assertEqual(new_img.wcs.fitsHeader.get(name),
gs_img.wcs.fitsHeader.get(name))
