def make_sensor_flat(det_name, wcs, counts_per_iter, niter, rng, overwrite=True): """Pickleable function to use with multiprocessing module.""" global camera_wrapper, phot_params, obs_md logger = desc.imsim.get_logger('INFO', name=det_name) visit = obs_md.OpsimMetaData['obshistID'] ccd_id = "R{}_S{}".format(det_name[2:5:2], det_name[8:11:2]) outfile = 'lsst_a_{}_{}_{}.fits'.format(visit, ccd_id, obs_md.bandpass) if not overwrite and os.path.isfile(outfile): logger.info("%s exists. Skipping.", outfile) return logger.info("running %s", det_name) gs_det = make_galsim_detector(camera_wrapper, det_name, phot_params, obs_md) desc.imsim.add_treering_info([gs_det]) wcs.set_keywords(outfile, det_name, obs_md, phot_params) my_flat = desc.imsim.make_flat(gs_det, counts_per_iter, niter, rng, logger=logger, wcs=wcs) exptime = wcs.header['EXPTIME'] with fits.open(wcs.eimage_file) as eimage: eimage[0].header.update(wcs.header) eimage[0].data = my_flat.array raw_image = desc.imsim.ImageSource(eimage[0].data, exptime, ccd_id, visit=visit) raw_image.eimage = eimage raw_image.eimage_data = eimage[0].data raw_image._read_pointing_info(None) raw_image.write_fits_file(outfile, image_type='FLAT')
def test_read_tree_rings(self): """Check reading of tree_ring_parameters file""" camera_wrapper = LSSTCameraWrapper() desc.imsim.read_config() needed_stuff = desc.imsim.parsePhoSimInstanceFile(self.instcat_file, ()) obs_md = needed_stuff.obs_metadata phot_params = needed_stuff.phot_params detector_list = [] for sensor in self.sensors: detector_list.append(make_galsim_detector(camera_wrapper, sensor, phot_params, obs_md)) gs_interpreter = GalSimInterpreter(detectors=detector_list) tr_filename = os.path.join(lsstUtils.getPackageDir('imsim'), 'data', 'tree_ring_data', 'tree_ring_parameters_19mar18.txt') desc.imsim.add_treering_info(gs_interpreter.detectors, tr_filename=tr_filename) for i, detector in enumerate(gs_interpreter.detectors): center = detector.tree_rings.center shifted_center = (center.x - detector._xCenterPix, center.y - detector._yCenterPix) self.assertAlmostEqual(shifted_center, self.centers[i], 1) r_value_test = detector.tree_rings.func(self.rtest) self.assertAlmostEqual(r_value_test, self.rvalues[i], 6)
def render_checkpoint(instcat, checkpoint_file, create_eimage=False): """ Render a checkpoint file as a raw file. """ config = desc.imsim.read_config() RS_digits = [x for x in checkpoint_file.split('-')[-1] if x.isdigit()] det_name = 'R:{},{} S:{},{}'.format(*RS_digits) obs_md, phot_params, _ \ = desc.imsim.parsePhoSimInstanceFile(instcat, [det_name], numRows=50, log_level='DEBUG') camera_wrapper = LSSTCameraWrapper() det = make_galsim_detector(camera_wrapper, det_name, phot_params, obs_md) gs_interpreter = make_gs_interpreter(obs_md, [det], None, None) gs_interpreter.checkpoint_file = checkpoint_file gs_interpreter.restore_checkpoint(camera_wrapper, phot_params, obs_md) desc.imsim.add_cosmic_rays(gs_interpreter, phot_params) full_well = int(config['ccd']['full_well']) desc.imsim.apply_channel_bleeding(gs_interpreter, full_well) visit = obs_md.OpsimMetaData['obshistID'] name_root = f'lsst_a_{visit}' for name, gs_image in gs_interpreter.detectorImages.items(): raw = desc.imsim.ImageSource.create_from_galsim_image(gs_image) outfile = '_'.join((name_root, name)) raw.write_fits_file(outfile, compress=True) if create_eimage: name_root = f'lsst_e_{visit}' gs_interpreter.writeImages(nameRoot=name_root)
def _make_gs_interpreters(self, seed, sensor_list, file_id): """ Create a separate GalSimInterpreter for each sensor so that they can be run in parallel and maintain separate checkpoint files. Also extract GsObjectLists from gs_obj_dict for only the sensors in sensor_list so that the memory in the underlying InstCatTrimmer object in gs_obj_dict can be recovered. """ bp_dict = BandpassDict.loadTotalBandpassesFromFiles(bandpassNames=self.obs_md.bandpass) disable_sky_model = self.config['sky_model']['disable_sky_model'] noise_and_background \ = make_sky_model(self.obs_md, self.phot_params, seed=seed, apply_sensor_model=self.apply_sensor_model, disable_sky_model=disable_sky_model) self.gs_interpreters = dict() for det in self.camera_wrapper.camera: det_type = det.getType() det_name = det.getName() if sensor_list is not None and det_name not in sensor_list: continue if det_type in (DetectorType.WAVEFRONT, DetectorType.GUIDER): continue gs_det = make_galsim_detector(self.camera_wrapper, det_name, self.phot_params, self.obs_md) self.gs_interpreters[det_name] \ = make_gs_interpreter(self.obs_md, [gs_det], bp_dict, noise_and_background, epoch=2000.0, seed=seed, apply_sensor_model=self.apply_sensor_model, bf_strength=self.config['ccd']['bf_strength']) self.gs_interpreters[det_name].sky_bg_per_pixel \ = noise_and_background.sky_counts(det_name) self.gs_interpreters[det_name].setPSF(PSF=self.psf) if self.apply_sensor_model: add_treering_info(self.gs_interpreters[det_name].detectors) if file_id is not None: self.gs_interpreters[det_name].checkpoint_file \ = self.checkpoint_file(file_id, det_name) self.gs_interpreters[det_name].nobj_checkpoint \ = self.config['checkpointing']['nobj'] self.gs_interpreters[det_name]\ .restore_checkpoint(self.camera_wrapper, self.phot_params, self.obs_md) if self.create_centroid_file: self.gs_interpreters[det_name].centroid_base_name = \ os.path.join(self.outdir, self.config['persistence']['centroid_prefix'])
def restore_checkpoint(self, camera_wrapper, phot_params, obs_metadata, epoch=2000.0): """ Restore self.detectorImages, self._rng, and self.drawn_objects states from the checkpoint file. Parameters ---------- camera_wrapper: lsst.sims.GalSimInterface.GalSimCameraWrapper An object representing the camera being simulated phot_params: lsst.sims.photUtils.PhotometricParameters An object containing the physical parameters representing the photometric properties of the system obs_metadata: lsst.sims.utils.ObservationMetaData Characterizing the pointing of the telescope epoch: float Representing the Julian epoch against which RA, Dec are reckoned (default = 2000) """ if (self.checkpoint_file is None or not os.path.isfile(self.checkpoint_file)): return with open(self.checkpoint_file, 'rb') as input_: image_state = pickle.load(input_) images = image_state['images'] for key in images: # Unmangle the detector name. detname = "R:{},{} S:{},{}".format(*tuple(key[1:3] + key[5:7])) # Create the galsim.Image from scratch as a blank image and # set the pixel data from the persisted image data array. detector = make_galsim_detector(camera_wrapper, detname, phot_params, obs_metadata, epoch=epoch) self.detectorImages[key] = self.blankImage(detector=detector) self.detectorImages[key] += image_state['images'][key] self._rng = image_state['rng'] self.drawn_objects = image_state['drawn_objects']
def compare_to_imsim(phosim_commands): bandpass = bandpass_all[int(phosim_commands['filter'].values[0])] obs_md = ObservationMetaData( pointingRA=float(phosim_commands['rightascension'].values[0]), pointingDec=float(phosim_commands['declination'].values[0]), mjd=float(phosim_commands['mjd'].values[0]), rotSkyPos=float(phosim_commands['rotskypos'].values[0]), bandpassName=bandpass, m5=LSSTdefaults().m5(bandpass), seeing=float(phosim_commands['seeing'].values[0])) noise_and_background = ESOSkyModel(obs_md, addNoise=True, addBackground=True) phot_params = PhotometricParameters( exptime=float(phosim_commands['vistime'].values[0]), nexp=int(phosim_commands['nsnap'].values[0]), gain=1, readnoise=0, darkcurrent=0, bandpass=bandpass) # We are going to check one sensor only detector_list = [ make_galsim_detector(camera_wrapper, "R:2,2 S:1,1", phot_params, obs_md) ] bp_dict = BandpassDict.loadTotalBandpassesFromFiles( bandpassNames=obs_md.bandpass) gs_interpreter = GalSimInterpreter(obs_metadata=obs_md, epoch=2000.0, detectors=detector_list, bandpassDict=bp_dict, noiseWrapper=noise_and_background, seed=1234) image = gs_interpreter.blankImage(detector=detector_list[0]) image_2 = noise_and_background.addNoiseAndBackground( image, bandpass=obs_md.bandpass, m5=obs_md.m5, FWHMeff=obs_md.seeing, photParams=phot_params, chipName=detector_list[0].name) return compute_bkg(image_2.array)
def _initializeGalSimInterpreter(self): """ This method creates the GalSimInterpreter (if it is None) This method reads in all of the data about the camera and pass it into the GalSimInterpreter. This method calls _getBandpasses to construct the paths to the files containing the bandpass data. """ if not isinstance(self.camera_wrapper, GalSimCameraWrapper): raise RuntimeError( "GalSimCatalog.camera_wrapper must be an instantiation of " "GalSimCameraWrapper or one of its daughter classes\n" "It is actually of type %s" % str(type(self.camera_wrapper))) if self.galSimInterpreter is None: # This list will contain instantiations of the GalSimDetector class # (see galSimInterpreter.py), which stores detector information in a way # that the GalSimInterpreter will understand detectors = [] for dd in self.camera_wrapper.camera: if dd.getType() == WAVEFRONT or dd.getType() == GUIDER: # This package does not yet handle the 90-degree rotation # in WCS that occurs for wavefront or guide sensors continue if self.allowed_chips is None or dd.getName( ) in self.allowed_chips: detectors.append( make_galsim_detector(self.camera_wrapper, dd.getName(), self.photParams, self.obs_metadata, epoch=self.db_obj.epoch)) if not hasattr(self, 'bandpassDict'): if self.noise_and_background is not None: if self.obs_metadata.m5 is None: raise RuntimeError( 'WARNING in GalSimCatalog; you did not specify m5 in your ' 'obs_metadata. m5 is required in order to ' 'add noise to your images') for name in self.bandpassNames: if name not in self.obs_metadata.m5: raise RuntimeError( 'WARNING in GalSimCatalog; your obs_metadata does not have ' + 'm5 values for all of your bandpasses \n' + 'bandpass has: %s \n' % self.bandpassNames.__repr__() + 'm5 has: %s ' % list(self.obs_metadata.m5.keys()).__repr__()) if self.obs_metadata.seeing is None: raise RuntimeError( 'WARNING in GalSimCatalog; you did not specify seeing in your ' 'obs_metadata. seeing is required in order to add ' 'noise to your images') for name in self.bandpassNames: if name not in self.obs_metadata.seeing: raise RuntimeError( 'WARNING in GalSimCatalog; your obs_metadata does not have ' + 'seeing values for all of your bandpasses \n' + 'bandpass has: %s \n' % self.bandpassNames.__repr__() + 'seeing has: %s ' % list(self.obs_metadata.seeing.keys()).__repr__( )) (self.bandpassDict, hardwareDict) = BandpassDict.loadBandpassesFromFiles( bandpassNames=self.bandpassNames, filedir=self.bandpassDir, bandpassRoot=self.bandpassRoot, componentList=self.componentList, atmoTransmission=os.path.join(self.bandpassDir, self.atmoTransmissionName)) self.galSimInterpreter = GalSimInterpreter( obs_metadata=self.obs_metadata, epoch=self.db_obj.epoch, detectors=detectors, bandpassDict=self.bandpassDict, noiseWrapper=self.noise_and_background, seed=self.seed) self.galSimInterpreter.setPSF(PSF=self.PSF)
def main(): """ Drive GalSim to simulate the LSST. """ # Setup a parser to take command line arguments parser = argparse.ArgumentParser() parser.add_argument('file', help="The instance catalog") parser.add_argument('-n', '--numrows', default=None, type=int, help="Read the first numrows of the file.") parser.add_argument('--outdir', type=str, default='fits', help='Output directory for eimage file') parser.add_argument( '--sensor', type=str, default=None, help='Sensor to simulate, e.g., "R:2,2 S:1,1".' + 'If None, then simulate all sensors with sources on them') parser.add_argument( '--config_file', type=str, default=None, help="Config file. If None, the default config will be used.") parser.add_argument('--log_level', type=str, choices=['DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL'], default='INFO', help='Logging level. Default: "INFO"') parser.add_argument( '--psf', type=str, default='Kolmogorov', choices=['DoubleGaussian', 'Kolmogorov'], help="PSF model to use; either the double Gaussian " "from LSE=40 (equation 30), or the Kolmogorov convolved " "with a Gaussian proposed by David Kirkby at the " "23 March 2017 SSims telecon") parser.add_argument('--checkpoint_file', type=str, default=None, help='Checkpoint file name.') parser.add_argument('--nobj_checkpoint', type=int, default=1000, help='# objects to process between checkpoints') parser.add_argument('--seed', type=int, default=267, help='integer used to seed random number generator') arguments = parser.parse_args() config = desc.imsim.read_config(arguments.config_file) logger = desc.imsim.get_logger(arguments.log_level) # Get the number of rows to read from the instance file. Use # default if not specified. numRows = arguments.numrows if numRows is not None: logger.info("Reading %i rows from the instance catalog %s.", numRows, arguments.file) else: logger.info("Reading all rows from the instance catalog %s.", arguments.file) camera_wrapper = LSSTCameraWrapper() catalog_contents = desc.imsim.parsePhoSimInstanceFile(arguments.file, numRows=numRows) obs_md = catalog_contents.obs_metadata phot_params = catalog_contents.phot_params sources = catalog_contents.sources gs_object_arr = sources[0] gs_object_dict = sources[1] # Sub-divide the source dataframe into stars and galaxies. if arguments.sensor is not None: detector_list = [ make_galsim_detector(camera_wrapper, arguments.sensor, phot_params, obs_md) ] else: detector_list = [] for det in camera_wrapper.camera: det_type = det.getType() if det_type != WAVEFRONT and det_type != GUIDER: detector_list.append( make_galsim_detector(camera_wrapper, det.getName(), phot_params, obs_md)) # Add noise and sky background # The simple code using the default lsst-GalSim interface would be: # # PhoSimStarCatalog.noise_and_background = ExampleCCDNoise(addNoise=True, # addBackground=True) # # But, we need a more realistic sky model and we need to pass more than # this basic info to use Peter Y's ESO sky model. # We must pass obs_metadata, chip information etc... noise_and_background \ = ESOSkyModel(obs_md, addNoise=True, addBackground=True) bp_dict = BandpassDict.loadTotalBandpassesFromFiles( bandpassNames=obs_md.bandpass) gs_interpreter = GalSimInterpreter(obs_metadata=obs_md, epoch=2000.0, detectors=detector_list, bandpassDict=bp_dict, noiseWrapper=noise_and_background, seed=arguments.seed) gs_interpreter.checkpoint_file = arguments.checkpoint_file gs_interpreter.nobj_checkpoint = arguments.nobj_checkpoint gs_interpreter.restore_checkpoint(camera_wrapper, phot_params, obs_md) # Add a PSF. if arguments.psf.lower() == "doublegaussian": # This one is taken from equation 30 of # www.astro.washington.edu/users/ivezic/Astr511/LSST_SNRdoc.pdf . # # Set seeing from self.obs_metadata. local_PSF = \ SNRdocumentPSF(obs_md.OpsimMetaData['FWHMgeom']) elif arguments.psf.lower() == "kolmogorov": # This PSF was presented by David Kirkby at the 23 March 2017 # Survey Simulations Working Group telecon # # https://confluence.slac.stanford.edu/pages/viewpage.action?spaceKey=LSSTDESC&title=SSim+2017-03-23 # equation 3 of Krisciunas and Schaefer 1991 airmass = 1.0 / np.sqrt( 1.0 - 0.96 * (np.sin(0.5 * np.pi - obs_md.OpsimMetaData['altitude']))**2) local_PSF = \ Kolmogorov_and_Gaussian_PSF(airmass=airmass, rawSeeing=obs_md.OpsimMetaData['rawSeeing'], band=obs_md.bandpass) else: raise RuntimeError("Do not know what to do with psf model: " "%s" % arguments.psf) gs_interpreter.setPSF(PSF=local_PSF) if arguments.sensor is not None: gs_objects_to_draw = gs_object_dict[arguments.sensor] else: gs_objects_to_draw = gs_object_arr for gs_obj in gs_objects_to_draw: if gs_obj.uniqueId in gs_interpreter.drawn_objects: continue gs_interpreter.drawObject(gs_obj) desc.imsim.add_cosmic_rays(gs_interpreter, phot_params) # Write out the fits files outdir = arguments.outdir if not os.path.isdir(outdir): os.makedirs(outdir) prefix = config['persistence']['eimage_prefix'] gs_interpreter.writeImages(nameRoot=os.path.join(outdir, prefix) + str(obs_md.OpsimMetaData['obshistID']))
rng = galsim.UniformDeviate(args.seed) niter = int(args.counts_total / args.counts_per_iter + 0.5) counts_per_iter = args.counts_total / niter logger = desc.imsim.get_logger(args.log_level, name='make_flats') visit = obs_md.OpsimMetaData['obshistID'] camera_wrapper = LSSTCameraWrapper() wcs = galsim.FitsWCS(args.wcs_file) if args.wcs_file is not None else None for det in camera_wrapper.camera: det_name = det.getName() if (det.getType() != cameraGeom.SCIENCE or (args.sensors is not None and det_name not in sensor_list)): continue logger.info("processing %s", det_name) gs_det = make_galsim_detector(camera_wrapper, det_name, phot_params, obs_md) desc.imsim.add_treering_info([gs_det]) my_flat = desc.imsim.make_flat(gs_det, counts_per_iter, niter, rng, logger=logger, wcs=wcs) ccd_id = "R{}_S{}".format(det_name[2:5:2], det_name[8:11:2]) prefix = config['persistence']['eimage_prefix'] my_flat.write('{}_{}_{}_{}.fits'.format(prefix, visit, ccd_id, obs_md.bandpass))
def test_checkpointing(self): "Test checkpointing of .detectorImages data." camera = camTestUtils.CameraWrapper().camera camera_wrapper = GalSimCameraWrapper(camera) phot_params = PhotometricParameters() obs_md = ObservationMetaData(pointingRA=23.0, pointingDec=12.0, rotSkyPos=13.2, mjd=59580.0, bandpassName='r') detectors = [ make_galsim_detector(camera_wrapper, dd.getName(), phot_params, obs_md) for dd in camera_wrapper.camera ] # Create a GalSimInterpreter object and set the checkpoint # attributes. gs_interpreter = GalSimInterpreter(detectors=detectors) gs_interpreter.checkpoint_file = self.cp_file nobj = 10 gs_interpreter.nobj_checkpoint = nobj # Set the image data by hand. key = "R00_S00_r.fits" detname = "R:0,0 S:0,0" detector = make_galsim_detector(camera_wrapper, detname, phot_params, obs_md) image = gs_interpreter.blankImage(detector=detector) image += 17 gs_interpreter.detectorImages[key] = image # Add some drawn objects and check that the checkpoint file is # written at the right cadence. for uniqueId in range(1, nobj + 1): gs_interpreter.drawn_objects.add(uniqueId) gs_interpreter.write_checkpoint() if uniqueId < nobj: self.assertFalse(os.path.isfile(self.cp_file)) else: self.assertTrue(os.path.isfile(self.cp_file)) # Verify that the checkpointed data has the expected content. with open(self.cp_file, 'rb') as input_: cp_data = pickle.load(input_) self.assertTrue(np.array_equal(cp_data['images'][key], image.array)) # Check the restore_checkpoint function. new_interpreter = GalSimInterpreter(detectors=detectors) new_interpreter.checkpoint_file = self.cp_file new_interpreter.restore_checkpoint(camera_wrapper, phot_params, obs_md) self.assertEqual(new_interpreter.drawn_objects, gs_interpreter.drawn_objects) self.assertEqual(set(new_interpreter.detectorImages.keys()), set(gs_interpreter.detectorImages.keys())) for det_name in new_interpreter.detectorImages.keys(): new_img = new_interpreter.detectorImages[det_name] gs_img = gs_interpreter.detectorImages[det_name] np.testing.assert_array_equal(new_img.array, gs_img.array) self.assertEqual(new_img.bounds, gs_img.bounds) self.assertEqual(new_img.wcs.crpix1, gs_img.wcs.crpix1) self.assertEqual(new_img.wcs.crpix2, gs_img.wcs.crpix2) self.assertEqual(new_img.wcs.crval1, gs_img.wcs.crval1) self.assertEqual(new_img.wcs.crval2, gs_img.wcs.crval2) self.assertEqual(new_img.wcs.detectorName, gs_img.wcs.detectorName) for name in new_img.wcs.fitsHeader.names(): self.assertEqual(new_img.wcs.fitsHeader.get(name), gs_img.wcs.fitsHeader.get(name))