def test_pixsim_cosmics(self):
night = '20150105'
expid = 124
camera = 'r0'
obs.new_exposure('arc', night=night, expid=expid, nspec=3)
pixsim.simulate(night, expid, camera, nspec=3, trimxy=True, cosmics=self.cosmics)
self.assertTrue(os.path.exists(io.findfile('simspec', night, expid)))
simspec = io.read_simspec(io.findfile('simspec', night, expid))
self.assertTrue(os.path.exists(io.findfile('simpix', night, expid, camera)))
self.assertTrue(os.path.exists(io.findfile('pix', night, expid, camera)))
def test_main_defaults(self):
night = self.night
expid = self.expid
camera = 'r0'
nspec = 3
ncpu = 3
obs.new_exposure('arc', night=night, expid=expid, nspec=nspec)
#- run pixsim
opts = ['--night', night, '--expid', expid]
if ncpu is not None:
opts.extend( ['--ncpu', ncpu] )
log.debug('testing pixsim.main({})'.format(opts))
pixsimargs = desisim.scripts.pixsim.parse(opts)
desisim.scripts.pixsim.main(pixsimargs)
#- verify outputs
simpixfile = io.findfile('simpix', night, expid)
self.assertTrue(os.path.exists(simpixfile))
rawfile = desispec.io.findfile('raw', night, expid)
self.assertTrue(os.path.exists(rawfile))
fx = fits.open(rawfile)
self.assertTrue('B0' in fx)
self.assertTrue('R0' in fx)
self.assertTrue('Z0' in fx)
fx.close()
#- cleanup as we go
os.remove(simpixfile)
os.remove(rawfile)
def tearDown(self):
rawfile = desispec.io.findfile('raw', self.night, self.expid)
if os.path.exists(rawfile):
os.remove(rawfile)
fibermap = desispec.io.findfile('fibermap', self.night, self.expid)
if os.path.exists(fibermap):
os.remove(fibermap)
simspecfile = io.findfile('simspec', self.night, self.expid)
if os.path.exists(simspecfile):
os.remove(simspecfile)
simpixfile = io.findfile('simpix', self.night, self.expid)
if os.path.exists(simpixfile):
os.remove(simpixfile)
for camera in ('b0', 'r0', 'z0'):
pixfile = desispec.io.findfile('preproc', self.night, self.expid, camera=camera)
if os.path.exists(pixfile):
os.remove(pixfile)
def test_pixsim_cosmics(self):
night = self.night
expid = self.expid
cameras = ['r0']
obs.new_exposure('arc', night=night, expid=expid, nspec=3)
simspecfile = io.findfile('simspec', night, expid)
rawfile = desispec.io.findfile('desi', night, expid)
simpixfile = io.findfile('simpix', night, expid, cameras)
self.assertFalse(os.path.exists(simpixfile))
self.assertFalse(os.path.exists(rawfile))
pixsim.simulate_exposure(simspecfile, rawfile, cameras,
addcosmics=True, ccdshape=self.ccdshape)
self.assertTrue(os.path.exists(rawfile))
#- No simpixfile option, shouldn't exist
self.assertFalse(os.path.exists(simpixfile))
def tearDown(self):
rawfile = desispec.io.findfile('raw', self.night, self.expid)
if os.path.exists(rawfile):
os.remove(rawfile)
fibermap = desispec.io.findfile('fibermap', self.night, self.expid)
if os.path.exists(fibermap):
os.remove(fibermap)
simspecfile = io.findfile('simspec', self.night, self.expid)
if os.path.exists(simspecfile):
os.remove(simspecfile)
simpixfile = io.findfile('simpix', self.night, self.expid)
if os.path.exists(simpixfile):
os.remove(simpixfile)
for camera in ('b0', 'r0', 'z0'):
pixfile = desispec.io.findfile('preproc',
self.night,
self.expid,
camera=camera)
if os.path.exists(pixfile):
os.remove(pixfile)
def test_parse(self):
night = self.night
expid = self.expid
opts = ['--night', night, '--expid', expid, '--cameras', 'b0,r1']
args = desisim.scripts.pixsim.parse(opts)
self.assertEqual(args.rawfile, desispec.io.findfile('raw', night, expid))
self.assertEqual(args.simspec, io.findfile('simspec', night, expid))
self.assertEqual(args.cameras, ['b0','r1'])
with self.assertRaises(ValueError):
desisim.scripts.pixsim.parse([])
def test_main_override(self):
night = self.night
expid = self.expid
camera = 'r0'
nspec = 3
ncpu = 3
obs.new_exposure('arc', night=night, expid=expid, nspec=nspec)
#- derive night from simspec input while overriding expid
#- Include wavelengths covering z, but only ask for b and r
simspecfile = io.findfile('simspec', night, expid)
altexpid = expid+1
altrawfile = desispec.io.findfile('raw', night, altexpid) + '.blat'
opts = [
'--simspec', simspecfile,
'--expid', altexpid,
'--rawfile', altrawfile,
'--cameras', 'b0,r0',
'--wavemin', 5500, '--wavemax', 7000.0,
'--ccd_npix_x', 2000,
]
if ncpu is not None:
opts.extend( ['--ncpu', ncpu] )
log.debug('testing pixsim.main({})'.format(opts))
pixsimargs = desisim.scripts.pixsim.parse(opts)
desisim.scripts.pixsim.main(pixsimargs)
simpixfile = io.findfile('simpix', night, altexpid)
self.assertTrue(os.path.exists(simpixfile))
self.assertTrue(os.path.exists(altrawfile))
fx = fits.open(altrawfile)
self.assertTrue('B0' in fx)
self.assertTrue('R0' in fx)
self.assertTrue('Z0' not in fx)
fx.close()
#- cleanup as we go
os.remove(simpixfile)
os.remove(altrawfile)
def test_pixsim_cosmics(self):
night = self.night
expid = self.expid
cameras = ['r0']
obs.new_exposure('arc', night=night, expid=expid, nspec=3)
simspecfile = io.findfile('simspec', night, expid)
rawfile = desispec.io.findfile('desi', night, expid)
simpixfile = io.findfile('simpix', night, expid, cameras)
self.assertFalse(os.path.exists(simpixfile))
self.assertFalse(os.path.exists(rawfile))
pixsim.simulate_exposure(simspecfile,
rawfile,
cameras,
addcosmics=True,
ccdshape=self.ccdshape)
self.assertTrue(os.path.exists(rawfile))
#- No simpixfile option, shouldn't exist
self.assertFalse(os.path.exists(simpixfile))
def test_parse(self):
night = self.night
expid = self.expid
opts = ['--night', night, '--expid', expid, '--cameras', 'b0,r1']
args = desisim.scripts.pixsim.parse(opts)
self.assertEqual(args.rawfile,
desispec.io.findfile('raw', night, expid))
self.assertEqual(args.simspec, io.findfile('simspec', night, expid))
self.assertEqual(args.cameras, ['b0', 'r1'])
with self.assertRaises(ValueError):
desisim.scripts.pixsim.parse([])
def expand_args(args):
'''expand camera string into list of cameras
'''
if args.simspec is None:
if args.night is None or args.expid is None:
msg = 'Must set --simspec or both --night and --expid'
log.error(msg)
raise ValueError(msg)
args.simspec = io.findfile('simspec', args.night, args.expid)
#- expand camera list
if args.cameras is not None:
args.cameras = args.cameras.split(',')
#- write to same directory as simspec
if args.rawfile is None:
rawfile = os.path.basename(desispec.io.findfile('raw', args.night, args.expid))
args.rawfile = os.path.join(os.path.dirname(args.simspec), rawfile)
if args.simpixfile is None:
outdir = os.path.dirname(os.path.abspath(args.rawfile))
args.simpixfile = io.findfile(
'simpix', night=args.night, expid=args.expid, outdir=outdir)
def test_main_override(self):
night = self.night
expid = self.expid
camera = 'r0'
nspec = 3
ncpu = 3
obs.new_exposure('arc', night=night, expid=expid, nspec=nspec)
#- derive night from simspec input while overriding expid
simspecfile = io.findfile('simspec', night, expid)
altrawfile = desispec.io.findfile('raw', night, expid) + '.blat'
opts = [
'--simspec', simspecfile,
'--expid', expid+1,
'--rawfile', altrawfile,
'--cameras', 'b0,r0',
'--preproc',
'--wavemin', 5000, '--wavemax', 7000.0,
'--ccd_npix_x', 2000,
]
if ncpu is not None:
opts.extend( ['--ncpu', ncpu] )
log.debug('testing pixsim.main({})'.format(opts))
desisim.scripts.pixsim.main(opts)
simpixfile = io.findfile('simpix', night, expid+1)
self.assertTrue(os.path.exists(simpixfile))
self.assertTrue(os.path.exists(altrawfile))
fx = fits.open(altrawfile)
self.assertTrue('B0' in fx)
self.assertTrue('R0' in fx)
self.assertTrue('Z0' not in fx)
fx.close()
#- cleanup as we go
os.remove(simpixfile)
os.remove(altrawfile)
def test_simulate(self):
import desispec.image
night = self.night
expid = self.expid
camera = 'r0'
nspec = 3
obs.new_exposure('arc', night=night, expid=expid, nspec=nspec)
simspec = io.read_simspec(io.findfile('simspec', night, expid))
psf = desimodel.io.load_psf(camera[0])
psf.npix_y, psf.npix_x = self.ccdshape
image, rawpix, truepix = pixsim.simulate(camera, simspec, psf, nspec=nspec)
self.assertTrue(isinstance(image, desispec.image.Image))
self.assertTrue(isinstance(rawpix, np.ndarray))
self.assertTrue(isinstance(truepix, np.ndarray))
self.assertEqual(image.pix.shape, truepix.shape)
self.assertEqual(image.pix.shape[0], rawpix.shape[0])
self.assertLess(image.pix.shape[1], rawpix.shape[1]) #- raw has overscan
def test_simulate(self):
import desispec.image
night = self.night
expid = self.expid
camera = 'r0'
nspec = 3
obs.new_exposure('arc', night=night, expid=expid, nspec=nspec)
simspec = io.read_simspec(io.findfile('simspec', night, expid))
psf = desimodel.io.load_psf(camera[0])
psf.npix_y, psf.npix_x = self.ccdshape
image, rawpix, truepix = pixsim.simulate(camera, simspec, psf, nspec=nspec)
self.assertTrue(isinstance(image, desispec.image.Image))
self.assertTrue(isinstance(rawpix, np.ndarray))
self.assertTrue(isinstance(truepix, np.ndarray))
self.assertEqual(image.pix.shape, truepix.shape)
self.assertEqual(image.pix.shape[0], rawpix.shape[0])
self.assertLess(image.pix.shape[1], rawpix.shape[1]) #- raw has overscan
import os
import desimodel
import fitsio
import numpy as np
import pylab as pl
import astropy.io.fits as fits
from astropy.table import Table, vstack
from desimodel.io import findfile
from desitarget.geomask import circles
from desimodel.focalplane import get_tile_radius_deg
tiles_file = findfile('footprint/desi-tiles.fits')
print(tiles_file)
tiles = Table(fits.open(tiles_file)[1].data)
tiles.pprint(max_width=-1)
extra_gray = Table(tiles[tiles['PASS'] == 8], copy=True)
extra_dark = Table(tiles[tiles['PASS'] == 9], copy=True)
extra_brt = Table(tiles[tiles['PASS'] == 8], copy=True)
example_gray = Table(tiles[tiles['PASS'] == 0], copy=True)
example_dark = Table(tiles[tiles['PASS'] == 1], copy=True)
example_brt = Table(tiles[tiles['PASS'] == 5], copy=True)
extra_gray['PASS'] = 8
extra_gray['PROGRAM'] = example_gray['PROGRAM'][0]
extra_gray['OBSCONDITIONS'] = example_gray['OBSCONDITIONS'][0]
def simulate(night, expid, camera, nspec=None, verbose=False, ncpu=None,
trimxy=False, cosmics=None):
"""
Run pixel-level simulation of input spectra
Args:
night (string) : YEARMMDD
expid (integer) : exposure id
camera (str) : e.g. b0, r1, z9
nspec (int) : number of spectra to simulate
verbose (boolean) : if True, print status messages
ncpu (int) : number of CPU cores to use in parallel
trimxy (boolean) : trim image to just pixels with input signal
cosmics (str) : filename with dark images with cosmics to add
Reads:
$DESI_SPECTRO_SIM/$PIXPROD/{night}/simspec-{expid}.fits
Writes:
$DESI_SPECTRO_SIM/$PIXPROD/{night}/simpix-{camera}-{expid}.fits
$DESI_SPECTRO_SIM/$PIXPROD/{night}/pix-{camera}-{expid}.fits
"""
if verbose:
print "Reading input files"
channel = camera[0].lower()
ispec = int(camera[1])
assert channel in 'brz'
assert 0 <= ispec < 10
#- Load DESI parameters
params = desimodel.io.load_desiparams()
nfibers = params['spectro']['nfibers']
#- Load simspec file
simfile = io.findfile('simspec', night=night, expid=expid)
simspec = io.read_simspec(simfile)
wave = simspec.wave[channel]
if simspec.skyphot is not None:
phot = simspec.phot[channel] + simspec.skyphot[channel]
else:
phot = simspec.phot[channel]
if ispec*nfibers >= simspec.nspec:
print "ERROR: camera {} not in the {} spectra in {}/{}".format(
camera, simspec.nspec, night, os.path.basename(simfile))
return
#- Load PSF
psf = desimodel.io.load_psf(channel)
#- Trim to just the spectra for this spectrograph
if nspec is None:
ii = slice(nfibers*ispec, nfibers*(ispec+1))
else:
ii = slice(nfibers*ispec, nfibers*ispec + nspec)
phot = phot[ii]
#- check if simulation has less than 500 input spectra
if phot.shape[0] < nspec:
nspec = phot.shape[0]
#- Project to image and append that to file
if verbose:
print "Projecting photons onto CCD"
img = parallel_project(psf, wave, phot, ncpu=ncpu)
if trimxy:
xmin, xmax, ymin, ymax = psf.xyrange((0,nspec), wave)
img = img[0:ymax, 0:xmax]
# img = img[ymin:ymax, xmin:xmax]
# hdr['CRVAL1'] = xmin+1
# hdr['CRVAL2'] = ymin+1
#- Prepare header
hdr = simspec.header
tmp = '/'.join(simfile.split('/')[-3:]) #- last 3 elements of path
hdr['SIMFILE'] = (tmp, 'Input simulation file')
#- Strip unnecessary keywords
for key in ('EXTNAME', 'LOGLAM', 'AIRORVAC', 'CRVAL1', 'CDELT1'):
if key in hdr:
del hdr[key]
#- Write noiseless output
simpixfile = io.findfile('simpix', night=night, expid=expid, camera=camera)
io.write_simpix(simpixfile, img, meta=hdr)
#- Add cosmics from library of dark images
#- in this case, don't add readnoise since the dark image already has it
if cosmics is not None:
cosmics = io.read_cosmics(cosmics, expid, shape=img.shape)
pix = np.random.poisson(img) + cosmics.pix
readnoise = cosmics.meta['RDNOISE']
ivar = 1.0/(pix.clip(0) + readnoise**2)
mask = cosmics.mask
#- Or just add noise
else:
params = desimodel.io.load_desiparams()
channel = camera[0].lower()
readnoise = params['ccd'][channel]['readnoise']
pix = np.random.poisson(img) + np.random.normal(scale=readnoise, size=img.shape)
ivar = 1.0/(pix.clip(0) + readnoise**2)
mask = np.zeros(img.shape, dtype=np.uint16)
#- Metadata to be included in pix file header is in the fibermap header
#- TODO: this is fragile; consider updating fibermap to use astropy Table
#- that includes the header rather than directly assuming FITS as the
#- underlying format.
fibermapfile = desispec.io.findfile('fibermap', night=night, expid=expid)
fm, fmhdr = desispec.io.read_fibermap(fibermapfile, header=True)
meta = dict()
try:
meta['TELRA'] = simspec.header['TELRA']
meta['TELDEC'] = simspec.header['TELDEC']
except KeyError: #- temporary backwards compatibilty
meta['TELRA'] = fmhdr['TELERA']
meta['TELDEC'] = fmhdr['TELEDEC']
meta['TILEID'] = simspec.header['TILEID']
meta['DATE-OBS'] = simspec.header['DATE-OBS']
meta['FLAVOR'] = simspec.header['FLAVOR']
meta['EXPTIME'] = simspec.header['EXPTIME']
meta['AIRMASS'] = simspec.header['AIRMASS']
image = Image(pix, ivar, mask, readnoise=readnoise, camera=camera, meta=meta)
pixfile = desispec.io.findfile('pix', night=night, camera=camera, expid=expid)
desispec.io.write_image(pixfile, image)
if verbose:
print "Wrote "+pixfile
def expand_args(args):
'''expand camera string into list of cameras
'''
# if simspec:
# if not night:
# get night from simspec
# if not expid:
# get expid from simspec
# else:
# assert night and expid are set
# get simspec from (night, expid)
#
# if not outrawfile:
# get outrawfile from (night, expid)
#
# if outpixfile or outsimpixfile:
# assert len(cameras) == 1
if args.simspec is None:
if args.night is None or args.expid is None:
msg = 'Must set --simspec or both --night and --expid'
log.error(msg)
raise ValueError(msg)
args.simspec = io.findfile('simspec', args.night, args.expid)
if (args.cameras is None) and (args.spectrographs is None):
from astropy.io import fits
try:
data = fits.getdata(args.simspec, 'B')
nspec = data['PHOT'].shape[1]
except KeyError:
#- Try old specsim format instead
hdr = fits.getheader(args.simspec, 'PHOT_B')
nspec = hdr['NAXIS2']
nspectrographs = (nspec - 1) // 500 + 1
args.spectrographs = list(range(nspectrographs))
if (args.night is None) or (args.expid is None):
from astropy.io import fits
hdr = fits.getheader(args.simspec)
if args.night is None:
args.night = str(hdr['NIGHT'])
if args.expid is None:
args.expid = int(hdr['EXPID'])
if isinstance(args.spectrographs, str):
args.spectrographs = [int(x) for x in args.spectrographs.split(',')]
#- expand camera list
if args.cameras is None:
args.cameras = list()
for arm in args.arms.split(','):
for ispec in args.spectrographs:
args.cameras.append(arm + str(ispec))
else:
args.cameras = args.cameras.split(',')
#- write to same directory as simspec
if args.rawfile is None:
rawfile = os.path.basename(
desispec.io.findfile('raw', args.night, args.expid))
args.rawfile = os.path.join(os.path.dirname(args.simspec), rawfile)
if args.preproc:
if args.preproc_dir is None:
args.preproc_dir = os.path.dirname(args.rawfile)
if args.simpixfile is None:
args.simpixfile = io.findfile('simpix',
night=args.night,
expid=args.expid,
outdir=os.path.dirname(
os.path.abspath(args.rawfile)))
def simulate(night,
expid,
camera,
nspec=None,
verbose=False,
ncpu=None,
trimxy=False,
cosmics=None):
"""
Run pixel-level simulation of input spectra
Args:
night (string) : YEARMMDD
expid (integer) : exposure id
camera (str) : e.g. b0, r1, z9
nspec (int) : number of spectra to simulate
verbose (boolean) : if True, print status messages
ncpu (int) : number of CPU cores to use in parallel
trimxy (boolean) : trim image to just pixels with input signal
cosmics (str) : filename with dark images with cosmics to add
Reads:
$DESI_SPECTRO_SIM/$PIXPROD/{night}/simspec-{expid}.fits
Writes:
$DESI_SPECTRO_SIM/$PIXPROD/{night}/simpix-{camera}-{expid}.fits
$DESI_SPECTRO_SIM/$PIXPROD/{night}/pix-{camera}-{expid}.fits
"""
if verbose:
print "Reading input files"
channel = camera[0].lower()
ispec = int(camera[1])
assert channel in 'brz'
assert 0 <= ispec < 10
#- Load DESI parameters
params = desimodel.io.load_desiparams()
nfibers = params['spectro']['nfibers']
#- Load simspec file
simfile = io.findfile('simspec', night=night, expid=expid)
simspec = io.read_simspec(simfile)
wave = simspec.wave[channel]
if simspec.skyphot is not None:
phot = simspec.phot[channel] + simspec.skyphot[channel]
else:
phot = simspec.phot[channel]
if ispec * nfibers >= simspec.nspec:
print "ERROR: camera {} not in the {} spectra in {}/{}".format(
camera, simspec.nspec, night, os.path.basename(simfile))
return
#- Load PSF
psf = desimodel.io.load_psf(channel)
#- Trim to just the spectra for this spectrograph
if nspec is None:
ii = slice(nfibers * ispec, nfibers * (ispec + 1))
else:
ii = slice(nfibers * ispec, nfibers * ispec + nspec)
phot = phot[ii]
#- check if simulation has less than 500 input spectra
if phot.shape[0] < nspec:
nspec = phot.shape[0]
#- Project to image and append that to file
if verbose:
print "Projecting photons onto CCD"
img = parallel_project(psf, wave, phot, ncpu=ncpu)
if trimxy:
xmin, xmax, ymin, ymax = psf.xyrange((0, nspec), wave)
img = img[0:ymax, 0:xmax]
# img = img[ymin:ymax, xmin:xmax]
# hdr['CRVAL1'] = xmin+1
# hdr['CRVAL2'] = ymin+1
#- Prepare header
hdr = simspec.header
tmp = '/'.join(simfile.split('/')[-3:]) #- last 3 elements of path
hdr['SIMFILE'] = (tmp, 'Input simulation file')
#- Strip unnecessary keywords
for key in ('EXTNAME', 'LOGLAM', 'AIRORVAC', 'CRVAL1', 'CDELT1'):
if key in hdr:
del hdr[key]
#- Write noiseless output
simpixfile = io.findfile('simpix', night=night, expid=expid, camera=camera)
io.write_simpix(simpixfile, img, meta=hdr)
#- Add cosmics from library of dark images
#- in this case, don't add readnoise since the dark image already has it
if cosmics is not None:
cosmics = io.read_cosmics(cosmics, expid, shape=img.shape)
pix = np.random.poisson(img) + cosmics.pix
readnoise = cosmics.meta['RDNOISE']
ivar = 1.0 / (pix.clip(0) + readnoise**2)
mask = cosmics.mask
#- Or just add noise
else:
params = desimodel.io.load_desiparams()
channel = camera[0].lower()
readnoise = params['ccd'][channel]['readnoise']
pix = np.random.poisson(img) + np.random.normal(scale=readnoise,
size=img.shape)
ivar = 1.0 / (pix.clip(0) + readnoise**2)
mask = np.zeros(img.shape, dtype=np.uint16)
#- Metadata to be included in pix file header is in the fibermap header
#- TODO: this is fragile; consider updating fibermap to use astropy Table
#- that includes the header rather than directly assuming FITS as the
#- underlying format.
fibermapfile = desispec.io.findfile('fibermap', night=night, expid=expid)
fm, fmhdr = desispec.io.read_fibermap(fibermapfile, header=True)
meta = dict()
try:
meta['TELRA'] = simspec.header['TELRA']
meta['TELDEC'] = simspec.header['TELDEC']
except KeyError: #- temporary backwards compatibilty
meta['TELRA'] = fmhdr['TELERA']
meta['TELDEC'] = fmhdr['TELEDEC']
meta['TILEID'] = simspec.header['TILEID']
meta['DATE-OBS'] = simspec.header['DATE-OBS']
meta['FLAVOR'] = simspec.header['FLAVOR']
meta['EXPTIME'] = simspec.header['EXPTIME']
meta['AIRMASS'] = simspec.header['AIRMASS']
image = Image(pix,
ivar,
mask,
readnoise=readnoise,
camera=camera,
meta=meta)
pixfile = desispec.io.findfile('pix',
night=night,
camera=camera,
expid=expid)
desispec.io.write_image(pixfile, image)
if verbose:
print "Wrote " + pixfile
def new_exposure(program, nspec=5000, night=None, expid=None, tileid=None,
nproc=None, seed=None, obsconditions=None,
specify_targets=dict(), testslit=False, exptime=None,
arc_lines_filename=None, flat_spectrum_filename=None):
"""
Create a new exposure and output input simulation files.
Does not generate pixel-level simulations or noisy spectra.
Args:
program: 'arc', 'flat', 'bright', 'dark', 'bgs', 'mws', ...
Options:
* nspec : integer number of spectra to simulate
* night : YEARMMDD string
* expid : positive integer exposure ID
* tileid : integer tile ID
* seed : random seed
* obsconditions: str or dict-like; see options below
* specify_targets: (dict of dicts) Define target properties like magnitude and redshift
for each target class. Each objtype has its own key,value pair
see simspec.templates.specify_galparams_dict()
or simsepc.templates.specify_starparams_dict()
* exptime: float exposure time [seconds], overrides obsconditions['EXPTIME']
* testslit : simulate test slit if True, default False; only for arc/flat
* arc_lines_filename : use alternate arc lines filename (used if program="arc")
* flat_spectrum_filename : use alternate flat spectrum filename (used if program="flat")
Writes:
* $DESI_SPECTRO_SIM/$PIXPROD/{night}/fibermap-{expid}.fits
* $DESI_SPECTRO_SIM/$PIXPROD/{night}/simspec-{expid}.fits
Returns:
* science: sim, fibermap, meta, obsconditions
input obsconditions can be a string 'dark', 'gray', 'bright', or dict-like
observation metadata with keys SEEING (arcsec), EXPTIME (sec), AIRMASS,
MOONFRAC (0-1), MOONALT (deg), MOONSEP (deg). Output obsconditions is
is expanded dict-like structure.
program is used to pick the sky brightness, and is propagated to
desisim.targets.sample_objtype() to get the correct distribution of
targets for a given program, e.g. ELGs, LRGs, QSOs for program='dark'.
if program is 'arc' or 'flat', then `sim` is truth table with keys
FLUX and WAVE; and meta=None and obsconditions=None.
Also see simexp.simarc(), .simflat(), and .simscience(), the last of
which simulates a science exposure given surveysim obsconditions input,
fiber assignments, and pre-generated mock target spectra.
"""
if expid is None:
expid = get_next_expid()
if tileid is None:
tileid = get_next_tileid()
if night is None:
#- simulation obs time = now, even if sun is up
dateobs = time.gmtime()
night = get_night(utc=dateobs)
else:
#- 10pm on night YEARMMDD
night = str(night) #- just in case we got an integer instead of string
dateobs = time.strptime(night+':22', '%Y%m%d:%H')
outsimspec = desisim.io.findfile('simspec', night, expid)
outfibermap = desisim.io.findfile('simfibermap', night, expid)
program = program.lower()
log.debug('Generating {} targets'.format(nspec))
header = dict(NIGHT=night, EXPID=expid, PROGRAM=program)
if program in ('arc', 'flat'):
header['FLAVOR'] = program
else:
header['FLAVOR'] = 'science'
#- ISO 8601 DATE-OBS year-mm-ddThh:mm:ss
header['DATE-OBS'] = time.strftime('%FT%T', dateobs)
if program == 'arc':
if arc_lines_filename is None :
infile = os.getenv('DESI_ROOT')+'/spectro/templates/calib/v0.3/arc-lines-average-in-vacuum.fits'
else :
infile = arc_lines_filename
arcdata = fits.getdata(infile, 1)
if exptime is None:
exptime = 5
wave, phot, fibermap = desisim.simexp.simarc(arcdata, nspec=nspec, testslit=testslit)
header['EXPTIME'] = exptime
desisim.io.write_simspec_arc(outsimspec, wave, phot, header, fibermap=fibermap)
fibermap.meta['NIGHT'] = night
fibermap.meta['EXPID'] = expid
fibermap.write(outfibermap)
truth = dict(WAVE=wave, PHOT=phot, UNITS='photon')
return truth, fibermap, None, None
elif program == 'flat':
if flat_spectrum_filename is None :
infile = os.getenv('DESI_ROOT')+'/spectro/templates/calib/v0.3/flat-3100K-quartz-iodine.fits'
else :
infile = flat_spectrum_filename
if exptime is None:
exptime = 10
sim, fibermap = desisim.simexp.simflat(infile, nspec=nspec, exptime=exptime, testslit=testslit)
header['EXPTIME'] = exptime
header['FLAVOR'] = 'flat'
desisim.io.write_simspec(sim, truth=None, fibermap=fibermap, obs=None,
expid=expid, night=night, header=header)
fibermap.meta['NIGHT'] = night
fibermap.meta['EXPID'] = expid
fibermap.write(outfibermap)
# fluxunits = 1e-17 * u.erg / (u.s * u.cm**2 * u.Angstrom)
fluxunits = '1e-17 erg/(s * cm2 * Angstrom)'
flux = sim.simulated['source_flux'].to(fluxunits)
wave = sim.simulated['wavelength'].to('Angstrom')
truth = dict(WAVE=wave, FLUX=flux, UNITS=str(fluxunits))
return truth, fibermap, None, None
#- all other programs
fibermap, (flux, wave, meta) = get_targets_parallel(nspec, program,
tileid=tileid, nproc=nproc, seed=seed, specify_targets=specify_targets)
if obsconditions is None:
if program in ['dark', 'lrg', 'qso']:
obsconditions = desisim.simexp.reference_conditions['DARK']
elif program in ['elg', 'gray', 'grey']:
obsconditions = desisim.simexp.reference_conditions['GRAY']
elif program in ['mws', 'bgs', 'bright']:
obsconditions = desisim.simexp.reference_conditions['BRIGHT']
else:
raise ValueError('unknown program {}'.format(program))
elif isinstance(obsconditions, str):
try:
obsconditions = desisim.simexp.reference_conditions[obsconditions.upper()]
except KeyError:
raise ValueError('obsconditions {} not in {}'.format(
obsconditions.upper(),
list(desisim.simexp.reference_conditions.keys())))
if exptime is not None:
obsconditions['EXPTIME'] = exptime
sim = simulate_spectra(wave, flux, fibermap=fibermap, obsconditions=obsconditions)
#- Override $DESI_SPECTRO_DATA in order to write to simulation area
datadir_orig = os.getenv('DESI_SPECTRO_DATA')
simbase = os.path.join(os.getenv('DESI_SPECTRO_SIM'), os.getenv('PIXPROD'))
os.environ['DESI_SPECTRO_DATA'] = simbase
#- Write fibermap
telera, teledec = io.get_tile_radec(tileid)
hdr = dict(
NIGHT = (night, 'Night of observation YEARMMDD'),
EXPID = (expid, 'DESI exposure ID'),
TILEID = (tileid, 'DESI tile ID'),
PROGRAM = (program, 'program [dark, bright, ...]'),
FLAVOR = ('science', 'Flavor [arc, flat, science, zero, ...]'),
TELRA = (telera, 'Telescope pointing RA [degrees]'),
TELDEC = (teledec, 'Telescope pointing dec [degrees]'),
AIRMASS = (obsconditions['AIRMASS'], 'Airmass at middle of exposure'),
EXPTIME = (obsconditions['EXPTIME'], 'Exposure time [sec]'),
SEEING = (obsconditions['SEEING'], 'Seeing FWHM [arcsec]'),
MOONFRAC = (obsconditions['MOONFRAC'], 'Moon illumination fraction 0-1; 1=full'),
MOONALT = (obsconditions['MOONALT'], 'Moon altitude [degrees]'),
MOONSEP = (obsconditions['MOONSEP'], 'Moon:tile separation angle [degrees]'),
)
hdr['DATE-OBS'] = (time.strftime('%FT%T', dateobs), 'Start of exposure')
simfile = io.write_simspec(sim, meta, fibermap, obsconditions, expid, night, header=hdr)
#- Write fibermap to $DESI_SPECTRO_SIM/$PIXPROD not $DESI_SPECTRO_DATA
fiberfile = io.findfile('simfibermap', night, expid)
desispec.io.write_fibermap(fiberfile, fibermap, header=hdr)
log.info('Wrote '+fiberfile)
update_obslog(obstype='science', program=program, expid=expid, dateobs=dateobs, tileid=tileid)
#- Restore $DESI_SPECTRO_DATA
if datadir_orig is not None:
os.environ['DESI_SPECTRO_DATA'] = datadir_orig
else:
del os.environ['DESI_SPECTRO_DATA']
return sim, fibermap, meta, obsconditions
def main(args, comm=None):
if args.verbose:
import logging
log.setLevel(logging.DEBUG)
#we do this so we can use operator.itemgetter
import operator
#we do this so our print statements can have timestamps
import time
rank = 0
nproc = 1
if comm is not None:
import mpi4py
rank = comm.rank
nproc = comm.size
if rank == 0:
log.info('Starting pixsim at {}'.format(asctime()))
#no preflight check here, too complicated.
#we'll assume the user knows what he or she is doing...
# Determine which nights we are using
nights = None
if args.nights is not None:
nights = args.nights.split(",")
else:
rawdir = os.path.abspath(specio.rawdata_root())
nights = []
nightpat = re.compile(r"\d{8}")
for root, dirs, files in os.walk(rawdir, topdown=True):
for d in dirs:
nightmat = nightpat.match(d)
if nightmat is not None:
nights.append(d)
# Get the list of exposures for each night
night_expid = {}
all_expid = []
exp_to_night = {}
for nt in nights:
night_expid[nt] = specio.get_exposures(nt, raw=True)
#get a list of tuples of (night,expid) that we can evenly divide between communicators
night_exposure_list=list()
if comm is None or comm.rank == 0:
for nt in nights:
for exp in night_expid[nt]:
rawfile = desispec.io.findfile('raw', nt, exp)
if not os.path.exists(rawfile):
night_exposure_list.append([nt,exp])
elif args.overwrite:
log.warning('Overwriting pre-existing {}'.format(os.path.basename(rawfile)))
os.remove(rawfile)
night_exposure_list.append([nt,exp])
else:
log.info('Skipping pre-existing {}'.format(os.path.basename(rawfile)))
if args.nexp is not None:
night_exposure_list = night_exposure_list[0:args.nexp]
if comm is not None:
night_exposure_list = comm.bcast(night_exposure_list, root=0)
if len(night_exposure_list) == 0:
if comm is None or comm.rank == 0:
log.error('No exposures to process')
sys.exit(1)
# Get the list of cameras and make sure it's in the right format
cams = []
if args.cameras is not None:
entry = args.cameras.split(',')
for i in entry:
cams.append(i)
else:
#do this with band first so we can avoid re-broadcasting cosmics
for band in ['b', 'r', 'z']:
for spec in range(10):
cams.append('{}{}'.format(band, spec))
#are we using cosmics?
if args.cosmics is not None:
addcosmics = True
else:
addcosmics = False
ncameras=len(cams)
nexposures=len(night_exposure_list)
#call ultity function to figure out how many nodes we have
nnodes=mpi_count_nodes(comm)
#call utility functions to divide our workload
if args.nodes_per_exp is not None:
user_specified_nodes=args.nodes_per_exp
else:
user_specified_nodes=None
nodes_per_comm_exp=get_nodes_per_exp(nnodes,nexposures,ncameras,user_specified_nodes)
#also figure out how many exposure communicators we have
num_exp_comm = nnodes // nodes_per_comm_exp
#split the communicator into exposure communicators
comm_exp, node_index_exp, num_nodes_exp = mpi_split_by_node(comm, nodes_per_comm_exp)
#further splitting will happen automatically in simulate_exposure
#based on this, figure out which simspecfiles and rawfiles are assigned to each communicator
#find all specfiles
#find all rawfiles
rawfile_list=[]
simspecfile_list=[]
night_list=[]
expid_list=[]
for i in range(len(night_exposure_list)):
night_list.append(night_exposure_list[i][0])
expid_list.append(night_exposure_list[i][1])
rawfile_list.append(desispec.io.findfile('raw', night_list[i], expid_list[i]))
simspecfile_list.append(io.findfile('simspec', night_list[i], expid_list[i]))
#now divy the rawfiles and specfiles between node communicators
#there is onerawfile and one specfile for each exposure
rawfile_comm_exp=[]
simspecfile_comm_exp=[]
for i in range(num_exp_comm):
if node_index_exp == i: #assign rawfile, simspec file to one communicator at a time
rawfile_comm_exp=rawfile_list[i::num_exp_comm]
simspecfile_comm_exp=simspecfile_list[i::num_exp_comm]
night_comm_exp=night_list[i::num_exp_comm]
expid_comm_exp=expid_list[i::num_exp_comm]
comm.Barrier()
#now wrap pixsim.simulate_exposure for each exposure (in desisim.pixsim)
if comm_exp.rank == 0:
log.info("Starting simulate_exposure for night {} expid {}".format(night_comm_exp, expid_comm_exp))
for i in range(len(rawfile_comm_exp)):
simulate_exposure(simspecfile_comm_exp[i], rawfile_comm_exp[i], cameras=cams,
ccdshape=None, simpixfile=None, addcosmics=addcosmics, comm=comm_exp)
comm.Barrier()
if rank == 0:
log.info('Finished pixsim nights {}'.format(args.nights, asctime()))
def load_hardware(fiberpos_file=None,
gfa_file=None,
rundate=None,
status_file=None):
"""Create a hardware class representing properties of the telescope.
Args:
fiberpos_file (str): Optional path to the fiber positioner FITS file.
If not specified, desimodel is used to get the location of the
default file.
gfa_file (str): Optional path to the GFA file.
rundate (str): ISO 8601 format time stamp as a string in the
format YYYY-MM-DDTHH:MM:SS. If None, uses current time.
status_file (str): Path to fiber status file. If not specified, all
fibers are assumed good.
Returns:
(Hardware): The hardware object.
"""
log = Logger.get()
# Read the fiber positioner data
if fiberpos_file is None:
fiberpos_file = dmio.findfile('focalplane/fiberpos-all.fits')
log.info("Reading fiber positions from {}".format(fiberpos_file))
fpdata = fitsio.read(fiberpos_file, ext=1)
pos_rows = np.where(fpdata["DEVICE_TYPE"] == b"POS")[0]
etc_rows = np.where(fpdata["DEVICE_TYPE"] == b"ETC")[0]
keep_rows = np.unique(np.concatenate((pos_rows, etc_rows)))
nfiber = len(keep_rows)
log.debug(" fiber position table keeping {} rows".format(nfiber))
device_type = np.empty(nfiber, dtype="a8")
device_type[:] = fpdata["DEVICE_TYPE"][keep_rows]
# For non-science positioners, no fiber ID is assigned in the positioner
# file. This is a pain, since all our quantities are indexed by fiber ID.
# Instead, we give every position a FIBER value which is unique and which
# is sorted by LOCATION. We start these fake FIBER values at 5000.
fiber = np.copy(fpdata["FIBER"][keep_rows])
location = np.copy(fpdata["LOCATION"][keep_rows])
missing_fiber = [x for x, y in enumerate(fiber) if y < 0]
if len(missing_fiber) > 0:
missing_locsorted = np.sort(location[missing_fiber])
locfiber = {y: x for x, y in enumerate(missing_locsorted)}
fiber[missing_fiber] = [(5000 + locfiber[x])
for x in location[missing_fiber]]
# Read the status file...
status = np.empty(nfiber, dtype=np.int32)
status[:] = FIBER_STATE_OK
if status_file is not None:
runtime = None
if rundate is None:
runtime = datetime.utcnow()
else:
runtime = datetime.strptime(rundate, "%Y-%m-%dT%H:%M:%S")
locindx = {y: x for x, y in enumerate(location)}
statdata = Table.read(status_file, format="ascii.ecsv")
for row in statdata:
loc = row["LOCATION"]
broken = row["BROKEN"]
stuck = row["STUCK"]
start = datetime.strptime(row["START_DATE"], "%Y-%m-%dT%H:%M:%S")
stop = datetime.strptime(row["END_DATE"], "%Y-%m-%dT%H:%M:%S")
# Check if this row applies to our current run
if (runtime >= start) and (runtime < stop):
# yep...
if broken > 0:
status[locindx[loc]] |= FIBER_STATE_BROKEN
if stuck > 0:
status[locindx[loc]] |= FIBER_STATE_STUCK
hw = Hardware(fiber, fpdata["PETAL"][keep_rows],
fpdata["SPECTRO"][keep_rows], location,
fpdata["SLIT"][keep_rows], fpdata["SLITBLOCK"][keep_rows],
fpdata["BLOCKFIBER"][keep_rows], fpdata["DEVICE"][keep_rows],
device_type, fpdata["X"][keep_rows], fpdata["Y"][keep_rows],
fpdata["Z"][keep_rows], fpdata["Q"][keep_rows],
fpdata["S"][keep_rows], status)
return hw