def test_freeze_iers_bad_format(self, mock_logger):
"""Test freezing from valid file with wrong format.
"""
with self.assertRaises(ValueError):
i.freeze_iers('census.yaml')
mock_logger().info.assert_has_calls(
[call('Freezing IERS table used by astropy time, coordinates.')])
def test_freeze_iers_again(self, mock_logger):
"""Test freezing twice.
"""
i._iers_is_frozen = True
i.freeze_iers()
mock_logger().debug.assert_has_calls(
[call('IERS table already frozen.')])
def test_freeze_iers_bad_name(self, mock_logger):
"""Test freezing from package data/ with bad filename.
"""
with self.assertRaises(ValueError):
i.freeze_iers('_non_existent_.ecsv')
mock_logger().info.assert_has_calls(
[call('Freezing IERS table used by astropy time, coordinates.')])
def test_update_iers_frozen(self, mock_logger):
"""Test attempt to update a frozen IERS table.
"""
save_name = os.path.join(self.tmpdir, 'iers.ecsv')
i.freeze_iers()
with self.assertRaises(ValueError):
i.update_iers(save_name)
mock_logger().info.assert_has_calls(
[call('Freezing IERS table used by astropy time, coordinates.')])
def test_freeze_iers(self, mock_logger):
"""Test freezing from package data/.
"""
i.freeze_iers()
future = Time('2024-01-01', location=self.location)
lst = future.sidereal_time('apparent')
self.assertFalse(astropy.utils.iers.conf.auto_download)
self.assertIsNone(astropy.utils.iers.conf.auto_max_age)
self.assertEqual(astropy.utils.iers.conf.iers_auto_url, 'frozen')
self.assertEqual(astropy.utils.iers.conf.iers_auto_url_mirror,
'frozen')
mock_logger().info.assert_has_calls(
[call('Freezing IERS table used by astropy time, coordinates.')])
def main(args=None):
'''
TODO: document
Note: this bypasses specsim since we don't have an arclamp model in
surface brightness units; we only have electrons on the CCD
'''
import desiutil.log
log = desiutil.log.get_logger()
from desiutil.iers import freeze_iers
freeze_iers()
if isinstance(args, (list, tuple, type(None))):
args = parse(args)
log.info('reading arc data from {}'.format(args.arcfile))
arcdata = astropy.table.Table.read(args.arcfile)
wave, phot, fibermap = \
desisim.simexp.simarc(arcdata, nspec=args.nspec, nonuniform=args.nonuniform)
log.info('Writing {}'.format(args.fibermap))
fibermap.meta['NIGHT'] = args.night
fibermap.meta['EXPID'] = args.expid
fibermap.meta['EXTNAME'] = 'FIBERMAP'
fibermap.write(args.fibermap, overwrite=args.clobber)
#- TODO: explain bypassing desisim.io.write_simspec
header = fits.Header()
desiutil.depend.add_dependencies(header)
header['EXPID'] = args.expid
header['NIGHT'] = args.night
header['FLAVOR'] = 'arc'
header['DOSVER'] = 'SIM'
header['EXPTIME'] = 5 #- TODO: add exptime support
#- TODO: DATE-OBS on night instead of now
tx = astropy.time.Time(datetime.datetime(*time.gmtime()[0:6]))
header['DATE-OBS'] = tx.utc.isot
desisim.io.write_simspec_arc(args.simspec,
wave,
phot,
header,
fibermap,
overwrite=args.clobber)
def main(args=None):
'''
Generates a new flat exposure; see newflat --help for usage options
'''
import desiutil.log
log = desiutil.log.get_logger()
from desiutil.iers import freeze_iers
freeze_iers()
if isinstance(args, (list, tuple, type(None))):
args = parse(args)
sim, fibermap = \
desisim.simexp.simflat(args.flatfile, nspec=args.nspec, nonuniform=args.nonuniform)
log.info('Writing {}'.format(args.fibermap))
fibermap.meta['NIGHT'] = args.night
fibermap.meta['EXPID'] = args.expid
fibermap.meta['EXTNAME'] = 'FIBERMAP'
fibermap.write(args.fibermap, overwrite=args.clobber)
header = fits.Header()
desiutil.depend.add_dependencies(header)
header['EXPID'] = args.expid
header['NIGHT'] = args.night
header['FLAVOR'] = 'flat'
header['DOSVER'] = 'SIM'
#- Set calibrations as happening at 15:00 AZ local time = 22:00 UTC
year = int(args.night[0:4])
month = int(args.night[4:6])
day = int(args.night[6:8])
tx = astropy.time.Time(datetime.datetime(year, month, day, 22, 0, 0))
header['DATE-OBS'] = tx.utc.isot
#- metadata truth and obs dictionary are None
desisim.io.write_simspec(sim,
None,
fibermap,
None,
args.expid,
args.night,
filename=args.simspec,
header=header,
overwrite=args.clobber)
def dateobs2night(dateobs):
'''
Convert UTC dateobs to KPNO YEARMMDD night string
Args:
dateobs:
float -> interpret as MJD
str -> interpret as ISO 8601 YEAR-MM-DDThh:mm:ss.s string
astropy.time.Time -> UTC
python datetime.datetime -> UTC
TODO: consider adding format option to pass to astropy.time.Time without
otherwise questioning the dateobs format
'''
# See pixsim.py
from desiutil.iers import freeze_iers
freeze_iers()
import astropy.time
import datetime
if isinstance(dateobs, float):
dateobs = astropy.time.Time(dateobs, format='mjd')
elif isinstance(dateobs, datetime.datetime):
dateobs = astropy.time.Time(dateobs, format='datetime')
elif isinstance(dateobs, str):
dateobs = astropy.time.Time(dateobs, format='isot')
elif not isinstance(dateobs, astropy.time.Time):
raise ValueError('dateobs must be float, str, datetime, or astropy time object')
import astropy.units as u
kpno_time = dateobs - 7*u.hour
#- "night" rolls over at local noon, not midnight, so subtract another 12 hours
yearmmdd = (kpno_time - 12*u.hour).isot[0:10].replace('-', '')
assert len(yearmmdd) == 8
return yearmmdd
def simulate_spectra(wave,
flux,
fibermap=None,
obsconditions=None,
redshift=None,
dwave_out=None,
seed=None,
psfconvolve=True,
specsim_config_file="desi"):
'''
Simulates an exposure without reading/writing data files
Args:
wave (array): 1D wavelengths in Angstroms
flux (array): 2D[nspec,nwave] flux in 1e-17 erg/s/cm2/Angstrom
or astropy Quantity with flux units
fibermap (Table, optional): table from fiberassign or fibermap; uses
X/YFOCAL_DESIGN, TARGETID, DESI_TARGET
obsconditions(dict-like, optional): observation metadata including
SEEING (arcsec), EXPTIME (sec), AIRMASS,
MOONFRAC (0-1), MOONALT (deg), MOONSEP (deg)
redshift (array-like, optional): list/array with each index being the redshifts for that target
seed (int, optional): random seed
psfconvolve (bool, optional): passed to simspec.simulator.Simulator camera_output.
if True, convolve with PSF and include per-camera outputs
specsim_config_file (str, optional): path to DESI instrument config file.
default is desi config in specsim package.
Returns:
A specsim.simulator.Simulator object
TODO: galsim support
'''
import specsim.simulator
import specsim.config
import astropy.units as u
from astropy.coordinates import SkyCoord
from desiutil.log import get_logger
log = get_logger('DEBUG')
freeze_iers()
# Input cosmology to calculate the angular diameter distance of the galaxy's redshift
from astropy.cosmology import FlatLambdaCDM
LCDM = FlatLambdaCDM(H0=70, Om0=0.3)
ang_diam_dist = LCDM.angular_diameter_distance
random_state = np.random.RandomState(seed)
nspec, nwave = flux.shape
#- Convert to unit-ful quantities for specsim
if not isinstance(flux, u.Quantity):
fluxunits = 1e-17 * u.erg / (u.Angstrom * u.s * u.cm**2)
flux = flux * fluxunits
if not isinstance(wave, u.Quantity):
wave = wave * u.Angstrom
log.debug('loading specsim desi config {}'.format(specsim_config_file))
config = _specsim_config_for_wave(wave.to('Angstrom').value,
dwave_out=dwave_out,
specsim_config_file=specsim_config_file)
#- Create simulator
log.debug('creating specsim desi simulator')
# desi = specsim.simulator.Simulator(config, num_fibers=nspec)
desi = desisim.specsim.get_simulator(config,
num_fibers=nspec,
camera_output=psfconvolve)
if obsconditions is None:
log.warning('Assuming DARK conditions')
obsconditions = reference_conditions['DARK']
elif isinstance(obsconditions, str):
obsconditions = reference_conditions[obsconditions.upper()]
desi.atmosphere.seeing_fwhm_ref = obsconditions['SEEING'] * u.arcsec
desi.observation.exposure_time = obsconditions['EXPTIME'] * u.s
desi.atmosphere.airmass = obsconditions['AIRMASS']
desi.atmosphere.moon.moon_phase = np.arccos(2 * obsconditions['MOONFRAC'] -
1) / np.pi
desi.atmosphere.moon.moon_zenith = (90 - obsconditions['MOONALT']) * u.deg
desi.atmosphere.moon.separation_angle = obsconditions['MOONSEP'] * u.deg
try:
desi.observation.exposure_start = astropy.time.Time(
obsconditions['MJD'], format='mjd')
log.info('exposure_start {}'.format(
desi.observation.exposure_start.utc.isot))
except KeyError:
log.info('MJD not in obsconditions, using DATE-OBS {}'.format(
desi.observation.exposure_start.utc.isot))
for obskey in reference_conditions['DARK'].keys():
obsval = obsconditions[obskey]
log.debug('obsconditions {} = {}'.format(obskey, obsval))
#- Set fiber locations from meta Table or default fiberpos
fiberpos = desimodel.io.load_fiberpos()
if fibermap is not None and len(fiberpos) != len(fibermap):
ii = np.in1d(fiberpos['FIBER'], fibermap['FIBER'])
fiberpos = fiberpos[ii]
if fibermap is None:
fibermap = astropy.table.Table()
fibermap['X'] = fiberpos['X'][0:nspec]
fibermap['Y'] = fiberpos['Y'][0:nspec]
fibermap['FIBER'] = fiberpos['FIBER'][0:nspec]
fibermap['LOCATION'] = fiberpos['LOCATION'][0:nspec]
#- Extract fiber locations from meta Table -> xy[nspec,2]
assert np.all(fibermap['FIBER'] == fiberpos['FIBER'][0:nspec])
if 'XFOCAL_DESIGN' in fibermap.dtype.names:
xy = np.vstack([fibermap['XFOCAL_DESIGN'], fibermap['YFOCAL_DESIGN']
]).T * u.mm
elif 'X' in fibermap.dtype.names:
xy = np.vstack([fibermap['X'], fibermap['Y']]).T * u.mm
else:
xy = np.vstack([fiberpos['X'], fiberpos['Y']]).T * u.mm
if 'TARGETID' in fibermap.dtype.names:
unassigned = (fibermap['TARGETID'] == -1)
if np.any(unassigned):
#- see https://github.com/astropy/astropy/issues/5961
#- for the units -> array -> units trick
xy[unassigned, 0] = np.asarray(fiberpos['X'][unassigned],
dtype=xy.dtype) * u.mm
xy[unassigned, 1] = np.asarray(fiberpos['Y'][unassigned],
dtype=xy.dtype) * u.mm
#- Determine source types
#- TODO: source shapes + galsim instead of fixed types + fiberloss table
source_types = get_source_types(fibermap)
# source types are sky elg lrg qso bgs star , they
# are only used in specsim.fiberloss for the desi.instrument.fiberloss_method="table" method
if specsim_config_file == "desi":
desi.instrument.fiberloss_method = 'fastsim'
log.debug('running simulation with {} fiber loss method'.format(
desi.instrument.fiberloss_method))
unique_source_types = set(source_types)
comment_line = "source types:"
for u in set(source_types):
comment_line += " {} {}".format(np.sum(source_types == u), u)
log.debug(comment_line)
source_fraction = None
source_half_light_radius = None
source_minor_major_axis_ratio = None
source_position_angle = None
if desi.instrument.fiberloss_method == 'fastsim' or desi.instrument.fiberloss_method == 'galsim':
# the following parameters are used only with fastsim and galsim methods
elgs = (source_types == "elg")
lrgs = (source_types == "lrg")
bgss = (source_types == "bgs")
if np.sum(lrgs) > 0 or np.sum(elgs) > 0:
log.warning(
"the half light radii are fixed here for LRGs and ELGs (and not magnitude or redshift dependent)"
)
if np.sum(bgss) > 0 and redshift is None:
log.warning(
"the half light radii are fixed here for BGS (as redshifts weren't supplied)"
)
# BGS parameters based on SDSS main sample, in g-band
# see analysis from J. Moustakas in
# https://github.com/desihub/desitarget/blob/master/doc/nb/bgs-morphology-properties.ipynb
# B/T (bulge-to-total ratio): 0.48 (0.36 - 0.59).
# Bulge Sersic n: 2.27 (1.12 - 3.60).
# log10 (Bulge Half-light radius): 0.11 (-0.077 - 0.307) arcsec
# log10 (Disk Half-light radius): 0.67 (0.54 - 0.82) arcsec
# This gives
# bulge_fraction = 0.48
# disk_fraction = 0.52
# bulge_half_light_radius = 1.3 arcsec
# disk_half_light_radius = 4.7 arcsec
# note we use De Vaucouleurs' law , which correspond to a Sersic index n=4
# source_fraction[:,0] is DISK profile (exponential) fraction
# source_fraction[:,1] is BULGE profile (devaucouleurs) fraction
# 1 - np.sum(source_fraction,axis=1) is POINT source profile fraction
# see specsim.GalsimFiberlossCalculator.create_source routine
source_fraction = np.zeros((nspec, 2))
source_fraction[elgs, 0] = 1. # ELG are disk only
source_fraction[lrgs, 1] = 1. # LRG are bulge only
source_fraction[bgss, 0] = 0.52 # disk comp in BGS
source_fraction[bgss, 1] = 0.48 # bulge comp in BGS
# source_half_light_radius[:,0] is the half light radius in arcsec for the DISK profile
# source_half_light_radius[:,1] is the half light radius in arcsec for the BULGE profile
# see specsim.GalsimFiberlossCalculator.create_source routine
source_half_light_radius = np.zeros((nspec, 2))
source_half_light_radius[elgs, 0] = 0.45 # ELG are disk only, arcsec
source_half_light_radius[lrgs, 1] = 1. # LRG are bulge only, arcsec
# 4.7 is angular size of z=0.1 disk, and 1.3 is angular size of z=0.1 bulge
bgs_disk_z01 = 4.7 # in arcsec
bgs_bulge_z01 = 1.3 # in arcsec
# Convert to angular size of the objects in this sample with given redshifts
if redshift is None:
angscales = np.ones(np.sum(bgss))
else:
bgs_redshifts = redshift[bgss]
# Avoid infinities
if np.any(bgs_redshifts <= 0.):
bgs_redshifts[bgs_redshifts <= 0.] = 0.0001
angscales = (ang_diam_dist(0.1) /
ang_diam_dist(bgs_redshifts)).value
source_half_light_radius[
bgss, 0] = bgs_disk_z01 * angscales # disk comp in BGS, arcsec
source_half_light_radius[
bgss, 1] = bgs_bulge_z01 * angscales # bulge comp in BGS, arcsec
if desi.instrument.fiberloss_method == 'galsim':
# the following parameters are used only with galsim method
# source_minor_major_axis_ratio[:,0] is the axis ratio for the DISK profile
# source_minor_major_axis_ratio[:,1] is the axis ratio for the BULGE profile
# see specsim.GalsimFiberlossCalculator.create_source routine
source_minor_major_axis_ratio = np.zeros((nspec, 2))
source_minor_major_axis_ratio[elgs, 0] = 0.7
source_minor_major_axis_ratio[lrgs, 1] = 0.7
source_minor_major_axis_ratio[bgss, 1] = 0.7
# the source position angle is in degrees
# see specsim.GalsimFiberlossCalculator.create_source routine
source_position_angle = np.zeros((nspec, 2))
random_angles = 360. * random_state.uniform(size=nspec)
source_position_angle[elgs, 0] = random_angles[elgs]
source_position_angle[lrgs, 1] = random_angles[lrgs]
source_position_angle[bgss, 1] = random_angles[bgss]
#- Work around randomness in specsim quickfiberloss calculations
#- while not impacting global random state.
#- See https://github.com/desihub/specsim/issues/83
randstate = np.random.get_state()
np.random.seed(seed)
desi.simulate(source_fluxes=flux,
focal_positions=xy,
source_types=source_types,
source_fraction=source_fraction,
source_half_light_radius=source_half_light_radius,
source_minor_major_axis_ratio=source_minor_major_axis_ratio,
source_position_angle=source_position_angle)
np.random.set_state(randstate)
return desi
def simscience(targets,
fiberassign,
obsconditions='DARK',
expid=None,
nspec=None,
psfconvolve=True):
'''
Simulates a new DESI exposure from surveysim+fiberassign+mock spectra
Args:
targets (tuple): tuple of (flux[nspec,nwave], wave[nwave], meta[nspec])
fiberassign (Table): fiber assignments table
obsconditions (object, optional): observation metadata as
str: DARK (default) or GRAY or BRIGHT
dict or row of Table with keys::
SEEING (arcsec), EXPTIME (sec), AIRMASS,
MOONFRAC (0-1), MOONALT (deg), MOONSEP (deg)
Table including EXPID for subselection of which row to use
filename with obsconditions Table; expid must also be set
expid (int, optional): exposure ID
nspec (int, optional): number of spectra to simulate
psfconvolve (bool, optional): passed to simspec.simulator.Simulator camera_output.
if True, convolve with PSF and include per-camera outputs
Returns: (sim, fibermap, meta)
sim: specsim.simulate.Simulator object
fibermap: Table
meta: target metadata truth table
See obs.new_exposure() for function to generate new random exposure,
independent from surveysim, fiberassignment, and pre-generated mocks.
'''
from desiutil.log import get_logger
log = get_logger()
freeze_iers()
flux, wave, meta = targets
if nspec is not None:
fiberassign = fiberassign[0:nspec]
flux = flux[0:nspec]
meta = meta[0:nspec]
assert np.all(fiberassign['TARGETID'] == meta['TARGETID'])
fibermap = fibermeta2fibermap(fiberassign, meta)
#- Parse multiple options for obsconditions
if isinstance(obsconditions, str):
#- DARK GRAY BRIGHT
if obsconditions.upper() in reference_conditions:
log.info('Using reference {} obsconditions'.format(
obsconditions.upper()))
obsconditions = reference_conditions[obsconditions.upper()]
#- filename
elif os.path.exists(obsconditions):
log.info('Loading obsconditions from {}'.format(
obsconditions.upper()))
if obsconditions.endswith('.ecsv'):
allobs = astropy.table.Table.read(obsconditions,
format='ascii.ecsv')
else:
allobs = astropy.table.Table.read(obsconditions)
#- trim down to just this exposure
if (expid is not None) and 'EXPID' in allobs.colnames:
obsconditions = allobs[allobs['EXPID'] == expid]
else:
raise ValueError(
'unable to select which exposure from obsconditions file')
else:
raise ValueError('bad obsconditions {}'.format(obsconditions))
elif isinstance(obsconditions, (astropy.table.Table, np.ndarray)):
#- trim down to just this exposure
if (expid is not None) and ('EXPID' in obsconditions):
obsconditions = allobs[allobs['EXPID'] == expid]
else:
raise ValueError(
'must provide expid when providing obsconditions as a Table')
#- Validate obsconditions keys
try:
obskeys = set(obsconditions.dtype.names)
except AttributeError:
obskeys = set(obsconditions.keys())
missing_keys = set(reference_conditions['DARK'].keys()) - obskeys
if len(missing_keys) > 0:
raise ValueError('obsconditions missing keys {}'.format(missing_keys))
sim = simulate_spectra(wave,
flux,
fibermap=fibermap,
obsconditions=obsconditions,
psfconvolve=psfconvolve)
return sim, fibermap
def simflat(flatfile,
nspec=5000,
nonuniform=False,
exptime=10,
testslit=False,
psfconvolve=True,
specsim_config_file="desi"):
'''
Simulates a flat lamp calibration exposure
Args:
flatfile (str): filename with flat lamp spectrum data
nspec (int, optional): number of spectra to simulate
nonuniform (bool, optional): include calibration screen non-uniformity
exptime (float, optional): exposure time in seconds
psfconvolve (bool, optional): passed to simspec.simulator.Simulator camera_output.
if True, convolve with PSF and include per-camera outputs
specsim_config_file (str, optional): path to DESI instrument config file.
default is desi config in specsim package.
Returns: (sim, fibermap)
sim: specsim Simulator object
fibermap: fibermap Table
'''
import astropy.units as u
import specsim.simulator
from desiutil.log import get_logger
log = get_logger()
freeze_iers()
log.info('Reading flat lamp spectrum from {}'.format(flatfile))
sbflux, hdr = fits.getdata(flatfile, header=True)
wave = desispec.io.util.header2wave(hdr)
assert len(wave) == len(sbflux)
#- Trim to DESI wavelength ranges
#- TODO: is there an easier way to get these parameters?
try:
params = desimodel.io.load_desiparams()
wavemin = params['ccd']['b']['wavemin']
wavemax = params['ccd']['z']['wavemax']
except KeyError:
wavemin = desimodel.io.load_throughput('b').wavemin
wavemax = desimodel.io.load_throughput('z').wavemax
ii = (wavemin <= wave) & (wave <= wavemax)
wave = wave[ii]
sbflux = sbflux[ii]
#- Downsample to 0.2A grid to not blow up memory
ww = np.arange(wave[0], wave[-1] + 0.1, 0.2)
sbflux = desispec.interpolation.resample_flux(ww, wave, sbflux)
wave = ww
if testslit:
fibermap = astropy.table.Table(testslit_fibermap()[0:nspec])
else:
fibermap = astropy.table.Table(desispec.io.empty_fibermap(nspec))
fibermap.meta['FLAVOR'] = 'flat'
fibermap['OBJTYPE'] = 'FLT'
x = fibermap['FIBERASSIGN_X']
y = fibermap['FIBERASSIGN_Y']
r = np.sqrt(x**2 + y**2)
xy = np.vstack([x, y]).T * u.mm
#- Convert to unit-ful 2D
sbunit = 1e-17 * u.erg / (u.Angstrom * u.s * u.cm**2 * u.arcsec**2)
sbflux = np.tile(sbflux, nspec).reshape(nspec, len(wave)) * sbunit
if nonuniform:
ratio = _calib_screen_uniformity(radius=r)
assert np.all(ratio <= 1) and np.all(ratio > 0.99)
sbflux = (sbflux.T * ratio).T
tmp = np.min(sbflux) / np.max(sbflux)
log.info(
'Adjusting for calibration screen non-uniformity {:.4f}'.format(
tmp))
log.debug('Creating specsim configuration')
config = _specsim_config_for_wave(wave,
specsim_config_file=specsim_config_file)
log.debug('Creating specsim simulator for {} spectra'.format(nspec))
# sim = specsim.simulator.Simulator(config, num_fibers=nspec)
sim = desisim.specsim.get_simulator(config,
num_fibers=nspec,
camera_output=psfconvolve)
sim.observation.exposure_time = exptime * u.s
log.debug('Simulating')
sim.simulate(calibration_surface_brightness=sbflux, focal_positions=xy)
return sim, fibermap
def integration_test(night=None, nspec=5, clobber=False):
"""Run an integration test from raw data simulations through redshifts
Args:
night (str, optional): YEARMMDD, defaults to current night
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
import argparse
parser = argparse.ArgumentParser(usage="{prog} [options]")
# parser.add_argument("-i", "--input", type=str, help="input data")
# parser.add_argument("-o", "--output", type=str, help="output data")
parser.add_argument("--skip-psf",
action="store_true",
help="Skip PSF fitting step")
args = parser.parse_args()
from desiutil.iers import freeze_iers
freeze_iers()
log = logging.get_logger()
# YEARMMDD string, rolls over at noon not midnight
if night is None:
night = "20160726"
# check for required environment variables
check_env()
# simulate inputs
sim(night, nspec=nspec, clobber=clobber)
# raw and production locations
rawdir = os.path.abspath(io.rawdata_root())
proddir = os.path.abspath(io.specprod_root())
# create production
if clobber and os.path.isdir(proddir):
shutil.rmtree(proddir)
dbfile = io.get_pipe_database()
if not os.path.exists(dbfile):
com = "desi_pipe create --db-sqlite"
log.info('Running {}'.format(com))
sp.check_call(com, shell=True)
else:
log.info("Using pre-existing production database {}".format(dbfile))
# Modify options file to restrict the spectral range
optpath = os.path.join(proddir, "run", "options.yaml")
opts = pipe.prod.yaml_read(optpath)
opts['extract']['specmin'] = 0
opts['extract']['nspec'] = nspec
opts['psf']['specmin'] = 0
opts['psf']['nspec'] = nspec
opts['traceshift']['nfibers'] = nspec
pipe.prod.yaml_write(optpath, opts)
if args.skip_psf:
#- Copy desimodel psf into this production instead of fitting psf
import shutil
for channel in ['b', 'r', 'z']:
refpsf = '{}/data/specpsf/psf-{}.fits'.format(
os.getenv('DESIMODEL'), channel)
nightpsf = io.findfile('psfnight', night, camera=channel + '0')
shutil.copy(refpsf, nightpsf)
for expid in [0, 1, 2]:
exppsf = io.findfile('psf', night, expid, camera=channel + '0')
shutil.copy(refpsf, exppsf)
#- Resync database to current state
dbpath = io.get_pipe_database()
db = pipe.load_db(dbpath, mode="w")
db.sync(night)
# Run the pipeline tasks in order
from desispec.pipeline.tasks.base import default_task_chain
for tasktype in default_task_chain:
#- if we skip psf/psfnight/traceshift, update state prior to extractions
if tasktype == 'traceshift' and args.skip_psf:
db.getready()
run_pipeline_step(tasktype)
# #-----
# #- Did it work?
# #- (this combination of fibermap, simspec, and zbest is a pain)
expid = 2
fmfile = io.findfile('fibermap', night=night, expid=expid)
fibermap = io.read_fibermap(fmfile)
simdir = os.path.dirname(fmfile)
simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid)
siminfo = fits.getdata(simspec, 'TRUTH')
try:
elginfo = fits.getdata(simspec, 'TRUTH_ELG')
except:
elginfo = None
from desimodel.footprint import radec2pix
nside = 64
pixels = np.unique(
radec2pix(nside, fibermap['TARGET_RA'], fibermap['TARGET_DEC']))
num_missing = 0
for pix in pixels:
zfile = io.findfile('zbest', groupname=pix)
if not os.path.exists(zfile):
log.error('Missing {}'.format(zfile))
num_missing += 1
if num_missing > 0:
log.critical('{} zbest files missing'.format(num_missing))
sys.exit(1)
print()
print("--------------------------------------------------")
print("Pixel True z -> Class z zwarn")
# print("3338p190 SKY 0.00000 -> QSO 1.60853 12 - ok")
for pix in pixels:
zfile = io.findfile('zbest', groupname=pix)
if not os.path.exists(zfile):
log.error('Missing {}'.format(zfile))
continue
zfx = fits.open(zfile, memmap=False)
zbest = zfx['ZBEST'].data
for i in range(len(zbest['Z'])):
objtype = zbest['SPECTYPE'][i]
z, zwarn = zbest['Z'][i], zbest['ZWARN'][i]
j = np.where(fibermap['TARGETID'] == zbest['TARGETID'][i])[0][0]
truetype = siminfo['OBJTYPE'][j]
oiiflux = 0.0
if truetype == 'ELG':
k = np.where(elginfo['TARGETID'] == zbest['TARGETID'][i])[0][0]
oiiflux = elginfo['OIIFLUX'][k]
truez = siminfo['REDSHIFT'][j]
dv = C_LIGHT * (z - truez) / (1 + truez)
status = None
if truetype == 'SKY' and zwarn > 0:
status = 'ok'
elif truetype == 'ELG' and zwarn > 0 and oiiflux < 8e-17:
status = 'ok ([OII] flux {:.2g})'.format(oiiflux)
elif zwarn == 0:
if truetype == 'LRG' and objtype == 'GALAXY' and abs(dv) < 150:
status = 'ok'
elif truetype == 'ELG' and objtype == 'GALAXY':
if abs(dv) < 150:
status = 'ok'
elif oiiflux < 8e-17:
status = 'ok ([OII] flux {:.2g})'.format(oiiflux)
else:
status = 'OOPS ([OII] flux {:.2g})'.format(oiiflux)
elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750:
status = 'ok'
elif truetype in ('STD', 'FSTD') and objtype == 'STAR':
status = 'ok'
else:
status = 'OOPS'
else:
status = 'OOPS'
print('{0:<8d} {1:4s} {2:8.5f} -> {3:5s} {4:8.5f} {5:4d} - {6}'.
format(pix, truetype, truez, objtype, z, zwarn, status))
print("--------------------------------------------------")
def integration_test(night=None, nspec=5, clobber=False):
"""Run an integration test from raw data simulations through redshifts.
Args:
night (str, optional): YEARMMDD, defaults to current night
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
from desiutil.iers import freeze_iers
freeze_iers()
log = get_logger()
#- YEARMMDD string, rolls over at noon not midnight
#- Simulate 8 years ago, prior to start of survey
if night is None:
night = time.strftime(
'%Y%m%d',
time.localtime(time.time() - 12 * 3600 - (8 * 365 * 24 * 3600)))
#- check for required environment variables
check_env()
#- parameter dictionary that will later be used for formatting commands
params = dict(night=night, nspec=nspec)
#-----
#- Input fibermaps, spectra, and pixel-level raw data
# raw_dict = {0: 'flat', 1: 'arc', 2: 'dark'}
programs = ('flat', 'arc', 'dark')
channels = ('b', 'r', 'z')
cameras = ('b0', 'r0', 'z0')
# for expid, program in raw_dict.items():
for expid, program in enumerate(programs):
cmd = "newexp-random --program {program} --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, program=program, **params)
fibermap = io.findfile('fibermap', night, expid)
simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap),
expid)
inputs = []
outputs = [fibermap, simspec]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError(
'pixsim newexp failed for {} exposure {}'.format(
program, expid))
cmd = "pixsim --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, **params)
inputs = [fibermap, simspec]
outputs = [
fibermap.replace('fibermap-', 'simpix-'),
]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('pixsim failed for {} exposure {}'.format(
program, expid))
#-----
#- Preproc
for expid, program in enumerate(programs):
rawfile = io.findfile('desi', night, expid)
outdir = os.path.dirname(io.findfile('preproc', night, expid, 'b0'))
cmd = "desi_preproc --cameras b0,r0,z0 --infile {} --outdir {} --ncpu 1".format(
rawfile, outdir)
inputs = [
rawfile,
]
outputs = list()
for camera in cameras:
outputs.append(io.findfile('preproc', night, expid, camera))
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('preproc failed for expid {}'.format(expid))
#-----
#- Extract
waverange = dict(b="3570,5940,1.0", r="5630,7740,1.0", z="7440,9830,1.0")
for expid, program in enumerate(programs):
for ic, channel in enumerate(channels):
pixfile = io.findfile('preproc', night, expid, cameras[ic])
fiberfile = io.findfile('fibermap', night, expid)
psffile = '{}/data/specpsf/psf-{}.fits'.format(
os.getenv('DESIMODEL'), channel)
framefile = io.findfile('frame', night, expid, cameras[ic])
# cmd = "exspec -i {pix} -p {psf} --specmin 0 --nspec {nspec} -w {wave} -o {frame}".format(
# pix=pixfile, psf=psffile, wave=waverange[channel], frame=framefile, **params)
cmd = "desi_extract_spectra -i {pix} -p {psf} -f {fibermap} --specmin 0 --nspec {nspec} -o {frame}".format(
pix=pixfile,
psf=psffile,
frame=framefile,
fibermap=fiberfile,
**params)
inputs = [pixfile, psffile, fiberfile]
outputs = [
framefile,
]
if runcmd(cmd, inputs=inputs, outputs=outputs,
clobber=clobber) != 0:
raise RuntimeError('extraction failed for {} expid {}'.format(
cameras[ic], expid))
#-----
#- Fiber flat
expid = 0
for ic, channel in enumerate(channels):
framefile = io.findfile('frame', night, expid, cameras[ic])
fiberflat = io.findfile('fiberflat', night, expid, cameras[ic])
fibermap = io.findfile('fibermap', night, expid) # for QA
qafile = io.findfile('qa_calib', night, expid, cameras[ic])
qafig = io.findfile('qa_flat_fig', night, expid, cameras[ic])
cmd = "desi_compute_fiberflat --infile {frame} --outfile {fiberflat} --qafile {qafile} --qafig {qafig}".format(
frame=framefile,
fiberflat=fiberflat,
qafile=qafile,
qafig=qafig,
**params)
inputs = [
framefile,
fibermap,
]
outputs = [
fiberflat,
qafile,
qafig,
]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('fiberflat failed for ' + cameras[ic])
#-----
#- Sky model
flat_expid = 0
expid = 2
for ic, channel in enumerate(channels):
framefile = io.findfile('frame', night, expid, cameras[ic])
fibermap = io.findfile('fibermap', night, expid)
fiberflat = io.findfile('fiberflat', night, flat_expid, cameras[ic])
skyfile = io.findfile('sky', night, expid, cameras[ic])
qafile = io.findfile('qa_data', night, expid, cameras[ic])
qafig = io.findfile('qa_sky_fig', night, expid, cameras[ic])
cmd = "desi_compute_sky --infile {frame} --fiberflat {fiberflat} --outfile {sky} --qafile {qafile} --qafig {qafig}".format(
frame=framefile,
fiberflat=fiberflat,
sky=skyfile,
qafile=qafile,
qafig=qafig,
**params)
inputs = [framefile, fibermap, fiberflat]
outputs = [
skyfile,
qafile,
qafig,
]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('sky model failed for ' + cameras[ic])
#-----
#- Fit standard stars
if 'STD_TEMPLATES' in os.environ:
std_templates = os.getenv('STD_TEMPLATES')
else:
std_templates = os.getenv(
'DESI_ROOT'
) + '/spectro/templates/star_templates/v1.1/star_templates_v1.1.fits'
stdstarfile = io.findfile('stdstars', night, expid, spectrograph=0)
flats = list()
frames = list()
skymodels = list()
for ic, channel in enumerate(channels):
frames.append(io.findfile('frame', night, expid, cameras[ic]))
flats.append(io.findfile('fiberflat', night, flat_expid, cameras[ic]))
skymodels.append(io.findfile('sky', night, expid, cameras[ic]))
frames = ' '.join(frames)
flats = ' '.join(flats)
skymodels = ' '.join(skymodels)
cmd = """desi_fit_stdstars \
--frames {frames} \
--fiberflats {flats} \
--skymodels {skymodels} \
--starmodels {std_templates} \
-o {stdstars}""".format(frames=frames,
flats=flats,
skymodels=skymodels,
std_templates=std_templates,
stdstars=stdstarfile)
inputs = [fibermap, std_templates]
outputs = [
stdstarfile,
]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('fitting stdstars failed')
#-----
#- Flux calibration
for ic, channel in enumerate(channels):
framefile = io.findfile('frame', night, expid, cameras[ic])
fibermap = io.findfile('fibermap', night, expid)
fiberflat = io.findfile('fiberflat', night, flat_expid, cameras[ic])
skyfile = io.findfile('sky', night, expid, cameras[ic])
calibfile = io.findfile('calib', night, expid, cameras[ic])
qafile = io.findfile('qa_data', night, expid, cameras[ic])
qafig = io.findfile('qa_flux_fig', night, expid, cameras[ic])
#- Compute flux calibration vector
cmd = """desi_compute_fluxcalibration \
--infile {frame} --fiberflat {fiberflat} --sky {sky} \
--models {stdstars} --outfile {calib} --qafile {qafile} --qafig {qafig}""".format(
frame=framefile,
fiberflat=fiberflat,
sky=skyfile,
stdstars=stdstarfile,
calib=calibfile,
qafile=qafile,
qafig=qafig)
inputs = [framefile, fibermap, fiberflat, skyfile, stdstarfile]
outputs = [calibfile, qafile, qafig]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('flux calibration failed for ' + cameras[ic])
#- Apply the flux calibration to write a cframe file
cframefile = io.findfile('cframe', night, expid, cameras[ic])
cmd = """desi_process_exposure \
--infile {frame} --fiberflat {fiberflat} --sky {sky} --calib {calib} \
--outfile {cframe}""".format(frame=framefile,
fibermap=fibermap,
fiberflat=fiberflat,
sky=skyfile,
calib=calibfile,
cframe=cframefile)
inputs = [framefile, fiberflat, skyfile, calibfile]
outputs = [
cframefile,
]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('combining calibration steps failed for ' +
cameras[ic])
#-----
#- Collate QA
# Collate data QA
program2flavor = dict(arc='arc', flat='flat')
for program in ('dark', 'gray', 'bright', 'elg', 'lrg', 'qso', 'bgs',
'mws'):
program2flavor[program] = 'science'
expid = 2
qafile = io.findfile('qa_data_exp', night, expid)
if clobber or not os.path.exists(qafile):
flavor = program2flavor[programs[expid]]
qaexp_data = QA_Exposure(expid, night, flavor) # Removes camera files
io.write_qa_exposure(os.path.splitext(qafile)[0], qaexp_data)
if not os.path.exists(qafile):
raise RuntimeError(
'FAILED data QA_Exposure({},{}, ...) -> {}'.format(
expid, night, qafile))
# Collate calib QA
calib_expid = [0, 1]
for expid in calib_expid:
qafile = io.findfile('qa_calib_exp', night, expid)
if clobber or not os.path.exists(qafile):
qaexp_calib = QA_Exposure(expid, night, programs[expid])
io.write_qa_exposure(os.path.splitext(qafile)[0], qaexp_calib)
if not os.path.exists(qafile):
raise RuntimeError(
'FAILED calib QA_Exposure({},{}, ...) -> {}'.format(
expid, night, qafile))
#-----
#- Regroup cframe -> spectra
expid = 2
inputs = list()
for camera in cameras:
inputs.append(io.findfile('cframe', night, expid, camera))
outputs = list()
fibermap = io.read_fibermap(io.findfile('fibermap', night, expid))
from desimodel.footprint import radec2pix
nside = 64
pixels = np.unique(
radec2pix(nside, fibermap['TARGET_RA'], fibermap['TARGET_DEC']))
for pix in pixels:
outputs.append(io.findfile('spectra', groupname=pix))
cmd = "desi_group_spectra"
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('spectra regrouping failed')
#-----
#- Redshifts!
for pix in pixels:
specfile = io.findfile('spectra', groupname=pix)
zbestfile = io.findfile('zbest', groupname=pix)
inputs = [
specfile,
]
outputs = [
zbestfile,
]
cmd = "rrdesi {} --zbest {}".format(specfile, zbestfile)
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('rrdesi failed for healpixel {}'.format(pix))
#
# Load redshifts into database
#
options = get_options(
'--overwrite', '--filename', 'dailytest.db',
os.path.join(os.environ['DESI_SPECTRO_REDUX'], os.environ['SPECPROD']))
postgresql = setup_db(options)
load_zbest(options.datapath)
# ztruth QA
# qafile = io.findfile('qa_ztruth', night)
# qafig = io.findfile('qa_ztruth_fig', night)
# cmd = "desi_qa_zfind --night {night} --qafile {qafile} --qafig {qafig} --verbose".format(
# night=night, qafile=qafile, qafig=qafig)
# inputs = []
# outputs = [qafile, qafig]
# if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
# raise RuntimeError('redshift QA failed for night '+night)
#-----
#- Did it work?
#- (this combination of fibermap, simspec, and zbest is a pain)
simdir = os.path.dirname(io.findfile('fibermap', night=night, expid=expid))
simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid)
siminfo = fits.getdata(simspec, 'TRUTH')
try:
elginfo = fits.getdata(simspec, 'TRUTH_ELG')
except:
elginfo = None
print()
print("--------------------------------------------------")
print("Pixel True z -> Class z zwarn")
# print("3338p190 SKY 0.00000 -> QSO 1.60853 12 - ok")
for pix in pixels:
zbest = fits.getdata(io.findfile('zbest', groupname=pix))
for i in range(len(zbest)):
objtype = zbest['SPECTYPE'][i]
z, zwarn = zbest['Z'][i], zbest['ZWARN'][i]
j = np.where(fibermap['TARGETID'] == zbest['TARGETID'][i])[0][0]
truetype = siminfo['OBJTYPE'][j]
oiiflux = 0.0
if truetype == 'ELG':
k = np.where(elginfo['TARGETID'] == zbest['TARGETID'][i])[0][0]
oiiflux = elginfo['OIIFLUX'][k]
truez = siminfo['REDSHIFT'][j]
dv = C_LIGHT * (z - truez) / (1 + truez)
if truetype == 'SKY' and zwarn > 0:
status = 'ok'
elif truetype == 'ELG' and zwarn > 0 and oiiflux < 8e-17:
status = 'ok ([OII] flux {:.2g})'.format(oiiflux)
elif zwarn == 0:
if truetype == 'LRG' and objtype == 'GALAXY' and abs(dv) < 150:
status = 'ok'
elif truetype == 'ELG' and objtype == 'GALAXY':
if abs(dv) < 150:
status = 'ok'
elif oiiflux < 8e-17:
status = 'ok ([OII] flux {:.2g})'.format(oiiflux)
else:
status = 'OOPS ([OII] flux {:.2g})'.format(oiiflux)
elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750:
status = 'ok'
elif truetype in ('STD', 'FSTD') and objtype == 'STAR':
status = 'ok'
else:
status = 'OOPS'
else:
status = 'OOPS'
print('{0:<8d} {1:4s} {2:8.5f} -> {3:6s} {4:8.5f} {5:4d} - {6}'.
format(pix, truetype, truez, objtype, z, zwarn, status))
print("--------------------------------------------------")
def main(args):
from desiutil.iers import freeze_iers
freeze_iers()
# Set up the logger
if args.verbose:
log = get_logger(DEBUG)
else:
log = get_logger()
# Make sure all necessary environment variables are set
DESI_SPECTRO_REDUX_DIR = "./quickGen"
if 'DESI_SPECTRO_REDUX' not in os.environ:
log.info('DESI_SPECTRO_REDUX environment is not set.')
else:
DESI_SPECTRO_REDUX_DIR = os.environ['DESI_SPECTRO_REDUX']
if os.path.exists(DESI_SPECTRO_REDUX_DIR):
if not os.path.isdir(DESI_SPECTRO_REDUX_DIR):
raise RuntimeError("Path %s Not a directory" %
DESI_SPECTRO_REDUX_DIR)
else:
try:
os.makedirs(DESI_SPECTRO_REDUX_DIR)
except:
raise
SPECPROD_DIR = 'specprod'
if 'SPECPROD' not in os.environ:
log.info('SPECPROD environment is not set.')
else:
SPECPROD_DIR = os.environ['SPECPROD']
prod_Dir = specprod_root()
if os.path.exists(prod_Dir):
if not os.path.isdir(prod_Dir):
raise RuntimeError("Path %s Not a directory" % prod_Dir)
else:
try:
os.makedirs(prod_Dir)
except:
raise
# Initialize random number generator to use.
np.random.seed(args.seed)
random_state = np.random.RandomState(args.seed)
# Derive spectrograph number from nstart if needed
if args.spectrograph is None:
args.spectrograph = args.nstart / 500
# Read fibermapfile to get object type, night and expid
if args.fibermap:
log.info("Reading fibermap file {}".format(args.fibermap))
fibermap = read_fibermap(args.fibermap)
objtype = get_source_types(fibermap)
stdindx = np.where(objtype == 'STD') # match STD with STAR
mwsindx = np.where(objtype == 'MWS_STAR') # match MWS_STAR with STAR
bgsindx = np.where(objtype == 'BGS') # match BGS with LRG
objtype[stdindx] = 'STAR'
objtype[mwsindx] = 'STAR'
objtype[bgsindx] = 'LRG'
NIGHT = fibermap.meta['NIGHT']
EXPID = fibermap.meta['EXPID']
else:
# Create a blank fake fibermap
fibermap = empty_fibermap(args.nspec)
targetids = random_state.randint(2**62, size=args.nspec)
fibermap['TARGETID'] = targetids
night = get_night()
expid = 0
log.info("Initializing SpecSim with config {}".format(args.config))
desiparams = load_desiparams()
qsim = get_simulator(args.config, num_fibers=1)
if args.simspec:
# Read the input file
log.info('Reading input file {}'.format(args.simspec))
simspec = desisim.io.read_simspec(args.simspec)
nspec = simspec.nspec
if simspec.flavor == 'arc':
log.warning("quickgen doesn't generate flavor=arc outputs")
return
else:
wavelengths = simspec.wave
spectra = simspec.flux
if nspec < args.nspec:
log.info("Only {} spectra in input file".format(nspec))
args.nspec = nspec
else:
# Initialize the output truth table.
spectra = []
wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value
npix = len(wavelengths)
truth = dict()
meta = Table()
truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10))
truth['FLUX'] = np.zeros((args.nspec, npix))
truth['WAVE'] = wavelengths
jj = list()
for thisobj in set(true_objtype):
ii = np.where(true_objtype == thisobj)[0]
nobj = len(ii)
truth['OBJTYPE'][ii] = thisobj
log.info('Generating {} template'.format(thisobj))
# Generate the templates
if thisobj == 'ELG':
elg = desisim.templates.ELG(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = elg.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_elg,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'LRG':
lrg = desisim.templates.LRG(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = lrg.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_lrg,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'QSO':
qso = desisim.templates.QSO(wave=wavelengths)
flux, tmpwave, meta1 = qso.make_templates(
nmodel=nobj, seed=args.seed, zrange=args.zrange_qso)
elif thisobj == 'BGS':
bgs = desisim.templates.BGS(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = bgs.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_bgs,
rmagrange=args.rmagrange_bgs,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'STD':
std = desisim.templates.STD(wave=wavelengths)
flux, tmpwave, meta1 = std.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'QSO_BAD': # use STAR template no color cuts
star = desisim.templates.STAR(wave=wavelengths)
flux, tmpwave, meta1 = star.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'MWS_STAR' or thisobj == 'MWS':
mwsstar = desisim.templates.MWS_STAR(wave=wavelengths)
flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'WD':
wd = desisim.templates.WD(wave=wavelengths)
flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'SKY':
flux = np.zeros((nobj, npix))
meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32)))
elif thisobj == 'TEST':
flux = np.zeros((args.nspec, npix))
indx = np.where(wave > 5800.0 - 1E-6)[0][0]
ref_integrated_flux = 1E-10
ref_cst_flux_density = 1E-17
single_line = (np.arange(args.nspec) % 2 == 0).astype(
np.float32)
continuum = (np.arange(args.nspec) % 2 == 1).astype(np.float32)
for spec in range(args.nspec):
flux[spec, indx] = single_line[
spec] * ref_integrated_flux / np.gradient(wavelengths)[
indx] # single line
flux[spec] += continuum[
spec] * ref_cst_flux_density # flat continuum
meta1 = Table(
dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32),
LINE=wave[indx] *
np.ones(args.nspec, dtype=np.float32),
LINEFLUX=single_line * ref_integrated_flux,
CONSTFLUXDENSITY=continuum * ref_cst_flux_density))
else:
log.fatal('Unknown object type {}'.format(thisobj))
sys.exit(1)
# Pack it in.
truth['FLUX'][ii] = flux
meta = vstack([meta, meta1])
jj.append(ii.tolist())
# Sanity check on units; templates currently return ergs, not 1e-17 ergs...
# assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6)
# Sort the metadata table.
jj = sum(jj, [])
meta_new = Table()
for k in range(args.nspec):
index = int(np.where(np.array(jj) == k)[0])
meta_new = vstack([meta_new, meta[index]])
meta = meta_new
# Add TARGETID and the true OBJTYPE to the metadata table.
meta.add_column(
Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE'))
meta.add_column(Column(targetids, name='TARGETID'))
# Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z
meta.rename_column('REDSHIFT', 'TRUEZ')
# explicitly set location on focal plane if needed to support airmass
# variations when using specsim v0.5
if qsim.source.focal_xy is None:
qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm'))
# Set simulation parameters from the simspec header or desiparams
bright_objects = ['bgs', 'mws', 'bright', 'BGS', 'MWS', 'BRIGHT_MIX']
gray_objects = ['gray', 'grey']
if args.simspec is None:
object_type = objtype
flavor = None
elif simspec.flavor == 'science':
object_type = None
flavor = simspec.header['PROGRAM']
else:
object_type = None
flavor = simspec.flavor
log.warning(
'Maybe using an outdated simspec file with flavor={}'.format(
flavor))
# Set airmass
if args.airmass is not None:
qsim.atmosphere.airmass = args.airmass
elif args.simspec and 'AIRMASS' in simspec.header:
qsim.atmosphere.airmass = simspec.header['AIRMASS']
else:
qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2
# Set exptime
if args.exptime is not None:
qsim.observation.exposure_time = args.exptime * u.s
elif args.simspec and 'EXPTIME' in simspec.header:
qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s
elif objtype in bright_objects:
qsim.observation.exposure_time = desiparams['exptime_bright'] * u.s
else:
qsim.observation.exposure_time = desiparams['exptime_dark'] * u.s
# Set Moon Phase
if args.moon_phase is not None:
qsim.atmosphere.moon.moon_phase = args.moon_phase
elif args.simspec and 'MOONFRAC' in simspec.header:
qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC']
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_phase = 0.7
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_phase = 0.1
else:
qsim.atmosphere.moon.moon_phase = 0.5
# Set Moon Zenith
if args.moon_zenith is not None:
qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg
elif args.simspec and 'MOONALT' in simspec.header:
qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_zenith = 30 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_zenith = 80 * u.deg
else:
qsim.atmosphere.moon.moon_zenith = 100 * u.deg
# Set Moon - Object Angle
if args.moon_angle is not None:
qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg
elif args.simspec and 'MOONSEP' in simspec.header:
qsim.atmosphere.moon.separation_angle = simspec.header[
'MOONSEP'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.separation_angle = 50 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
else:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
# Initialize per-camera output arrays that will be saved
waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {}
maxbin = 0
nmax = args.nspec
for camera in qsim.instrument.cameras:
# Lookup this camera's resolution matrix and convert to the sparse
# format used in desispec.
R = Resolution(camera.get_output_resolution_matrix())
resolution[camera.name] = np.tile(R.to_fits_array(),
[args.nspec, 1, 1])
waves[camera.name] = (camera.output_wavelength.to(
u.Angstrom).value.astype(np.float32))
nwave = len(waves[camera.name])
maxbin = max(maxbin, len(waves[camera.name]))
nobj = np.zeros((nmax, 3, maxbin)) # object photons
nsky = np.zeros((nmax, 3, maxbin)) # sky photons
nivar = np.zeros((nmax, 3, maxbin)) # inverse variance (object+sky)
cframe_observedflux = np.zeros(
(nmax, 3, maxbin)) # calibrated object flux
cframe_ivar = np.zeros(
(nmax, 3, maxbin)) # inverse variance of calibrated object flux
cframe_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to calibrated flux
sky_ivar = np.zeros((nmax, 3, maxbin)) # inverse variance of sky
sky_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to sky only
frame_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to nobj+nsky
trueflux[camera.name] = np.empty(
(args.nspec, nwave)) # calibrated flux
noisyflux[camera.name] = np.empty(
(args.nspec, nwave)) # observed flux with noise
obsivar[camera.name] = np.empty(
(args.nspec, nwave)) # inverse variance of flux
if args.simspec:
for i in range(10):
cn = camera.name + str(i)
if cn in simspec.cameras:
dw = np.gradient(simspec.cameras[cn].wave)
break
else:
raise RuntimeError(
'Unable to find a {} camera in input simspec'.format(
camera))
else:
sflux = np.empty((args.nspec, npix))
#- Check if input simspec is for a continuum flat lamp instead of science
#- This does not convolve to per-fiber resolution
if args.simspec:
if simspec.flavor == 'flat':
log.info("Simulating flat lamp exposure")
for i, camera in enumerate(qsim.instrument.cameras):
channel = camera.name #- from simspec, b/r/z not b0/r1/z9
assert camera.output_wavelength.unit == u.Angstrom
num_pixels = len(waves[channel])
phot = list()
for j in range(10):
cn = camera.name + str(j)
if cn in simspec.cameras:
camwave = simspec.cameras[cn].wave
dw = np.gradient(camwave)
phot.append(simspec.cameras[cn].phot)
if len(phot) == 0:
raise RuntimeError(
'Unable to find a {} camera in input simspec'.format(
camera))
else:
phot = np.vstack(phot)
meanspec = resample_flux(waves[channel], camwave,
np.average(phot / dw, axis=0))
fiberflat = random_state.normal(loc=1.0,
scale=1.0 / np.sqrt(meanspec),
size=(nspec, num_pixels))
ivar = np.tile(meanspec, [nspec, 1])
mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32)
for kk in range((args.nspec + args.nstart - 1) // 500 + 1):
camera = channel + str(kk)
outfile = desispec.io.findfile('fiberflat', NIGHT, EXPID,
camera)
start = max(500 * kk, args.nstart)
end = min(500 * (kk + 1), nmax)
if (args.spectrograph <= kk):
log.info(
"Writing files for channel:{}, spectrograph:{}, spectra:{} to {}"
.format(channel, kk, start, end))
ff = FiberFlat(waves[channel],
fiberflat[start:end, :],
ivar[start:end, :],
mask[start:end, :],
meanspec,
header=dict(CAMERA=camera))
write_fiberflat(outfile, ff)
filePath = desispec.io.findfile("fiberflat", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(filePath))
sys.exit(0)
# Repeat the simulation for all spectra
fluxunits = 1e-17 * u.erg / (u.s * u.cm**2 * u.Angstrom)
for j in range(args.nspec):
thisobjtype = objtype[j]
sys.stdout.flush()
if flavor == 'arc':
qsim.source.update_in('Quickgen source {0}'.format, 'perfect',
wavelengths * u.Angstrom,
spectra * fluxunits)
else:
qsim.source.update_in('Quickgen source {0}'.format(j),
thisobjtype.lower(),
wavelengths * u.Angstrom,
spectra[j, :] * fluxunits)
qsim.source.update_out()
qsim.simulate()
qsim.generate_random_noise(random_state)
for i, output in enumerate(qsim.camera_output):
assert output['observed_flux'].unit == 1e17 * fluxunits
# Extract the simulation results needed to create our uncalibrated
# frame output file.
num_pixels = len(output)
nobj[j, i, :num_pixels] = output['num_source_electrons'][:, 0]
nsky[j, i, :num_pixels] = output['num_sky_electrons'][:, 0]
nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:, 0]
# Get results for our flux-calibrated output file.
cframe_observedflux[
j, i, :num_pixels] = 1e17 * output['observed_flux'][:, 0]
cframe_ivar[
j,
i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:, 0]
# Fill brick arrays from the results.
camera = output.meta['name']
trueflux[camera][j][:] = 1e17 * output['observed_flux'][:, 0]
noisyflux[camera][j][:] = 1e17 * (
output['observed_flux'][:, 0] +
output['flux_calibration'][:, 0] *
output['random_noise_electrons'][:, 0])
obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:,
0]
# Use the same noise realization in the cframe and frame, without any
# additional noise from sky subtraction for now.
frame_rand_noise[
j, i, :num_pixels] = output['random_noise_electrons'][:, 0]
cframe_rand_noise[j, i, :num_pixels] = 1e17 * (
output['flux_calibration'][:, 0] *
output['random_noise_electrons'][:, 0])
# The sky output file represents a model fit to ~40 sky fibers.
# We reduce the variance by a factor of 25 to account for this and
# give the sky an independent (Gaussian) noise realization.
sky_ivar[
j,
i, :num_pixels] = 25.0 / (output['variance_electrons'][:, 0] -
output['num_source_electrons'][:, 0])
sky_rand_noise[j, i, :num_pixels] = random_state.normal(
scale=1.0 / np.sqrt(sky_ivar[j, i, :num_pixels]),
size=num_pixels)
armName = {"b": 0, "r": 1, "z": 2}
for channel in 'brz':
#Before writing, convert from counts/bin to counts/A (as in Pixsim output)
#Quicksim Default:
#FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang]
#COUNTS_OBJ - object counts in 0.5 Ang bin
#COUNTS_SKY - sky counts in 0.5 Ang bin
num_pixels = len(waves[channel])
dwave = np.gradient(waves[channel])
nobj[:, armName[channel], :num_pixels] /= dwave
frame_rand_noise[:, armName[channel], :num_pixels] /= dwave
nivar[:, armName[channel], :num_pixels] *= dwave**2
nsky[:, armName[channel], :num_pixels] /= dwave
sky_rand_noise[:, armName[channel], :num_pixels] /= dwave
sky_ivar[:, armName[channel], :num_pixels] /= dwave**2
# Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra
# Looping over spectrograph
for ii in range((args.nspec + args.nstart - 1) // 500 + 1):
start = max(500 * ii,
args.nstart) # first spectrum for a given spectrograph
end = min(500 * (ii + 1),
nmax) # last spectrum for the spectrograph
if (args.spectrograph <= ii):
camera = "{}{}".format(channel, ii)
log.info(
"Writing files for channel:{}, spectrograph:{}, spectra:{} to {}"
.format(channel, ii, start, end))
num_pixels = len(waves[channel])
# Write frame file
framefileName = desispec.io.findfile("frame", NIGHT, EXPID,
camera)
frame_flux=nobj[start:end,armName[channel],:num_pixels]+ \
nsky[start:end,armName[channel],:num_pixels] + \
frame_rand_noise[start:end,armName[channel],:num_pixels]
frame_ivar = nivar[start:end, armName[channel], :num_pixels]
sh1 = frame_flux.shape[
0] # required for slicing the resolution metric, resolusion matrix has (nspec,ndiag,wave)
# for example if nstart =400, nspec=150: two spectrographs:
# 400-499=> 0 spectrograph, 500-549 => 1
if (args.nstart == start):
resol = resolution[channel][:sh1, :, :]
else:
resol = resolution[channel][-sh1:, :, :]
# must create desispec.Frame object
frame=Frame(waves[channel], frame_flux, frame_ivar,\
resolution_data=resol, spectrograph=ii, \
fibermap=fibermap[start:end], \
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) )
desispec.io.write_frame(framefileName, frame)
framefilePath = desispec.io.findfile("frame", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(framefilePath))
if args.frameonly or simspec.flavor == 'arc':
continue
# Write cframe file
cframeFileName = desispec.io.findfile("cframe", NIGHT, EXPID,
camera)
cframeFlux = cframe_observedflux[
start:end,
armName[channel], :num_pixels] + cframe_rand_noise[
start:end, armName[channel], :num_pixels]
cframeIvar = cframe_ivar[start:end,
armName[channel], :num_pixels]
# must create desispec.Frame object
cframe = Frame(waves[channel], cframeFlux, cframeIvar, \
resolution_data=resol, spectrograph=ii,
fibermap=fibermap[start:end],
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) )
desispec.io.frame.write_frame(cframeFileName, cframe)
cframefilePath = desispec.io.findfile("cframe", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(cframefilePath))
# Write sky file
skyfileName = desispec.io.findfile("sky", NIGHT, EXPID, camera)
skyflux=nsky[start:end,armName[channel],:num_pixels] + \
sky_rand_noise[start:end,armName[channel],:num_pixels]
skyivar = sky_ivar[start:end, armName[channel], :num_pixels]
skymask = np.zeros(skyflux.shape, dtype=np.uint32)
# must create desispec.Sky object
skymodel = SkyModel(waves[channel],
skyflux,
skyivar,
skymask,
header=dict(CAMERA=camera))
desispec.io.sky.write_sky(skyfileName, skymodel)
skyfilePath = desispec.io.findfile("sky", NIGHT, EXPID, camera)
log.info("Wrote file {}".format(skyfilePath))
# Write calib file
calibVectorFile = desispec.io.findfile("calib", NIGHT, EXPID,
camera)
flux = cframe_observedflux[start:end,
armName[channel], :num_pixels]
phot = nobj[start:end, armName[channel], :num_pixels]
calibration = np.zeros_like(phot)
jj = (flux > 0)
calibration[jj] = phot[jj] / flux[jj]
#- TODO: what should calibivar be?
#- For now, model it as the noise of combining ~10 spectra
calibivar = 10 / cframe_ivar[start:end,
armName[channel], :num_pixels]
#mask=(1/calibivar>0).astype(int)??
mask = np.zeros(calibration.shape, dtype=np.uint32)
# write flux calibration
fluxcalib = FluxCalib(waves[channel], calibration, calibivar,
mask)
write_flux_calibration(calibVectorFile, fluxcalib)
calibfilePath = desispec.io.findfile("calib", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(calibfilePath))
def main_mpi(args, comm=None, timing=None):
freeze_iers()
nproc = 1
rank = 0
if comm is not None:
nproc = comm.size
rank = comm.rank
mark_start = time.time()
log = get_logger()
psf_file = args.psf
input_file = args.input
# these parameters are interpreted as the *global* spec range,
# to be divided among processes.
specmin = args.specmin
nspec = args.nspec
#- Load input files and broadcast
# FIXME: after we have fixed the serialization
# of the PSF, read and broadcast here, to reduce
# disk contention.
img = None
if rank == 0:
img = io.read_image(input_file)
if comm is not None:
img = comm.bcast(img, root=0)
psf = load_psf(psf_file)
mark_read_input = time.time()
# get spectral range
if nspec is None:
nspec = psf.nspec
if args.fibermap is not None:
fibermap = io.read_fibermap(args.fibermap)
else:
try:
fibermap = io.read_fibermap(args.input)
except (AttributeError, IOError, KeyError):
fibermap = None
if fibermap is not None:
fibermap = fibermap[specmin:specmin + nspec]
if nspec > len(fibermap):
log.warning(
"nspec {} > len(fibermap) {}; reducing nspec to {}".format(
nspec, len(fibermap), len(fibermap)))
nspec = len(fibermap)
fibers = fibermap['FIBER']
else:
fibers = np.arange(specmin, specmin + nspec)
specmax = specmin + nspec
#- Get wavelength grid from options
if args.wavelength is not None:
raw_wstart, raw_wstop, raw_dw = [
float(tmp) for tmp in args.wavelength.split(',')
]
else:
raw_wstart = np.ceil(psf.wmin_all)
raw_wstop = np.floor(psf.wmax_all)
raw_dw = 0.7
raw_wave = np.arange(raw_wstart, raw_wstop + raw_dw / 2.0, raw_dw)
nwave = len(raw_wave)
bundlesize = args.bundlesize
if args.barycentric_correction:
if ('RA' in img.meta) or ('TARGTRA' in img.meta):
barycentric_correction_factor = \
barycentric_correction_multiplicative_factor(img.meta)
#- Early commissioning has RA/TARGTRA in fibermap but not HDU 0
elif fibermap is not None and \
(('RA' in fibermap.meta) or ('TARGTRA' in fibermap.meta)):
barycentric_correction_factor = \
barycentric_correction_multiplicative_factor(fibermap.meta)
else:
msg = 'Barycentric corr requires (TARGT)RA in HDU 0 or fibermap'
log.critical(msg)
raise KeyError(msg)
else:
barycentric_correction_factor = 1.
# Explictly define the correct wavelength values to avoid confusion of reference frame
# If correction applied, otherwise divide by 1 and use the same raw values
wstart = raw_wstart / barycentric_correction_factor
wstop = raw_wstop / barycentric_correction_factor
dw = raw_dw / barycentric_correction_factor
wave = raw_wave / barycentric_correction_factor
#- Confirm that this PSF covers these wavelengths for these spectra
psf_wavemin = np.max(psf.wavelength(list(range(specmin, specmax)), y=-0.5))
psf_wavemax = np.min(
psf.wavelength(list(range(specmin, specmax)), y=psf.npix_y - 0.5))
if psf_wavemin - 5 > wstart:
raise ValueError(
'Start wavelength {:.2f} < min wavelength {:.2f} for these fibers'.
format(wstart, psf_wavemin))
if psf_wavemax + 5 < wstop:
raise ValueError(
'Stop wavelength {:.2f} > max wavelength {:.2f} for these fibers'.
format(wstop, psf_wavemax))
if rank == 0:
#- Print parameters
log.info("extract: input = {}".format(input_file))
log.info("extract: psf = {}".format(psf_file))
log.info("extract: specmin = {}".format(specmin))
log.info("extract: nspec = {}".format(nspec))
log.info("extract: wavelength = {},{},{}".format(wstart, wstop, dw))
log.info("extract: nwavestep = {}".format(args.nwavestep))
log.info("extract: regularize = {}".format(args.regularize))
if barycentric_correction_factor != 1.:
img.meta['HELIOCOR'] = barycentric_correction_factor
#- Augment input image header for output
img.meta['NSPEC'] = (nspec, 'Number of spectra')
img.meta['WAVEMIN'] = (raw_wstart, 'First wavelength [Angstroms]')
img.meta['WAVEMAX'] = (raw_wstop, 'Last wavelength [Angstroms]')
img.meta['WAVESTEP'] = (raw_dw, 'Wavelength step size [Angstroms]')
img.meta['SPECTER'] = (specter.__version__,
'https://github.com/desihub/specter')
img.meta['IN_PSF'] = (io.shorten_filename(psf_file), 'Input spectral PSF')
img.meta['IN_IMG'] = io.shorten_filename(input_file)
depend.add_dependencies(img.meta)
#- Check if input PSF was itself a traceshifted version of another PSF
orig_psf = None
if rank == 0:
try:
psfhdr = fits.getheader(psf_file, 'PSF')
orig_psf = psfhdr['IN_PSF']
except KeyError:
#- could happen due to PSF format not having "PSF" extension,
#- or due to PSF header not having 'IN_PSF' keyword. Either is OK
pass
if comm is not None:
orig_psf = comm.bcast(orig_psf, root=0)
if orig_psf is not None:
img.meta['ORIG_PSF'] = orig_psf
#- If not using MPI, use a single call to each of these and then end this function call
# Otherwise, continue on to splitting things up for the different ranks
if comm is None:
_extract_and_save(img, psf, specmin, nspec, specmin, wave, raw_wave,
fibers, fibermap, args.output, args.model,
bundlesize, args, log)
#- This is it if we aren't running MPI, so return
return
#else:
# # Continue to the MPI section, which could go under this else statment
# # But to save on indentation we'll just pass on to the rest of the function
# # since the alternative has already returned
# pass
# Now we divide our spectra into bundles
checkbundles = set()
checkbundles.update(
np.floor_divide(np.arange(specmin, specmax),
bundlesize * np.ones(nspec)).astype(int))
bundles = sorted(checkbundles)
nbundle = len(bundles)
bspecmin = {}
bnspec = {}
for b in bundles:
if specmin > b * bundlesize:
bspecmin[b] = specmin
else:
bspecmin[b] = b * bundlesize
if (b + 1) * bundlesize > specmax:
bnspec[b] = specmax - bspecmin[b]
else:
bnspec[b] = bundlesize
# Now we assign bundles to processes
mynbundle = int(nbundle // nproc)
myfirstbundle = 0
leftover = nbundle % nproc
if rank < leftover:
mynbundle += 1
myfirstbundle = rank * mynbundle
else:
myfirstbundle = ((mynbundle + 1) * leftover) + (mynbundle *
(rank - leftover))
# get the root output file
outpat = re.compile(r'(.*)\.fits')
outmat = outpat.match(args.output)
if outmat is None:
raise RuntimeError(
"extraction output file should have .fits extension")
outroot = outmat.group(1)
outdir = os.path.normpath(os.path.dirname(outroot))
if rank == 0:
if not os.path.isdir(outdir):
os.makedirs(outdir)
if comm is not None:
comm.barrier()
mark_preparation = time.time()
time_total_extraction = 0.0
time_total_write_output = 0.0
failcount = 0
for b in range(myfirstbundle, myfirstbundle + mynbundle):
mark_iteration_start = time.time()
outbundle = "{}_{:02d}.fits".format(outroot, b)
outmodel = "{}_model_{:02d}.fits".format(outroot, b)
log.info('extract: Rank {} extracting {} spectra {}:{} at {}'.format(
rank,
os.path.basename(input_file),
bspecmin[b],
bspecmin[b] + bnspec[b],
time.asctime(),
))
sys.stdout.flush()
#- The actual extraction
try:
mark_extraction = _extract_and_save(img, psf, bspecmin[b],
bnspec[b], specmin, wave,
raw_wave, fibers, fibermap,
outbundle, outmodel,
bundlesize, args, log)
mark_write_output = time.time()
time_total_extraction += mark_extraction - mark_iteration_start
time_total_write_output += mark_write_output - mark_extraction
except:
# Log the error and increment the number of failures
log.error(
"extract: FAILED bundle {}, spectrum range {}:{}".format(
b, bspecmin[b], bspecmin[b] + bnspec[b]))
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value,
exc_traceback)
log.error(''.join(lines))
failcount += 1
sys.stdout.flush()
if comm is not None:
failcount = comm.allreduce(failcount)
if failcount > 0:
# all processes throw
raise RuntimeError("some extraction bundles failed")
time_merge = None
if rank == 0:
mark_merge_start = time.time()
mergeopts = ['--output', args.output, '--force', '--delete']
mergeopts.extend(
["{}_{:02d}.fits".format(outroot, b) for b in bundles])
mergeargs = mergebundles.parse(mergeopts)
mergebundles.main(mergeargs)
if args.model is not None:
model = None
for b in bundles:
outmodel = "{}_model_{:02d}.fits".format(outroot, b)
if model is None:
model = fits.getdata(outmodel)
else:
#- TODO: test and warn if models overlap for pixels with
#- non-zero values
model += fits.getdata(outmodel)
os.remove(outmodel)
fits.writeto(args.model, model)
mark_merge_end = time.time()
time_merge = mark_merge_end - mark_merge_start
# Resolve difference timer data
if type(timing) is dict:
timing["read_input"] = mark_read_input - mark_start
timing["preparation"] = mark_preparation - mark_read_input
timing["total_extraction"] = time_total_extraction
timing["total_write_output"] = time_total_write_output
timing["merge"] = time_merge
import matplotlib.pyplot as plt
import astropy.units as units
from astropy.table import Table, vstack
from scipy import ndimage
from multiprocessing import Pool, Array
from desisurvey.utils import get_location
from astropy.time import Time
from astropy.coordinates import SkyCoord, EarthLocation, AltAz
from specsim.atmosphere import krisciunas_schaefer, Moon
from get_sky import get_sky
from pkg_resources import resource_filename
from desiutil.iers import freeze_iers
freeze_iers()
mayall = get_location()
emayall = ephem.Observer()
emayall.lon = ephem.degrees(mayall.lon.value * np.pi / 180.)
emayall.lat = ephem.degrees(mayall.lat.value * np.pi / 180.)
emayall.elevation = mayall.height.value
moon = ephem.Moon()
sun = ephem.Sun()
def airmass(zd):
# Airmass at given zenith distance.
return (1. - 0.96 * np.sin(zd * np.pi / 180.)**2.)**-0.5
def simulate(camera,
simspec,
psf,
nspec=None,
ncpu=None,
cosmics=None,
wavemin=None,
wavemax=None,
preproc=True,
comm=None):
"""Run pixel-level simulation of input spectra
Args:
camera (string) : b0, r1, .. z9
simspec : desispec.io.SimSpec object from desispec.io.read_simspec()
psf : subclass of specter.psf.psf.PSF, e.g. from desimodel.io.load_psf()
Options:
nspec (int): number of spectra to simulate
ncpu (int): number of CPU cores to use in parallel
cosmics (desispec.image.Image): e.g. from desisim.io.read_cosmics()
wavemin (float): minimum wavelength range to simulate
wavemax (float): maximum wavelength range to simulate
preproc (boolean, optional) : also preprocess raw data (default True)
Returns:
(image, rawpix, truepix) tuple, where image is the preproc Image object
(only header is meaningful if preproc=False), rawpix is a 2D
ndarray of unprocessed raw pixel data, and truepix is a 2D ndarray
of truth for image.pix
"""
freeze_iers()
if (comm is None) or (comm.rank == 0):
log.info('Starting pixsim.simulate camera {} at {}'.format(
camera, asctime()))
#- parse camera name into channel and spectrograph number
channel = camera[0].lower()
ispec = int(camera[1])
assert channel in 'brz', \
'unrecognized channel {} camera {}'.format(channel, camera)
assert 0 <= ispec < 10, \
'unrecognized spectrograph {} camera {}'.format(ispec, camera)
assert len(camera) == 2, \
'unrecognized camera {}'.format(camera)
#- Load DESI parameters
params = desimodel.io.load_desiparams()
#- this is not necessarily true, the truth in is the fibermap
nfibers = params['spectro']['nfibers']
phot = simspec.cameras[camera].phot
if simspec.cameras[camera].skyphot is not None:
phot += simspec.cameras[camera].skyphot
if nspec is not None:
phot = phot[0:nspec]
else:
nspec = phot.shape[0]
#- Trim wavelengths if needed
wave = simspec.cameras[camera].wave
if wavemin is not None:
ii = (wave >= wavemin)
phot = phot[:, ii]
wave = wave[ii]
if wavemax is not None:
ii = (wave <= wavemax)
phot = phot[:, ii]
wave = wave[ii]
#- Project to image and append that to file
if (comm is None) or (comm.rank == 0):
log.info('Starting {} projection at {}'.format(camera, asctime()))
# The returned true pixel values will only exist on rank 0 in the
# MPI case. Otherwise it will be None.
truepix = parallel_project(psf, wave, phot, ncpu=ncpu, comm=comm)
if (comm is None) or (comm.rank == 0):
log.info('Finished {} projection at {}'.format(camera, asctime()))
image = None
rawpix = None
if (comm is None) or (comm.rank == 0):
#- Start metadata header
header = simspec.header.copy()
header['CAMERA'] = camera
header['DOSVER'] = 'SIM'
header['FEEVER'] = 'SIM'
header['DETECTOR'] = 'SIM'
#- Add cosmics from library of dark images
ny = truepix.shape[0] // 2
nx = truepix.shape[1] // 2
if cosmics is not None:
# set to zeros values with mask bit 0 (= dead column or hot pixels)
cosmics_pix = cosmics.pix * ((cosmics.mask & 1) == 0)
pix = np.random.poisson(truepix) + cosmics_pix
try: #- cosmics templates >= v0.3
rdnoiseA = cosmics.meta['OBSRDNA']
rdnoiseB = cosmics.meta['OBSRDNB']
rdnoiseC = cosmics.meta['OBSRDNC']
rdnoiseD = cosmics.meta['OBSRDND']
except KeyError: #- cosmics templates <= v0.2
print(cosmic.meta)
rdnoiseA = cosmics.meta['RDNOISE0']
rdnoiseB = cosmics.meta['RDNOISE1']
rdnoiseC = cosmics.meta['RDNOISE2']
rdnoiseD = cosmics.meta['RDNOISE3']
else:
pix = truepix
readnoise = params['ccd'][channel]['readnoise']
rdnoiseA = rdnoiseB = rdnoiseC = rdnoiseD = readnoise
#- data already has noise if cosmics were added
noisydata = (cosmics is not None)
#- Split by amplifier and expand into raw data
nprescan = params['ccd'][channel]['prescanpixels']
if 'overscanpixels' in params['ccd'][channel]:
noverscan = params['ccd'][channel]['overscanpixels']
else:
noverscan = 50
#- Reproducibly random overscan bias level offsets across diff exp
assert channel in 'brz'
if channel == 'b':
irand = ispec
elif channel == 'r':
irand = 10 + ispec
elif channel == 'z':
irand = 20 + ispec
seeds = np.random.RandomState(0).randint(2**32 - 1, size=30)
rand = np.random.RandomState(seeds[irand])
nyraw = ny
nxraw = nx + nprescan + noverscan
rawpix = np.empty((nyraw * 2, nxraw * 2), dtype=np.int32)
gain = params['ccd'][channel]['gain']
#- Amp A/1 Lower Left
rawpix[0:nyraw, 0:nxraw] = \
photpix2raw(pix[0:ny, 0:nx], gain, rdnoiseA,
readorder='lr', nprescan=nprescan, noverscan=noverscan,
offset=rand.uniform(100, 200),
noisydata=noisydata)
#- Amp B/2 Lower Right
rawpix[0:nyraw, nxraw:nxraw+nxraw] = \
photpix2raw(pix[0:ny, nx:nx+nx], gain, rdnoiseB,
readorder='rl', nprescan=nprescan, noverscan=noverscan,
offset=rand.uniform(100, 200),
noisydata=noisydata)
#- Amp C/3 Upper Left
rawpix[nyraw:nyraw+nyraw, 0:nxraw] = \
photpix2raw(pix[ny:ny+ny, 0:nx], gain, rdnoiseC,
readorder='lr', nprescan=nprescan, noverscan=noverscan,
offset=rand.uniform(100, 200),
noisydata=noisydata)
#- Amp D/4 Upper Right
rawpix[nyraw:nyraw+nyraw, nxraw:nxraw+nxraw] = \
photpix2raw(pix[ny:ny+ny, nx:nx+nx], gain, rdnoiseD,
readorder='rl', nprescan=nprescan, noverscan=noverscan,
offset=rand.uniform(100, 200),
noisydata=noisydata)
def xyslice2header(xyslice):
'''
convert 2D slice into IRAF style [a:b,c:d] header value
e.g. xyslice2header(np.s_[0:10, 5:20]) -> '[6:20,1:10]'
'''
yy, xx = xyslice
value = '[{}:{},{}:{}]'.format(xx.start + 1, xx.stop, yy.start + 1,
yy.stop)
return value
#- Amp order from DESI-1964 (previously 1-4 instead of A-D)
#- C D
#- A B
xoffset = nprescan + nx + noverscan
header['PRESECA'] = xyslice2header(np.s_[0:nyraw, 0:0 + nprescan])
header['DATASECA'] = xyslice2header(np.s_[0:nyraw,
nprescan:nprescan + nx])
header['BIASSECA'] = xyslice2header(
np.s_[0:nyraw, nprescan + nx:nprescan + nx + noverscan])
header['CCDSECA'] = xyslice2header(np.s_[0:ny, 0:nx])
header['PRESECB'] = xyslice2header(
np.s_[0:nyraw, xoffset + noverscan + nx:xoffset + noverscan + nx +
nprescan])
header['DATASECB'] = xyslice2header(
np.s_[0:nyraw, xoffset + noverscan:xoffset + noverscan + nx])
header['BIASSECB'] = xyslice2header(np.s_[0:nyraw,
xoffset:xoffset + noverscan])
header['CCDSECB'] = xyslice2header(np.s_[0:ny, nx:2 * nx])
header['PRESECC'] = xyslice2header(np.s_[nyraw:2 * nyraw,
0:0 + nprescan])
header['DATASECC'] = xyslice2header(np.s_[nyraw:2 * nyraw,
nprescan:nprescan + nx])
header['BIASSECC'] = xyslice2header(
np.s_[nyraw:2 * nyraw, nprescan + nx:nprescan + nx + noverscan])
header['CCDSECC'] = xyslice2header(np.s_[ny:2 * ny, 0:nx])
header['PRESECD'] = xyslice2header(
np.s_[nyraw:2 * nyraw, xoffset + noverscan + nx:xoffset +
noverscan + nx + nprescan])
header['DATASECD'] = xyslice2header(
np.s_[nyraw:2 * nyraw,
xoffset + noverscan:xoffset + noverscan + nx])
header['BIASSECD'] = xyslice2header(np.s_[nyraw:2 * nyraw,
xoffset:xoffset + noverscan])
header['CCDSECD'] = xyslice2header(np.s_[ny:2 * ny, nx:2 * nx])
#- Add additional keywords to mimic real raw data
header['INSTRUME'] = 'DESI'
header['PROCTYPE'] = 'RAW'
header['PRODTYPE'] = 'image'
header['EXPFRAME'] = 0
header['REQTIME'] = simspec.header['EXPTIME']
header['TIMESYS'] = 'UTC'
#- DATE-OBS format YEAR-MM-DDThh:mm:ss.sss -> OBSID kpnoYEARMMDDthhmmss
header['OBSID'] = 'kp4m' + header['DATE-OBS'][0:19].replace(
'-', '').replace(':', '').lower()
header['TIME-OBS'] = header['DATE-OBS'].split('T')[1]
header['DELTARA'] = 0.0
header['DELTADEC'] = 0.0
header['SPECGRPH'] = ispec
header['CCDNAME'] = 'CCDS' + str(ispec) + str(channel).upper()
header['CCDPREP'] = 'purge,clear'
header['CCDSIZE'] = str(rawpix.shape)
header['CCDTEMP'] = 850.0
header['CPUTEMP'] = 63.7
header['CASETEMP'] = 62.8
header['CCDTMING'] = 'sim_timing.txt'
header['CCDCFG'] = 'sim.cfg'
header['SETTINGS'] = 'sim_detectors.json'
header['VESSEL'] = 7 #- I don't know what this is
header['FEEBOX'] = 'sim097'
header['PGAGAIN'] = 5
header['OCSVER'] = 'SIM'
header['CONSTVER'] = 'SIM'
header['BLDTIME'] = 0.35
header['DIGITIME'] = 61.9
#- Remove some spurious header keywords from upstream
if 'BUNIT' in header and header['BUNIT'] == 'Angstrom':
del header['BUNIT']
if 'MJD' in header and 'MJD-OBS' not in header:
header['MJD-OBS'] = header['MJD']
del header['MJD']
for key in ['RA', 'DEC']:
if key in header:
del header[key]
#- Drive MJD-OBS from DATE-OBS if needed
if 'MJD-OBS' not in header:
header['MJD-OBS'] = Time(header['DATE-OBS']).mjd
#- from http://www-kpno.kpno.noao.edu/kpno-misc/mayall_params.html
kpno_longitude = -(111. + 35 / 60. + 59.6 / 3600) * u.deg
#- Convert DATE-OBS to sexigesimal (sigh) Local Sidereal Time
#- Use mean ST as close enough for sims to avoid nutation calc
t = Time(header['DATE-OBS'])
st = t.sidereal_time('mean', kpno_longitude).to('deg').value
hour = st / 15
minute = (hour % 1) * 60
second = (minute % 1) * 60
header['ST'] = '{:02d}:{:02d}:{:0.3f}'.format(int(hour), int(minute),
second)
if preproc:
log.debug('Running preprocessing at {}'.format(asctime()))
image = desispec.preproc.preproc(rawpix,
header,
primary_header=simspec.header)
else:
log.debug('Skipping preprocessing')
image = Image(np.zeros(truepix.shape),
np.zeros(truepix.shape),
meta=header)
if (comm is None) or (comm.rank == 0):
log.info('Finished pixsim.simulate for camera {} at {}'.format(
camera, asctime()))
return image, rawpix, truepix