def setUp(self):
self.fileio = "test_spectra.fits"
self.fileappend = "test_spectra_append.fits"
self.filebuild = "test_spectra_build.fits"
self.meta = {
"KEY1" : "VAL1",
"KEY2" : "VAL2"
}
self.nwave = 100
self.nspec = 5
self.ndiag = 3
fmap = empty_fibermap(self.nspec)
fmap = add_columns(fmap,
['NIGHT', 'EXPID', 'TILEID'],
[np.int32(0), np.int32(0), np.int32(0)],
)
for s in range(self.nspec):
fmap[s]["TARGETID"] = 456 + s
fmap[s]["FIBER"] = 123 + s
fmap[s]["NIGHT"] = s
fmap[s]["EXPID"] = s
self.fmap1 = encode_table(fmap)
fmap = empty_fibermap(self.nspec)
fmap = add_columns(fmap,
['NIGHT', 'EXPID', 'TILEID'],
[np.int32(0), np.int32(0), np.int32(0)],
)
for s in range(self.nspec):
fmap[s]["TARGETID"] = 789 + s
fmap[s]["FIBER"] = 200 + s
fmap[s]["NIGHT"] = 1000
fmap[s]["EXPID"] = s
self.fmap2 = encode_table(fmap)
self.bands = ["b", "r", "z"]
self.wave = {}
self.flux = {}
self.ivar = {}
self.mask = {}
self.res = {}
self.extra = {}
for s in range(self.nspec):
for b in self.bands:
self.wave[b] = np.arange(self.nwave)
self.flux[b] = np.repeat(np.arange(self.nspec),
self.nwave).reshape( (self.nspec, self.nwave) ) + 3.0
self.ivar[b] = 1.0 / self.flux[b]
self.mask[b] = np.tile(np.arange(2, dtype=np.uint32),
(self.nwave * self.nspec) // 2).reshape( (self.nspec, self.nwave) )
self.res[b] = np.zeros( (self.nspec, self.ndiag, self.nwave),
dtype=np.float64)
self.res[b][:,1,:] = 1.0
self.extra[b] = {}
self.extra[b]["FOO"] = self.flux[b]
def _write_fibermap(self):
"""Write a fake fiberflat"""
fibermap = io.empty_fibermap(self.nspec)
for i in range(0, self.nspec, 3):
fibermap['OBJTYPE'][i] = 'SKY'
io.write_fibermap(self.fibermapfile, fibermap)
def get_frame_data(nspec=10, objtype=None):
"""
Return basic test data for desispec.frame object:
"""
nwave = 100
wavemin, wavemax = 4000, 4100
wave, model_flux = get_models(nspec,
nwave,
wavemin=wavemin,
wavemax=wavemax)
resol_data = set_resolmatrix(nspec, nwave)
calib = np.sin((wave - wavemin) * np.pi / np.max(wave))
flux = np.zeros((nspec, nwave))
for i in range(nspec):
flux[i] = Resolution(resol_data[i]).dot(model_flux[i] * calib)
sigma = 0.01
# flux += np.random.normal(scale=sigma, size=flux.shape)
ivar = np.ones(flux.shape) / sigma**2
mask = np.zeros(flux.shape, dtype=int)
fibermap = empty_fibermap(nspec, 1500)
if objtype is None:
fibermap['OBJTYPE'] = 'QSO'
fibermap['OBJTYPE'][0:3] = 'STD' # For flux tests
else:
fibermap['OBJTYPE'] = objtype
frame = Frame(wave, flux, ivar, mask, resol_data, fibermap=fibermap)
frame.meta = {}
frame.meta['EXPTIME'] = 1. # For flux tests
return frame
def get_frame_data(nspec=10):
"""
Return basic test data for desispec.frame object:
"""
nwave = 100
wavemin, wavemax = 4000, 4100
wave, model_flux = get_models(nspec, nwave, wavemin=wavemin, wavemax=wavemax)
resol_data=set_resolmatrix(nspec,nwave)
calib = np.sin((wave-wavemin) * np.pi / np.max(wave))
flux = np.zeros((nspec, nwave))
for i in range(nspec):
flux[i] = Resolution(resol_data[i]).dot(model_flux[i] * calib)
sigma = 0.01
# flux += np.random.normal(scale=sigma, size=flux.shape)
ivar = np.ones(flux.shape) / sigma**2
mask = np.zeros(flux.shape, dtype=int)
fibermap = empty_fibermap(nspec, 1500)
fibermap['OBJTYPE'] = 'TGT'
fibermap['DESI_TARGET'] = desi_mask.QSO
fibermap['DESI_TARGET'][0:3] = desi_mask.STD_FAINT # For flux tests
meta = dict(EXPTIME = 1.0)
frame = Frame(wave, flux, ivar, mask,resol_data,fibermap=fibermap, meta=meta)
return frame
def _get_fibermap(self):
fibermap = io.empty_fibermap(self.nspec, 1500)
for i in range(0, self.nspec, 3):
fibermap['OBJTYPE'][i] = 'SKY'
fibermap['DESI_TARGET'][i] = desi_mask.SKY
fibermap['OBJTYPE'][i + 1] = 'TGT'
fibermap['DESI_TARGET'][i + 1] = desi_mask.STD_FAINT
return fibermap
def _get_fibermap(self):
fibermap = io.empty_fibermap(self.nspec, 1500)
for i in range(0, self.nspec, 3):
fibermap['OBJTYPE'][i] = 'SKY'
fibermap['DESI_TARGET'][i] = desi_mask.SKY
fibermap['OBJTYPE'][i+1] = 'TGT'
fibermap['DESI_TARGET'][i+1] = desi_mask.STD_FAINT
return fibermap
def _get_spectra(self, with_gradient=False):
#- Setup data for a Resolution matrix
sigma2 = 4.0
ndiag = 21
xx = np.linspace(-(ndiag - 1) / 2.0, +(ndiag - 1) / 2.0, ndiag)
Rdata = np.zeros((self.nspec, ndiag, self.nwave))
for i in range(self.nspec):
kernel = np.exp(-(xx + float(i) / self.nspec * 0.3)**2 /
(2 * sigma2))
#kernel = np.exp(-xx**2/(2*sigma2))
kernel /= sum(kernel)
for j in range(self.nwave):
Rdata[i, :, j] = kernel
flux = np.zeros((self.nspec, self.nwave), dtype=float)
ivar = np.ones((self.nspec, self.nwave), dtype=float)
# Add a random component
for i in range(self.nspec):
ivar[i] += 0.4 * np.random.uniform(size=self.nwave)
mask = np.zeros((self.nspec, self.nwave), dtype=int)
fibermap = empty_fibermap(self.nspec, 1500)
fibermap['OBJTYPE'][0::2] = 'SKY'
x = fibermap["FIBERASSIGN_X"]
y = fibermap["FIBERASSIGN_Y"]
x = x - np.mean(x)
y = y - np.mean(y)
if np.std(x) > 0: x /= np.std(x)
if np.std(y) > 0: y /= np.std(y)
w = (self.wave - self.wave[0]) / (self.wave[-1] -
self.wave[0]) * 2. - 1
for i in range(self.nspec):
R = Resolution(Rdata[i])
if with_gradient:
scale = 1. + (0.1 * x[i] + 0.2 * y[i]) * (1 + 0.4 * w)
flux[i] = R.dot(scale * self.flux)
else:
flux[i] = R.dot(self.flux)
meta = {"camera": "r2"}
return Frame(self.wave,
flux,
ivar,
mask,
Rdata,
spectrograph=2,
fibermap=fibermap,
meta=meta)
def _random_spectra(self, ns=3, nw=10):
wave = np.linspace(5000, 5100, nw)
flux = np.random.uniform(0, 1, size=(ns,nw))
ivar = np.random.uniform(0, 1, size=(ns,nw))
#mask = np.zeros((ns,nw),dtype=int)
mask = None
rdat = np.ones((ns,3,nw))
rdat[:,0] *= 0.25
rdat[:,1] *= 0.5
rdat[:,2] *= 0.25
fmap = empty_fibermap(ns)
fmap["TARGETID"][:]=12 # same target
return Spectra(bands=["x"],wave={"x":wave},flux={"x":flux},ivar={"x":ivar}, mask=None, resolution_data={"x":rdat} , fibermap=fmap)
def test_main(self):
"""
Test the main program.
"""
# generate the frame data
wave, flux, ivar, mask = _get_data()
nspec, nwave = flux.shape
#- Setup data for a Resolution matrix
sigma = 4.0
ndiag = 11
xx = np.linspace(-(ndiag - 1) / 2.0, +(ndiag - 1) / 2.0, ndiag)
Rdata = np.zeros((nspec, ndiag, nwave))
kernel = np.exp(-xx**2 / (2 * sigma))
kernel /= sum(kernel)
for i in range(nspec):
for j in range(nwave):
Rdata[i, :, j] = kernel
#- Convolve the data with the resolution matrix
convflux = np.empty_like(flux)
for i in range(nspec):
convflux[i] = Resolution(Rdata[i]).dot(flux[i])
# create a fake fibermap
fibermap = io.empty_fibermap(nspec, nwave)
for i in range(0, nspec):
fibermap['OBJTYPE'][i] = 'FAKE'
io.write_fibermap(self.testfibermap, fibermap)
#- write out the frame
frame = Frame(wave,
convflux,
ivar,
mask,
Rdata,
spectrograph=0,
fibermap=fibermap,
meta=dict(FLAVOR='flat'))
write_frame(self.testframe, frame, fibermap=fibermap)
# set program arguments
argstr = ['--infile', self.testframe, '--outfile', self.testflat]
# run it
args = ffscript.parse(options=argstr)
ffscript.main(args)
def _get_fibermap(self, gaia_only=False):
fibermap = io.empty_fibermap(self.nspec, 1500)
for i in range(0, self.nspec, 3):
fibermap['OBJTYPE'][i] = 'SKY'
fibermap['DESI_TARGET'][i] = desi_mask.SKY
fibermap['OBJTYPE'][i + 1] = 'TGT'
fibermap['DESI_TARGET'][i + 1] = desi_mask.STD_FAINT
fibermap['GAIA_PHOT_G_MEAN_MAG'][i + 1] = 15
fibermap['GAIA_PHOT_BP_MEAN_MAG'][i + 1] = 15
fibermap['GAIA_PHOT_RP_MEAN_MAG'][i + 1] = 15
if gaia_only:
fibermap['PHOTSYS'] = 'G'
else:
fibermap['FLUX_G'][i + 1] = 100
fibermap['FLUX_R'][i + 1] = 100
fibermap['FLUX_Z'][i + 1] = 100
return fibermap
def test_main(self):
"""
Test the main program.
"""
# generate the frame data
wave, flux, ivar, mask = _get_data()
nspec, nwave = flux.shape
#- Setup data for a Resolution matrix
sigma = 4.0
ndiag = 11
xx = np.linspace(-(ndiag-1)/2.0, +(ndiag-1)/2.0, ndiag)
Rdata = np.zeros( (nspec, ndiag, nwave) )
kernel = np.exp(-xx**2/(2*sigma))
kernel /= sum(kernel)
for i in range(nspec):
for j in range(nwave):
Rdata[i,:,j] = kernel
#- Convolve the data with the resolution matrix
convflux = np.empty_like(flux)
for i in range(nspec):
convflux[i] = Resolution(Rdata[i]).dot(flux[i])
# create a fake fibermap
fibermap = io.empty_fibermap(nspec, nwave)
for i in range(0, nspec):
fibermap['OBJTYPE'][i] = 'FAKE'
io.write_fibermap(self.testfibermap, fibermap)
#- write out the frame
frame = Frame(wave, convflux, ivar, mask, Rdata, spectrograph=0, fibermap=fibermap)
write_frame(self.testframe, frame, fibermap=fibermap)
# set program arguments
argstr = [
'--infile', self.testframe,
'--outfile', self.testflat
]
# run it
args = ffscript.parse(options=argstr)
ffscript.main(args)
def get_frame_data(nspec=10, wavemin=4000, wavemax=4100, nwave=100, meta={}):
"""
Return basic test data for desispec.frame object:
"""
wave, model_flux = get_models(nspec,
nwave,
wavemin=wavemin,
wavemax=wavemax)
resol_data = set_resolmatrix(nspec, nwave)
calib = np.sin((wave - wavemin) * np.pi / np.max(wave))
flux = np.zeros((nspec, nwave))
for i in range(nspec):
flux[i] = Resolution(resol_data[i]).dot(model_flux[i] * calib)
sigma = 0.01
# flux += np.random.normal(scale=sigma, size=flux.shape)
ivar = np.ones(flux.shape) / sigma**2
mask = np.zeros(flux.shape, dtype=int)
fibermap = empty_fibermap(nspec, 1500)
fibermap['OBJTYPE'] = 'TGT'
fibermap['DESI_TARGET'] = desi_mask.QSO
fibermap['DESI_TARGET'][0:3] = desi_mask.STD_FAINT # For flux tests
fibermap['FIBER_X'] = np.arange(nspec) * 400. / nspec #mm
fibermap['FIBER_Y'] = np.arange(nspec) * 400. / nspec #mm
fibermap['DELTA_X'] = 0.005 * np.ones(nspec) #mm
fibermap['DELTA_Y'] = 0.003 * np.ones(nspec) #mm
if "EXPTIME" not in meta.keys():
meta['EXPTIME'] = 1.0
frame = Frame(wave,
flux,
ivar,
mask,
resol_data,
fibermap=fibermap,
meta=meta)
return frame
def _get_fibermap(self):
fibermap = io.empty_fibermap(self.nspec, 1500)
for i in range(0, self.nspec, 3):
fibermap['OBJTYPE'][i] = 'SKY'
fibermap['OBJTYPE'][i+1] = 'STD'
return fibermap
def setUp(cls):
cls.program = program = 'dark'
cls.flavor = flavor = 'bias'
cls.night = night = '20150105'
cls.camera = camera = 'r0'
cls.expid = expid = 314
cls.psfExpid = psfExpid = 313
cls.flatExpid = flatExpid = 312
cls.templateExpid = templateExpid = 311
cls.nspec = nspec = 5
cls.exptime = exptime = 100
#- Setup environment and override default environment variables
#- python 2.7 location:
cls.topDir = os.path.dirname( # top-level
os.path.dirname( # py/
os.path.dirname( # desispec/
os.path.dirname(os.path.abspath(__file__)) # test/
)
)
)
cls.binDir = os.path.join(cls.topDir,'bin')
if not os.path.isdir(cls.binDir):
#- python 3.x setup.py test location:
cls.topDir = os.path.dirname( # top-level
os.path.dirname( # build/
os.path.dirname( # lib/
os.path.dirname( # desispec/
os.path.dirname(os.path.abspath(__file__)) # test/
)
)
)
)
cls.binDir = os.path.join(cls.topDir,'bin')
#- last attempt
if not os.path.isdir(cls.binDir):
cls.topDir = os.getcwd()
cls.binDir = os.path.join(cls.topDir, 'bin')
if not os.path.isdir(cls.binDir):
raise RuntimeError('Unable to auto-locate desispec/bin from {}'.format(__file__))
id = uuid4().hex
cls.fibermapfile = 'fibermap-'+id+'.fits'
cls.framefile = 'frame-'+id+'.fits'
cls.testDir = testDir = os.path.join(os.environ['HOME'],'ql_test_io')
datanightDir = os.path.join(testDir,night)
dataDir = os.path.join(datanightDir,'{:08d}'.format(expid))
expDir = os.path.join(testDir,'exposures')
expnightDir = os.path.join(expDir,night)
reduxDir = os.path.join(expnightDir,'{:08d}'.format(expid))
calibDir = os.path.join(testDir, 'ql_calib')
configDir = os.path.join(testDir, 'ql_config')
os.environ['QL_CALIB_DIR'] = calibDir
os.environ['QL_CONFIG_DIR'] = configDir
if not os.path.exists(testDir):
os.makedirs(testDir)
os.makedirs(datanightDir)
os.makedirs(dataDir)
os.makedirs(expDir)
os.makedirs(expnightDir)
os.makedirs(reduxDir)
os.makedirs(calibDir)
os.makedirs(configDir)
#- Write dummy configuration and input files to test merging
configdict = {'name': 'Test Configuration',
'Program': program,
'Flavor': flavor,
'PSFExpid': psfExpid,
'PSFType': 'psf',
'FiberflatExpid': flatExpid,
'TemplateExpid': templateExpid,
'TemplateNight': night,
'WritePreprocfile': False,
'WriteSkyModelfile': False,
'WriteIntermediatefiles': False,
'WriteStaticPlots': False,
'Debuglevel': 20,
'UseResolution': False,
'Period': 5.0,
'Timeout': 120.0,
'Pipeline': ['Initialize','Preproc'],
'Algorithms': {'Initialize':{
'QA':{'Check_HDUs':{'PARAMS':{}}
}},
'Preproc':{
'QA':{'Bias_From_Overscan':{'PARAMS':{'BIAS_AMP_NORMAL_RANGE':[-100.0,100.0],'BIAS_AMP_WARN_RANGE':[-200.0,200.0]}},
'Get_RMS':{'PARAMS':{'PERCENTILES':[68.2,95.4,99.7],'NOISE_AMP_NORMAL_RANGE':[-1.0,1.0],'NOISE_AMP_WARN_RANGE':[-2.0,2.0]}},
'Count_Pixels':{'PARAMS':{'CUTPIX':500,'LITFRAC_NORMAL_RANGE':[-0.1,0.1],'LITFRAC_WARN_RANGE':[-0.2,0.2]}}}}}
}
with open('{}/test_config.yaml'.format(testDir),'w') as config:
yaml.dump(configdict,config)
cls.configfile = '{}/test_config.yaml'.format(testDir)
#- Generate raw file
rawfile = os.path.join(dataDir,'desi-00000314.fits.fz')
raw_hdr = {}
raw_hdr['DATE-OBS'] = '2015-01-05T08:17:03.988'
raw_hdr['NIGHT'] = night
raw_hdr['PROGRAM'] = program
raw_hdr['FLAVOR'] = flavor
raw_hdr['CAMERA'] = camera
raw_hdr['EXPID'] = expid
raw_hdr['EXPTIME'] = exptime
raw_hdr['DOSVER'] = 'SIM'
raw_hdr['FEEVER'] = 'SIM'
raw_hdr['DETECTOR'] = 'SIM'
raw_hdr['PRESEC1'] = '[1:4,1:2048]'
raw_hdr['DATASEC1'] = '[5:2052,1:2048]'
raw_hdr['BIASSEC1'] = '[2053:2102,1:2048]'
raw_hdr['CCDSEC1'] = '[1:2048,1:2048]'
raw_hdr['PRESEC2'] = '[4201:4204,1:2048]'
raw_hdr['DATASEC2'] = '[2153:4200,1:2048]'
raw_hdr['BIASSEC2'] = '[2103:2152,1:2048]'
raw_hdr['CCDSEC2'] = '[2049:4096,1:2048]'
raw_hdr['PRESEC3'] = '[1:4,2049:4096]'
raw_hdr['DATASEC3'] = '[5:2052,2049:4096]'
raw_hdr['BIASSEC3'] = '[2053:2102,2049:4096]'
raw_hdr['CCDSEC3'] = '[1:2048,2049:4096]'
raw_hdr['PRESEC4'] = '[4201:4204,2049:4096]'
raw_hdr['DATASEC4'] = '[2153:4200,2049:4096]'
raw_hdr['BIASSEC4'] = '[2103:2152,2049:4096]'
raw_hdr['CCDSEC4'] = '[2049:4096,2049:4096]'
raw_hdr['GAIN1'] = 1.0
raw_hdr['GAIN2'] = 1.0
raw_hdr['GAIN3'] = 1.0
raw_hdr['GAIN4'] = 1.0
raw_hdr['RDNOISE1'] = 3.0
raw_hdr['RDNOISE2'] = 3.0
raw_hdr['RDNOISE3'] = 3.0
raw_hdr['RDNOISE4'] = 3.0
primary_header={'PROGRAM':program}
data=np.zeros((4096,4204))+200.
raw_data=data.astype(int)
write_raw(rawfile,raw_data,raw_hdr,primary_header=primary_header)
#- Generate fibermap file
fibermapfile = os.path.join(dataDir,'fibermap-00000314.fits')
fibermap = empty_fibermap(nspec)
write_fibermap(fibermapfile,fibermap)
#- Generate calib data
for camera in ['b0', 'r0', 'z0']:
#- Fiberflat has to exist but can be a dummpy file
filename = '{}/fiberflat-{}.fits'.format(calibDir, camera)
fx = open(filename, 'w'); fx.write('fiberflat file'); fx.close()
#- PSF has to be real file
psffile = '{}/psf-{}.fits'.format(calibDir, camera)
example_psf = resource_filename('desispec', 'test/data/ql/psf-{}.fits'.format(camera))
shutil.copy(example_psf, psffile)
#- Copy test calibration-data.yaml file
specdir=calibDir+"spec/sp0"
if not os.path.isdir(specdir) :
os.makedirs(specdir)
for c in "brz" :
shutil.copy(resource_filename('desispec', 'test/data/ql/{}0.yaml'.format(c)),os.path.join(specdir,"{}0.yaml".format(c)))
#- Set calibration environment variable
os.environ['DESI_SPECTRO_CALIB'] = calibDir
def main(args):
# Set up the logger
if args.verbose:
log = get_logger(DEBUG)
else:
log = get_logger()
# Make sure all necessary environment variables are set
DESI_SPECTRO_REDUX_DIR="./quickGen"
if 'DESI_SPECTRO_REDUX' not in os.environ:
log.info('DESI_SPECTRO_REDUX environment is not set.')
else:
DESI_SPECTRO_REDUX_DIR=os.environ['DESI_SPECTRO_REDUX']
if os.path.exists(DESI_SPECTRO_REDUX_DIR):
if not os.path.isdir(DESI_SPECTRO_REDUX_DIR):
raise RuntimeError("Path %s Not a directory"%DESI_SPECTRO_REDUX_DIR)
else:
try:
os.makedirs(DESI_SPECTRO_REDUX_DIR)
except:
raise
SPECPROD_DIR='specprod'
if 'SPECPROD' not in os.environ:
log.info('SPECPROD environment is not set.')
else:
SPECPROD_DIR=os.environ['SPECPROD']
prod_Dir=specprod_root()
if os.path.exists(prod_Dir):
if not os.path.isdir(prod_Dir):
raise RuntimeError("Path %s Not a directory"%prod_Dir)
else:
try:
os.makedirs(prod_Dir)
except:
raise
# Initialize random number generator to use.
np.random.seed(args.seed)
random_state = np.random.RandomState(args.seed)
# Derive spectrograph number from nstart if needed
if args.spectrograph is None:
args.spectrograph = args.nstart / 500
# Read fibermapfile to get object type, night and expid
if args.fibermap:
log.info("Reading fibermap file {}".format(args.fibermap))
fibermap=read_fibermap(args.fibermap)
objtype = get_source_types(fibermap)
stdindx=np.where(objtype=='STD') # match STD with STAR
mwsindx=np.where(objtype=='MWS_STAR') # match MWS_STAR with STAR
bgsindx=np.where(objtype=='BGS') # match BGS with LRG
objtype[stdindx]='STAR'
objtype[mwsindx]='STAR'
objtype[bgsindx]='LRG'
NIGHT=fibermap.meta['NIGHT']
EXPID=fibermap.meta['EXPID']
else:
# Create a blank fake fibermap
fibermap = empty_fibermap(args.nspec)
targetids = random_state.randint(2**62, size=args.nspec)
fibermap['TARGETID'] = targetids
night = get_night()
expid = 0
log.info("Initializing SpecSim with config {}".format(args.config))
desiparams = load_desiparams()
qsim = get_simulator(args.config, num_fibers=1)
if args.simspec:
# Read the input file
log.info('Reading input file {}'.format(args.simspec))
simspec = desisim.io.read_simspec(args.simspec)
nspec = simspec.nspec
if simspec.flavor == 'arc':
log.warning("quickgen doesn't generate flavor=arc outputs")
return
else:
wavelengths = simspec.wave
spectra = simspec.flux
if nspec < args.nspec:
log.info("Only {} spectra in input file".format(nspec))
args.nspec = nspec
else:
# Initialize the output truth table.
spectra = []
wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value
npix = len(wavelengths)
truth = dict()
meta = Table()
truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10))
truth['FLUX'] = np.zeros((args.nspec, npix))
truth['WAVE'] = wavelengths
jj = list()
for thisobj in set(true_objtype):
ii = np.where(true_objtype == thisobj)[0]
nobj = len(ii)
truth['OBJTYPE'][ii] = thisobj
log.info('Generating {} template'.format(thisobj))
# Generate the templates
if thisobj == 'ELG':
elg = desisim.templates.ELG(wave=wavelengths, add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = elg.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_elg,sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'LRG':
lrg = desisim.templates.LRG(wave=wavelengths, add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = lrg.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_lrg,sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'QSO':
qso = desisim.templates.QSO(wave=wavelengths)
flux, tmpwave, meta1 = qso.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_qso)
elif thisobj == 'BGS':
bgs = desisim.templates.BGS(wave=wavelengths, add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = bgs.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_bgs,rmagrange=args.rmagrange_bgs,sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj =='STD':
std = desisim.templates.STD(wave=wavelengths)
flux, tmpwave, meta1 = std.make_templates(nmodel=nobj, seed=args.seed)
elif thisobj == 'QSO_BAD': # use STAR template no color cuts
star = desisim.templates.STAR(wave=wavelengths)
flux, tmpwave, meta1 = star.make_templates(nmodel=nobj, seed=args.seed)
elif thisobj == 'MWS_STAR' or thisobj == 'MWS':
mwsstar = desisim.templates.MWS_STAR(wave=wavelengths)
flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj, seed=args.seed)
elif thisobj == 'WD':
wd = desisim.templates.WD(wave=wavelengths)
flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj, seed=args.seed)
elif thisobj == 'SKY':
flux = np.zeros((nobj, npix))
meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32)))
elif thisobj == 'TEST':
flux = np.zeros((args.nspec, npix))
indx = np.where(wave>5800.0-1E-6)[0][0]
ref_integrated_flux = 1E-10
ref_cst_flux_density = 1E-17
single_line = (np.arange(args.nspec)%2 == 0).astype(np.float32)
continuum = (np.arange(args.nspec)%2 == 1).astype(np.float32)
for spec in range(args.nspec) :
flux[spec,indx] = single_line[spec]*ref_integrated_flux/np.gradient(wavelengths)[indx] # single line
flux[spec] += continuum[spec]*ref_cst_flux_density # flat continuum
meta1 = Table(dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32),
LINE=wave[indx]*np.ones(args.nspec, dtype=np.float32),
LINEFLUX=single_line*ref_integrated_flux,
CONSTFLUXDENSITY=continuum*ref_cst_flux_density))
else:
log.fatal('Unknown object type {}'.format(thisobj))
sys.exit(1)
# Pack it in.
truth['FLUX'][ii] = flux
meta = vstack([meta, meta1])
jj.append(ii.tolist())
# Sanity check on units; templates currently return ergs, not 1e-17 ergs...
# assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6)
# Sort the metadata table.
jj = sum(jj,[])
meta_new = Table()
for k in range(args.nspec):
index = int(np.where(np.array(jj) == k)[0])
meta_new = vstack([meta_new, meta[index]])
meta = meta_new
# Add TARGETID and the true OBJTYPE to the metadata table.
meta.add_column(Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE'))
meta.add_column(Column(targetids, name='TARGETID'))
# Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z
meta.rename_column('REDSHIFT', 'TRUEZ')
# explicitly set location on focal plane if needed to support airmass
# variations when using specsim v0.5
if qsim.source.focal_xy is None:
qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm'))
# Set simulation parameters from the simspec header or desiparams
bright_objects = ['bgs','mws','bright','BGS','MWS','BRIGHT_MIX']
gray_objects = ['gray','grey']
if args.simspec is None:
object_type = objtype
flavor = None
elif simspec.flavor == 'science':
object_type = None
flavor = simspec.header['PROGRAM']
else:
object_type = None
flavor = simspec.flavor
log.warning('Maybe using an outdated simspec file with flavor={}'.format(flavor))
# Set airmass
if args.airmass is not None:
qsim.atmosphere.airmass = args.airmass
elif args.simspec and 'AIRMASS' in simspec.header:
qsim.atmosphere.airmass = simspec.header['AIRMASS']
else:
qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2
# Set exptime
if args.exptime is not None:
qsim.observation.exposure_time = args.exptime * u.s
elif args.simspec and 'EXPTIME' in simspec.header:
qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s
elif objtype in bright_objects:
qsim.observation.exposure_time = desiparams['exptime_bright'] * u.s
else:
qsim.observation.exposure_time = desiparams['exptime_dark'] * u.s
# Set Moon Phase
if args.moon_phase is not None:
qsim.atmosphere.moon.moon_phase = args.moon_phase
elif args.simspec and 'MOONFRAC' in simspec.header:
qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC']
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_phase = 0.7
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_phase = 0.1
else:
qsim.atmosphere.moon.moon_phase = 0.5
# Set Moon Zenith
if args.moon_zenith is not None:
qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg
elif args.simspec and 'MOONALT' in simspec.header:
qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_zenith = 30 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_zenith = 80 * u.deg
else:
qsim.atmosphere.moon.moon_zenith = 100 * u.deg
# Set Moon - Object Angle
if args.moon_angle is not None:
qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg
elif args.simspec and 'MOONSEP' in simspec.header:
qsim.atmosphere.moon.separation_angle = simspec.header['MOONSEP'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.separation_angle = 50 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
else:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
# Initialize per-camera output arrays that will be saved
waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {}
maxbin = 0
nmax= args.nspec
for camera in qsim.instrument.cameras:
# Lookup this camera's resolution matrix and convert to the sparse
# format used in desispec.
R = Resolution(camera.get_output_resolution_matrix())
resolution[camera.name] = np.tile(R.to_fits_array(), [args.nspec, 1, 1])
waves[camera.name] = (camera.output_wavelength.to(u.Angstrom).value.astype(np.float32))
nwave = len(waves[camera.name])
maxbin = max(maxbin, len(waves[camera.name]))
nobj = np.zeros((nmax,3,maxbin)) # object photons
nsky = np.zeros((nmax,3,maxbin)) # sky photons
nivar = np.zeros((nmax,3,maxbin)) # inverse variance (object+sky)
cframe_observedflux = np.zeros((nmax,3,maxbin)) # calibrated object flux
cframe_ivar = np.zeros((nmax,3,maxbin)) # inverse variance of calibrated object flux
cframe_rand_noise = np.zeros((nmax,3,maxbin)) # random Gaussian noise to calibrated flux
sky_ivar = np.zeros((nmax,3,maxbin)) # inverse variance of sky
sky_rand_noise = np.zeros((nmax,3,maxbin)) # random Gaussian noise to sky only
frame_rand_noise = np.zeros((nmax,3,maxbin)) # random Gaussian noise to nobj+nsky
trueflux[camera.name] = np.empty((args.nspec, nwave)) # calibrated flux
noisyflux[camera.name] = np.empty((args.nspec, nwave)) # observed flux with noise
obsivar[camera.name] = np.empty((args.nspec, nwave)) # inverse variance of flux
if args.simspec:
for i in range(10):
cn = camera.name + str(i)
if cn in simspec.cameras:
dw = np.gradient(simspec.cameras[cn].wave)
break
else:
raise RuntimeError('Unable to find a {} camera in input simspec'.format(camera))
else:
sflux = np.empty((args.nspec, npix))
#- Check if input simspec is for a continuum flat lamp instead of science
#- This does not convolve to per-fiber resolution
if args.simspec:
if simspec.flavor == 'flat':
log.info("Simulating flat lamp exposure")
for i,camera in enumerate(qsim.instrument.cameras):
channel = camera.name #- from simspec, b/r/z not b0/r1/z9
assert camera.output_wavelength.unit == u.Angstrom
num_pixels = len(waves[channel])
phot = list()
for j in range(10):
cn = camera.name + str(j)
if cn in simspec.cameras:
camwave = simspec.cameras[cn].wave
dw = np.gradient(camwave)
phot.append(simspec.cameras[cn].phot)
if len(phot) == 0:
raise RuntimeError('Unable to find a {} camera in input simspec'.format(camera))
else:
phot = np.vstack(phot)
meanspec = resample_flux(
waves[channel], camwave, np.average(phot/dw, axis=0))
fiberflat = random_state.normal(loc=1.0,
scale=1.0 / np.sqrt(meanspec), size=(nspec, num_pixels))
ivar = np.tile(meanspec, [nspec, 1])
mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32)
for kk in range((args.nspec+args.nstart-1)//500+1):
camera = channel+str(kk)
outfile = desispec.io.findfile('fiberflat', NIGHT, EXPID, camera)
start=max(500*kk,args.nstart)
end=min(500*(kk+1),nmax)
if (args.spectrograph <= kk):
log.info("Writing files for channel:{}, spectrograph:{}, spectra:{} to {}".format(channel,kk,start,end))
ff = FiberFlat(
waves[channel], fiberflat[start:end,:],
ivar[start:end,:], mask[start:end,:], meanspec,
header=dict(CAMERA=camera))
write_fiberflat(outfile, ff)
filePath=desispec.io.findfile("fiberflat",NIGHT,EXPID,camera)
log.info("Wrote file {}".format(filePath))
sys.exit(0)
# Repeat the simulation for all spectra
fluxunits = 1e-17 * u.erg / (u.s * u.cm ** 2 * u.Angstrom)
for j in range(args.nspec):
thisobjtype = objtype[j]
sys.stdout.flush()
if flavor == 'arc':
qsim.source.update_in(
'Quickgen source {0}'.format, 'perfect',
wavelengths * u.Angstrom, spectra * fluxunits)
else:
qsim.source.update_in(
'Quickgen source {0}'.format(j), thisobjtype.lower(),
wavelengths * u.Angstrom, spectra[j, :] * fluxunits)
qsim.source.update_out()
qsim.simulate()
qsim.generate_random_noise(random_state)
for i, output in enumerate(qsim.camera_output):
assert output['observed_flux'].unit == 1e17 * fluxunits
# Extract the simulation results needed to create our uncalibrated
# frame output file.
num_pixels = len(output)
nobj[j, i, :num_pixels] = output['num_source_electrons'][:,0]
nsky[j, i, :num_pixels] = output['num_sky_electrons'][:,0]
nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:,0]
# Get results for our flux-calibrated output file.
cframe_observedflux[j, i, :num_pixels] = 1e17 * output['observed_flux'][:,0]
cframe_ivar[j, i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:,0]
# Fill brick arrays from the results.
camera = output.meta['name']
trueflux[camera][j][:] = 1e17 * output['observed_flux'][:,0]
noisyflux[camera][j][:] = 1e17 * (output['observed_flux'][:,0] +
output['flux_calibration'][:,0] * output['random_noise_electrons'][:,0])
obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:,0]
# Use the same noise realization in the cframe and frame, without any
# additional noise from sky subtraction for now.
frame_rand_noise[j, i, :num_pixels] = output['random_noise_electrons'][:,0]
cframe_rand_noise[j, i, :num_pixels] = 1e17 * (
output['flux_calibration'][:,0] * output['random_noise_electrons'][:,0])
# The sky output file represents a model fit to ~40 sky fibers.
# We reduce the variance by a factor of 25 to account for this and
# give the sky an independent (Gaussian) noise realization.
sky_ivar[j, i, :num_pixels] = 25.0 / (
output['variance_electrons'][:,0] - output['num_source_electrons'][:,0])
sky_rand_noise[j, i, :num_pixels] = random_state.normal(
scale=1.0 / np.sqrt(sky_ivar[j,i,:num_pixels]),size=num_pixels)
armName={"b":0,"r":1,"z":2}
for channel in 'brz':
#Before writing, convert from counts/bin to counts/A (as in Pixsim output)
#Quicksim Default:
#FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang]
#COUNTS_OBJ - object counts in 0.5 Ang bin
#COUNTS_SKY - sky counts in 0.5 Ang bin
num_pixels = len(waves[channel])
dwave=np.gradient(waves[channel])
nobj[:,armName[channel],:num_pixels]/=dwave
frame_rand_noise[:,armName[channel],:num_pixels]/=dwave
nivar[:,armName[channel],:num_pixels]*=dwave**2
nsky[:,armName[channel],:num_pixels]/=dwave
sky_rand_noise[:,armName[channel],:num_pixels]/=dwave
sky_ivar[:,armName[channel],:num_pixels]/=dwave**2
# Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra
# Looping over spectrograph
for ii in range((args.nspec+args.nstart-1)//500+1):
start=max(500*ii,args.nstart) # first spectrum for a given spectrograph
end=min(500*(ii+1),nmax) # last spectrum for the spectrograph
if (args.spectrograph <= ii):
camera = "{}{}".format(channel, ii)
log.info("Writing files for channel:{}, spectrograph:{}, spectra:{} to {}".format(channel,ii,start,end))
num_pixels = len(waves[channel])
# Write frame file
framefileName=desispec.io.findfile("frame",NIGHT,EXPID,camera)
frame_flux=nobj[start:end,armName[channel],:num_pixels]+ \
nsky[start:end,armName[channel],:num_pixels] + \
frame_rand_noise[start:end,armName[channel],:num_pixels]
frame_ivar=nivar[start:end,armName[channel],:num_pixels]
sh1=frame_flux.shape[0] # required for slicing the resolution metric, resolusion matrix has (nspec,ndiag,wave)
# for example if nstart =400, nspec=150: two spectrographs:
# 400-499=> 0 spectrograph, 500-549 => 1
if (args.nstart==start):
resol=resolution[channel][:sh1,:,:]
else:
resol=resolution[channel][-sh1:,:,:]
# must create desispec.Frame object
frame=Frame(waves[channel], frame_flux, frame_ivar,\
resolution_data=resol, spectrograph=ii, \
fibermap=fibermap[start:end], \
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) )
desispec.io.write_frame(framefileName, frame)
framefilePath=desispec.io.findfile("frame",NIGHT,EXPID,camera)
log.info("Wrote file {}".format(framefilePath))
if args.frameonly or simspec.flavor == 'arc':
continue
# Write cframe file
cframeFileName=desispec.io.findfile("cframe",NIGHT,EXPID,camera)
cframeFlux=cframe_observedflux[start:end,armName[channel],:num_pixels]+cframe_rand_noise[start:end,armName[channel],:num_pixels]
cframeIvar=cframe_ivar[start:end,armName[channel],:num_pixels]
# must create desispec.Frame object
cframe = Frame(waves[channel], cframeFlux, cframeIvar, \
resolution_data=resol, spectrograph=ii,
fibermap=fibermap[start:end],
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) )
desispec.io.frame.write_frame(cframeFileName,cframe)
cframefilePath=desispec.io.findfile("cframe",NIGHT,EXPID,camera)
log.info("Wrote file {}".format(cframefilePath))
# Write sky file
skyfileName=desispec.io.findfile("sky",NIGHT,EXPID,camera)
skyflux=nsky[start:end,armName[channel],:num_pixels] + \
sky_rand_noise[start:end,armName[channel],:num_pixels]
skyivar=sky_ivar[start:end,armName[channel],:num_pixels]
skymask=np.zeros(skyflux.shape, dtype=np.uint32)
# must create desispec.Sky object
skymodel = SkyModel(waves[channel], skyflux, skyivar, skymask,
header=dict(CAMERA=camera))
desispec.io.sky.write_sky(skyfileName, skymodel)
skyfilePath=desispec.io.findfile("sky",NIGHT,EXPID,camera)
log.info("Wrote file {}".format(skyfilePath))
# Write calib file
calibVectorFile=desispec.io.findfile("calib",NIGHT,EXPID,camera)
flux = cframe_observedflux[start:end,armName[channel],:num_pixels]
phot = nobj[start:end,armName[channel],:num_pixels]
calibration = np.zeros_like(phot)
jj = (flux>0)
calibration[jj] = phot[jj] / flux[jj]
#- TODO: what should calibivar be?
#- For now, model it as the noise of combining ~10 spectra
calibivar=10/cframe_ivar[start:end,armName[channel],:num_pixels]
#mask=(1/calibivar>0).astype(int)??
mask=np.zeros(calibration.shape, dtype=np.uint32)
# write flux calibration
fluxcalib = FluxCalib(waves[channel], calibration, calibivar, mask)
write_flux_calibration(calibVectorFile, fluxcalib)
calibfilePath=desispec.io.findfile("calib",NIGHT,EXPID,camera)
log.info("Wrote file {}".format(calibfilePath))
def main(args):
# Set up the logger
if args.verbose:
log = get_logger(DEBUG)
else:
log = get_logger()
# Make sure all necessary environment variables are set
DESI_SPECTRO_REDUX_DIR = "./quickGen"
if 'DESI_SPECTRO_REDUX' not in os.environ:
log.info('DESI_SPECTRO_REDUX environment is not set.')
else:
DESI_SPECTRO_REDUX_DIR = os.environ['DESI_SPECTRO_REDUX']
if os.path.exists(DESI_SPECTRO_REDUX_DIR):
if not os.path.isdir(DESI_SPECTRO_REDUX_DIR):
raise RuntimeError("Path %s Not a directory" %
DESI_SPECTRO_REDUX_DIR)
else:
try:
os.makedirs(DESI_SPECTRO_REDUX_DIR)
except:
raise
SPECPROD_DIR = 'specprod'
if 'SPECPROD' not in os.environ:
log.info('SPECPROD environment is not set.')
else:
SPECPROD_DIR = os.environ['SPECPROD']
prod_Dir = specprod_root()
if os.path.exists(prod_Dir):
if not os.path.isdir(prod_Dir):
raise RuntimeError("Path %s Not a directory" % prod_Dir)
else:
try:
os.makedirs(prod_Dir)
except:
raise
# Initialize random number generator to use.
np.random.seed(args.seed)
random_state = np.random.RandomState(args.seed)
# Derive spectrograph number from nstart if needed
if args.spectrograph is None:
args.spectrograph = args.nstart / 500
# Read fibermapfile to get object type, night and expid
if args.fibermap:
log.info("Reading fibermap file {}".format(args.fibermap))
fibermap = read_fibermap(args.fibermap)
objtype = get_source_types(fibermap)
stdindx = np.where(objtype == 'STD') # match STD with STAR
mwsindx = np.where(objtype == 'MWS_STAR') # match MWS_STAR with STAR
bgsindx = np.where(objtype == 'BGS') # match BGS with LRG
objtype[stdindx] = 'STAR'
objtype[mwsindx] = 'STAR'
objtype[bgsindx] = 'LRG'
NIGHT = fibermap.meta['NIGHT']
EXPID = fibermap.meta['EXPID']
else:
# Create a blank fake fibermap
fibermap = empty_fibermap(args.nspec)
targetids = random_state.randint(2**62, size=args.nspec)
fibermap['TARGETID'] = targetids
night = get_night()
expid = 0
log.info("Initializing SpecSim with config {}".format(args.config))
desiparams = load_desiparams()
qsim = get_simulator(args.config, num_fibers=1)
if args.simspec:
# Read the input file
log.info('Reading input file {}'.format(args.simspec))
simspec = desisim.io.read_simspec(args.simspec)
nspec = simspec.nspec
if simspec.flavor == 'arc':
log.warning("quickgen doesn't generate flavor=arc outputs")
return
else:
wavelengths = simspec.wave
spectra = simspec.flux
if nspec < args.nspec:
log.info("Only {} spectra in input file".format(nspec))
args.nspec = nspec
else:
# Initialize the output truth table.
spectra = []
wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value
npix = len(wavelengths)
truth = dict()
meta = Table()
truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10))
truth['FLUX'] = np.zeros((args.nspec, npix))
truth['WAVE'] = wavelengths
jj = list()
for thisobj in set(true_objtype):
ii = np.where(true_objtype == thisobj)[0]
nobj = len(ii)
truth['OBJTYPE'][ii] = thisobj
log.info('Generating {} template'.format(thisobj))
# Generate the templates
if thisobj == 'ELG':
elg = desisim.templates.ELG(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = elg.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_elg,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'LRG':
lrg = desisim.templates.LRG(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = lrg.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_lrg,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'QSO':
qso = desisim.templates.QSO(wave=wavelengths)
flux, tmpwave, meta1 = qso.make_templates(
nmodel=nobj, seed=args.seed, zrange=args.zrange_qso)
elif thisobj == 'BGS':
bgs = desisim.templates.BGS(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = bgs.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_bgs,
rmagrange=args.rmagrange_bgs,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'STD':
std = desisim.templates.STD(wave=wavelengths)
flux, tmpwave, meta1 = std.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'QSO_BAD': # use STAR template no color cuts
star = desisim.templates.STAR(wave=wavelengths)
flux, tmpwave, meta1 = star.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'MWS_STAR' or thisobj == 'MWS':
mwsstar = desisim.templates.MWS_STAR(wave=wavelengths)
flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'WD':
wd = desisim.templates.WD(wave=wavelengths)
flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'SKY':
flux = np.zeros((nobj, npix))
meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32)))
elif thisobj == 'TEST':
flux = np.zeros((args.nspec, npix))
indx = np.where(wave > 5800.0 - 1E-6)[0][0]
ref_integrated_flux = 1E-10
ref_cst_flux_density = 1E-17
single_line = (np.arange(args.nspec) % 2 == 0).astype(
np.float32)
continuum = (np.arange(args.nspec) % 2 == 1).astype(np.float32)
for spec in range(args.nspec):
flux[spec, indx] = single_line[
spec] * ref_integrated_flux / np.gradient(wavelengths)[
indx] # single line
flux[spec] += continuum[
spec] * ref_cst_flux_density # flat continuum
meta1 = Table(
dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32),
LINE=wave[indx] *
np.ones(args.nspec, dtype=np.float32),
LINEFLUX=single_line * ref_integrated_flux,
CONSTFLUXDENSITY=continuum * ref_cst_flux_density))
else:
log.fatal('Unknown object type {}'.format(thisobj))
sys.exit(1)
# Pack it in.
truth['FLUX'][ii] = flux
meta = vstack([meta, meta1])
jj.append(ii.tolist())
# Sanity check on units; templates currently return ergs, not 1e-17 ergs...
# assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6)
# Sort the metadata table.
jj = sum(jj, [])
meta_new = Table()
for k in range(args.nspec):
index = int(np.where(np.array(jj) == k)[0])
meta_new = vstack([meta_new, meta[index]])
meta = meta_new
# Add TARGETID and the true OBJTYPE to the metadata table.
meta.add_column(
Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE'))
meta.add_column(Column(targetids, name='TARGETID'))
# Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z
meta.rename_column('REDSHIFT', 'TRUEZ')
# explicitly set location on focal plane if needed to support airmass
# variations when using specsim v0.5
if qsim.source.focal_xy is None:
qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm'))
# Set simulation parameters from the simspec header or desiparams
bright_objects = ['bgs', 'mws', 'bright', 'BGS', 'MWS', 'BRIGHT_MIX']
gray_objects = ['gray', 'grey']
if args.simspec is None:
object_type = objtype
flavor = None
elif simspec.flavor == 'science':
object_type = None
flavor = simspec.header['PROGRAM']
else:
object_type = None
flavor = simspec.flavor
log.warning(
'Maybe using an outdated simspec file with flavor={}'.format(
flavor))
# Set airmass
if args.airmass is not None:
qsim.atmosphere.airmass = args.airmass
elif args.simspec and 'AIRMASS' in simspec.header:
qsim.atmosphere.airmass = simspec.header['AIRMASS']
else:
qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2
# Set exptime
if args.exptime is not None:
qsim.observation.exposure_time = args.exptime * u.s
elif args.simspec and 'EXPTIME' in simspec.header:
qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s
elif objtype in bright_objects:
qsim.observation.exposure_time = desiparams['exptime_bright'] * u.s
else:
qsim.observation.exposure_time = desiparams['exptime_dark'] * u.s
# Set Moon Phase
if args.moon_phase is not None:
qsim.atmosphere.moon.moon_phase = args.moon_phase
elif args.simspec and 'MOONFRAC' in simspec.header:
qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC']
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_phase = 0.7
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_phase = 0.1
else:
qsim.atmosphere.moon.moon_phase = 0.5
# Set Moon Zenith
if args.moon_zenith is not None:
qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg
elif args.simspec and 'MOONALT' in simspec.header:
qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_zenith = 30 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_zenith = 80 * u.deg
else:
qsim.atmosphere.moon.moon_zenith = 100 * u.deg
# Set Moon - Object Angle
if args.moon_angle is not None:
qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg
elif args.simspec and 'MOONSEP' in simspec.header:
qsim.atmosphere.moon.separation_angle = simspec.header[
'MOONSEP'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.separation_angle = 50 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
else:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
# Initialize per-camera output arrays that will be saved
waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {}
maxbin = 0
nmax = args.nspec
for camera in qsim.instrument.cameras:
# Lookup this camera's resolution matrix and convert to the sparse
# format used in desispec.
R = Resolution(camera.get_output_resolution_matrix())
resolution[camera.name] = np.tile(R.to_fits_array(),
[args.nspec, 1, 1])
waves[camera.name] = (camera.output_wavelength.to(
u.Angstrom).value.astype(np.float32))
nwave = len(waves[camera.name])
maxbin = max(maxbin, len(waves[camera.name]))
nobj = np.zeros((nmax, 3, maxbin)) # object photons
nsky = np.zeros((nmax, 3, maxbin)) # sky photons
nivar = np.zeros((nmax, 3, maxbin)) # inverse variance (object+sky)
cframe_observedflux = np.zeros(
(nmax, 3, maxbin)) # calibrated object flux
cframe_ivar = np.zeros(
(nmax, 3, maxbin)) # inverse variance of calibrated object flux
cframe_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to calibrated flux
sky_ivar = np.zeros((nmax, 3, maxbin)) # inverse variance of sky
sky_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to sky only
frame_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to nobj+nsky
trueflux[camera.name] = np.empty(
(args.nspec, nwave)) # calibrated flux
noisyflux[camera.name] = np.empty(
(args.nspec, nwave)) # observed flux with noise
obsivar[camera.name] = np.empty(
(args.nspec, nwave)) # inverse variance of flux
if args.simspec:
for i in range(10):
cn = camera.name + str(i)
if cn in simspec.cameras:
dw = np.gradient(simspec.cameras[cn].wave)
break
else:
raise RuntimeError(
'Unable to find a {} camera in input simspec'.format(
camera))
else:
sflux = np.empty((args.nspec, npix))
#- Check if input simspec is for a continuum flat lamp instead of science
#- This does not convolve to per-fiber resolution
if args.simspec:
if simspec.flavor == 'flat':
log.info("Simulating flat lamp exposure")
for i, camera in enumerate(qsim.instrument.cameras):
channel = camera.name #- from simspec, b/r/z not b0/r1/z9
assert camera.output_wavelength.unit == u.Angstrom
num_pixels = len(waves[channel])
phot = list()
for j in range(10):
cn = camera.name + str(j)
if cn in simspec.cameras:
camwave = simspec.cameras[cn].wave
dw = np.gradient(camwave)
phot.append(simspec.cameras[cn].phot)
if len(phot) == 0:
raise RuntimeError(
'Unable to find a {} camera in input simspec'.format(
camera))
else:
phot = np.vstack(phot)
meanspec = resample_flux(waves[channel], camwave,
np.average(phot / dw, axis=0))
fiberflat = random_state.normal(loc=1.0,
scale=1.0 / np.sqrt(meanspec),
size=(nspec, num_pixels))
ivar = np.tile(meanspec, [nspec, 1])
mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32)
for kk in range((args.nspec + args.nstart - 1) // 500 + 1):
camera = channel + str(kk)
outfile = desispec.io.findfile('fiberflat', NIGHT, EXPID,
camera)
start = max(500 * kk, args.nstart)
end = min(500 * (kk + 1), nmax)
if (args.spectrograph <= kk):
log.info(
"Writing files for channel:{}, spectrograph:{}, spectra:{} to {}"
.format(channel, kk, start, end))
ff = FiberFlat(waves[channel],
fiberflat[start:end, :],
ivar[start:end, :],
mask[start:end, :],
meanspec,
header=dict(CAMERA=camera))
write_fiberflat(outfile, ff)
filePath = desispec.io.findfile("fiberflat", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(filePath))
sys.exit(0)
# Repeat the simulation for all spectra
fluxunits = 1e-17 * u.erg / (u.s * u.cm**2 * u.Angstrom)
for j in range(args.nspec):
thisobjtype = objtype[j]
sys.stdout.flush()
if flavor == 'arc':
qsim.source.update_in('Quickgen source {0}'.format, 'perfect',
wavelengths * u.Angstrom,
spectra * fluxunits)
else:
qsim.source.update_in('Quickgen source {0}'.format(j),
thisobjtype.lower(),
wavelengths * u.Angstrom,
spectra[j, :] * fluxunits)
qsim.source.update_out()
qsim.simulate()
qsim.generate_random_noise(random_state)
for i, output in enumerate(qsim.camera_output):
assert output['observed_flux'].unit == 1e17 * fluxunits
# Extract the simulation results needed to create our uncalibrated
# frame output file.
num_pixels = len(output)
nobj[j, i, :num_pixels] = output['num_source_electrons'][:, 0]
nsky[j, i, :num_pixels] = output['num_sky_electrons'][:, 0]
nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:, 0]
# Get results for our flux-calibrated output file.
cframe_observedflux[
j, i, :num_pixels] = 1e17 * output['observed_flux'][:, 0]
cframe_ivar[
j,
i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:, 0]
# Fill brick arrays from the results.
camera = output.meta['name']
trueflux[camera][j][:] = 1e17 * output['observed_flux'][:, 0]
noisyflux[camera][j][:] = 1e17 * (
output['observed_flux'][:, 0] +
output['flux_calibration'][:, 0] *
output['random_noise_electrons'][:, 0])
obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:,
0]
# Use the same noise realization in the cframe and frame, without any
# additional noise from sky subtraction for now.
frame_rand_noise[
j, i, :num_pixels] = output['random_noise_electrons'][:, 0]
cframe_rand_noise[j, i, :num_pixels] = 1e17 * (
output['flux_calibration'][:, 0] *
output['random_noise_electrons'][:, 0])
# The sky output file represents a model fit to ~40 sky fibers.
# We reduce the variance by a factor of 25 to account for this and
# give the sky an independent (Gaussian) noise realization.
sky_ivar[
j,
i, :num_pixels] = 25.0 / (output['variance_electrons'][:, 0] -
output['num_source_electrons'][:, 0])
sky_rand_noise[j, i, :num_pixels] = random_state.normal(
scale=1.0 / np.sqrt(sky_ivar[j, i, :num_pixels]),
size=num_pixels)
armName = {"b": 0, "r": 1, "z": 2}
for channel in 'brz':
#Before writing, convert from counts/bin to counts/A (as in Pixsim output)
#Quicksim Default:
#FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang]
#COUNTS_OBJ - object counts in 0.5 Ang bin
#COUNTS_SKY - sky counts in 0.5 Ang bin
num_pixels = len(waves[channel])
dwave = np.gradient(waves[channel])
nobj[:, armName[channel], :num_pixels] /= dwave
frame_rand_noise[:, armName[channel], :num_pixels] /= dwave
nivar[:, armName[channel], :num_pixels] *= dwave**2
nsky[:, armName[channel], :num_pixels] /= dwave
sky_rand_noise[:, armName[channel], :num_pixels] /= dwave
sky_ivar[:, armName[channel], :num_pixels] /= dwave**2
# Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra
# Looping over spectrograph
for ii in range((args.nspec + args.nstart - 1) // 500 + 1):
start = max(500 * ii,
args.nstart) # first spectrum for a given spectrograph
end = min(500 * (ii + 1),
nmax) # last spectrum for the spectrograph
if (args.spectrograph <= ii):
camera = "{}{}".format(channel, ii)
log.info(
"Writing files for channel:{}, spectrograph:{}, spectra:{} to {}"
.format(channel, ii, start, end))
num_pixels = len(waves[channel])
# Write frame file
framefileName = desispec.io.findfile("frame", NIGHT, EXPID,
camera)
frame_flux=nobj[start:end,armName[channel],:num_pixels]+ \
nsky[start:end,armName[channel],:num_pixels] + \
frame_rand_noise[start:end,armName[channel],:num_pixels]
frame_ivar = nivar[start:end, armName[channel], :num_pixels]
sh1 = frame_flux.shape[
0] # required for slicing the resolution metric, resolusion matrix has (nspec,ndiag,wave)
# for example if nstart =400, nspec=150: two spectrographs:
# 400-499=> 0 spectrograph, 500-549 => 1
if (args.nstart == start):
resol = resolution[channel][:sh1, :, :]
else:
resol = resolution[channel][-sh1:, :, :]
# must create desispec.Frame object
frame=Frame(waves[channel], frame_flux, frame_ivar,\
resolution_data=resol, spectrograph=ii, \
fibermap=fibermap[start:end], \
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) )
desispec.io.write_frame(framefileName, frame)
framefilePath = desispec.io.findfile("frame", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(framefilePath))
if args.frameonly or simspec.flavor == 'arc':
continue
# Write cframe file
cframeFileName = desispec.io.findfile("cframe", NIGHT, EXPID,
camera)
cframeFlux = cframe_observedflux[
start:end,
armName[channel], :num_pixels] + cframe_rand_noise[
start:end, armName[channel], :num_pixels]
cframeIvar = cframe_ivar[start:end,
armName[channel], :num_pixels]
# must create desispec.Frame object
cframe = Frame(waves[channel], cframeFlux, cframeIvar, \
resolution_data=resol, spectrograph=ii,
fibermap=fibermap[start:end],
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) )
desispec.io.frame.write_frame(cframeFileName, cframe)
cframefilePath = desispec.io.findfile("cframe", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(cframefilePath))
# Write sky file
skyfileName = desispec.io.findfile("sky", NIGHT, EXPID, camera)
skyflux=nsky[start:end,armName[channel],:num_pixels] + \
sky_rand_noise[start:end,armName[channel],:num_pixels]
skyivar = sky_ivar[start:end, armName[channel], :num_pixels]
skymask = np.zeros(skyflux.shape, dtype=np.uint32)
# must create desispec.Sky object
skymodel = SkyModel(waves[channel],
skyflux,
skyivar,
skymask,
header=dict(CAMERA=camera))
desispec.io.sky.write_sky(skyfileName, skymodel)
skyfilePath = desispec.io.findfile("sky", NIGHT, EXPID, camera)
log.info("Wrote file {}".format(skyfilePath))
# Write calib file
calibVectorFile = desispec.io.findfile("calib", NIGHT, EXPID,
camera)
flux = cframe_observedflux[start:end,
armName[channel], :num_pixels]
phot = nobj[start:end, armName[channel], :num_pixels]
calibration = np.zeros_like(phot)
jj = (flux > 0)
calibration[jj] = phot[jj] / flux[jj]
#- TODO: what should calibivar be?
#- For now, model it as the noise of combining ~10 spectra
calibivar = 10 / cframe_ivar[start:end,
armName[channel], :num_pixels]
#mask=(1/calibivar>0).astype(int)??
mask = np.zeros(calibration.shape, dtype=np.uint32)
# write flux calibration
fluxcalib = FluxCalib(waves[channel], calibration, calibivar,
mask)
write_flux_calibration(calibVectorFile, fluxcalib)
calibfilePath = desispec.io.findfile("calib", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(calibfilePath))
def fake_raw():
dirhash = uuid.uuid4()
night = fake_night()
rawdir = "test_pipe_raw_{}".format(dirhash)
if os.path.exists(rawdir):
shutil.rmtree(rawdir)
os.makedirs(rawdir)
nightdir = os.path.join(rawdir, night)
# set up one spectrograph (500 fibers)
nspec = 500
# arc
expid = "00000000"
tileid = "0"
flavor = "arc"
telera = "0.0"
teledec = "0.0"
fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid))
hdr = dict(
NIGHT=(night, 'Night of observation YEARMMDD'),
EXPID=(expid, 'DESI exposure ID'),
TILEID=(tileid, 'DESI tile ID'),
FLAVOR=(flavor, 'Flavor [arc, flat, science, ...]'),
TELRA=(telera, 'Telescope pointing RA [degrees]'),
TELDEC=(teledec, 'Telescope pointing dec [degrees]'),
)
fibermap = io.empty_fibermap(nspec)
fibermap['OBJTYPE'] = 'ARC'
io.write_fibermap(fmfile, fibermap, header=hdr)
for cam in ["r0", "b0", "z0"]:
pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid))
pix = np.random.normal(0, 3.0, size=(10, 10))
ivar = np.ones_like(pix) / 3.0**2
mask = np.zeros(pix.shape, dtype=np.uint32)
img = dimg.Image(pix, ivar, mask, camera=cam)
io.write_image(pfile, img)
# flat
expid = "00000001"
tileid = "1"
flavor = "flat"
telera = "0.0"
teledec = "0.0"
fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid))
hdr = dict(
NIGHT=(night, 'Night of observation YEARMMDD'),
EXPID=(expid, 'DESI exposure ID'),
TILEID=(tileid, 'DESI tile ID'),
FLAVOR=(flavor, 'Flavor [arc, flat, science, ...]'),
TELRA=(telera, 'Telescope pointing RA [degrees]'),
TELDEC=(teledec, 'Telescope pointing dec [degrees]'),
)
fibermap = io.empty_fibermap(nspec)
fibermap['OBJTYPE'] = 'FLAT'
io.write_fibermap(fmfile, fibermap, header=hdr)
for cam in ["r0", "b0", "z0"]:
pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid))
pix = np.random.normal(0, 3.0, size=(10, 10))
ivar = np.ones_like(pix) / 3.0**2
mask = np.zeros(pix.shape, dtype=np.uint32)
img = dimg.Image(pix, ivar, mask, camera=cam)
io.write_image(pfile, img)
# science
expid = "00000002"
tileid = "2"
flavor = "dark"
telera = "0.0"
teledec = "0.0"
fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid))
hdr = dict(
NIGHT=(night, 'Night of observation YEARMMDD'),
EXPID=(expid, 'DESI exposure ID'),
TILEID=(tileid, 'DESI tile ID'),
FLAVOR=(flavor, 'Flavor [arc, flat, science, ...]'),
TELRA=(telera, 'Telescope pointing RA [degrees]'),
TELDEC=(teledec, 'Telescope pointing dec [degrees]'),
)
fibermap = io.empty_fibermap(nspec)
fibermap['OBJTYPE'] = 'ELG'
fibermap['FIBER'] = np.arange(nspec, dtype='i4')
fibermap['TARGETID'] = np.random.randint(sys.maxsize, size=nspec)
fibermap['BRICKNAME'] = ['3412p195' for x in range(nspec)]
io.write_fibermap(fmfile, fibermap, header=hdr)
for cam in ["r0", "b0", "z0"]:
pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid))
pix = np.random.normal(0, 3.0, size=(10, 10))
ivar = np.ones_like(pix) / 3.0**2
mask = np.zeros(pix.shape, dtype=np.uint32)
img = dimg.Image(pix, ivar, mask, camera=cam)
io.write_image(pfile, img)
return rawdir
def fake_raw():
dirhash = uuid.uuid4()
night = fake_night()
rawdir = "test_pipe_raw_{}".format(dirhash)
if os.path.exists(rawdir):
shutil.rmtree(rawdir)
os.makedirs(rawdir)
nightdir = os.path.join(rawdir, night)
# set up one spectrograph (500 fibers)
nspec = 500
# arc
expid = "00000000"
tileid = "0"
flavor = "arc"
telera = "0.0"
teledec = "0.0"
fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid))
hdr = dict(
NIGHT = (night, 'Night of observation YEARMMDD'),
EXPID = (expid, 'DESI exposure ID'),
TILEID = (tileid, 'DESI tile ID'),
FLAVOR = (flavor, 'Flavor [arc, flat, science, ...]'),
TELRA = (telera, 'Telescope pointing RA [degrees]'),
TELDEC = (teledec, 'Telescope pointing dec [degrees]'),
)
fibermap = io.empty_fibermap(nspec)
fibermap['OBJTYPE'] = 'ARC'
io.write_fibermap(fmfile, fibermap, header=hdr)
for cam in ["r0", "b0", "z0"]:
pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid))
pix = np.random.normal(0, 3.0, size=(10,10))
ivar = np.ones_like(pix) / 3.0**2
mask = np.zeros(pix.shape, dtype=np.uint32)
img = dimg.Image(pix, ivar, mask, camera=cam)
io.write_image(pfile, img)
# flat
expid = "00000001"
tileid = "1"
flavor = "flat"
telera = "0.0"
teledec = "0.0"
fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid))
hdr = dict(
NIGHT = (night, 'Night of observation YEARMMDD'),
EXPID = (expid, 'DESI exposure ID'),
TILEID = (tileid, 'DESI tile ID'),
FLAVOR = (flavor, 'Flavor [arc, flat, science, ...]'),
TELRA = (telera, 'Telescope pointing RA [degrees]'),
TELDEC = (teledec, 'Telescope pointing dec [degrees]'),
)
fibermap = io.empty_fibermap(nspec)
fibermap['OBJTYPE'] = 'FLAT'
io.write_fibermap(fmfile, fibermap, header=hdr)
for cam in ["r0", "b0", "z0"]:
pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid))
pix = np.random.normal(0, 3.0, size=(10,10))
ivar = np.ones_like(pix) / 3.0**2
mask = np.zeros(pix.shape, dtype=np.uint32)
img = dimg.Image(pix, ivar, mask, camera=cam)
io.write_image(pfile, img)
# science
expid = "00000002"
tileid = "2"
flavor = "dark"
telera = "0.0"
teledec = "0.0"
fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid))
hdr = dict(
NIGHT = (night, 'Night of observation YEARMMDD'),
EXPID = (expid, 'DESI exposure ID'),
TILEID = (tileid, 'DESI tile ID'),
FLAVOR = (flavor, 'Flavor [arc, flat, science, ...]'),
TELRA = (telera, 'Telescope pointing RA [degrees]'),
TELDEC = (teledec, 'Telescope pointing dec [degrees]'),
)
fibermap = io.empty_fibermap(nspec)
fibermap['OBJTYPE'] = 'ELG'
fibermap['FIBER'] = np.arange(nspec, dtype='i4')
fibermap['TARGETID'] = np.random.randint(sys.maxsize, size=nspec)
fibermap['BRICKNAME'] = [ '3412p195' for x in range(nspec) ]
io.write_fibermap(fmfile, fibermap, header=hdr)
for cam in ["r0", "b0", "z0"]:
pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid))
pix = np.random.normal(0, 3.0, size=(10,10))
ivar = np.ones_like(pix) / 3.0**2
mask = np.zeros(pix.shape, dtype=np.uint32)
img = dimg.Image(pix, ivar, mask, camera=cam)
io.write_image(pfile, img)
return rawdir
def setUp(self):
#- catch specific warnings so that we can find and fix
# warnings.filterwarnings("error", ".*did not parse as fits unit.*")
#- Test data and files to work with
self.fileio = "test_spectra.fits"
self.fileappend = "test_spectra_append.fits"
self.filebuild = "test_spectra_build.fits"
self.meta = {"KEY1": "VAL1", "KEY2": "VAL2"}
self.nwave = 100
self.nspec = 5
self.ndiag = 3
fmap = empty_fibermap(self.nspec)
fmap = add_columns(
fmap,
['NIGHT', 'EXPID', 'TILEID'],
[np.int32(0), np.int32(0), np.int32(0)],
)
for s in range(self.nspec):
fmap[s]["TARGETID"] = 456 + s
fmap[s]["FIBER"] = 123 + s
fmap[s]["NIGHT"] = s
fmap[s]["EXPID"] = s
self.fmap1 = encode_table(fmap)
fmap = empty_fibermap(self.nspec)
fmap = add_columns(
fmap,
['NIGHT', 'EXPID', 'TILEID'],
[np.int32(0), np.int32(0), np.int32(0)],
)
for s in range(self.nspec):
fmap[s]["TARGETID"] = 789 + s
fmap[s]["FIBER"] = 200 + s
fmap[s]["NIGHT"] = 1000
fmap[s]["EXPID"] = 1000 + s
self.fmap2 = encode_table(fmap)
for s in range(self.nspec):
fmap[s]["TARGETID"] = 1234 + s
fmap[s]["FIBER"] = 300 + s
fmap[s]["NIGHT"] = 2000
fmap[s]["EXPID"] = 2000 + s
self.fmap3 = encode_table(fmap)
self.bands = ["b", "r", "z"]
self.wave = {}
self.flux = {}
self.ivar = {}
self.mask = {}
self.res = {}
self.extra = {}
for s in range(self.nspec):
for b in self.bands:
self.wave[b] = np.arange(self.nwave)
self.flux[b] = np.repeat(np.arange(self.nspec),
self.nwave).reshape(
(self.nspec, self.nwave)) + 3.0
self.ivar[b] = 1.0 / self.flux[b]
self.mask[b] = np.tile(np.arange(2, dtype=np.uint32),
(self.nwave * self.nspec) // 2).reshape(
(self.nspec, self.nwave))
self.res[b] = np.zeros((self.nspec, self.ndiag, self.nwave),
dtype=np.float64)
self.res[b][:, 1, :] = 1.0
self.extra[b] = {}
self.extra[b]["FOO"] = self.flux[b]
self.scores = dict(BLAT=np.arange(self.nspec), FOO=np.ones(self.nspec))
self.extra_catalog = Table()
self.extra_catalog['A'] = np.arange(self.nspec)
self.extra_catalog['B'] = np.ones(self.nspec)
def setUp(cls):
cls.program = program = 'dark'
cls.flavor = flavor = 'bias'
cls.night = night = '20150105'
cls.camera = camera = 'r0'
cls.expid = expid = 314
cls.psfExpid = psfExpid = 313
cls.flatExpid = flatExpid = 312
cls.templateExpid = templateExpid = 311
cls.nspec = nspec = 5
cls.exptime = exptime = 100
#- Seup environment and override default environment variables
#- python 2.7 location:
cls.topDir = os.path.dirname( # top-level
os.path.dirname( # py/
os.path.dirname( # desispec/
os.path.dirname(os.path.abspath(__file__)) # test/
)
)
)
cls.binDir = os.path.join(cls.topDir,'bin')
if not os.path.isdir(cls.binDir):
#- python 3.x setup.py test location:
cls.topDir = os.path.dirname( # top-level
os.path.dirname( # build/
os.path.dirname( # lib/
os.path.dirname( # desispec/
os.path.dirname(os.path.abspath(__file__)) # test/
)
)
)
)
cls.binDir = os.path.join(cls.topDir,'bin')
#- last attempt
if not os.path.isdir(cls.binDir):
cls.topDir = os.getcwd()
cls.binDir = os.path.join(cls.topDir, 'bin')
if not os.path.isdir(cls.binDir):
raise RuntimeError('Unable to auto-locate desispec/bin from {}'.format(__file__))
id = uuid4().hex
cls.fibermapfile = 'fibermap-'+id+'.fits'
cls.framefile = 'frame-'+id+'.fits'
cls.testDir = testDir = os.path.join(os.environ['HOME'],'ql_test_io')
dataDir = os.path.join(testDir,night)
expDir = os.path.join(testDir,'exposures')
nightDir = os.path.join(expDir,night)
reduxDir = os.path.join(nightDir,'{:08d}'.format(expid))
if not os.path.exists(testDir):
os.makedirs(testDir)
os.makedirs(dataDir)
os.makedirs(expDir)
os.makedirs(nightDir)
os.makedirs(reduxDir)
#- Write dummy configuration and input files to test merging
configdict = {'name': 'Test Configuration',
'Program': program,
'Flavor': flavor,
'PSFExpid': psfExpid,
'PSFType': 'psf',
'FiberflatExpid': flatExpid,
'TemplateExpid': templateExpid,
'WritePixfile': False,
'WriteSkyModelfile': False,
'WriteIntermediatefiles': False,
'WriteStaticPlots': False,
'Debuglevel': 20,
'UseResolution': False,
'Period': 5.0,
'Timeout': 120.0,
'Pipeline': ['Initialize','Preproc'],
'Algorithms': {'Initialize':{
'QA':{
'Bias_From_Overscan':{'PARAMS':{'PERCENTILES':[68.2,95.4,99.7],'DIFF_WARN_RANGE':[-1.0,1.0],'DIFF_ALARM_RANGE':[-2.0,2.0]}}}},
'Preproc':{
'QA':{
'Get_RMS':{'PARAMS':{'RMS_WARN_RANGE':[-1.0,1.0],'RMS_ALARM_RANGE':[-2.0,2.0]}},
'Count_Pixels':{'PARAMS':{'CUTPIX':500,'LITFRAC_NORMAL_RANGE':[200.0,500.0],'LITFRAC_WARN_RANGE':[50.0,650.0]}}}}}
}
with open('{}/test_config.yaml'.format(testDir),'w') as config:
yaml.dump(configdict,config)
cls.configfile = '{}/test_config.yaml'.format(testDir)
#- Generate raw file
rawfile = os.path.join(dataDir,'desi-00000314.fits.fz')
raw_hdr = {}
raw_hdr['DATE-OBS'] = '2015-01-05T08:17:03.988'
raw_hdr['NIGHT'] = night
raw_hdr['PROGRAM'] = program
raw_hdr['FLAVOR'] = flavor
raw_hdr['CAMERA'] = camera
raw_hdr['EXPID'] = expid
raw_hdr['EXPTIME'] = exptime
raw_hdr['DOSVER'] = 'SIM'
raw_hdr['FEEVER'] = 'SIM'
raw_hdr['DETECTOR'] = 'SIM'
raw_hdr['PRESEC1'] = '[1:4,1:2048]'
raw_hdr['DATASEC1'] = '[5:2052,1:2048]'
raw_hdr['BIASSEC1'] = '[2053:2102,1:2048]'
raw_hdr['CCDSEC1'] = '[1:2048,1:2048]'
raw_hdr['PRESEC2'] = '[4201:4204,1:2048]'
raw_hdr['DATASEC2'] = '[2153:4200,1:2048]'
raw_hdr['BIASSEC2'] = '[2103:2152,1:2048]'
raw_hdr['CCDSEC2'] = '[2049:4096,1:2048]'
raw_hdr['PRESEC3'] = '[1:4,2049:4096]'
raw_hdr['DATASEC3'] = '[5:2052,2049:4096]'
raw_hdr['BIASSEC3'] = '[2053:2102,2049:4096]'
raw_hdr['CCDSEC3'] = '[1:2048,2049:4096]'
raw_hdr['PRESEC4'] = '[4201:4204,2049:4096]'
raw_hdr['DATASEC4'] = '[2153:4200,2049:4096]'
raw_hdr['BIASSEC4'] = '[2103:2152,2049:4096]'
raw_hdr['CCDSEC4'] = '[2049:4096,2049:4096]'
raw_hdr['GAIN1'] = 1.0
raw_hdr['GAIN2'] = 1.0
raw_hdr['GAIN3'] = 1.0
raw_hdr['GAIN4'] = 1.0
raw_hdr['RDNOISE1'] = 3.0
raw_hdr['RDNOISE2'] = 3.0
raw_hdr['RDNOISE3'] = 3.0
raw_hdr['RDNOISE4'] = 3.0
data=np.zeros((4096,4204))+200.
raw_data=data.astype(int)
write_raw(rawfile,raw_data,raw_hdr)
#- Generate fibermap file
fibermapfile = os.path.join(dataDir,'fibermap-00000314.fits')
fibermap = empty_fibermap(nspec)
write_fibermap(fibermapfile,fibermap)
def simulate(airmass=None,
exptime=None,
seeing=None,
moon_frac=None,
moon_sep=None,
moon_alt=None,
seed=1234,
nspec=5000,
brickname='testbrick',
galsim=False,
ra=None,
dec=None):
#- construct the simulator
qsim = simulator.Simulator('desi')
# Initialize random number generator to use.
random_state = np.random.RandomState(seed)
#- Create a blank fake fibermap for bricks
fibermap = empty_fibermap(nspec)
targetids = random_state.randint(2**62, size=nspec)
fibermap['TARGETID'] = targetids
night = get_night()
expid = 0
#- working out only ELG
objtype = 'ELG'
true_objtype = np.tile(np.array([objtype]), (nspec))
#- Initialize the output truth table.
spectra = []
wavemin = desimodel.io.load_throughput('b').wavemin
wavemax = desimodel.io.load_throughput('z').wavemax
dw = 0.2
wavelengths = np.arange(round(wavemin, 1), wavemax, dw)
npix = len(wavelengths)
truth = dict()
meta = Table()
truth['OBJTYPE'] = 'ELG' * nspec
truth['FLUX'] = np.zeros((nspec, npix))
truth['WAVE'] = wavelengths
#- get the templates
flux, tmpwave, meta1 = get_templates(wavelengths, seed=seed, nmodel=nspec)
truth['FLUX'] = flux
meta = vstack([meta, meta1])
#- Add TARGETID and the true OBJTYPE to the metadata table.
meta.add_column(Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE'))
meta.add_column(Column(targetids, name='TARGETID'))
#- Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z
meta.rename_column('REDSHIFT', 'TRUEZ')
waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {}
#- Now simulate
maxbin = 0
nmax = nspec
for camera in qsim.instrument.cameras:
# Lookup this camera's resolution matrix and convert to the sparse
# format used in desispec.
R = Resolution(camera.get_output_resolution_matrix())
resolution[camera.name] = np.tile(R.to_fits_array(), [nspec, 1, 1])
waves[camera.name] = (camera.output_wavelength.to(
u.Angstrom).value.astype(np.float32))
nwave = len(waves[camera.name])
maxbin = max(maxbin, len(waves[camera.name]))
nobj = np.zeros((nmax, 3, maxbin)) # object photons
nsky = np.zeros((nmax, 3, maxbin)) # sky photons
nivar = np.zeros((nmax, 3, maxbin)) # inverse variance (object+sky)
cframe_observedflux = np.zeros(
(nmax, 3, maxbin)) # calibrated object flux
cframe_ivar = np.zeros(
(nmax, 3, maxbin)) # inverse variance of calibrated object flux
cframe_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to calibrated flux
sky_ivar = np.zeros((nmax, 3, maxbin)) # inverse variance of sky
sky_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to sky only
frame_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to nobj+nsky
trueflux[camera.name] = np.empty(
(nspec, nwave)) # calibrated brick flux
noisyflux[camera.name] = np.empty(
(nspec, nwave)) # brick flux with noise
obsivar[camera.name] = np.empty(
(nspec, nwave)) # inverse variance of brick flux
sflux = np.empty((nspec, npix))
#- Repeat the simulation for all spectra
fluxunits = 1e-17 * u.erg / (u.s * u.cm**2 * u.Angstrom)
spectra = truth['FLUX'] * 1.0e17
print("Simulating Spectra")
for j in range(nspec):
print("Simulating %s/%s spectra" % (j, nspec), end='\r')
thisobjtype = 'ELG'
sys.stdout.flush()
#- update qsim using conditions
if airmass is None:
thisairmass = None
else:
thisairmass = airmass[j]
if seeing is None:
thisseeing = None
else:
thisseeing = seeing[j]
if moon_frac is None:
thismoon_frac = None
else:
thismoon_frac = moon_frac[j]
if moon_sep is None:
thismoon_sep = None
else:
thismoon_sep = moon_sep[j]
if moon_alt is None:
thismoon_alt = None
else:
thismoon_alt = moon_alt[j]
if exptime is None:
thisexptime = None
else:
thisexptime = exptime[j]
nqsim = update_simulator(qsim,
airmass=thisairmass,
exptime=thisexptime,
seeing=thisseeing,
moon_frac=thismoon_frac,
moon_sep=thismoon_sep,
moon_alt=thismoon_alt,
galsim=galsim)
nqsim.source.update_in('Quickgen source {0}'.format(j),
thisobjtype.lower(), wavelengths * u.Angstrom,
spectra[j, :] * fluxunits)
nqsim.source.update_out()
nqsim.simulate()
nqsim.generate_random_noise(random_state)
sflux[j][:] = 1e17 * qsim.source.flux_in.to(fluxunits).value
for i, output in enumerate(nqsim.camera_output):
assert output['observed_flux'].unit == 1e17 * fluxunits
# Extract the simulation results needed to create our uncalibrated
# frame output file.
num_pixels = len(output)
nobj[j, i, :num_pixels] = output['num_source_electrons'][:, 0]
nsky[j, i, :num_pixels] = output['num_sky_electrons'][:, 0]
nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:, 0]
# Get results for our flux-calibrated output file.
cframe_observedflux[
j, i, :num_pixels] = 1e17 * output['observed_flux'][:, 0]
cframe_ivar[
j,
i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:, 0]
# Fill brick arrays from the results.
camera = output.meta['name']
trueflux[camera][j][:] = 1e17 * output['observed_flux'][:, 0]
noisyflux[camera][j][:] = 1e17 * (
output['observed_flux'][:, 0] +
output['flux_calibration'][:, 0] *
output['random_noise_electrons'][:, 0])
#return output
obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:,
0]
# Use the same noise realization in the cframe and frame, without any
# additional noise from sky subtraction for now.
frame_rand_noise[
j, i, :num_pixels] = output['random_noise_electrons'][:, 0]
cframe_rand_noise[j, i, :num_pixels] = 1e17 * (
output['flux_calibration'][:, 0] *
output['random_noise_electrons'][:, 0])
# The sky output file represents a model fit to ~40 sky fibers.
# We reduce the variance by a factor of 25 to account for this and
# give the sky an independent (Gaussian) noise realization.
sky_ivar[
j,
i, :num_pixels] = 25.0 / (output['variance_electrons'][:, 0] -
output['num_source_electrons'][:, 0])
sky_rand_noise[j, i, :num_pixels] = random_state.normal(
scale=1.0 / np.sqrt(sky_ivar[j, i, :num_pixels]),
size=num_pixels)
cframe_flux = cframe_observedflux[
j, i, :num_pixels] + cframe_rand_noise[j, i, :num_pixels]
armName = {"b": 0, "r": 1, "z": 2}
for channel in 'brz':
num_pixels = len(waves[channel])
dwave = np.gradient(waves[channel])
nobj[:, armName[channel], :num_pixels] /= dwave
frame_rand_noise[:, armName[channel], :num_pixels] /= dwave
nivar[:, armName[channel], :num_pixels] *= dwave**2
nsky[:, armName[channel], :num_pixels] /= dwave
sky_rand_noise[:, armName[channel], :num_pixels] /= dwave
sky_ivar[:, armName[channel], :num_pixels] /= dwave**2
# Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra
# Output brick files
if ra is None or dec is None:
filename = 'brick-{}-{}.fits'.format(channel, brickname)
filepath = os.path.normpath(
os.path.join('{}'.format(brickname), filename))
if os.path.exists(filepath):
os.remove(filepath)
print('Writing {}'.format(filepath))
header = dict(BRICKNAM=brickname, CHANNEL=channel)
brick = Brick(filepath, mode='update', header=header)
brick.add_objects(noisyflux[channel], obsivar[channel],
waves[channel], resolution[channel], fibermap,
night, expid)
brick.close()
"""
# Append truth to the file. Note: we add the resolution-convolved true
# flux, not the high resolution source flux, which makes chi2
# calculations easier.
header = fitsheader(header)
fx = fits.open(filepath, mode='append')
_add_truth(fx, header, meta, trueflux, sflux, wavelengths, channel)
fx.flush()
fx.close()
#sys.stdout.close()
"""
print("Wrote file {}".format(filepath))
else:
bricknames = get_bricknames(ra, dec)
fibermap['BRICKNAME'] = bricknames
bricknames = set(bricknames)
print("No. of bricks: {}".format(len(bricknames)))
print("Writing brick files")
for brick_name in bricknames:
thisbrick = (fibermap['BRICKNAME'] == brick_name)
brickdata = fibermap[thisbrick]
fibers = brickdata['FIBER'] #np.mod(brickdata['FIBER'],nspec)
filename = 'brick-{}-{}.fits'.format(channel, brick_name)
filepath = os.path.normpath(
os.path.join('./{}'.format(brick_name), filename))
if os.path.exists(filepath):
os.remove(filepath)
#print('Writing {}'.format(filepath))
header = dict(BRICKNAM=brick_name, CHANNEL=channel)
brick = Brick(filepath, mode='update', header=header)
brick.add_objects(noisyflux[channel][fibers],
obsivar[channel][fibers], waves[channel],
resolution[channel][fibers], brickdata,
night, expid)
brick.close()
print("Finished writing brick files for {} bricks".format(
len(bricknames)))
#- make a truth file
header = fitsheader(header)
make_truthfile(header, meta, trueflux, sflux, wavelengths)
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
bias=True
if args.bias : bias=args.bias
if args.nobias : bias=False
dark=True
if args.dark : dark=args.dark
if args.nodark : dark=False
pixflat=True
if args.pixflat : pixflat=args.pixflat
if args.nopixflat : pixflat=False
mask=True
if args.mask : mask=args.mask
if args.nomask : mask=False
if args.cameras is None:
args.cameras = [c+str(i) for c in 'brz' for i in range(10)]
else:
args.cameras = args.cameras.split(',')
if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None) or (args.dark is not None):
if len(args.cameras) > 1:
raise ValueError('must use only one camera with --bias, --dark, --pixflat, --mask options')
if (args.outfile is not None) and len(args.cameras) > 1:
raise ValueError('must use only one camera with --outfile option')
if args.outdir is None:
args.outdir = os.getcwd()
log.warning('--outdir not specified; using {}'.format(args.outdir))
ccd_calibration_filename = None
if args.no_ccd_calib_filename :
ccd_calibration_filename = False
elif args.ccd_calib_filename is not None :
ccd_calibration_filename = args.ccd_calib_filename
if args.fibermap and not os.path.exists(fibermap):
raise ValueError('--fibermap {} not found'.format(args.fibermap))
if args.fibermap is None:
datadir, infile = os.path.split(os.path.abspath(args.infile))
fibermapfile = infile.replace('desi-', 'fibermap-').replace('.fits.fz', '.fits')
args.fibermap = os.path.join(datadir, fibermapfile)
if args.nofibermap:
fibermap = None
elif os.path.exists(args.fibermap):
fibermap = io.read_fibermap(args.fibermap)
else:
log.warning('fibermap file not found; creating blank fibermap')
fibermap = io.empty_fibermap(5000)
for camera in args.cameras:
try:
img = io.read_raw(args.infile, camera,
bias=bias, dark=dark, pixflat=pixflat, mask=mask, bkgsub=args.bkgsub,
nocosmic=args.nocosmic,
cosmics_nsig=args.cosmics_nsig,
cosmics_cfudge=args.cosmics_cfudge,
cosmics_c2fudge=args.cosmics_c2fudge,
ccd_calibration_filename=ccd_calibration_filename,
nocrosstalk=args.nocrosstalk,
nogain=args.nogain,
nodarktrail=args.nodarktrail,
fill_header=args.fill_header,
)
except IOError:
log.error('Error while reading or preprocessing camera {} in {}'.format(camera, args.infile))
continue
if(args.zero_masked) :
img.pix *= (img.mask==0)
if args.outfile is None:
night = img.meta['NIGHT']
expid = img.meta['EXPID']
outfile = io.findfile('preproc', night=night, expid=expid, camera=camera,
outdir=args.outdir)
else:
outfile = args.outfile
if fibermap:
petal_loc = int(img.camera[1])
ii = (fibermap['PETAL_LOC'] == petal_loc)
img.fibermap = fibermap[ii]
io.write_image(outfile, img)
log.info("Wrote {}".format(outfile))
def setUp(cls):
cls.program = program = 'dark'
cls.flavor = flavor = 'bias'
cls.night = night = '20150105'
cls.camera = camera = 'r0'
cls.expid = expid = 314
cls.psfExpid = psfExpid = 313
cls.flatExpid = flatExpid = 312
cls.templateExpid = templateExpid = 311
cls.nspec = nspec = 5
cls.exptime = exptime = 100
#- Setup environment and override default environment variables
#- python 2.7 location:
cls.topDir = os.path.dirname( # top-level
os.path.dirname( # py/
os.path.dirname( # desispec/
os.path.dirname(os.path.abspath(__file__)) # test/
)
)
)
cls.binDir = os.path.join(cls.topDir,'bin')
if not os.path.isdir(cls.binDir):
#- python 3.x setup.py test location:
cls.topDir = os.path.dirname( # top-level
os.path.dirname( # build/
os.path.dirname( # lib/
os.path.dirname( # desispec/
os.path.dirname(os.path.abspath(__file__)) # test/
)
)
)
)
cls.binDir = os.path.join(cls.topDir,'bin')
#- last attempt
if not os.path.isdir(cls.binDir):
cls.topDir = os.getcwd()
cls.binDir = os.path.join(cls.topDir, 'bin')
if not os.path.isdir(cls.binDir):
raise RuntimeError('Unable to auto-locate desispec/bin from {}'.format(__file__))
id = uuid4().hex
cls.fibermapfile = 'fibermap-'+id+'.fits'
cls.framefile = 'frame-'+id+'.fits'
cls.testDir = testDir = os.path.join(os.environ['HOME'],'ql_test_io')
datanightDir = os.path.join(testDir,night)
dataDir = os.path.join(datanightDir,'{:08d}'.format(expid))
expDir = os.path.join(testDir,'exposures')
expnightDir = os.path.join(expDir,night)
reduxDir = os.path.join(expnightDir,'{:08d}'.format(expid))
calibDir = os.path.join(testDir, 'ql_calib')
configDir = os.path.join(testDir, 'ql_config')
os.environ['QL_CALIB_DIR'] = calibDir
os.environ['QL_CONFIG_DIR'] = configDir
if not os.path.exists(testDir):
os.makedirs(testDir)
os.makedirs(datanightDir)
os.makedirs(dataDir)
os.makedirs(expDir)
os.makedirs(expnightDir)
os.makedirs(reduxDir)
os.makedirs(calibDir)
os.makedirs(configDir)
#- Write dummy configuration and input files to test merging
configdict = {'name': 'Test Configuration',
'Program': program,
'Flavor': flavor,
'PSFExpid': psfExpid,
'PSFType': 'psf',
'FiberflatExpid': flatExpid,
'TemplateExpid': templateExpid,
'TemplateNight': night,
'WritePreprocfile': False,
'WriteSkyModelfile': False,
'WriteIntermediatefiles': False,
'WriteStaticPlots': False,
'Debuglevel': 20,
'UseResolution': False,
'Period': 5.0,
'Timeout': 120.0,
'Pipeline': ['Initialize','Preproc'],
'Algorithms': {'Initialize':{
'QA':{'Check_HDUs':{'PARAMS':{}}
}},
'Preproc':{
'QA':{'Bias_From_Overscan':{'PARAMS':{'BIAS_AMP_NORMAL_RANGE':[-100.0,100.0],'BIAS_AMP_WARN_RANGE':[-200.0,200.0]}},
'Get_RMS':{'PARAMS':{'PERCENTILES':[68.2,95.4,99.7],'NOISE_AMP_NORMAL_RANGE':[-1.0,1.0],'NOISE_AMP_WARN_RANGE':[-2.0,2.0]}},
'Count_Pixels':{'PARAMS':{'CUTPIX':500,'LITFRAC_NORMAL_RANGE':[-0.1,0.1],'LITFRAC_WARN_RANGE':[-0.2,0.2]}}}}}
}
with open('{}/test_config.yaml'.format(testDir),'w') as config:
yaml.dump(configdict,config)
cls.configfile = '{}/test_config.yaml'.format(testDir)
#- Generate raw file
rawfile = os.path.join(dataDir,'desi-00000314.fits.fz')
raw_hdr = {}
raw_hdr['DATE-OBS'] = '2015-01-05T08:17:03.988'
raw_hdr['NIGHT'] = night
raw_hdr['PROGRAM'] = program
raw_hdr['FLAVOR'] = flavor
raw_hdr['CAMERA'] = camera
raw_hdr['EXPID'] = expid
raw_hdr['EXPTIME'] = exptime
raw_hdr['DOSVER'] = 'SIM'
raw_hdr['FEEVER'] = 'SIM'
raw_hdr['DETECTOR'] = 'SIM'
raw_hdr['PRESECA'] = '[1:4,1:2048]'
raw_hdr['DATASECA'] = '[5:2052,1:2048]'
raw_hdr['BIASSECA'] = '[2053:2102,1:2048]'
raw_hdr['CCDSECA'] = '[1:2048,1:2048]'
raw_hdr['PRESECB'] = '[4201:4204,1:2048]'
raw_hdr['DATASECB'] = '[2153:4200,1:2048]'
raw_hdr['BIASSECB'] = '[2103:2152,1:2048]'
raw_hdr['CCDSECB'] = '[2049:4096,1:2048]'
raw_hdr['PRESECC'] = '[1:4,2049:4096]'
raw_hdr['DATASECC'] = '[5:2052,2049:4096]'
raw_hdr['BIASSECC'] = '[2053:2102,2049:4096]'
raw_hdr['CCDSECC'] = '[1:2048,2049:4096]'
raw_hdr['PRESECD'] = '[4201:4204,2049:4096]'
raw_hdr['DATASECD'] = '[2153:4200,2049:4096]'
raw_hdr['BIASSECD'] = '[2103:2152,2049:4096]'
raw_hdr['CCDSECD'] = '[2049:4096,2049:4096]'
raw_hdr['GAINA'] = 1.0
raw_hdr['GAINB'] = 1.0
raw_hdr['GAINC'] = 1.0
raw_hdr['GAIND'] = 1.0
raw_hdr['RDNOISEA'] = 3.0
raw_hdr['RDNOISEB'] = 3.0
raw_hdr['RDNOISEC'] = 3.0
raw_hdr['RDNOISED'] = 3.0
primary_header={'PROGRAM':program}
data=np.zeros((4096,4204))+200.
raw_data=data.astype(int)
write_raw(rawfile,raw_data,raw_hdr,primary_header=primary_header)
#- Generate fibermap file
fibermapfile = os.path.join(dataDir,'fibermap-00000314.fits')
fibermap = empty_fibermap(nspec)
write_fibermap(fibermapfile,fibermap)
#- Generate calib data
for camera in ['b0', 'r0', 'z0']:
#- Fiberflat has to exist but can be a dummpy file
filename = '{}/fiberflat-{}.fits'.format(calibDir, camera)
fx = open(filename, 'w'); fx.write('fiberflat file'); fx.close()
#- PSF has to be real file
psffile = '{}/psf-{}.fits'.format(calibDir, camera)
example_psf = resource_filename('desispec', 'test/data/ql/psf-{}.fits'.format(camera))
shutil.copy(example_psf, psffile)
#- Copy test calibration-data.yaml file
specdir=calibDir+"spec/sp0"
if not os.path.isdir(specdir) :
os.makedirs(specdir)
for c in "brz" :
shutil.copy(resource_filename('desispec', 'test/data/ql/{}0.yaml'.format(c)),os.path.join(specdir,"{}0.yaml".format(c)))
#- Set calibration environment variable
os.environ['DESI_SPECTRO_CALIB'] = calibDir
