def test_quickcat(self):
#- First round of obs: perfect input z -> output z
zcat1 = quickcat(self.tilefiles[0:2],
self.targets,
truth=self.truth,
perfect=True)
nx = self.nspec // self.ntiles
self.assertTrue(
np.all(zcat1['TARGETID'] == self.truth['TARGETID'][0:2 * nx]))
self.assertTrue(np.all(zcat1['Z'] == self.truth['TRUEZ'][0:2 * nx]))
self.assertTrue(np.all(zcat1['ZWARN'] == 0))
#- Now observe with random redshift errors
zcat2 = quickcat(self.tilefiles[0:2],
self.targets,
truth=self.truth,
perfect=False)
self.assertTrue(
np.all(zcat2['TARGETID'] == self.truth['TARGETID'][0:2 * nx]))
self.assertTrue(np.all(zcat2['Z'] != self.truth['TRUEZ'][0:2 * nx]))
self.assertTrue(np.any(zcat2['ZWARN'] != 0))
#- And add a second round of observations
zcat3 = quickcat(self.tilefiles[2:4],
self.targets,
truth=self.truth,
zcat=zcat2)
self.assertTrue(np.all(zcat3['TARGETID'] == self.truth['TARGETID']))
self.assertTrue(np.all(zcat3['Z'] != self.truth['TRUEZ']))
#- successful targets in the first round of observations shouldn't be updated
ii2 = np.in1d(zcat2['TARGETID'], zcat3['TARGETID']) & (zcat2['ZWARN']
== 0)
ii3 = np.in1d(zcat3['TARGETID'], zcat2['TARGETID'][ii2])
self.assertTrue(np.all(zcat2['Z'][ii2] == zcat3['Z'][ii3]))
#- Observe the last tile again
zcat3copy = zcat3.copy()
zcat4 = quickcat(self.tilefiles[3:4],
self.targets,
truth=self.truth,
zcat=zcat3)
self.assertTrue(np.all(zcat3copy == zcat3)) #- original unmodified
self.assertTrue(np.all(
zcat4['TARGETID'] == self.truth['TARGETID'])) #- order preserved
self.assertTrue(np.all(zcat4['Z'] != self.truth['TRUEZ']))
#- Check that NUMOBS was incremented
i3 = np.in1d(zcat3['TARGETID'], self.targets_in_tile[self.tileids[3]])
i4 = np.in1d(zcat4['TARGETID'], self.targets_in_tile[self.tileids[3]])
self.assertTrue(np.all(zcat4['NUMOBS'][i4] == zcat3['NUMOBS'][i3] + 1))
#- ZWARN==0 targets should be preserved, while ZWARN!=0 updated
nx = self.nspec // self.ntiles
z3 = zcat3[-nx:]
z4 = zcat4[-nx:]
ii = (z3['ZWARN'] != 0)
self.assertTrue(np.all(z3['Z'][ii] != z4['Z'][ii]))
self.assertTrue(np.all(z3['Z'][~ii] == z4['Z'][~ii]))
def test_multiobs(self):
# Earlier targets got more observations so should have higher efficiency
nx = self.nspec // self.ntiles
zcat = quickcat(self.tilefiles_multiobs, self.targets, truth=self.truth, perfect=False)
n1 = np.count_nonzero(zcat['ZWARN'][0:nx] == 0)
n2 = np.count_nonzero(zcat['ZWARN'][-nx:] == 0)
self.assertGreater(n1, n2)
def test_quickcat(self):
#- First round of obs: perfect input z -> output z
zcat1 = quickcat(self.tilefiles[0:2], self.targets, truth=self.truth, perfect=True)
nx = self.nspec // self.ntiles
self.assertTrue(np.all(zcat1['TARGETID'] == self.truth['TARGETID'][0:2*nx]))
self.assertTrue(np.all(zcat1['Z'] == self.truth['TRUEZ'][0:2*nx]))
self.assertTrue(np.all(zcat1['ZWARN'] == 0))
#- Now observe with random redshift errors
zcat2 = quickcat(self.tilefiles[0:2], self.targets, truth=self.truth, perfect=False)
self.assertTrue(np.all(zcat2['TARGETID'] == self.truth['TARGETID'][0:2*nx]))
self.assertTrue(np.all(zcat2['Z'] != self.truth['TRUEZ'][0:2*nx]))
self.assertTrue(np.any(zcat2['ZWARN'] != 0))
#- And add a second round of observations
zcat3 = quickcat(self.tilefiles[2:4], self.targets, truth=self.truth, zcat=zcat2)
self.assertTrue(np.all(zcat3['TARGETID'] == self.truth['TARGETID']))
self.assertTrue(np.all(zcat3['Z'] != self.truth['TRUEZ']))
#- successful targets in the first round of observations shouldn't be updated
ii2 = np.in1d(zcat2['TARGETID'], zcat3['TARGETID']) & (zcat2['ZWARN'] == 0)
ii3 = np.in1d(zcat3['TARGETID'], zcat2['TARGETID'][ii2])
self.assertTrue(np.all(zcat2['Z'][ii2] == zcat3['Z'][ii3]))
#- Observe the last tile again
zcat3copy = zcat3.copy()
zcat4 = quickcat(self.tilefiles[3:4], self.targets, truth=self.truth, zcat=zcat3)
self.assertTrue(np.all(zcat3copy == zcat3)) #- original unmodified
self.assertTrue(np.all(zcat4['TARGETID'] == self.truth['TARGETID'])) #- order preserved
self.assertTrue(np.all(zcat4['Z'] != self.truth['TRUEZ']))
#- Check that NUMOBS was incremented
i3 = np.in1d(zcat3['TARGETID'], self.targets_in_tile[self.tileids[3]])
i4 = np.in1d(zcat4['TARGETID'], self.targets_in_tile[self.tileids[3]])
self.assertTrue(np.all(zcat4['NUMOBS'][i4] == zcat3['NUMOBS'][i3]+1))
#- ZWARN==0 targets should be preserved, while ZWARN!=0 updated
nx = self.nspec // self.ntiles
z3 = zcat3[-nx:]
z4 = zcat4[-nx:]
ii = (z3['ZWARN'] != 0)
self.assertTrue(np.all(z3['Z'][ii] != z4['Z'][ii]))
self.assertTrue(np.all(z3['Z'][~ii] == z4['Z'][~ii]))
def test_multiobs(self):
# Earlier targets got more observations so should have higher efficiency
nx = self.nspec // self.ntiles
zcat = quickcat(self.tilefiles_multiobs,
self.targets,
truth=self.truth,
perfect=False)
n1 = np.count_nonzero(zcat['ZWARN'][0:nx] == 0)
n2 = np.count_nonzero(zcat['ZWARN'][-nx:] == 0)
self.assertGreater(n1, n2)
def test_multiobs(self):
# Targets with more observations should have a better efficiency
zcat = quickcat(self.tilefiles_multiobs,
self.targets,
truth=self.truth,
perfect=False)
oneobs = (zcat['NUMOBS'] == 1)
manyobs = (zcat['NUMOBS'] == np.max(zcat['NUMOBS']))
goodz = (zcat['ZWARN'] == 0)
p1 = np.count_nonzero(oneobs & goodz) / np.count_nonzero(oneobs)
p2 = np.count_nonzero(manyobs & goodz) / np.count_nonzero(manyobs)
self.assertGreater(p2, p1)
def simulate_epoch(self, epoch, truth, targets, perfect=False, zcat=None):
"""Core routine simulating a DESI epoch,
Args:
epoch (int): epoch to simulate
perfect (boolean): Default: False. Selects whether how redshifts are taken from the truth file.
True: redshifts are taken without any error from the truth file.
False: redshifts include uncertainties.
truth (Table): Truth data
targets (Table): Targets data
zcat (Table): Redshift Catalog Data
Notes:
This routine simulates three steps:
* Merged target list creation
* Fiber allocation
* Redshift catalogue construction
"""
# create the MTL file
print("{} Starting MTL".format(asctime()))
self.mtl_file = os.path.join(self.tmp_output_path, 'mtl.fits')
if zcat is None:
mtl = desitarget.mtl.make_mtl(targets)
else:
mtl = desitarget.mtl.make_mtl(targets, zcat)
mtl.write(self.mtl_file, overwrite=True)
del mtl
gc.collect()
print("{} Finished MTL".format(asctime()))
# clean files and prepare fiberasign inputs
tilefiles = sorted(glob.glob(self.tmp_fiber_path+'/tile*.fits'))
if tilefiles:
for tilefile in tilefiles:
os.remove(tilefile)
# setup the tileids for the current observation epoch
self.create_surveyfile(epoch)
# launch fiberassign
print("{} Launching fiberassign".format(asctime()))
f = open('fiberassign.log','a')
p = subprocess.call([self.fiberassign,
'--mtl', os.path.join(self.tmp_output_path, 'mtl.fits'),
'--stdstar', self.stdfile,
'--sky', self.skyfile,
'--surveytiles', self.surveyfile,
'--outdir',os.path.join(self.tmp_output_path, 'fiberassign'),
'--fibstatusfile', self.fibstatusfile],
stdout=f)
print("{} Finished fiberassign".format(asctime()))
f.close()
# create a list of fiberassign tiles to read and update zcat
self.update_observed_tiles(epoch)
#update obsconditions from progress_data
# progress_data = Table.read(self.progress_files[epoch + 1])
# ii = np.in1d(progress_data['TILEID'], self.observed_tiles)
# obsconditions = progress_data[ii]
obsconditions = None
print('tilefiles', len(self.tilefiles))
# write the zcat, it uses the tilesfiles constructed in the last step
self.zcat_file = os.path.join(self.tmp_output_path, 'zcat.fits')
print("{} starting quickcat".format(asctime()))
newzcat = quickcat(self.tilefiles, targets, truth, zcat=zcat,
obsconditions=obsconditions, perfect=perfect)
print("{} writing zcat".format(asctime()))
newzcat.write(self.zcat_file, format='fits', overwrite=True)
print("{} Finished zcat".format(asctime()))
del newzcat
gc.collect()
return
targets[iiobj].write('targets.fits', overwrite=True)
targets[iisky].write('sky.fits', overwrite=True)
targets[iistd].write('stdstars.fits', overwrite=True)
truth[iiobj].write('truth.fits', overwrite=True)
# this part tests another PR https://github.com/desihub/desitarget/pull/32
mtl = desitarget.mtl.make_mtl(targets)
mtl.write('mtl.fits', overwrite=True)
! cp *.fits /project/projectdirs/desi/users/forero/mtl
# this now tests another PR https://github.com/desihub/fiberassign/pull/24
# I had to go first into fiberassign to run make && make install
! ./fiberassign/bin/./fiberassign fa_features.txt
#find the tiles
from glob import glob
tiledir = '/project/projectdirs/desi/users/forero/fiberassign'
tilefiles = sorted(glob(tiledir+'/tile*.fits'))
len(tilefiles)
# this ugly hack avoids import from desi stack
import os
os.chdir('/global/homes/f/forero/desisim/py')
#testing this PR: https://github.com/desihub/desisim/pull/85
from desisim.quickcat import quickcat
zcat = quickcat(tilefiles, targets, truth, zcat=None, perfect=False)
#zcat.write('zcat.fits', overwrite=True)
def simulate_epoch(self,
epoch,
truth,
targets,
perfect=False,
mtl=None,
zcat=None):
"""Core routine simulating a DESI epoch,
Args:
epoch (int): epoch to simulate
perfect (boolean): Default: False. Selects whether how redshifts are taken from the truth file.
True: redshifts are taken without any error from the truth file.
False: redshifts include uncertainties.
truth (Table): Truth data
targets (Table): Targets data
mtl (Table): Merged Targets List data
zcat (Table): Redshift Catalog Data
Notes:
This routine simulates three steps:
* Merged target list creation
* Fiber allocation
* Redshift catalogue construction
"""
# create the MTL file
print("{} Starting MTL".format(asctime()))
self.mtl_file = os.path.join(self.tmp_output_path, 'mtl.fits')
mtl = desitarget.mtl.make_mtl(targets, zcat)
mtl.write(self.mtl_file, overwrite=True)
print("{} Finished MTL".format(asctime()))
# clean files and prepare fiberasign inputs
tilefiles = sorted(glob.glob(self.tmp_fiber_path + '/tile*.fits'))
if tilefiles:
for tilefile in tilefiles:
os.remove(tilefile)
# setup the tileids for the current observation epoch
self.create_surveyfile(epoch)
self.create_fiberassign_input()
# launch fiberassign
print("{} Launching fiberassign".format(asctime()))
f = open('fiberassign.log', 'a')
p = subprocess.call([
self.fiberassign_exec,
os.path.join(self.tmp_output_path, 'fa_features.txt')
],
stdout=f) # stdout=subprocess.PIPE)
print("{} Finished fiberassign".format(asctime()))
f.close()
# create a list of fiberassign tiles to read and update zcat
self.update_observed_tiles(epoch)
#update obsconditions from progress_data
# progress_data = Table.read(self.progress_files[epoch + 1])
# ii = np.in1d(progress_data['TILEID'], self.observed_tiles)
# obsconditions = progress_data[ii]
obsconditions = None
# write the zcat, it uses the tilesfiles constructed in the last step
self.zcat_file = os.path.join(self.tmp_output_path, 'zcat.fits')
print("{} starting quickcat".format(asctime()))
newzcat = quickcat(self.tilefiles,
targets,
truth,
zcat=zcat,
obsconditions=obsconditions,
perfect=perfect)
print("{} writing zcat".format(asctime()))
newzcat.write(self.zcat_file, format='fits', overwrite=True)
print("{} Finished zcat".format(asctime()))
return mtl, newzcat
print('starting fiberassign for dark tiles')
subprocess.call(cmd.split())
print('finished fiberassign for dark tiles')
zcat_dark = 'data/zcat_dark.fits'
if not os.path.exists(zcat_dark):
tile_files = glob.glob('output/dark/tile_*.fits')
print('{} files to gather'.format(len(tile_files)))
print('reading mtl')
mtl = Table.read('data/mtl.fits')
print('reading truth')
truth = Table.read('data/truth.fits')
print('making zcat')
zcat = quickcat(tile_files, mtl, truth, perfect=True)
print('writing zcat')
zcat.write(zcat_dark, overwrite=True)
print('finished zcat')
zcat_bright = 'data/zcat_bright.fits'
if not os.path.exists(zcat_bright):
tile_files = glob.glob('output/bright/tile_*.fits')
print('{} files to gather'.format(len(tile_files)))
print('reading mtl')
mtl = Table.read('data/mtl.fits')
print('reading truth')
truth = Table.read('data/truth.fits')
print('making zcat')
import glob
from desisim.quickcat import quickcat
from astropy.table import Table
fiberassigndir = "/global/cscratch1/sd/forero/DR5FiberAssign/fiberassign/tileresults/"
tile_files = glob.glob(fiberassigndir+"tile_*.fits")
mtl = Table.read("mtl.fits")
truth = Table.read("truth.fits")
zcat = quickcat(tile_files, mtl, truth, perfect=True)
zcat.write("zcat.fits", overwrite=True)
