def mkfa(targetf,tileff,rd,fad,srun,nrun,DESIMODEL):
'''
make the fiberassign files needed
targetf is the target file (e.g., an mtl file)
tilef is the root string for the tile files produced for each epoch
rd is the output directory
fad is the directory for the data fiberassign files
srun is the initial epoch
nrun is the number of epochs
DESIMODEL is the directory for where to find the focal plane model for running these
'''
os.environ['DESIMODEL'] = DESIMODEL
#targetf = e2eout +program+'/randoms_mtl_cuttod.fits' #above file, cut to ~e2e area with significant padding
#use fiberassign tools to read in randoms to be assigned
tgs = Targets()
load_target_file(tgs,targetf)
print('loaded target file '+targetf)
tree = TargetTree(tgs, 0.01)
for run in range(srun,srun+nrun):
#make the tile file for this run
#e2e.mke2etiles(run,program=program)
tilef = tileff+str(run)+'.fits'
#+str(run)+'.fits'
randir = rd +str(run)
#+str(run)
if os.path.isdir(randir):
ofls = glob.glob(randir+'/*')
for fl in ofls:
os.remove(fl) #remove the old files if we are rerunning
else:
os.mkdir(randir)
fafls = glob.glob(fad+str(run)+'/fiberassign/fiberassign*')
#+str(run)+'/fiberassign/fiberassign*')
hd = fitsio.read_header(fafls[0])
dt = hd['FA_RUN']
hw = load_hardware(rundate=dt)
tiles = load_tiles(tiles_file=tilef)
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
favail = LocationsAvailable(tgsavail)
#del tree
asgn = Assignment(tgs, tgsavail, favail)
asgn.assign_unused(TARGET_TYPE_SCIENCE)
write_assignment_fits(tiles, asgn, out_dir=randir, all_targets=True)
print('wrote assignment files to '+randir)
def getfatiles(targetf, tilef, dirout='', dt='2020-03-10T00:00:00'):
'''
will write out fiberassignment files for each tile with the FASSIGN, FTARGETS, FAVAIL HDUS
these are what are required to determine the geometry of what fiberassign thinks could have been observed and also match to actual observations (though FASSIGN is not really necessary)
targetf is file with all targets to be run through
tilef lists the tiles to "assign"
dirout is the directory where this all gets written out !make sure this is unique for every different target!
'''
tgs = Targets()
load_target_file(tgs, targetf)
print('loaded target file ' + targetf)
tree = TargetTree(tgs, 0.01)
hw = load_hardware(rundate=dt)
tiles = load_tiles(tiles_file=tilef)
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
favail = LocationsAvailable(tgsavail)
del tree
asgn = Assignment(tgs, tgsavail, favail)
asgn.assign_unused(TARGET_TYPE_SCIENCE)
write_assignment_fits(tiles, asgn, out_dir=dirout, all_targets=True)
print('wrote assignment files to ' + dirout)
def test_available(self):
test_dir = test_subdir_create("targets_test_available")
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
nscience = sim_targets(input_mtl, TARGET_TYPE_SCIENCE, 0)
nstd = sim_targets(input_std, TARGET_TYPE_STANDARD, nscience)
nsky = sim_targets(input_sky, TARGET_TYPE_SKY, (nscience + nstd))
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
print(tgs)
# Create a hierarchical triangle mesh lookup of the targets positions
tree = TargetTree(tgs, 0.01)
# Compute the targets available to each fiber for each tile.
hw = load_hardware()
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Free the tree
del tree
# Compute the fibers on all tiles available for each target
favail = FibersAvailable(tgsavail)
return
def test_available(self):
test_dir = test_subdir_create("targets_test_available")
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
input_suppsky = os.path.join(test_dir, "suppsky.fits")
tgoff = 0
nscience = sim_targets(input_mtl, TARGET_TYPE_SCIENCE, tgoff)
tgoff += nscience
nstd = sim_targets(input_std, TARGET_TYPE_STANDARD, tgoff)
tgoff += nstd
nsky = sim_targets(input_sky, TARGET_TYPE_SKY, tgoff)
tgoff += nsky
nsuppsky = sim_targets(input_suppsky, TARGET_TYPE_SUPPSKY, tgoff)
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
load_target_file(tgs, input_suppsky)
print(tgs)
# Test access
ids = tgs.ids()
tt = tgs.get(ids[0])
tt.ra += 1.0e-5
tt.dec += 1.0e-5
tt.subpriority = 0.99
# Create a hierarchical triangle mesh lookup of the targets positions
tree = TargetTree(tgs, 0.01)
# Compute the targets available to each fiber for each tile.
hw = load_hardware()
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Free the tree
del tree
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
return
program = str(sys.argv[3])
#test
run = srun
fafls = glob.glob(e2ein + 'run/quicksurvey/' + program + '/' + str(run) +
'/fiberassign/fiberassign*')
hd = fitsio.read_header(fafls[0])
dt = hd['FA_RUN']
#targetf = '/project/projectdirs/desi/users/ajross/catalogs/minisv2/random/random_mtl.fits' #2e8 randoms from dr8 randoms, with columns needs for fa,
targetf = '/project/projectdirs/desi/users/ajross/catalogs/e2eoneper/' + program + '/randoms/randoms_mtl_cuttod.fits' #above file, cut to ~e2e area with significant padding
#random are only for DARK right now, need to work something out for BRIGHT and GRAY
#use fiberassign tools to read in randoms to be assigned
tgs = Targets()
load_target_file(tgs, targetf)
print('loaded target file ' + targetf)
tree = TargetTree(tgs, 0.01)
for run in range(srun, srun + nrun):
#make the tile file for this run
mke2etiles(run, program=program)
tilef = e2eout + 'e2etiles_run' + str(run) + '.fits'
randir = e2eout + program + '/randoms/' + str(run)
if os.path.isdir(randir):
pass
else:
os.mkdir(randir)
fafls = glob.glob(e2ein + 'run/quicksurvey/' + program + '/' + str(run) +
def test_science(self):
set_matplotlib_pdf_backend()
import matplotlib.pyplot as plt
test_dir = test_subdir_create("qa_test_science")
log_file = os.path.join(test_dir, "log.txt")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
# For this test, we will use just 2 science target classes, in order to verify
# we get approximately the correct distribution
sdist = [(3000, 1, 0.25, "QSO"), (2000, 1, 0.75, "ELG")]
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
0,
density=self.density_science,
science_frac=sdist)
log_msg = "Simulated {} science targets\n".format(nscience)
tgs = Targets()
load_target_file(tgs, input_mtl)
# Read hardware properties
fp, exclude, state = sim_focalplane(rundate=test_assign_date)
hw = load_hardware(focalplane=(fp, exclude, state))
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
# Precompute target positions
tile_targetids, tile_x, tile_y = targets_in_tiles(hw, tgs, tiles)
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tiles, tile_targetids, tile_x, tile_y)
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Pass empty map of STUCK positioners that land on good sky
stucksky = {}
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail, stucksky)
# First-pass assignment of science targets
asgn.assign_unused(TARGET_TYPE_SCIENCE)
# Redistribute
asgn.redistribute_science()
write_assignment_fits(tiles, asgn, out_dir=test_dir, all_targets=True)
tile_ids = list(tiles.id)
merge_results([input_mtl],
list(),
tile_ids,
result_dir=test_dir,
copy_fba=False)
# FIXME: In order to use the qa_targets function, we need to know the
# starting requested number of observations (NUMOBS_INIT). Then we can use
# that value for each target and compare to the number actually assigned.
# However, the NUMOBS_INIT column was removed from the merged TARGET table.
# If we are ever able to reach consensus on restoring that column, then we
# can re-enable these tests below.
#
# qa_targets(
# hw,
# tiles,
# result_dir=test_dir,
# result_prefix="fiberassign-"
# )
#
# # Load the target catalog so that we have access to the target properties
#
# fd = fitsio.FITS(input_mtl, "r")
# scidata = np.array(np.sort(fd[1].read(), order="TARGETID"))
# fd.close()
# del fd
#
# # How many possible positioner assignments did we have?
# nassign = 5000 * len(tile_ids)
#
# possible = dict()
# achieved = dict()
#
# namepat = re.compile(r".*/qa_target_count_(.*)_init-(.*)\.fits")
# for qafile in glob.glob("{}/qa_target_count_*.fits".format(test_dir)):
# namemat = namepat.match(qafile)
# name = namemat.group(1)
# obs = int(namemat.group(2))
# if obs == 0:
# continue
# fd = fitsio.FITS(qafile, "r")
# fdata = fd["COUNTS"].read()
# # Sort by target ID so we can select easily
# fdata = np.sort(fdata, order="TARGETID")
# tgid = np.array(fdata["TARGETID"])
# counts = np.array(fdata["NUMOBS_DONE"])
# avail = np.array(fdata["NUMOBS_AVAIL"])
# del fdata
# fd.close()
#
# # Select target properties. BOTH TARGET LISTS MUST BE SORTED.
# rows = np.where(np.isin(scidata["TARGETID"], tgid, assume_unique=True))[0]
#
# ra = np.array(scidata["RA"][rows])
# dec = np.array(scidata["DEC"][rows])
# dtarget = np.array(scidata["DESI_TARGET"][rows])
# init = np.array(scidata["NUMOBS_INIT"][rows])
#
# requested = obs * np.ones_like(avail)
#
# under = np.where(avail < requested)[0]
# over = np.where(avail > requested)[0]
#
# limavail = np.array(avail)
# limavail[over] = obs
#
# deficit = np.zeros(len(limavail), dtype=np.int)
#
# deficit[:] = limavail - counts
# deficit[avail == 0] = 0
#
# possible[name] = np.sum(limavail)
# achieved[name] = np.sum(counts)
#
# log_msg += "{}-{}:\n".format(name, obs)
#
# pindx = np.where(deficit > 0)[0]
# poor_tgid = tgid[pindx]
# poor_dtarget = dtarget[pindx]
# log_msg += " Deficit > 0: {}\n".format(len(poor_tgid))
# poor_ra = ra[pindx]
# poor_dec = dec[pindx]
# poor_deficit = deficit[pindx]
#
# # Plot Target availability
# # Commented out by default, since in the case of high target density
# # needed for maximizing assignments, there are far more targets than
# # the number of available fiber placements.
#
# # marksize = 4 * np.ones_like(deficit)
# #
# # fig = plt.figure(figsize=(12, 12))
# # ax = fig.add_subplot(1, 1, 1)
# # ax.scatter(ra, dec, s=2, c="black", marker="o")
# # for pt, pr, pd, pdef in zip(poor_tgid, poor_ra, poor_dec, poor_deficit):
# # ploc = plt.Circle(
# # (pr, pd), radius=(0.05*pdef), fc="none", ec="red"
# # )
# # ax.add_artist(ploc)
# # ax.set_xlabel("RA", fontsize="large")
# # ax.set_ylabel("DEC", fontsize="large")
# # ax.set_title(
# # "Target \"{}\": (min(avail, requested) - counts) > 0".format(
# # name, obs
# # )
# # )
# # #ax.legend(handles=lg, framealpha=1.0, loc="upper right")
# # plt.savefig(os.path.join(test_dir, "{}-{}_deficit.pdf".format(name, obs)), dpi=300, format="pdf")
#
# log_msg += \
# "Assigned {} tiles for total of {} possible target observations\n".format(
# len(tile_ids), nassign
# )
# ach = 0
# for nm in possible.keys():
# ach += achieved[nm]
# log_msg += \
# " type {} had {} possible target obs and achieved {}\n".format(
# nm, possible[nm], achieved[nm]
# )
# frac = 100.0 * ach / nassign
# log_msg += \
# " {} / {} = {:0.2f}% of fibers were assigned\n".format(
# ach, nassign, frac
# )
# for nm in possible.keys():
# log_msg += \
# " type {} had {:0.2f}% of achieved observations\n".format(
# nm, achieved[nm] / ach
# )
# with open(log_file, "w") as f:
# f.write(log_msg)
#
# self.assertGreaterEqual(frac, 99.0)
# Test if qa-fiberassign script runs without crashing
script = os.path.join(self.binDir, "qa-fiberassign")
if os.path.exists(script):
fafiles = glob.glob(f"{test_dir}/fiberassign-*.fits")
cmd = "{} --targets {}".format(script, " ".join(fafiles))
err = subprocess.call(cmd.split())
self.assertEqual(err, 0, f"FAILED ({err}): {cmd}")
else:
print(f"ERROR: didn't find {script}")
def test_io(self):
np.random.seed(123456789)
test_dir = test_subdir_create("assign_test_io")
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
input_suppsky = os.path.join(test_dir, "suppsky.fits")
tgoff = 0
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
tgoff,
density=self.density_science)
tgoff += nscience
nstd = sim_targets(input_std,
TARGET_TYPE_STANDARD,
tgoff,
density=self.density_standards)
tgoff += nstd
nsky = sim_targets(input_sky,
TARGET_TYPE_SKY,
tgoff,
density=self.density_sky)
tgoff += nsky
nsuppsky = sim_targets(input_suppsky,
TARGET_TYPE_SUPPSKY,
tgoff,
density=self.density_suppsky)
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
load_target_file(tgs, input_suppsky)
# Create a hierarchical triangle mesh lookup of the targets positions
tree = TargetTree(tgs, 0.01)
# Compute the targets available to each fiber for each tile.
fp, exclude, state = sim_focalplane(rundate=test_assign_date)
hw = load_hardware(focalplane=(fp, exclude, state))
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Free the tree
del tree
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Pass empty map of STUCK positioners that land on good sky
stucksky = {}
# First pass assignment
asgn = Assignment(tgs, tgsavail, favail, stucksky)
asgn.assign_unused(TARGET_TYPE_SCIENCE)
# Write out, merge, read back in and verify
write_assignment_ascii(tiles,
asgn,
out_dir=test_dir,
out_prefix="test_io_ascii_")
write_assignment_fits(tiles,
asgn,
out_dir=test_dir,
out_prefix="basic_",
all_targets=False)
write_assignment_fits(tiles,
asgn,
out_dir=test_dir,
out_prefix="full_",
all_targets=True)
plotpetals = [0]
# plotpetals = None
plot_tiles(hw,
tiles,
result_dir=test_dir,
result_prefix="basic_",
plot_dir=test_dir,
plot_prefix="basic_",
result_split_dir=False,
petals=plotpetals,
serial=True)
plot_tiles(hw,
tiles,
result_dir=test_dir,
result_prefix="full_",
plot_dir=test_dir,
plot_prefix="full_",
result_split_dir=False,
petals=plotpetals,
serial=True)
target_files = [input_mtl, input_sky, input_std]
tile_ids = list(tiles.id)
merge_results(target_files,
list(),
tile_ids,
result_dir=test_dir,
result_prefix="basic_",
out_dir=test_dir,
out_prefix="basic_tile-",
copy_fba=False)
merge_results(target_files,
list(),
tile_ids,
result_dir=test_dir,
result_prefix="full_",
out_dir=test_dir,
out_prefix="full_tile-",
copy_fba=False)
# Here we test reading with the standard reading function
for tid in tile_ids:
tdata = asgn.tile_location_target(tid)
avail = tgsavail.tile_data(tid)
# Check basic format
infile = os.path.join(test_dir,
"basic_tile-{:06d}.fits".format(tid))
inhead, fiber_data, targets_data, avail_data, gfa_targets = \
read_assignment_fits_tile((tid, infile))
for lid, tgid, tgra, tgdec in zip(fiber_data["LOCATION"],
fiber_data["TARGETID"],
fiber_data["TARGET_RA"],
fiber_data["TARGET_DEC"]):
if tgid >= 0:
self.assertEqual(tgid, tdata[lid])
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
# Check full format
infile = os.path.join(test_dir,
"full_tile-{:06d}.fits".format(tid))
inhead, fiber_data, targets_data, avail_data, gfa_targets = \
read_assignment_fits_tile((tid, infile))
for lid, tgid, tgra, tgdec in zip(fiber_data["LOCATION"],
fiber_data["TARGETID"],
fiber_data["TARGET_RA"],
fiber_data["TARGET_DEC"]):
if tgid >= 0:
self.assertEqual(tgid, tdata[lid])
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
# Now read the files directly with fitsio and verify against the input
# target data.
for tid in tile_ids:
tdata = asgn.tile_location_target(tid)
avail = tgsavail.tile_data(tid)
# Check basic format
infile = os.path.join(test_dir,
"basic_tile-{:06d}.fits".format(tid))
fdata = fitsio.FITS(infile, "r")
fassign = fdata["FIBERASSIGN"].read()
ftargets = fdata["TARGETS"].read()
for lid, tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip(
fassign["LOCATION"], fassign["TARGETID"],
fassign["TARGET_RA"], fassign["TARGET_DEC"],
fassign["SUBPRIORITY"], fassign["PRIORITY"],
fassign["OBSCONDITIONS"]):
if tgid >= 0:
self.assertEqual(tgid, tdata[lid])
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
self.assertEqual(tgsub, props.subpriority)
self.assertEqual(tgprior, props.priority)
self.assertEqual(tgobs, props.obscond)
for tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip(
ftargets["TARGETID"], ftargets["RA"], ftargets["DEC"],
ftargets["SUBPRIORITY"], ftargets["PRIORITY"],
ftargets["OBSCONDITIONS"]):
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
self.assertEqual(tgsub, props.subpriority)
self.assertEqual(tgprior, props.priority)
self.assertEqual(tgobs, props.obscond)
# Check full format
infile = os.path.join(test_dir,
"full_tile-{:06d}.fits".format(tid))
fdata = fitsio.FITS(infile, "r")
fassign = fdata["FIBERASSIGN"].read()
ftargets = fdata["TARGETS"].read()
for lid, tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip(
fassign["LOCATION"], fassign["TARGETID"],
fassign["TARGET_RA"], fassign["TARGET_DEC"],
fassign["SUBPRIORITY"], fassign["PRIORITY"],
fassign["OBSCONDITIONS"]):
if tgid >= 0:
self.assertEqual(tgid, tdata[lid])
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
self.assertEqual(tgsub, props.subpriority)
self.assertEqual(tgprior, props.priority)
self.assertEqual(tgobs, props.obscond)
for tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip(
ftargets["TARGETID"], ftargets["RA"], ftargets["DEC"],
ftargets["SUBPRIORITY"], ftargets["PRIORITY"],
ftargets["OBSCONDITIONS"]):
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
self.assertEqual(tgsub, props.subpriority)
self.assertEqual(tgprior, props.priority)
self.assertEqual(tgobs, props.obscond)
plot_tiles(hw,
tiles,
result_dir=test_dir,
result_prefix="basic_tile-",
plot_dir=test_dir,
plot_prefix="basic_tile-",
result_split_dir=False,
petals=plotpetals,
serial=True)
plot_tiles(hw,
tiles,
result_dir=test_dir,
result_prefix="full_tile-",
plot_dir=test_dir,
plot_prefix="full_tile-",
result_split_dir=False,
petals=plotpetals,
serial=True)
return
def test_fieldrot(self):
test_dir = test_subdir_create("assign_test_fieldrot")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
input_suppsky = os.path.join(test_dir, "suppsky.fits")
tgoff = 0
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
tgoff,
density=self.density_science)
tgoff += nscience
nstd = sim_targets(input_std,
TARGET_TYPE_STANDARD,
tgoff,
density=self.density_standards)
tgoff += nstd
nsky = sim_targets(input_sky,
TARGET_TYPE_SKY,
tgoff,
density=self.density_sky)
tgoff += nsky
nsuppsky = sim_targets(input_suppsky,
TARGET_TYPE_SUPPSKY,
tgoff,
density=self.density_suppsky)
# Simulate the tiles
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
# petal mapping
rotator = petal_rotation(1, reverse=False)
rots = [0, 36]
tile_ids = None
for rt in rots:
odir = "theta_{:02d}".format(rt)
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
load_target_file(tgs, input_suppsky)
# Create a hierarchical triangle mesh lookup of the targets
# positions
tree = TargetTree(tgs, 0.01)
# Manually override the field rotation
tiles = load_tiles(tiles_file=tfile, obstheta=float(rt))
if tile_ids is None:
tile_ids = list(tiles.id)
# Simulate a fake focalplane
fp, exclude, state = sim_focalplane(rundate=test_assign_date,
fakepos=True)
# Load the focalplane
hw = load_hardware(focalplane=(fp, exclude, state))
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Pass empty map of STUCK positioners that land on good sky
stucksky = {}
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail, stucksky)
# First-pass assignment of science targets
asgn.assign_unused(TARGET_TYPE_SCIENCE)
out = os.path.join(test_dir, odir)
write_assignment_fits(tiles, asgn, out_dir=out, all_targets=True)
ppet = 6
if odir == "theta_36":
ppet = rotator[6]
plot_tiles(hw,
tiles,
result_dir=out,
plot_dir=out,
real_shapes=True,
petals=[ppet],
serial=True)
# Explicitly free everything
del asgn
del favail
del tgsavail
del hw
del tiles
del tree
del tgs
# For each tile, compare the assignment output and verify that they
# agree with a one-petal rotation.
# NOTE: The comparison below will NOT pass, since we are still
# Sorting by highest priority available target and then (in case
# of a tie) by fiber ID. See line 333 of assign.cpp. Re-enable this
# test after that is changed to sort by location in case of a tie.
# for tl in tile_ids:
# orig_path = os.path.join(
# test_dir, "theta_00", "fiberassign_{:06d}.fits".format(tl)
# )
# orig_header, orig_data, _, _, _ = \
# read_assignment_fits_tile((tl, orig_path))
# rot_path = os.path.join(
# test_dir, "theta_36", "fiberassign_{:06d}.fits".format(tl)
# )
# rot_header, rot_data, _, _, _ = \
# read_assignment_fits_tile((tl, rot_path))
# comppath = os.path.join(
# test_dir, "comp_00-36_{:06d}.txt".format(tl)
# )
# with open(comppath, "w") as fc:
# for dev, petal, tg in zip(
# orig_data["DEVICE_LOC"], orig_data["PETAL_LOC"],
# orig_data["TARGETID"]
# ):
# for newdev, newpetal, newtg in zip(
# rot_data["DEVICE_LOC"], rot_data["PETAL_LOC"],
# rot_data["TARGETID"]
# ):
# rpet = rotator[newpetal]
# if (newdev == dev) and (rpet == petal):
# fc.write(
# "{}, {} = {} : {}, {} = {}\n"
# .format(petal, dev, tg, rpet, newdev, newtg)
# )
# # self.assertEqual(newtg, tg)
return
def test_full(self, do_stucksky=False):
test_dir = test_subdir_create("assign_test_full")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
input_suppsky = os.path.join(test_dir, "suppsky.fits")
tgoff = 0
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
tgoff,
density=self.density_science)
tgoff += nscience
nstd = sim_targets(input_std,
TARGET_TYPE_STANDARD,
tgoff,
density=self.density_standards)
tgoff += nstd
nsky = sim_targets(input_sky,
TARGET_TYPE_SKY,
tgoff,
density=self.density_sky)
tgoff += nsky
nsuppsky = sim_targets(input_suppsky,
TARGET_TYPE_SUPPSKY,
tgoff,
density=self.density_suppsky)
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
load_target_file(tgs, input_suppsky)
# Create a hierarchical triangle mesh lookup of the targets positions
tree = TargetTree(tgs, 0.01)
# Read hardware properties
fp, exclude, state = sim_focalplane(rundate=test_assign_date)
hw = load_hardware(focalplane=(fp, exclude, state))
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
if do_stucksky:
sim_stuck_sky(test_dir, hw, tiles)
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Free the tree
del tree
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Pass empty map of STUCK positioners that land on good sky
stucksky = None
if do_stucksky:
stucksky = stuck_on_sky(hw, tiles)
if stucksky is None:
# (the pybind code doesn't like None when a dict is expected...)
stucksky = {}
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail, stucksky)
run(asgn)
write_assignment_fits(tiles,
asgn,
out_dir=test_dir,
all_targets=True,
stucksky=stucksky)
plotpetals = [0]
#plotpetals = None
plot_tiles(hw,
tiles,
result_dir=test_dir,
plot_dir=test_dir,
result_prefix="fba-",
real_shapes=True,
petals=plotpetals,
serial=True)
qa_tiles(hw, tiles, result_dir=test_dir)
qadata = None
with open(os.path.join(test_dir, "qa.json"), "r") as f:
qadata = json.load(f)
for tile, props in qadata.items():
self.assertTrue(props["assign_science"] >= 4485)
self.assertEqual(100, props["assign_std"])
if do_stucksky:
# We get 3 stuck positioners landing on good sky!
self.assertTrue(
(props["assign_sky"] + props["assign_suppsky"]) >= 397)
else:
self.assertTrue(
(props["assign_sky"] + props["assign_suppsky"]) >= 400)
plot_qa(qadata,
os.path.join(test_dir, "qa"),
outformat="pdf",
labels=True)
return
def test_full(self):
test_dir = test_subdir_create("assign_test_full")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
nscience = sim_targets(input_mtl, TARGET_TYPE_SCIENCE, 0)
nstd = sim_targets(input_std, TARGET_TYPE_STANDARD, nscience)
nsky = sim_targets(input_sky, TARGET_TYPE_SKY, (nscience + nstd))
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
# Create a hierarchical triangle mesh lookup of the targets positions
tree = TargetTree(tgs, 0.01)
# Read hardware properties
fstatus = os.path.join(test_dir, "fiberstatus.ecsv")
sim_status(fstatus)
hw = load_hardware(status_file=fstatus)
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Free the tree
del tree
# Compute the fibers on all tiles available for each target
favail = FibersAvailable(tgsavail)
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail)
# First-pass assignment of science targets
asgn.assign_unused(TARGET_TYPE_SCIENCE)
# Redistribute science targets
asgn.redistribute_science()
# Assign standards, 10 per petal
asgn.assign_unused(TARGET_TYPE_STANDARD, 10)
asgn.assign_force(TARGET_TYPE_STANDARD, 10)
# Assign sky to unused fibers, up to 40 per petal
asgn.assign_unused(TARGET_TYPE_SKY, 40)
asgn.assign_force(TARGET_TYPE_SKY, 40)
# If there are any unassigned fibers, try to place them somewhere.
asgn.assign_unused(TARGET_TYPE_SCIENCE)
asgn.assign_unused(TARGET_TYPE_SKY)
write_assignment_fits(tiles, asgn, out_dir=test_dir, all_targets=True)
plot_tiles(hw,
tiles,
result_dir=test_dir,
plot_dir=test_dir,
petals=[0],
serial=True)
qa_tiles(hw, tiles, result_dir=test_dir)
qadata = None
with open(os.path.join(test_dir, "qa.json"), "r") as f:
qadata = json.load(f)
for tile, props in qadata.items():
self.assertEqual(4495, props["assign_science"])
self.assertEqual(100, props["assign_std"])
self.assertEqual(400, props["assign_sky"])
plot_qa(qadata,
os.path.join(test_dir, "qa"),
outformat="pdf",
labels=True)
return
def test_science(self):
set_matplotlib_pdf_backend()
import matplotlib.pyplot as plt
test_dir = test_subdir_create("qa_test_science")
log_file = os.path.join(test_dir, "log.txt")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
# For this test, we will use just 2 science target classes, in order to verify
# we get approximately the correct distribution
sdist = [(3000, 1, 0.25, "QSO"), (2000, 1, 0.75, "ELG")]
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
0,
density=self.density_science,
science_frac=sdist)
log_msg = "Simulated {} science targets\n".format(nscience)
tgs = Targets()
load_target_file(tgs, input_mtl)
# Create a hierarchical triangle mesh lookup of the targets positions
tree = TargetTree(tgs, 0.01)
# Read hardware properties
fp, exclude, state = sim_focalplane(rundate=test_assign_date)
hw = load_hardware(focalplane=(fp, exclude, state))
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Free the tree
del tree
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail)
# First-pass assignment of science targets
asgn.assign_unused(TARGET_TYPE_SCIENCE)
# Redistribute
asgn.redistribute_science()
write_assignment_fits(tiles, asgn, out_dir=test_dir, all_targets=True)
tile_ids = list(tiles.id)
merge_results([input_mtl],
list(),
tile_ids,
result_dir=test_dir,
copy_fba=False)
# if "TRAVIS" not in os.environ:
# plot_tiles(
# hw,
# tiles,
# result_dir=test_dir,
# plot_dir=test_dir,
# real_shapes=True,
# serial=True
# )
qa_targets(hw,
tiles,
result_dir=test_dir,
result_prefix="fiberassign-")
# Load the target catalog so that we have access to the target properties
fd = fitsio.FITS(input_mtl, "r")
scidata = np.array(np.sort(fd[1].read(), order="TARGETID"))
fd.close()
del fd
# How many possible positioner assignments did we have?
nassign = 5000 * len(tile_ids)
possible = dict()
achieved = dict()
namepat = re.compile(r".*/qa_target_count_(.*)_init-(.*)\.fits")
for qafile in glob.glob("{}/qa_target_count_*.fits".format(test_dir)):
namemat = namepat.match(qafile)
name = namemat.group(1)
obs = int(namemat.group(2))
if obs == 0:
continue
fd = fitsio.FITS(qafile, "r")
fdata = fd["COUNTS"].read()
# Sort by target ID so we can select easily
fdata = np.sort(fdata, order="TARGETID")
tgid = np.array(fdata["TARGETID"])
counts = np.array(fdata["NUMOBS_DONE"])
avail = np.array(fdata["NUMOBS_AVAIL"])
del fdata
fd.close()
# Select target properties. BOTH TARGET LISTS MUST BE SORTED.
rows = np.where(
np.isin(scidata["TARGETID"], tgid, assume_unique=True))[0]
ra = np.array(scidata["RA"][rows])
dec = np.array(scidata["DEC"][rows])
dtarget = np.array(scidata["DESI_TARGET"][rows])
init = np.array(scidata["NUMOBS_MORE"][rows])
requested = obs * np.ones_like(avail)
under = np.where(avail < requested)[0]
over = np.where(avail > requested)[0]
limavail = np.array(avail)
limavail[over] = obs
deficit = np.zeros(len(limavail), dtype=np.int)
deficit[:] = limavail - counts
deficit[avail == 0] = 0
possible[name] = np.sum(limavail)
achieved[name] = np.sum(counts)
log_msg += "{}-{}:\n".format(name, obs)
pindx = np.where(deficit > 0)[0]
poor_tgid = tgid[pindx]
poor_dtarget = dtarget[pindx]
log_msg += " Deficit > 0: {}\n".format(len(poor_tgid))
poor_ra = ra[pindx]
poor_dec = dec[pindx]
poor_deficit = deficit[pindx]
# Plot Target availability
# Commented out by default, since in the case of high target density
# needed for maximizing assignments, there are far more targets than
# the number of available fiber placements.
# marksize = 4 * np.ones_like(deficit)
#
# fig = plt.figure(figsize=(12, 12))
# ax = fig.add_subplot(1, 1, 1)
# ax.scatter(ra, dec, s=2, c="black", marker="o")
# for pt, pr, pd, pdef in zip(poor_tgid, poor_ra, poor_dec, poor_deficit):
# ploc = plt.Circle(
# (pr, pd), radius=(0.05*pdef), fc="none", ec="red"
# )
# ax.add_artist(ploc)
# ax.set_xlabel("RA", fontsize="large")
# ax.set_ylabel("DEC", fontsize="large")
# ax.set_title(
# "Target \"{}\": (min(avail, requested) - counts) > 0".format(
# name, obs
# )
# )
# #ax.legend(handles=lg, framealpha=1.0, loc="upper right")
# plt.savefig(os.path.join(test_dir, "{}-{}_deficit.pdf".format(name, obs)), dpi=300, format="pdf")
log_msg += \
"Assigned {} tiles for total of {} possible target observations\n".format(
len(tile_ids), nassign
)
ach = 0
for nm in possible.keys():
ach += achieved[nm]
log_msg += \
" type {} had {} possible target obs and achieved {}\n".format(
nm, possible[nm], achieved[nm]
)
frac = 100.0 * ach / nassign
log_msg += \
" {} / {} = {:0.2f}% of fibers were assigned\n".format(
ach, nassign, frac
)
for nm in possible.keys():
log_msg += \
" type {} had {:0.2f}% of achieved observations\n".format(
nm, achieved[nm] / ach
)
with open(log_file, "w") as f:
f.write(log_msg)
self.assertGreaterEqual(frac, 99.0)
#- Test if qa-fiberassign script runs without crashing
bindir = os.path.join(os.path.dirname(fiberassign.__file__), '..',
'..', 'bin')
script = os.path.join(os.path.abspath(bindir), 'qa-fiberassign')
if os.path.exists(script):
fafiles = glob.glob(f"{test_dir}/fiberassign-*.fits")
cmd = "{} --targets {}".format(script, " ".join(fafiles))
err = subprocess.call(cmd.split())
self.assertEqual(err, 0, f"FAILED ({err}): {cmd}")
else:
print(f"ERROR: didn't find {script}")
def test_full(self):
test_dir = test_subdir_create("assign_test_full")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
input_suppsky = os.path.join(test_dir, "suppsky.fits")
tgoff = 0
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
tgoff,
density=self.density_science)
tgoff += nscience
nstd = sim_targets(input_std,
TARGET_TYPE_STANDARD,
tgoff,
density=self.density_standards)
tgoff += nstd
nsky = sim_targets(input_sky,
TARGET_TYPE_SKY,
tgoff,
density=self.density_sky)
tgoff += nsky
nsuppsky = sim_targets(input_suppsky,
TARGET_TYPE_SUPPSKY,
tgoff,
density=self.density_suppsky)
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
load_target_file(tgs, input_suppsky)
# Create a hierarchical triangle mesh lookup of the targets positions
tree = TargetTree(tgs, 0.01)
# Read hardware properties
fp, exclude, state = sim_focalplane(rundate=test_assign_date)
hw = load_hardware(focalplane=(fp, exclude, state))
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Free the tree
del tree
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail)
# First-pass assignment of science targets
asgn.assign_unused(TARGET_TYPE_SCIENCE)
# Redistribute science targets
asgn.redistribute_science()
# Assign standards, 10 per petal
asgn.assign_unused(TARGET_TYPE_STANDARD, 10)
asgn.assign_force(TARGET_TYPE_STANDARD, 10)
# Assign sky to unused fibers, up to 40 per petal
asgn.assign_unused(TARGET_TYPE_SKY, 40)
asgn.assign_force(TARGET_TYPE_SKY, 40)
# Use supplemental sky to meet our requirements
asgn.assign_unused(TARGET_TYPE_SUPPSKY, 40)
asgn.assign_force(TARGET_TYPE_SUPPSKY, 40)
# If there are any unassigned fibers, try to place them somewhere.
asgn.assign_unused(TARGET_TYPE_SCIENCE)
asgn.assign_unused(TARGET_TYPE_SKY)
asgn.assign_unused(TARGET_TYPE_SUPPSKY)
write_assignment_fits(tiles, asgn, out_dir=test_dir, all_targets=True)
plotpetals = [0]
#plotpetals = None
plot_tiles(hw,
tiles,
result_dir=test_dir,
plot_dir=test_dir,
result_prefix="fba-",
real_shapes=True,
petals=plotpetals,
serial=True)
qa_tiles(hw, tiles, result_dir=test_dir)
qadata = None
with open(os.path.join(test_dir, "qa.json"), "r") as f:
qadata = json.load(f)
for tile, props in qadata.items():
self.assertTrue(props["assign_science"] >= 4485)
self.assertEqual(100, props["assign_std"])
self.assertTrue(
(props["assign_sky"] + props["assign_suppsky"]) >= 400)
plot_qa(qadata,
os.path.join(test_dir, "qa"),
outformat="pdf",
labels=True)
return
def main():
log = Logger.get()
mpi_procs = MPI.COMM_WORLD.size
mpi_rank = MPI.COMM_WORLD.rank
parser = argparse.ArgumentParser()
parser.add_argument(
"--survey_log",
type=str,
required=False,
help="Eventually we would pass in a file containing the log"
" of when each fiber assignment was run and for which tiles, "
"along with the options that were used.")
parser.add_argument(
"--sky",
type=str,
required=False,
help="Input file with sky or supp_sky targets. "
"These target files are assumed to be constant and not "
"tracked by the MTL ledger.")
parser.add_argument(
"--mtl",
type=str,
required=True,
help="The MTL ledger. This is still a work in progress and"
" I am not sure what the interface will be, but given the "
"fiber assignment dates in the survey log, we should be able"
" to get the MTL state at that time. For now, this option"
" is just one or more target files.")
parser.add_argument("--footprint",
type=str,
required=False,
default=None,
help="Optional FITS file defining the footprint. If"
" not specified, the default footprint from desimodel"
" is used.")
parser.add_argument("--tiles",
type=str,
required=False,
default=None,
help="Optional text file containing a subset of the"
" tile IDs to use in the footprint, one ID per line."
" Default uses all tiles in the footprint.")
parser.add_argument("--out",
type=str,
required=False,
default=None,
help="Output directory.")
parser.add_argument("--realizations",
type=int,
required=False,
default=10,
help="Number of realizations.")
args = parser.parse_args()
if args.sky is None:
args.sky = list()
# Set output directory
if args.out is None:
args.out = "."
# Read tiles we are using
tileselect = None
if args.tiles is not None:
tileselect = list()
with open(args.tiles, "r") as f:
for line in f:
# Try to convert the first column to an integer.
try:
tileselect.append(int(line.split()[0]))
except ValueError:
pass
tiles = load_tiles(
tiles_file=args.footprint,
select=tileselect,
)
# Create empty target list
tgs = Targets()
# Append each input target file. These target files must all be of the
# same survey type, and will set the Targets object to be of that survey.
print(args.mtl)
print(args.sky)
#for tgfile in args.targets:
# load_target_file(tgs, tgfile)
load_target_file(tgs, args.mtl)
# Just the science target IDs
tg_science = tgs.ids()
tg_science2indx = {y: x for x, y in enumerate(tg_science)}
# Now load the sky target files.
survey = tgs.survey()
#for tgfile in args.sky:
# load_target_file(tgs, tgfile)
load_target_file(tgs, args.sky)
# Divide up realizations among the processes.
n_realization = args.realizations
realizations = np.arange(n_realization, dtype=np.int32)
my_realizations = np.array_split(realizations, mpi_procs)[mpi_rank]
# Bitarray for all targets and realizations
#tgarray = bitarray(len(tg_science) * n_realization)
#tgarray.setall(False)
tgarray = np.zeros(len(tg_science) * n_realization, dtype='bool')
# Target tree
tree = TargetTree(tgs)
hw = load_hardware()
for realization in my_realizations:
# Set the seed based on the realization, so that the result is reproducible
# regardless of which process is working on the realization.
np.random.seed(realization)
# Comment out the next block to avoid randomizing subpriority
# ----
# Randomize science target subpriority for this realization
new_subpriority = np.random.random_sample(size=len(tg_science))
for indx, tgid in enumerate(tg_science):
tg = tgs.get(tgid)
tg.subpriority = new_subpriority[indx]
# Comment out the next block to avoid dithering tiles
# ----
# Dither tiles centers by the same
# Compute available targets / locations
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
favail = LocationsAvailable(tgsavail)
asgn = Assignment(tgs, tgsavail, favail)
# Replay the survey log for each time fiber assignment was run. For now, this
# is just doing the tiles all at once.
for assign_event in range(1):
# In the future, load MTL updates to the obs remaining for each target here
# Read hardware properties- in the future, pass in the assignment run date
# to this function.
hw = load_hardware()
# Run assignment for this event.
run(asgn)
# Update bit arrays for assigned science targets
for tile_id in tiles.id: #():
adata = asgn.tile_location_target(tile_id)
for loc, tgid in adata.items():
try:
idx = tg_science2indx[tgid]
tgarray[idx * n_realization + realization] = True
except KeyError:
# Not a science target
pass
# Reduce bitarrays to root process. The bitarray type conforms to the
# buffer protocol.
tgall = None
#if mpi_rank == 0:
# tgall = bitarray(tgarray)
# tgall.setall(False)
MPI.COMM_WORLD.Reduce(tgarray, tgall, op=MPI.BOR, root=0)
# Write it out
if mpi_rank == 0:
#pass
print(len(tgall))
else:
nominal_tiles = load_tiles(tiles_file=tilesfile)
ntiles = len(nominal_tiles.id)
# Container for the Mersenne Twister pseudo-random number generator. Since every MPI rank is seeded
# with a different number, size number of different subpriorities are generated.
#seed = 62*rank_group + rank
seed = rank
random_generator = RandomState(seed=seed)
# Load Target data.
tgs = Targets()
# First load the MTL file and compute the target IDs.
load_target_file(tgs, mtlfile, random_generator=random_generator)
# --------------------------------------------------------------------------------------------------
# TARGET IDs IN PARALLEL
#
# Target IDs for each tracer in parallel.
# The total number of targets is split almost evenly between all processes. Each MPI task takes a
# portion of the total targets and extract the corresponding target IDs for each type of tracer.
# --------------------------------------------------------------------------------------------------
# Total number of targets.
ntargets = len(tgs.ids())
# Dictionary with DESI bitmask values for LRGs, ELGs and QSOs.
desi_bitmask = {'lrg': 65537, 'elg': 131074, 'qso': 262148}
