def test_available(self):
test_dir = test_subdir_create("targets_test_available")
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
input_suppsky = os.path.join(test_dir, "suppsky.fits")
tgoff = 0
nscience = sim_targets(input_mtl, TARGET_TYPE_SCIENCE, tgoff)
tgoff += nscience
nstd = sim_targets(input_std, TARGET_TYPE_STANDARD, tgoff)
tgoff += nstd
nsky = sim_targets(input_sky, TARGET_TYPE_SKY, tgoff)
tgoff += nsky
nsuppsky = sim_targets(input_suppsky, TARGET_TYPE_SUPPSKY, tgoff)
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
load_target_file(tgs, input_suppsky)
print(tgs)
# Test access
ids = tgs.ids()
tt = tgs.get(ids[0])
tt.ra += 1.0e-5
tt.dec += 1.0e-5
tt.subpriority = 0.99
# Compute the targets available to each fiber for each tile.
hw = load_hardware()
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
# Precompute target positions
tile_targetids, tile_x, tile_y = targets_in_tiles(hw, tgs, tiles)
tgsavail = TargetsAvailable(hw, tiles, tile_targetids, tile_x, tile_y)
# Free the tree
del tile_targetids, tile_x, tile_y
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
return
def test_science(self):
set_matplotlib_pdf_backend()
import matplotlib.pyplot as plt
test_dir = test_subdir_create("qa_test_science")
log_file = os.path.join(test_dir, "log.txt")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
# For this test, we will use just 2 science target classes, in order to verify
# we get approximately the correct distribution
sdist = [(3000, 1, 0.25, "QSO"), (2000, 1, 0.75, "ELG")]
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
0,
density=self.density_science,
science_frac=sdist)
log_msg = "Simulated {} science targets\n".format(nscience)
tgs = Targets()
load_target_file(tgs, input_mtl)
# Read hardware properties
fp, exclude, state = sim_focalplane(rundate=test_assign_date)
hw = load_hardware(focalplane=(fp, exclude, state))
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
# Precompute target positions
tile_targetids, tile_x, tile_y = targets_in_tiles(hw, tgs, tiles)
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tiles, tile_targetids, tile_x, tile_y)
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Pass empty map of STUCK positioners that land on good sky
stucksky = {}
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail, stucksky)
# First-pass assignment of science targets
asgn.assign_unused(TARGET_TYPE_SCIENCE)
# Redistribute
asgn.redistribute_science()
write_assignment_fits(tiles, asgn, out_dir=test_dir, all_targets=True)
tile_ids = list(tiles.id)
merge_results([input_mtl],
list(),
tile_ids,
result_dir=test_dir,
copy_fba=False)
# FIXME: In order to use the qa_targets function, we need to know the
# starting requested number of observations (NUMOBS_INIT). Then we can use
# that value for each target and compare to the number actually assigned.
# However, the NUMOBS_INIT column was removed from the merged TARGET table.
# If we are ever able to reach consensus on restoring that column, then we
# can re-enable these tests below.
#
# qa_targets(
# hw,
# tiles,
# result_dir=test_dir,
# result_prefix="fiberassign-"
# )
#
# # Load the target catalog so that we have access to the target properties
#
# fd = fitsio.FITS(input_mtl, "r")
# scidata = np.array(np.sort(fd[1].read(), order="TARGETID"))
# fd.close()
# del fd
#
# # How many possible positioner assignments did we have?
# nassign = 5000 * len(tile_ids)
#
# possible = dict()
# achieved = dict()
#
# namepat = re.compile(r".*/qa_target_count_(.*)_init-(.*)\.fits")
# for qafile in glob.glob("{}/qa_target_count_*.fits".format(test_dir)):
# namemat = namepat.match(qafile)
# name = namemat.group(1)
# obs = int(namemat.group(2))
# if obs == 0:
# continue
# fd = fitsio.FITS(qafile, "r")
# fdata = fd["COUNTS"].read()
# # Sort by target ID so we can select easily
# fdata = np.sort(fdata, order="TARGETID")
# tgid = np.array(fdata["TARGETID"])
# counts = np.array(fdata["NUMOBS_DONE"])
# avail = np.array(fdata["NUMOBS_AVAIL"])
# del fdata
# fd.close()
#
# # Select target properties. BOTH TARGET LISTS MUST BE SORTED.
# rows = np.where(np.isin(scidata["TARGETID"], tgid, assume_unique=True))[0]
#
# ra = np.array(scidata["RA"][rows])
# dec = np.array(scidata["DEC"][rows])
# dtarget = np.array(scidata["DESI_TARGET"][rows])
# init = np.array(scidata["NUMOBS_INIT"][rows])
#
# requested = obs * np.ones_like(avail)
#
# under = np.where(avail < requested)[0]
# over = np.where(avail > requested)[0]
#
# limavail = np.array(avail)
# limavail[over] = obs
#
# deficit = np.zeros(len(limavail), dtype=np.int)
#
# deficit[:] = limavail - counts
# deficit[avail == 0] = 0
#
# possible[name] = np.sum(limavail)
# achieved[name] = np.sum(counts)
#
# log_msg += "{}-{}:\n".format(name, obs)
#
# pindx = np.where(deficit > 0)[0]
# poor_tgid = tgid[pindx]
# poor_dtarget = dtarget[pindx]
# log_msg += " Deficit > 0: {}\n".format(len(poor_tgid))
# poor_ra = ra[pindx]
# poor_dec = dec[pindx]
# poor_deficit = deficit[pindx]
#
# # Plot Target availability
# # Commented out by default, since in the case of high target density
# # needed for maximizing assignments, there are far more targets than
# # the number of available fiber placements.
#
# # marksize = 4 * np.ones_like(deficit)
# #
# # fig = plt.figure(figsize=(12, 12))
# # ax = fig.add_subplot(1, 1, 1)
# # ax.scatter(ra, dec, s=2, c="black", marker="o")
# # for pt, pr, pd, pdef in zip(poor_tgid, poor_ra, poor_dec, poor_deficit):
# # ploc = plt.Circle(
# # (pr, pd), radius=(0.05*pdef), fc="none", ec="red"
# # )
# # ax.add_artist(ploc)
# # ax.set_xlabel("RA", fontsize="large")
# # ax.set_ylabel("DEC", fontsize="large")
# # ax.set_title(
# # "Target \"{}\": (min(avail, requested) - counts) > 0".format(
# # name, obs
# # )
# # )
# # #ax.legend(handles=lg, framealpha=1.0, loc="upper right")
# # plt.savefig(os.path.join(test_dir, "{}-{}_deficit.pdf".format(name, obs)), dpi=300, format="pdf")
#
# log_msg += \
# "Assigned {} tiles for total of {} possible target observations\n".format(
# len(tile_ids), nassign
# )
# ach = 0
# for nm in possible.keys():
# ach += achieved[nm]
# log_msg += \
# " type {} had {} possible target obs and achieved {}\n".format(
# nm, possible[nm], achieved[nm]
# )
# frac = 100.0 * ach / nassign
# log_msg += \
# " {} / {} = {:0.2f}% of fibers were assigned\n".format(
# ach, nassign, frac
# )
# for nm in possible.keys():
# log_msg += \
# " type {} had {:0.2f}% of achieved observations\n".format(
# nm, achieved[nm] / ach
# )
# with open(log_file, "w") as f:
# f.write(log_msg)
#
# self.assertGreaterEqual(frac, 99.0)
# Test if qa-fiberassign script runs without crashing
script = os.path.join(self.binDir, "qa-fiberassign")
if os.path.exists(script):
fafiles = glob.glob(f"{test_dir}/fiberassign-*.fits")
cmd = "{} --targets {}".format(script, " ".join(fafiles))
err = subprocess.call(cmd.split())
self.assertEqual(err, 0, f"FAILED ({err}): {cmd}")
else:
print(f"ERROR: didn't find {script}")
def test_fieldrot(self):
test_dir = test_subdir_create("assign_test_fieldrot")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
input_suppsky = os.path.join(test_dir, "suppsky.fits")
tgoff = 0
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
tgoff,
density=self.density_science)
tgoff += nscience
nstd = sim_targets(input_std,
TARGET_TYPE_STANDARD,
tgoff,
density=self.density_standards)
tgoff += nstd
nsky = sim_targets(input_sky,
TARGET_TYPE_SKY,
tgoff,
density=self.density_sky)
tgoff += nsky
nsuppsky = sim_targets(input_suppsky,
TARGET_TYPE_SUPPSKY,
tgoff,
density=self.density_suppsky)
# Simulate the tiles
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
# petal mapping
rotator = petal_rotation(1, reverse=False)
rots = [0, 36]
tile_ids = None
for rt in rots:
odir = "theta_{:02d}".format(rt)
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
load_target_file(tgs, input_suppsky)
# Manually override the field rotation
tiles = load_tiles(tiles_file=tfile, obstheta=float(rt))
if tile_ids is None:
tile_ids = list(tiles.id)
# Simulate a fake focalplane
fp, exclude, state = sim_focalplane(rundate=test_assign_date,
fakepos=True)
# Load the focalplane
hw = load_hardware(focalplane=(fp, exclude, state),
rundate=test_assign_date)
# Precompute target positions
tile_targetids, tile_x, tile_y = targets_in_tiles(hw, tgs, tiles)
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tiles, tile_targetids, tile_x,
tile_y)
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Pass empty map of STUCK positioners that land on good sky
stucksky = {}
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail, stucksky)
# First-pass assignment of science targets
asgn.assign_unused(TARGET_TYPE_SCIENCE)
out = os.path.join(test_dir, odir)
write_assignment_fits(tiles, asgn, out_dir=out, all_targets=True)
ppet = 6
if odir == "theta_36":
ppet = rotator[6]
plot_tiles(glob.glob(os.path.join(out, "fba-*.fits")),
real_shapes=True,
petals=[ppet],
serial=True)
# Explicitly free everything
del asgn
del favail
del tgsavail
del hw
del tiles
del tile_targetids, tile_x, tile_y
del tgs
# For each tile, compare the assignment output and verify that they
# agree with a one-petal rotation.
# NOTE: The comparison below will NOT pass, since we are still
# Sorting by highest priority available target and then (in case
# of a tie) by fiber ID. See line 333 of assign.cpp. Re-enable this
# test after that is changed to sort by location in case of a tie.
# for tl in tile_ids:
# orig_path = os.path.join(
# test_dir, "theta_00", "fiberassign_{:06d}.fits".format(tl)
# )
# orig_header, orig_data, _, _, _ = \
# read_assignment_fits_tile((tl, orig_path))
# rot_path = os.path.join(
# test_dir, "theta_36", "fiberassign_{:06d}.fits".format(tl)
# )
# rot_header, rot_data, _, _, _ = \
# read_assignment_fits_tile((tl, rot_path))
# comppath = os.path.join(
# test_dir, "comp_00-36_{:06d}.txt".format(tl)
# )
# with open(comppath, "w") as fc:
# for dev, petal, tg in zip(
# orig_data["DEVICE_LOC"], orig_data["PETAL_LOC"],
# orig_data["TARGETID"]
# ):
# for newdev, newpetal, newtg in zip(
# rot_data["DEVICE_LOC"], rot_data["PETAL_LOC"],
# rot_data["TARGETID"]
# ):
# rpet = rotator[newpetal]
# if (newdev == dev) and (rpet == petal):
# fc.write(
# "{}, {} = {} : {}, {} = {}\n"
# .format(petal, dev, tg, rpet, newdev, newtg)
# )
# # self.assertEqual(newtg, tg)
return
def test_io(self):
np.random.seed(123456789)
test_dir = test_subdir_create("assign_test_io")
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
input_suppsky = os.path.join(test_dir, "suppsky.fits")
tgoff = 0
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
tgoff,
density=self.density_science)
tgoff += nscience
nstd = sim_targets(input_std,
TARGET_TYPE_STANDARD,
tgoff,
density=self.density_standards)
tgoff += nstd
nsky = sim_targets(input_sky,
TARGET_TYPE_SKY,
tgoff,
density=self.density_sky)
tgoff += nsky
nsuppsky = sim_targets(input_suppsky,
TARGET_TYPE_SUPPSKY,
tgoff,
density=self.density_suppsky)
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
load_target_file(tgs, input_suppsky)
# Compute the targets available to each fiber for each tile.
fp, exclude, state = sim_focalplane(rundate=test_assign_date)
hw = load_hardware(focalplane=(fp, exclude, state),
rundate=test_assign_date)
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
# Precompute target positions
tile_targetids, tile_x, tile_y = targets_in_tiles(hw, tgs, tiles)
tgsavail = TargetsAvailable(hw, tiles, tile_targetids, tile_x, tile_y)
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Pass empty map of STUCK positioners that land on good sky
stucksky = {}
# First pass assignment
asgn = Assignment(tgs, tgsavail, favail, stucksky)
asgn.assign_unused(TARGET_TYPE_SCIENCE)
# Write out, merge, read back in and verify
write_assignment_ascii(tiles,
asgn,
out_dir=test_dir,
out_prefix="test_io_ascii_")
write_assignment_fits(tiles,
asgn,
out_dir=test_dir,
out_prefix="basic_",
all_targets=False)
write_assignment_fits(tiles,
asgn,
out_dir=test_dir,
out_prefix="full_",
all_targets=True)
plotpetals = [0]
# plotpetals = None
plot_tiles(glob.glob(os.path.join(test_dir, "basic_*.fits")),
petals=plotpetals,
serial=True)
plot_tiles(glob.glob(os.path.join(test_dir, "full_*.fits")),
petals=plotpetals,
serial=True)
target_files = [input_mtl, input_sky, input_std]
tile_ids = list(tiles.id)
merge_results(target_files,
list(),
tile_ids,
result_dir=test_dir,
result_prefix="basic_",
out_dir=test_dir,
out_prefix="basic_tile-",
copy_fba=False)
merge_results(target_files,
list(),
tile_ids,
result_dir=test_dir,
result_prefix="full_",
out_dir=test_dir,
out_prefix="full_tile-",
copy_fba=False)
# Here we test reading with the standard reading function
for tid in tile_ids:
tdata = asgn.tile_location_target(tid)
avail = tgsavail.tile_data(tid)
# Check basic format
infile = os.path.join(test_dir,
"basic_tile-{:06d}.fits".format(tid))
inhead, fiber_data, targets_data, avail_data, gfa_targets = \
read_assignment_fits_tile((infile))
for lid, tgid, tgra, tgdec in zip(fiber_data["LOCATION"],
fiber_data["TARGETID"],
fiber_data["TARGET_RA"],
fiber_data["TARGET_DEC"]):
if tgid >= 0:
self.assertEqual(tgid, tdata[lid])
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
# Check full format
infile = os.path.join(test_dir,
"full_tile-{:06d}.fits".format(tid))
inhead, fiber_data, targets_data, avail_data, gfa_targets = \
read_assignment_fits_tile((infile))
for lid, tgid, tgra, tgdec in zip(fiber_data["LOCATION"],
fiber_data["TARGETID"],
fiber_data["TARGET_RA"],
fiber_data["TARGET_DEC"]):
if tgid >= 0:
self.assertEqual(tgid, tdata[lid])
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
# Now read the files directly with fitsio and verify against the input
# target data.
for tid in tile_ids:
tdata = asgn.tile_location_target(tid)
avail = tgsavail.tile_data(tid)
# Check basic format
infile = os.path.join(test_dir,
"basic_tile-{:06d}.fits".format(tid))
fdata = fitsio.FITS(infile, "r")
fassign = fdata["FIBERASSIGN"].read()
ftargets = fdata["TARGETS"].read()
for lid, tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip(
fassign["LOCATION"], fassign["TARGETID"],
fassign["TARGET_RA"], fassign["TARGET_DEC"],
fassign["SUBPRIORITY"], fassign["PRIORITY"],
fassign["OBSCONDITIONS"]):
if tgid >= 0:
self.assertEqual(tgid, tdata[lid])
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
self.assertEqual(tgsub, props.subpriority)
self.assertEqual(tgprior, props.priority)
self.assertEqual(tgobs, props.obscond)
for tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip(
ftargets["TARGETID"], ftargets["RA"], ftargets["DEC"],
ftargets["SUBPRIORITY"], ftargets["PRIORITY"],
ftargets["OBSCONDITIONS"]):
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
self.assertEqual(tgsub, props.subpriority)
self.assertEqual(tgprior, props.priority)
self.assertEqual(tgobs, props.obscond)
# Check full format
infile = os.path.join(test_dir,
"full_tile-{:06d}.fits".format(tid))
fdata = fitsio.FITS(infile, "r")
fassign = fdata["FIBERASSIGN"].read()
ftargets = fdata["TARGETS"].read()
for lid, tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip(
fassign["LOCATION"], fassign["TARGETID"],
fassign["TARGET_RA"], fassign["TARGET_DEC"],
fassign["SUBPRIORITY"], fassign["PRIORITY"],
fassign["OBSCONDITIONS"]):
if tgid >= 0:
self.assertEqual(tgid, tdata[lid])
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
self.assertEqual(tgsub, props.subpriority)
self.assertEqual(tgprior, props.priority)
self.assertEqual(tgobs, props.obscond)
for tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip(
ftargets["TARGETID"], ftargets["RA"], ftargets["DEC"],
ftargets["SUBPRIORITY"], ftargets["PRIORITY"],
ftargets["OBSCONDITIONS"]):
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
self.assertEqual(tgsub, props.subpriority)
self.assertEqual(tgprior, props.priority)
self.assertEqual(tgobs, props.obscond)
plot_tiles(glob.glob(os.path.join(test_dir, "basic_tile-*")),
petals=plotpetals,
serial=True)
plot_tiles(glob.glob(os.path.join(test_dir, "full_tile-*")),
petals=plotpetals,
serial=True)
return
def test_full(self, do_stucksky=False):
test_dir = test_subdir_create("assign_test_full")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
input_suppsky = os.path.join(test_dir, "suppsky.fits")
tgoff = 0
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
tgoff,
density=self.density_science)
tgoff += nscience
nstd = sim_targets(input_std,
TARGET_TYPE_STANDARD,
tgoff,
density=self.density_standards)
tgoff += nstd
nsky = sim_targets(input_sky,
TARGET_TYPE_SKY,
tgoff,
density=self.density_sky)
tgoff += nsky
nsuppsky = sim_targets(input_suppsky,
TARGET_TYPE_SUPPSKY,
tgoff,
density=self.density_suppsky)
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
load_target_file(tgs, input_suppsky)
# Read hardware properties
fp, exclude, state = sim_focalplane(rundate=test_assign_date)
hw = load_hardware(focalplane=(fp, exclude, state),
rundate=test_assign_date)
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
if do_stucksky:
sim_stuck_sky(test_dir, hw, tiles)
# Precompute target positions
tile_targetids, tile_x, tile_y = targets_in_tiles(hw, tgs, tiles)
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tiles, tile_targetids, tile_x, tile_y)
del tile_targetids, tile_x, tile_y
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Pass empty map of STUCK positioners that land on good sky
stucksky = None
if do_stucksky:
stucksky = stuck_on_sky(hw, tiles)
if stucksky is None:
# (the pybind code doesn't like None when a dict is expected...)
stucksky = {}
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail, stucksky)
run(asgn)
write_assignment_fits(tiles,
asgn,
out_dir=test_dir,
all_targets=True,
stucksky=stucksky)
plotpetals = [0]
#plotpetals = None
plot_tiles(glob.glob(os.path.join(test_dir, "fba-*.fits")),
real_shapes=True,
petals=plotpetals,
serial=True)
qa_tiles(hw, tiles, result_dir=test_dir)
qadata = None
with open(os.path.join(test_dir, "qa.json"), "r") as f:
qadata = json.load(f)
for tile, props in qadata.items():
self.assertTrue(props["assign_science"] >= 4485)
self.assertEqual(100, props["assign_std"])
if do_stucksky:
# We get 3 stuck positioners landing on good sky!
self.assertTrue(
(props["assign_sky"] + props["assign_suppsky"]) >= 397)
else:
self.assertTrue(
(props["assign_sky"] + props["assign_suppsky"]) >= 400)
plot_qa(qadata,
os.path.join(test_dir, "qa"),
outformat="pdf",
labels=True)
return