def mkfa(targetf,tileff,rd,fad,srun,nrun,DESIMODEL):
'''
make the fiberassign files needed
targetf is the target file (e.g., an mtl file)
tilef is the root string for the tile files produced for each epoch
rd is the output directory
fad is the directory for the data fiberassign files
srun is the initial epoch
nrun is the number of epochs
DESIMODEL is the directory for where to find the focal plane model for running these
'''
os.environ['DESIMODEL'] = DESIMODEL
#targetf = e2eout +program+'/randoms_mtl_cuttod.fits' #above file, cut to ~e2e area with significant padding
#use fiberassign tools to read in randoms to be assigned
tgs = Targets()
load_target_file(tgs,targetf)
print('loaded target file '+targetf)
tree = TargetTree(tgs, 0.01)
for run in range(srun,srun+nrun):
#make the tile file for this run
#e2e.mke2etiles(run,program=program)
tilef = tileff+str(run)+'.fits'
#+str(run)+'.fits'
randir = rd +str(run)
#+str(run)
if os.path.isdir(randir):
ofls = glob.glob(randir+'/*')
for fl in ofls:
os.remove(fl) #remove the old files if we are rerunning
else:
os.mkdir(randir)
fafls = glob.glob(fad+str(run)+'/fiberassign/fiberassign*')
#+str(run)+'/fiberassign/fiberassign*')
hd = fitsio.read_header(fafls[0])
dt = hd['FA_RUN']
hw = load_hardware(rundate=dt)
tiles = load_tiles(tiles_file=tilef)
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
favail = LocationsAvailable(tgsavail)
#del tree
asgn = Assignment(tgs, tgsavail, favail)
asgn.assign_unused(TARGET_TYPE_SCIENCE)
write_assignment_fits(tiles, asgn, out_dir=randir, all_targets=True)
print('wrote assignment files to '+randir)
def getfatiles(targetf, tilef, dirout='', dt='2020-03-10T00:00:00'):
'''
will write out fiberassignment files for each tile with the FASSIGN, FTARGETS, FAVAIL HDUS
these are what are required to determine the geometry of what fiberassign thinks could have been observed and also match to actual observations (though FASSIGN is not really necessary)
targetf is file with all targets to be run through
tilef lists the tiles to "assign"
dirout is the directory where this all gets written out !make sure this is unique for every different target!
'''
tgs = Targets()
load_target_file(tgs, targetf)
print('loaded target file ' + targetf)
tree = TargetTree(tgs, 0.01)
hw = load_hardware(rundate=dt)
tiles = load_tiles(tiles_file=tilef)
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
favail = LocationsAvailable(tgsavail)
del tree
asgn = Assignment(tgs, tgsavail, favail)
asgn.assign_unused(TARGET_TYPE_SCIENCE)
write_assignment_fits(tiles, asgn, out_dir=dirout, all_targets=True)
print('wrote assignment files to ' + dirout)
tgs = Targets()
load_target_file(tgs, targetf)
print('loaded target file ' + targetf)
tree = TargetTree(tgs, 0.01)
for run in range(srun, srun + nrun):
#make the tile file for this run
mke2etiles(run, program=program)
tilef = e2eout + 'e2etiles_run' + str(run) + '.fits'
randir = e2eout + program + '/randoms/' + str(run)
if os.path.isdir(randir):
pass
else:
os.mkdir(randir)
fafls = glob.glob(e2ein + 'run/quicksurvey/' + program + '/' + str(run) +
'/fiberassign/fiberassign*')
hd = fitsio.read_header(fafls[0])
dt = hd['FA_RUN']
hw = load_hardware(rundate=dt)
tiles = load_tiles(tiles_file=tilef)
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
favail = LocationsAvailable(tgsavail)
#del tree
asgn = Assignment(tgs, tgsavail, favail)
asgn.assign_unused(TARGET_TYPE_SCIENCE)
write_assignment_fits(tiles, asgn, out_dir=randir, all_targets=True)
print('wrote assignment files to ' + randir)
def test_science(self):
set_matplotlib_pdf_backend()
import matplotlib.pyplot as plt
test_dir = test_subdir_create("qa_test_science")
log_file = os.path.join(test_dir, "log.txt")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
# For this test, we will use just 2 science target classes, in order to verify
# we get approximately the correct distribution
sdist = [(3000, 1, 0.25, "QSO"), (2000, 1, 0.75, "ELG")]
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
0,
density=self.density_science,
science_frac=sdist)
log_msg = "Simulated {} science targets\n".format(nscience)
tgs = Targets()
load_target_file(tgs, input_mtl)
# Read hardware properties
fp, exclude, state = sim_focalplane(rundate=test_assign_date)
hw = load_hardware(focalplane=(fp, exclude, state))
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
# Precompute target positions
tile_targetids, tile_x, tile_y = targets_in_tiles(hw, tgs, tiles)
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tiles, tile_targetids, tile_x, tile_y)
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Pass empty map of STUCK positioners that land on good sky
stucksky = {}
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail, stucksky)
# First-pass assignment of science targets
asgn.assign_unused(TARGET_TYPE_SCIENCE)
# Redistribute
asgn.redistribute_science()
write_assignment_fits(tiles, asgn, out_dir=test_dir, all_targets=True)
tile_ids = list(tiles.id)
merge_results([input_mtl],
list(),
tile_ids,
result_dir=test_dir,
copy_fba=False)
# FIXME: In order to use the qa_targets function, we need to know the
# starting requested number of observations (NUMOBS_INIT). Then we can use
# that value for each target and compare to the number actually assigned.
# However, the NUMOBS_INIT column was removed from the merged TARGET table.
# If we are ever able to reach consensus on restoring that column, then we
# can re-enable these tests below.
#
# qa_targets(
# hw,
# tiles,
# result_dir=test_dir,
# result_prefix="fiberassign-"
# )
#
# # Load the target catalog so that we have access to the target properties
#
# fd = fitsio.FITS(input_mtl, "r")
# scidata = np.array(np.sort(fd[1].read(), order="TARGETID"))
# fd.close()
# del fd
#
# # How many possible positioner assignments did we have?
# nassign = 5000 * len(tile_ids)
#
# possible = dict()
# achieved = dict()
#
# namepat = re.compile(r".*/qa_target_count_(.*)_init-(.*)\.fits")
# for qafile in glob.glob("{}/qa_target_count_*.fits".format(test_dir)):
# namemat = namepat.match(qafile)
# name = namemat.group(1)
# obs = int(namemat.group(2))
# if obs == 0:
# continue
# fd = fitsio.FITS(qafile, "r")
# fdata = fd["COUNTS"].read()
# # Sort by target ID so we can select easily
# fdata = np.sort(fdata, order="TARGETID")
# tgid = np.array(fdata["TARGETID"])
# counts = np.array(fdata["NUMOBS_DONE"])
# avail = np.array(fdata["NUMOBS_AVAIL"])
# del fdata
# fd.close()
#
# # Select target properties. BOTH TARGET LISTS MUST BE SORTED.
# rows = np.where(np.isin(scidata["TARGETID"], tgid, assume_unique=True))[0]
#
# ra = np.array(scidata["RA"][rows])
# dec = np.array(scidata["DEC"][rows])
# dtarget = np.array(scidata["DESI_TARGET"][rows])
# init = np.array(scidata["NUMOBS_INIT"][rows])
#
# requested = obs * np.ones_like(avail)
#
# under = np.where(avail < requested)[0]
# over = np.where(avail > requested)[0]
#
# limavail = np.array(avail)
# limavail[over] = obs
#
# deficit = np.zeros(len(limavail), dtype=np.int)
#
# deficit[:] = limavail - counts
# deficit[avail == 0] = 0
#
# possible[name] = np.sum(limavail)
# achieved[name] = np.sum(counts)
#
# log_msg += "{}-{}:\n".format(name, obs)
#
# pindx = np.where(deficit > 0)[0]
# poor_tgid = tgid[pindx]
# poor_dtarget = dtarget[pindx]
# log_msg += " Deficit > 0: {}\n".format(len(poor_tgid))
# poor_ra = ra[pindx]
# poor_dec = dec[pindx]
# poor_deficit = deficit[pindx]
#
# # Plot Target availability
# # Commented out by default, since in the case of high target density
# # needed for maximizing assignments, there are far more targets than
# # the number of available fiber placements.
#
# # marksize = 4 * np.ones_like(deficit)
# #
# # fig = plt.figure(figsize=(12, 12))
# # ax = fig.add_subplot(1, 1, 1)
# # ax.scatter(ra, dec, s=2, c="black", marker="o")
# # for pt, pr, pd, pdef in zip(poor_tgid, poor_ra, poor_dec, poor_deficit):
# # ploc = plt.Circle(
# # (pr, pd), radius=(0.05*pdef), fc="none", ec="red"
# # )
# # ax.add_artist(ploc)
# # ax.set_xlabel("RA", fontsize="large")
# # ax.set_ylabel("DEC", fontsize="large")
# # ax.set_title(
# # "Target \"{}\": (min(avail, requested) - counts) > 0".format(
# # name, obs
# # )
# # )
# # #ax.legend(handles=lg, framealpha=1.0, loc="upper right")
# # plt.savefig(os.path.join(test_dir, "{}-{}_deficit.pdf".format(name, obs)), dpi=300, format="pdf")
#
# log_msg += \
# "Assigned {} tiles for total of {} possible target observations\n".format(
# len(tile_ids), nassign
# )
# ach = 0
# for nm in possible.keys():
# ach += achieved[nm]
# log_msg += \
# " type {} had {} possible target obs and achieved {}\n".format(
# nm, possible[nm], achieved[nm]
# )
# frac = 100.0 * ach / nassign
# log_msg += \
# " {} / {} = {:0.2f}% of fibers were assigned\n".format(
# ach, nassign, frac
# )
# for nm in possible.keys():
# log_msg += \
# " type {} had {:0.2f}% of achieved observations\n".format(
# nm, achieved[nm] / ach
# )
# with open(log_file, "w") as f:
# f.write(log_msg)
#
# self.assertGreaterEqual(frac, 99.0)
# Test if qa-fiberassign script runs without crashing
script = os.path.join(self.binDir, "qa-fiberassign")
if os.path.exists(script):
fafiles = glob.glob(f"{test_dir}/fiberassign-*.fits")
cmd = "{} --targets {}".format(script, " ".join(fafiles))
err = subprocess.call(cmd.split())
self.assertEqual(err, 0, f"FAILED ({err}): {cmd}")
else:
print(f"ERROR: didn't find {script}")
def test_io(self):
np.random.seed(123456789)
test_dir = test_subdir_create("assign_test_io")
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
input_suppsky = os.path.join(test_dir, "suppsky.fits")
tgoff = 0
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
tgoff,
density=self.density_science)
tgoff += nscience
nstd = sim_targets(input_std,
TARGET_TYPE_STANDARD,
tgoff,
density=self.density_standards)
tgoff += nstd
nsky = sim_targets(input_sky,
TARGET_TYPE_SKY,
tgoff,
density=self.density_sky)
tgoff += nsky
nsuppsky = sim_targets(input_suppsky,
TARGET_TYPE_SUPPSKY,
tgoff,
density=self.density_suppsky)
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
load_target_file(tgs, input_suppsky)
# Create a hierarchical triangle mesh lookup of the targets positions
tree = TargetTree(tgs, 0.01)
# Compute the targets available to each fiber for each tile.
fp, exclude, state = sim_focalplane(rundate=test_assign_date)
hw = load_hardware(focalplane=(fp, exclude, state))
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Free the tree
del tree
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Pass empty map of STUCK positioners that land on good sky
stucksky = {}
# First pass assignment
asgn = Assignment(tgs, tgsavail, favail, stucksky)
asgn.assign_unused(TARGET_TYPE_SCIENCE)
# Write out, merge, read back in and verify
write_assignment_ascii(tiles,
asgn,
out_dir=test_dir,
out_prefix="test_io_ascii_")
write_assignment_fits(tiles,
asgn,
out_dir=test_dir,
out_prefix="basic_",
all_targets=False)
write_assignment_fits(tiles,
asgn,
out_dir=test_dir,
out_prefix="full_",
all_targets=True)
plotpetals = [0]
# plotpetals = None
plot_tiles(hw,
tiles,
result_dir=test_dir,
result_prefix="basic_",
plot_dir=test_dir,
plot_prefix="basic_",
result_split_dir=False,
petals=plotpetals,
serial=True)
plot_tiles(hw,
tiles,
result_dir=test_dir,
result_prefix="full_",
plot_dir=test_dir,
plot_prefix="full_",
result_split_dir=False,
petals=plotpetals,
serial=True)
target_files = [input_mtl, input_sky, input_std]
tile_ids = list(tiles.id)
merge_results(target_files,
list(),
tile_ids,
result_dir=test_dir,
result_prefix="basic_",
out_dir=test_dir,
out_prefix="basic_tile-",
copy_fba=False)
merge_results(target_files,
list(),
tile_ids,
result_dir=test_dir,
result_prefix="full_",
out_dir=test_dir,
out_prefix="full_tile-",
copy_fba=False)
# Here we test reading with the standard reading function
for tid in tile_ids:
tdata = asgn.tile_location_target(tid)
avail = tgsavail.tile_data(tid)
# Check basic format
infile = os.path.join(test_dir,
"basic_tile-{:06d}.fits".format(tid))
inhead, fiber_data, targets_data, avail_data, gfa_targets = \
read_assignment_fits_tile((tid, infile))
for lid, tgid, tgra, tgdec in zip(fiber_data["LOCATION"],
fiber_data["TARGETID"],
fiber_data["TARGET_RA"],
fiber_data["TARGET_DEC"]):
if tgid >= 0:
self.assertEqual(tgid, tdata[lid])
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
# Check full format
infile = os.path.join(test_dir,
"full_tile-{:06d}.fits".format(tid))
inhead, fiber_data, targets_data, avail_data, gfa_targets = \
read_assignment_fits_tile((tid, infile))
for lid, tgid, tgra, tgdec in zip(fiber_data["LOCATION"],
fiber_data["TARGETID"],
fiber_data["TARGET_RA"],
fiber_data["TARGET_DEC"]):
if tgid >= 0:
self.assertEqual(tgid, tdata[lid])
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
# Now read the files directly with fitsio and verify against the input
# target data.
for tid in tile_ids:
tdata = asgn.tile_location_target(tid)
avail = tgsavail.tile_data(tid)
# Check basic format
infile = os.path.join(test_dir,
"basic_tile-{:06d}.fits".format(tid))
fdata = fitsio.FITS(infile, "r")
fassign = fdata["FIBERASSIGN"].read()
ftargets = fdata["TARGETS"].read()
for lid, tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip(
fassign["LOCATION"], fassign["TARGETID"],
fassign["TARGET_RA"], fassign["TARGET_DEC"],
fassign["SUBPRIORITY"], fassign["PRIORITY"],
fassign["OBSCONDITIONS"]):
if tgid >= 0:
self.assertEqual(tgid, tdata[lid])
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
self.assertEqual(tgsub, props.subpriority)
self.assertEqual(tgprior, props.priority)
self.assertEqual(tgobs, props.obscond)
for tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip(
ftargets["TARGETID"], ftargets["RA"], ftargets["DEC"],
ftargets["SUBPRIORITY"], ftargets["PRIORITY"],
ftargets["OBSCONDITIONS"]):
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
self.assertEqual(tgsub, props.subpriority)
self.assertEqual(tgprior, props.priority)
self.assertEqual(tgobs, props.obscond)
# Check full format
infile = os.path.join(test_dir,
"full_tile-{:06d}.fits".format(tid))
fdata = fitsio.FITS(infile, "r")
fassign = fdata["FIBERASSIGN"].read()
ftargets = fdata["TARGETS"].read()
for lid, tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip(
fassign["LOCATION"], fassign["TARGETID"],
fassign["TARGET_RA"], fassign["TARGET_DEC"],
fassign["SUBPRIORITY"], fassign["PRIORITY"],
fassign["OBSCONDITIONS"]):
if tgid >= 0:
self.assertEqual(tgid, tdata[lid])
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
self.assertEqual(tgsub, props.subpriority)
self.assertEqual(tgprior, props.priority)
self.assertEqual(tgobs, props.obscond)
for tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip(
ftargets["TARGETID"], ftargets["RA"], ftargets["DEC"],
ftargets["SUBPRIORITY"], ftargets["PRIORITY"],
ftargets["OBSCONDITIONS"]):
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
self.assertEqual(tgsub, props.subpriority)
self.assertEqual(tgprior, props.priority)
self.assertEqual(tgobs, props.obscond)
plot_tiles(hw,
tiles,
result_dir=test_dir,
result_prefix="basic_tile-",
plot_dir=test_dir,
plot_prefix="basic_tile-",
result_split_dir=False,
petals=plotpetals,
serial=True)
plot_tiles(hw,
tiles,
result_dir=test_dir,
result_prefix="full_tile-",
plot_dir=test_dir,
plot_prefix="full_tile-",
result_split_dir=False,
petals=plotpetals,
serial=True)
return
def test_fieldrot(self):
test_dir = test_subdir_create("assign_test_fieldrot")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
input_suppsky = os.path.join(test_dir, "suppsky.fits")
tgoff = 0
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
tgoff,
density=self.density_science)
tgoff += nscience
nstd = sim_targets(input_std,
TARGET_TYPE_STANDARD,
tgoff,
density=self.density_standards)
tgoff += nstd
nsky = sim_targets(input_sky,
TARGET_TYPE_SKY,
tgoff,
density=self.density_sky)
tgoff += nsky
nsuppsky = sim_targets(input_suppsky,
TARGET_TYPE_SUPPSKY,
tgoff,
density=self.density_suppsky)
# Simulate the tiles
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
# petal mapping
rotator = petal_rotation(1, reverse=False)
rots = [0, 36]
tile_ids = None
for rt in rots:
odir = "theta_{:02d}".format(rt)
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
load_target_file(tgs, input_suppsky)
# Create a hierarchical triangle mesh lookup of the targets
# positions
tree = TargetTree(tgs, 0.01)
# Manually override the field rotation
tiles = load_tiles(tiles_file=tfile, obstheta=float(rt))
if tile_ids is None:
tile_ids = list(tiles.id)
# Simulate a fake focalplane
fp, exclude, state = sim_focalplane(rundate=test_assign_date,
fakepos=True)
# Load the focalplane
hw = load_hardware(focalplane=(fp, exclude, state))
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Pass empty map of STUCK positioners that land on good sky
stucksky = {}
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail, stucksky)
# First-pass assignment of science targets
asgn.assign_unused(TARGET_TYPE_SCIENCE)
out = os.path.join(test_dir, odir)
write_assignment_fits(tiles, asgn, out_dir=out, all_targets=True)
ppet = 6
if odir == "theta_36":
ppet = rotator[6]
plot_tiles(hw,
tiles,
result_dir=out,
plot_dir=out,
real_shapes=True,
petals=[ppet],
serial=True)
# Explicitly free everything
del asgn
del favail
del tgsavail
del hw
del tiles
del tree
del tgs
# For each tile, compare the assignment output and verify that they
# agree with a one-petal rotation.
# NOTE: The comparison below will NOT pass, since we are still
# Sorting by highest priority available target and then (in case
# of a tie) by fiber ID. See line 333 of assign.cpp. Re-enable this
# test after that is changed to sort by location in case of a tie.
# for tl in tile_ids:
# orig_path = os.path.join(
# test_dir, "theta_00", "fiberassign_{:06d}.fits".format(tl)
# )
# orig_header, orig_data, _, _, _ = \
# read_assignment_fits_tile((tl, orig_path))
# rot_path = os.path.join(
# test_dir, "theta_36", "fiberassign_{:06d}.fits".format(tl)
# )
# rot_header, rot_data, _, _, _ = \
# read_assignment_fits_tile((tl, rot_path))
# comppath = os.path.join(
# test_dir, "comp_00-36_{:06d}.txt".format(tl)
# )
# with open(comppath, "w") as fc:
# for dev, petal, tg in zip(
# orig_data["DEVICE_LOC"], orig_data["PETAL_LOC"],
# orig_data["TARGETID"]
# ):
# for newdev, newpetal, newtg in zip(
# rot_data["DEVICE_LOC"], rot_data["PETAL_LOC"],
# rot_data["TARGETID"]
# ):
# rpet = rotator[newpetal]
# if (newdev == dev) and (rpet == petal):
# fc.write(
# "{}, {} = {} : {}, {} = {}\n"
# .format(petal, dev, tg, rpet, newdev, newtg)
# )
# # self.assertEqual(newtg, tg)
return
def test_full(self):
test_dir = test_subdir_create("assign_test_full")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
nscience = sim_targets(input_mtl, TARGET_TYPE_SCIENCE, 0)
nstd = sim_targets(input_std, TARGET_TYPE_STANDARD, nscience)
nsky = sim_targets(input_sky, TARGET_TYPE_SKY, (nscience + nstd))
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
# Create a hierarchical triangle mesh lookup of the targets positions
tree = TargetTree(tgs, 0.01)
# Read hardware properties
fstatus = os.path.join(test_dir, "fiberstatus.ecsv")
sim_status(fstatus)
hw = load_hardware(status_file=fstatus)
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Free the tree
del tree
# Compute the fibers on all tiles available for each target
favail = FibersAvailable(tgsavail)
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail)
# First-pass assignment of science targets
asgn.assign_unused(TARGET_TYPE_SCIENCE)
# Redistribute science targets
asgn.redistribute_science()
# Assign standards, 10 per petal
asgn.assign_unused(TARGET_TYPE_STANDARD, 10)
asgn.assign_force(TARGET_TYPE_STANDARD, 10)
# Assign sky to unused fibers, up to 40 per petal
asgn.assign_unused(TARGET_TYPE_SKY, 40)
asgn.assign_force(TARGET_TYPE_SKY, 40)
# If there are any unassigned fibers, try to place them somewhere.
asgn.assign_unused(TARGET_TYPE_SCIENCE)
asgn.assign_unused(TARGET_TYPE_SKY)
write_assignment_fits(tiles, asgn, out_dir=test_dir, all_targets=True)
plot_tiles(hw,
tiles,
result_dir=test_dir,
plot_dir=test_dir,
petals=[0],
serial=True)
qa_tiles(hw, tiles, result_dir=test_dir)
qadata = None
with open(os.path.join(test_dir, "qa.json"), "r") as f:
qadata = json.load(f)
for tile, props in qadata.items():
self.assertEqual(4495, props["assign_science"])
self.assertEqual(100, props["assign_std"])
self.assertEqual(400, props["assign_sky"])
plot_qa(qadata,
os.path.join(test_dir, "qa"),
outformat="pdf",
labels=True)
return
def test_science(self):
set_matplotlib_pdf_backend()
import matplotlib.pyplot as plt
test_dir = test_subdir_create("qa_test_science")
log_file = os.path.join(test_dir, "log.txt")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
# For this test, we will use just 2 science target classes, in order to verify
# we get approximately the correct distribution
sdist = [(3000, 1, 0.25, "QSO"), (2000, 1, 0.75, "ELG")]
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
0,
density=self.density_science,
science_frac=sdist)
log_msg = "Simulated {} science targets\n".format(nscience)
tgs = Targets()
load_target_file(tgs, input_mtl)
# Create a hierarchical triangle mesh lookup of the targets positions
tree = TargetTree(tgs, 0.01)
# Read hardware properties
fp, exclude, state = sim_focalplane(rundate=test_assign_date)
hw = load_hardware(focalplane=(fp, exclude, state))
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Free the tree
del tree
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail)
# First-pass assignment of science targets
asgn.assign_unused(TARGET_TYPE_SCIENCE)
# Redistribute
asgn.redistribute_science()
write_assignment_fits(tiles, asgn, out_dir=test_dir, all_targets=True)
tile_ids = list(tiles.id)
merge_results([input_mtl],
list(),
tile_ids,
result_dir=test_dir,
copy_fba=False)
# if "TRAVIS" not in os.environ:
# plot_tiles(
# hw,
# tiles,
# result_dir=test_dir,
# plot_dir=test_dir,
# real_shapes=True,
# serial=True
# )
qa_targets(hw,
tiles,
result_dir=test_dir,
result_prefix="fiberassign-")
# Load the target catalog so that we have access to the target properties
fd = fitsio.FITS(input_mtl, "r")
scidata = np.array(np.sort(fd[1].read(), order="TARGETID"))
fd.close()
del fd
# How many possible positioner assignments did we have?
nassign = 5000 * len(tile_ids)
possible = dict()
achieved = dict()
namepat = re.compile(r".*/qa_target_count_(.*)_init-(.*)\.fits")
for qafile in glob.glob("{}/qa_target_count_*.fits".format(test_dir)):
namemat = namepat.match(qafile)
name = namemat.group(1)
obs = int(namemat.group(2))
if obs == 0:
continue
fd = fitsio.FITS(qafile, "r")
fdata = fd["COUNTS"].read()
# Sort by target ID so we can select easily
fdata = np.sort(fdata, order="TARGETID")
tgid = np.array(fdata["TARGETID"])
counts = np.array(fdata["NUMOBS_DONE"])
avail = np.array(fdata["NUMOBS_AVAIL"])
del fdata
fd.close()
# Select target properties. BOTH TARGET LISTS MUST BE SORTED.
rows = np.where(
np.isin(scidata["TARGETID"], tgid, assume_unique=True))[0]
ra = np.array(scidata["RA"][rows])
dec = np.array(scidata["DEC"][rows])
dtarget = np.array(scidata["DESI_TARGET"][rows])
init = np.array(scidata["NUMOBS_MORE"][rows])
requested = obs * np.ones_like(avail)
under = np.where(avail < requested)[0]
over = np.where(avail > requested)[0]
limavail = np.array(avail)
limavail[over] = obs
deficit = np.zeros(len(limavail), dtype=np.int)
deficit[:] = limavail - counts
deficit[avail == 0] = 0
possible[name] = np.sum(limavail)
achieved[name] = np.sum(counts)
log_msg += "{}-{}:\n".format(name, obs)
pindx = np.where(deficit > 0)[0]
poor_tgid = tgid[pindx]
poor_dtarget = dtarget[pindx]
log_msg += " Deficit > 0: {}\n".format(len(poor_tgid))
poor_ra = ra[pindx]
poor_dec = dec[pindx]
poor_deficit = deficit[pindx]
# Plot Target availability
# Commented out by default, since in the case of high target density
# needed for maximizing assignments, there are far more targets than
# the number of available fiber placements.
# marksize = 4 * np.ones_like(deficit)
#
# fig = plt.figure(figsize=(12, 12))
# ax = fig.add_subplot(1, 1, 1)
# ax.scatter(ra, dec, s=2, c="black", marker="o")
# for pt, pr, pd, pdef in zip(poor_tgid, poor_ra, poor_dec, poor_deficit):
# ploc = plt.Circle(
# (pr, pd), radius=(0.05*pdef), fc="none", ec="red"
# )
# ax.add_artist(ploc)
# ax.set_xlabel("RA", fontsize="large")
# ax.set_ylabel("DEC", fontsize="large")
# ax.set_title(
# "Target \"{}\": (min(avail, requested) - counts) > 0".format(
# name, obs
# )
# )
# #ax.legend(handles=lg, framealpha=1.0, loc="upper right")
# plt.savefig(os.path.join(test_dir, "{}-{}_deficit.pdf".format(name, obs)), dpi=300, format="pdf")
log_msg += \
"Assigned {} tiles for total of {} possible target observations\n".format(
len(tile_ids), nassign
)
ach = 0
for nm in possible.keys():
ach += achieved[nm]
log_msg += \
" type {} had {} possible target obs and achieved {}\n".format(
nm, possible[nm], achieved[nm]
)
frac = 100.0 * ach / nassign
log_msg += \
" {} / {} = {:0.2f}% of fibers were assigned\n".format(
ach, nassign, frac
)
for nm in possible.keys():
log_msg += \
" type {} had {:0.2f}% of achieved observations\n".format(
nm, achieved[nm] / ach
)
with open(log_file, "w") as f:
f.write(log_msg)
self.assertGreaterEqual(frac, 99.0)
#- Test if qa-fiberassign script runs without crashing
bindir = os.path.join(os.path.dirname(fiberassign.__file__), '..',
'..', 'bin')
script = os.path.join(os.path.abspath(bindir), 'qa-fiberassign')
if os.path.exists(script):
fafiles = glob.glob(f"{test_dir}/fiberassign-*.fits")
cmd = "{} --targets {}".format(script, " ".join(fafiles))
err = subprocess.call(cmd.split())
self.assertEqual(err, 0, f"FAILED ({err}): {cmd}")
else:
print(f"ERROR: didn't find {script}")
def test_full(self):
test_dir = test_subdir_create("assign_test_full")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
input_suppsky = os.path.join(test_dir, "suppsky.fits")
tgoff = 0
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
tgoff,
density=self.density_science)
tgoff += nscience
nstd = sim_targets(input_std,
TARGET_TYPE_STANDARD,
tgoff,
density=self.density_standards)
tgoff += nstd
nsky = sim_targets(input_sky,
TARGET_TYPE_SKY,
tgoff,
density=self.density_sky)
tgoff += nsky
nsuppsky = sim_targets(input_suppsky,
TARGET_TYPE_SUPPSKY,
tgoff,
density=self.density_suppsky)
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
load_target_file(tgs, input_suppsky)
# Create a hierarchical triangle mesh lookup of the targets positions
tree = TargetTree(tgs, 0.01)
# Read hardware properties
fp, exclude, state = sim_focalplane(rundate=test_assign_date)
hw = load_hardware(focalplane=(fp, exclude, state))
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Free the tree
del tree
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail)
# First-pass assignment of science targets
asgn.assign_unused(TARGET_TYPE_SCIENCE)
# Redistribute science targets
asgn.redistribute_science()
# Assign standards, 10 per petal
asgn.assign_unused(TARGET_TYPE_STANDARD, 10)
asgn.assign_force(TARGET_TYPE_STANDARD, 10)
# Assign sky to unused fibers, up to 40 per petal
asgn.assign_unused(TARGET_TYPE_SKY, 40)
asgn.assign_force(TARGET_TYPE_SKY, 40)
# Use supplemental sky to meet our requirements
asgn.assign_unused(TARGET_TYPE_SUPPSKY, 40)
asgn.assign_force(TARGET_TYPE_SUPPSKY, 40)
# If there are any unassigned fibers, try to place them somewhere.
asgn.assign_unused(TARGET_TYPE_SCIENCE)
asgn.assign_unused(TARGET_TYPE_SKY)
asgn.assign_unused(TARGET_TYPE_SUPPSKY)
write_assignment_fits(tiles, asgn, out_dir=test_dir, all_targets=True)
plotpetals = [0]
#plotpetals = None
plot_tiles(hw,
tiles,
result_dir=test_dir,
plot_dir=test_dir,
result_prefix="fba-",
real_shapes=True,
petals=plotpetals,
serial=True)
qa_tiles(hw, tiles, result_dir=test_dir)
qadata = None
with open(os.path.join(test_dir, "qa.json"), "r") as f:
qadata = json.load(f)
for tile, props in qadata.items():
self.assertTrue(props["assign_science"] >= 4485)
self.assertEqual(100, props["assign_std"])
self.assertTrue(
(props["assign_sky"] + props["assign_suppsky"]) >= 400)
plot_qa(qadata,
os.path.join(test_dir, "qa"),
outformat="pdf",
labels=True)
return
tgsavail = TargetsAvailable(hw, tgs, nominal_tiles, tree) # Compute the fibers on all tiles available for each target favail = FibersAvailable(tgsavail) # Create assignment object asgn = Assignment(tgs, tgsavail, favail) # -------------------------------------------------------------------------------------------------- # FIBER ASSIGNMENT PROCESS # # Fiber assignment process is carried out here. Results are stored in the Assignment type object. # -------------------------------------------------------------------------------------------------- # First-pass assignment of science targets asgn.assign_unused(TARGET_TYPE_SCIENCE) # Redistribute science targets across available petals asgn.redistribute_science() # Assign standards, 10 per petal asgn.assign_unused(TARGET_TYPE_STANDARD, 10) asgn.assign_force(TARGET_TYPE_STANDARD, 10) # Assign sky, up to 40 per petal asgn.assign_unused(TARGET_TYPE_SKY, 40) asgn.assign_force(TARGET_TYPE_SKY, 40) # If there are any unassigned fibers, try to place them somewhere. asgn.assign_unused(TARGET_TYPE_SCIENCE) asgn.assign_unused(TARGET_TYPE_SKY)