def test_available(self):
test_dir = test_subdir_create("targets_test_available")
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
nscience = sim_targets(input_mtl, TARGET_TYPE_SCIENCE, 0)
nstd = sim_targets(input_std, TARGET_TYPE_STANDARD, nscience)
nsky = sim_targets(input_sky, TARGET_TYPE_SKY, (nscience + nstd))
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
print(tgs)
# Create a hierarchical triangle mesh lookup of the targets positions
tree = TargetTree(tgs, 0.01)
# Compute the targets available to each fiber for each tile.
hw = load_hardware()
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Free the tree
del tree
# Compute the fibers on all tiles available for each target
favail = FibersAvailable(tgsavail)
return
def test_read(self):
hw = load_hardware(rundate=test_assign_date)
print(hw)
locs = hw.locations
cs5 = hw.loc_pos_cs5_mm
curved = hw.loc_pos_curved_mm
# for p in locs:
# print("{}: curved = [{}, {}] cs5 = [{}, {}]".format(
# p, curved[p][0], curved[p][1], cs5[p][0], cs5[p][1]
# ))
return
def mkfa(targetf,tileff,rd,fad,srun,nrun,DESIMODEL):
'''
make the fiberassign files needed
targetf is the target file (e.g., an mtl file)
tilef is the root string for the tile files produced for each epoch
rd is the output directory
fad is the directory for the data fiberassign files
srun is the initial epoch
nrun is the number of epochs
DESIMODEL is the directory for where to find the focal plane model for running these
'''
os.environ['DESIMODEL'] = DESIMODEL
#targetf = e2eout +program+'/randoms_mtl_cuttod.fits' #above file, cut to ~e2e area with significant padding
#use fiberassign tools to read in randoms to be assigned
tgs = Targets()
load_target_file(tgs,targetf)
print('loaded target file '+targetf)
tree = TargetTree(tgs, 0.01)
for run in range(srun,srun+nrun):
#make the tile file for this run
#e2e.mke2etiles(run,program=program)
tilef = tileff+str(run)+'.fits'
#+str(run)+'.fits'
randir = rd +str(run)
#+str(run)
if os.path.isdir(randir):
ofls = glob.glob(randir+'/*')
for fl in ofls:
os.remove(fl) #remove the old files if we are rerunning
else:
os.mkdir(randir)
fafls = glob.glob(fad+str(run)+'/fiberassign/fiberassign*')
#+str(run)+'/fiberassign/fiberassign*')
hd = fitsio.read_header(fafls[0])
dt = hd['FA_RUN']
hw = load_hardware(rundate=dt)
tiles = load_tiles(tiles_file=tilef)
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
favail = LocationsAvailable(tgsavail)
#del tree
asgn = Assignment(tgs, tgsavail, favail)
asgn.assign_unused(TARGET_TYPE_SCIENCE)
write_assignment_fits(tiles, asgn, out_dir=randir, all_targets=True)
print('wrote assignment files to '+randir)
def test_plotfp(self):
test_dir = test_subdir_create("vis_test_plotfp")
time = test_assign_date
hw = load_hardware(rundate=time)
suffix = "{}_simple".format(time)
self._load_and_plotfp(hw, test_dir, suffix, simple=True)
suffix = "{}".format(time)
self._load_and_plotfp(hw, test_dir, suffix, simple=False)
# time = "2012-12-12T00:00:00"
# hw = load_hardware(rundate=time)
# suffix = "{}_simple".format(time)
# self._load_and_plotfp(hw, test_dir, suffix, simple=True)
# suffix = "{}".format(time)
# self._load_and_plotfp(hw, test_dir, suffix, simple=False)
return
def test_plot_fakefp(self):
test_dir = test_subdir_create("vis_test_fakefp")
time = test_assign_date
# Simulate a fake focalplane
fp, exclude, state = sim_focalplane(rundate=test_assign_date, fakepos=True)
# Load the focalplane
hw = load_hardware(focalplane=(fp, exclude, state))
suffix = "{}".format(time)
self._load_and_plotfp(hw, test_dir, suffix, simple=False)
time = "2012-12-12T00:00:00"
suffix = "{}".format(time)
self._load_and_plotfp(hw, test_dir, suffix, simple=False)
return
def test_available(self):
test_dir = test_subdir_create("targets_test_available")
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
input_suppsky = os.path.join(test_dir, "suppsky.fits")
tgoff = 0
nscience = sim_targets(input_mtl, TARGET_TYPE_SCIENCE, tgoff)
tgoff += nscience
nstd = sim_targets(input_std, TARGET_TYPE_STANDARD, tgoff)
tgoff += nstd
nsky = sim_targets(input_sky, TARGET_TYPE_SKY, tgoff)
tgoff += nsky
nsuppsky = sim_targets(input_suppsky, TARGET_TYPE_SUPPSKY, tgoff)
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
load_target_file(tgs, input_suppsky)
print(tgs)
# Test access
ids = tgs.ids()
tt = tgs.get(ids[0])
tt.ra += 1.0e-5
tt.dec += 1.0e-5
tt.subpriority = 0.99
# Create a hierarchical triangle mesh lookup of the targets positions
tree = TargetTree(tgs, 0.01)
# Compute the targets available to each fiber for each tile.
hw = load_hardware()
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Free the tree
del tree
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
return
def test_collision_thetaphi(self):
hw = load_hardware()
fiber_id = hw.fiber_id
ntheta = 10
nphi = 10
thetaincr = 2 * np.pi / ntheta
phiincr = np.pi / nphi
tm = Timer()
tm.start()
for thetarot in range(ntheta):
for phirot in range(nphi):
theta = [thetarot * thetaincr for x in fiber_id]
phi = [phirot * phiincr for x in fiber_id]
result = hw.check_collisions_thetaphi(fiber_id, theta, phi, 0)
tm.stop()
tm.report("check_collisions_thetaphi 100 configurations")
return
def test_collision_thetaphi(self):
hw = load_hardware(rundate=test_assign_date)
locs = hw.locations
ntheta = 10
nphi = 10
thetaincr = 2 * np.pi / ntheta
phiincr = np.pi / nphi
tm = Timer()
tm.start()
for thetarot in range(ntheta):
for phirot in range(nphi):
theta = [thetarot * thetaincr for x in locs]
phi = [phirot * phiincr for x in locs]
result = hw.check_collisions_thetaphi(locs, theta, phi, 0)
tm.stop()
tm.report("check_collisions_thetaphi 100 configurations")
return
def test_collision_xy(self):
hw = load_hardware(rundate=test_assign_date)
center_mm = hw.loc_pos_curved_mm
locs = hw.locations
nrot = 100
rotrad = 0.5
rotincr = 2 * np.pi / nrot
tm = Timer()
tm.start()
for rot in range(nrot):
xoff = rotrad * np.cos(rot * rotincr)
yoff = rotrad * np.sin(rot * rotincr)
xy = [(center_mm[p][0] + xoff, center_mm[p][1] + yoff)
for p in locs]
result = hw.check_collisions_xy(locs, xy, 0)
tm.stop()
tm.report("check_collisions_xy 100 configurations")
return
def test_collision_xy(self):
hw = load_hardware()
center_mm = hw.fiber_pos_xy_mm
fiber_id = hw.fiber_id
nrot = 100
rotrad = 0.5
rotincr = 2 * np.pi / nrot
tm = Timer()
tm.start()
for rot in range(nrot):
xoff = rotrad * np.cos(rot * rotincr)
yoff = rotrad * np.sin(rot * rotincr)
xy = [(center_mm[p][0] + xoff, center_mm[p][1] + yoff)
for p in fiber_id]
result = hw.check_collisions_xy(fiber_id, xy, 0)
tm.stop()
tm.report("check_collisions_xy 100 configurations")
return
def test_read(self):
test_dir = test_subdir_create("tiles_test_read")
print('test_dir', test_dir)
hw = load_hardware()
tfile = os.path.join(test_dir, "footprint.fits")
sfile = os.path.join(test_dir, "footprint_keep.txt")
sim_tiles(tfile, selectfile=sfile)
stiles = list()
with open(sfile, "r") as f:
for line in f:
# Try to convert the first column to an integer.
try:
stiles.append(int(line.split()[0]))
except ValueError:
pass
tls = load_tiles(tiles_file=tfile, select=stiles)
print(tls)
indx = 0
for st in stiles:
self.assertEqual(tls.order[st], indx)
indx += 1
#- Syntax / coverage tests
tls = load_tiles(tiles_file=tfile, select=stiles, obstime='2020-10-20')
self.assertEqual(tls.obstime[0], Time('2020-10-20T00:00:00').isot)
tls = load_tiles(tiles_file=tfile,
select=stiles,
obstime='2020-10-20T10:20:30')
self.assertEqual(tls.obstime[0], Time('2020-10-20T10:20:30').isot)
#- including obsdate in tile file
tiles = Table.read(tfile)
tiles['OBSDATE'] = '2020-11-22' # creates full column
tiles['OBSDATE'][1] = '2020-11-23' # override second entry
tfile = os.path.join(test_dir, "footprint-obsdate.fits")
tiles.write(tfile)
tls = load_tiles(tiles_file=tfile, select=stiles)
self.assertEqual(tls.obstime[0], Time('2020-11-22').isot)
self.assertEqual(tls.obstime[1], Time('2020-11-23').isot)
return
def getfatiles(targetf, tilef, dirout='', dt='2020-03-10T00:00:00'):
'''
will write out fiberassignment files for each tile with the FASSIGN, FTARGETS, FAVAIL HDUS
these are what are required to determine the geometry of what fiberassign thinks could have been observed and also match to actual observations (though FASSIGN is not really necessary)
targetf is file with all targets to be run through
tilef lists the tiles to "assign"
dirout is the directory where this all gets written out !make sure this is unique for every different target!
'''
tgs = Targets()
load_target_file(tgs, targetf)
print('loaded target file ' + targetf)
tree = TargetTree(tgs, 0.01)
hw = load_hardware(rundate=dt)
tiles = load_tiles(tiles_file=tilef)
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
favail = LocationsAvailable(tgsavail)
del tree
asgn = Assignment(tgs, tgsavail, favail)
asgn.assign_unused(TARGET_TYPE_SCIENCE)
write_assignment_fits(tiles, asgn, out_dir=dirout, all_targets=True)
print('wrote assignment files to ' + dirout)
tilesfile = args.tiles
outdir = args.outdir
label = args.label
#rank_group = args.rank_group
# python --cpu-bind=verbose parallel_pip_bianchi_general_testing.py --mtl /global/project/projectdirs/desi/users/arroyoc/pip_scripts/parallel_pip_bianchi/mtlz-patch-thesis-version.fits --stdstar-dark /global/project/projectdirs/desi/users/jguy/mocks/darksky-v1.0.1-v3/targets-0/standards-dark.fits --stdstar-bright /global/project/projectdirs/desi/users/jguy/mocks/darksky-v1.0.1-v3/targets-0/standards-bright.fits --sky /global/project/projectdirs/desi/users/jguy/mocks/darksky-v1.0.1-v3/targets-0/sky.fits --tiles /global/project/projectdirs/desi/users/arroyoc/pip_scripts/parallel_pip_bianchi/test-tiles.fits --rank-group 0
# --------------------------------------------------------------------------------------------------
# READING AND LOADING THE RELEVANT DATA (FIRST PART)
#
# In this part of the code the main objects to be used are defined and the data to make the
# assignation process is loaded.
# --------------------------------------------------------------------------------------------------
# Read hardware properties.
hw = load_hardware()
# Read the tiles to be used and define the number of tiles. Path obtained from command line
# argument.
if (tilesfile is None):
nominal_tiles = load_tiles()
else:
nominal_tiles = load_tiles(tiles_file=tilesfile)
ntiles = len(nominal_tiles.id)
# Container for the Mersenne Twister pseudo-random number generator. Since every MPI rank is seeded
# with a different number, size number of different subpriorities are generated.
#seed = 62*rank_group + rank
seed = rank
random_generator = RandomState(seed=seed)
def main():
os.environ['DESIMODEL'] = '/global/homes/d/dstn/desimodel-data'
global hw
global stuck_x
global stuck_y
global stuck_loc
global starkd
hw = load_hardware()
# From fiberassign/stucksky.py: find X,Y positions of stuck positioners.
# (grab the hw dictionaries once -- these are python wrappers over C++ so not simple accessors)
state = hw.state
devtype = hw.loc_device_type
stuck_loc = [
loc for loc in hw.locations
if (((state[loc] & (FIBER_STATE_STUCK | FIBER_STATE_BROKEN)
) == FIBER_STATE_STUCK) and (devtype[loc] == 'POS'))
]
print(len(stuck_loc), 'stuck positioners')
theta_pos = hw.loc_theta_pos
theta_off = hw.loc_theta_offset
phi_pos = hw.loc_phi_pos
phi_off = hw.loc_phi_offset
stuck_theta = [theta_pos[loc] + theta_off[loc] for loc in stuck_loc]
stuck_phi = [phi_pos[loc] + phi_off[loc] for loc in stuck_loc]
curved_mm = hw.loc_pos_curved_mm
theta_arm = hw.loc_theta_arm
phi_arm = hw.loc_phi_arm
theta_min = hw.loc_theta_min
theta_max = hw.loc_theta_max
phi_min = hw.loc_phi_min
phi_max = hw.loc_phi_max
# Convert positioner angle orientations to curved focal surface X / Y (not CS5)
# Note: we could add some methods to the python bindings to vectorize this or make it less clunky...
stuck_x = np.zeros(len(stuck_loc))
stuck_y = np.zeros(len(stuck_loc))
for iloc, (loc, theta,
phi) in enumerate(zip(stuck_loc, stuck_theta, stuck_phi)):
loc_x, loc_y = hw.thetaphi_to_xy(curved_mm[loc], theta, phi,
theta_arm[loc], phi_arm[loc],
theta_off[loc], phi_off[loc],
theta_min[loc], phi_min[loc],
theta_max[loc], phi_max[loc], True)
stuck_x[iloc] = loc_x
stuck_y[iloc] = loc_y
tiles = Table.read(
'/global/cfs/cdirs/desi/target/surveyops/ops/tiles-main.ecsv')
print(len(tiles), 'tiles')
# Deduplicate tiles with same RA,Dec center
tilera = tiles['RA']
tiledec = tiles['DEC']
tileid = tiles['TILEID']
rdtile = {}
tilemap = {}
for tid, r, d in zip(tileid, tilera, tiledec):
key = r, d
if key in rdtile:
# already seen a tile with this RA,Dec; point to it
tilemap[tid] = rdtile[key]
else:
rdtile[key] = tid
del rdtile
tnow = datetime.now()
tile_obstime = tnow.isoformat(timespec='seconds')
mjd = Time(tnow).mjd
stars = fits_table(
'/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/masking/gaia-mask-dr9.fits.gz'
)
print(len(stars), 'stars for masking')
print('Moving to MJD', mjd)
ra, dec = radec_at_mjd(stars.ra, stars.dec, stars.ref_epoch.astype(float),
stars.pmra, stars.pmdec, stars.parallax, mjd)
assert (np.all(np.isfinite(ra)))
assert (np.all(np.isfinite(dec)))
stars.ra = ra
stars.dec = dec
print('Building kd-tree...')
starkd = tree_build_radec(stars.ra, stars.dec)
match_radius = deg2dist(30. / 3600.)
stuck_loc = np.array(stuck_loc)
allresults = {}
mp = multiproc(32)
print('Building arg lists...')
args = []
for tid, tile_ra, tile_dec, tile_obsha in zip(tileid, tilera, tiledec,
tiles['DESIGNHA']):
# skip duplicate tiles
if tid in tilemap:
continue
# "fieldrot"
tile_theta = field_rotation_angle(tile_ra, tile_dec, mjd)
args.append((tid, tile_ra, tile_dec, tile_obstime, tile_theta,
tile_obsha, match_radius))
print('Matching', len(args), 'unique tile RA,Decs in parallel...')
res = mp.map(_match_tile, args)
print('Organizing results...')
T = fits_table()
T.tileid = []
T.loc = []
T.petal = []
T.device = []
T.fiber = []
T.pos_ra = []
T.pos_dec = []
T.star_ra = []
T.star_dec = []
T.dist_arcsec = []
T.mask_mag = []
loc_to_petal = hw.loc_petal
loc_to_device = hw.loc_device
loc_to_fiber = hw.loc_fiber
for vals in res:
if vals is None:
continue
tileid, I, pos_ra, pos_dec, pos_loc, dists = vals
T.tileid.extend([tileid] * len(I))
T.loc.extend(pos_loc)
for loc in pos_loc:
T.petal.append(loc_to_petal[loc])
T.device.append(loc_to_device[loc])
T.fiber.append(loc_to_fiber[loc])
T.pos_ra.extend(pos_ra)
T.pos_dec.extend(pos_dec)
T.star_ra.extend(stars.ra[I])
T.star_dec.extend(stars.dec[I])
T.dist_arcsec.extend(dists)
T.mask_mag.extend(stars.mask_mag[I])
T.to_np_arrays()
T.writeto('stuck-on-stars.fits')
tgs = Targets()
load_target_file(tgs, targetf)
print('loaded target file ' + targetf)
tree = TargetTree(tgs, 0.01)
for run in range(srun, srun + nrun):
#make the tile file for this run
mke2etiles(run, program=program)
tilef = e2eout + 'e2etiles_run' + str(run) + '.fits'
randir = e2eout + program + '/randoms/' + str(run)
if os.path.isdir(randir):
pass
else:
os.mkdir(randir)
fafls = glob.glob(e2ein + 'run/quicksurvey/' + program + '/' + str(run) +
'/fiberassign/fiberassign*')
hd = fitsio.read_header(fafls[0])
dt = hd['FA_RUN']
hw = load_hardware(rundate=dt)
tiles = load_tiles(tiles_file=tilef)
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
favail = LocationsAvailable(tgsavail)
#del tree
asgn = Assignment(tgs, tgsavail, favail)
asgn.assign_unused(TARGET_TYPE_SCIENCE)
write_assignment_fits(tiles, asgn, out_dir=randir, all_targets=True)
print('wrote assignment files to ' + randir)
def test_read(self):
hw = load_hardware()
print(hw)
return
def main():
log = Logger.get()
mpi_procs = MPI.COMM_WORLD.size
mpi_rank = MPI.COMM_WORLD.rank
parser = argparse.ArgumentParser()
parser.add_argument(
"--survey_log",
type=str,
required=False,
help="Eventually we would pass in a file containing the log"
" of when each fiber assignment was run and for which tiles, "
"along with the options that were used.")
parser.add_argument(
"--sky",
type=str,
required=False,
help="Input file with sky or supp_sky targets. "
"These target files are assumed to be constant and not "
"tracked by the MTL ledger.")
parser.add_argument(
"--mtl",
type=str,
required=True,
help="The MTL ledger. This is still a work in progress and"
" I am not sure what the interface will be, but given the "
"fiber assignment dates in the survey log, we should be able"
" to get the MTL state at that time. For now, this option"
" is just one or more target files.")
parser.add_argument("--footprint",
type=str,
required=False,
default=None,
help="Optional FITS file defining the footprint. If"
" not specified, the default footprint from desimodel"
" is used.")
parser.add_argument("--tiles",
type=str,
required=False,
default=None,
help="Optional text file containing a subset of the"
" tile IDs to use in the footprint, one ID per line."
" Default uses all tiles in the footprint.")
parser.add_argument("--out",
type=str,
required=False,
default=None,
help="Output directory.")
parser.add_argument("--realizations",
type=int,
required=False,
default=10,
help="Number of realizations.")
args = parser.parse_args()
if args.sky is None:
args.sky = list()
# Set output directory
if args.out is None:
args.out = "."
# Read tiles we are using
tileselect = None
if args.tiles is not None:
tileselect = list()
with open(args.tiles, "r") as f:
for line in f:
# Try to convert the first column to an integer.
try:
tileselect.append(int(line.split()[0]))
except ValueError:
pass
tiles = load_tiles(
tiles_file=args.footprint,
select=tileselect,
)
# Create empty target list
tgs = Targets()
# Append each input target file. These target files must all be of the
# same survey type, and will set the Targets object to be of that survey.
print(args.mtl)
print(args.sky)
#for tgfile in args.targets:
# load_target_file(tgs, tgfile)
load_target_file(tgs, args.mtl)
# Just the science target IDs
tg_science = tgs.ids()
tg_science2indx = {y: x for x, y in enumerate(tg_science)}
# Now load the sky target files.
survey = tgs.survey()
#for tgfile in args.sky:
# load_target_file(tgs, tgfile)
load_target_file(tgs, args.sky)
# Divide up realizations among the processes.
n_realization = args.realizations
realizations = np.arange(n_realization, dtype=np.int32)
my_realizations = np.array_split(realizations, mpi_procs)[mpi_rank]
# Bitarray for all targets and realizations
#tgarray = bitarray(len(tg_science) * n_realization)
#tgarray.setall(False)
tgarray = np.zeros(len(tg_science) * n_realization, dtype='bool')
# Target tree
tree = TargetTree(tgs)
hw = load_hardware()
for realization in my_realizations:
# Set the seed based on the realization, so that the result is reproducible
# regardless of which process is working on the realization.
np.random.seed(realization)
# Comment out the next block to avoid randomizing subpriority
# ----
# Randomize science target subpriority for this realization
new_subpriority = np.random.random_sample(size=len(tg_science))
for indx, tgid in enumerate(tg_science):
tg = tgs.get(tgid)
tg.subpriority = new_subpriority[indx]
# Comment out the next block to avoid dithering tiles
# ----
# Dither tiles centers by the same
# Compute available targets / locations
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
favail = LocationsAvailable(tgsavail)
asgn = Assignment(tgs, tgsavail, favail)
# Replay the survey log for each time fiber assignment was run. For now, this
# is just doing the tiles all at once.
for assign_event in range(1):
# In the future, load MTL updates to the obs remaining for each target here
# Read hardware properties- in the future, pass in the assignment run date
# to this function.
hw = load_hardware()
# Run assignment for this event.
run(asgn)
# Update bit arrays for assigned science targets
for tile_id in tiles.id: #():
adata = asgn.tile_location_target(tile_id)
for loc, tgid in adata.items():
try:
idx = tg_science2indx[tgid]
tgarray[idx * n_realization + realization] = True
except KeyError:
# Not a science target
pass
# Reduce bitarrays to root process. The bitarray type conforms to the
# buffer protocol.
tgall = None
#if mpi_rank == 0:
# tgall = bitarray(tgarray)
# tgall.setall(False)
MPI.COMM_WORLD.Reduce(tgarray, tgall, op=MPI.BOR, root=0)
# Write it out
if mpi_rank == 0:
#pass
print(len(tgall))
def test_plotpos(self):
test_dir = test_subdir_create("vis_test_plotpos")
hw = load_hardware()
patrol_mm = hw.patrol_mm
fiber_id = hw.fiber_id
center_mm = hw.fiber_pos_xy_mm
# Plot data range in mm
width = 1.2 * (2.0 * patrol_mm)
height = 1.2 * (2.0 * patrol_mm)
# Plot size in inches
xfigsize = 8
yfigsize = 8
figdpi = 75
# Compute the font size to use for detector labels
fontpix = 0.2 * figdpi
fontpt = int(0.75 * fontpix)
# Plot the first fiber in a variety of positions
fid = fiber_id[0]
nincr = 8
configincr = 2.0 * np.pi / nincr
center = center_mm[fid]
for configindx, (configrad, col) in \
enumerate(zip([0.5*patrol_mm, patrol_mm], ["r", "b"])):
fig = plt.figure(figsize=(xfigsize, yfigsize), dpi=figdpi)
ax = fig.add_subplot(1, 1, 1)
ax.set_aspect("equal")
cb, fh = hw.fiber_position(fid, center_mm[fid])
plot_positioner(ax, patrol_mm, fid, center, cb, fh, color="k")
for inc in range(nincr):
ang = inc * configincr
xoff = configrad * np.cos(ang) + center_mm[fid][0]
yoff = configrad * np.sin(ang) + center_mm[fid][1]
cb, fh = hw.fiber_position(fid, (xoff, yoff))
plot_positioner(ax, patrol_mm, fid, center, cb, fh, color=col)
xend = xoff
yend = yoff
ax.plot([center_mm[fid][0], xend], [center_mm[fid][1], yend],
color="k",
linewidth="0.5")
ax.text(xend,
yend,
"{}".format(inc),
color='k',
fontsize=fontpt,
horizontalalignment='center',
verticalalignment='center',
bbox=dict(fc='w', ec='none', pad=1, alpha=1.0))
pxcent = center_mm[fid][0]
pycent = center_mm[fid][1]
half_width = 0.5 * width
half_height = 0.5 * height
ax.set_xlabel("Millimeters", fontsize="large")
ax.set_ylabel("Millimeters", fontsize="large")
ax.set_xlim([pxcent - half_width, pxcent + half_width])
ax.set_ylim([pycent - half_height, pycent + half_height])
outfile = os.path.join(test_dir,
"test_plotpos_{}.png".format(configindx))
plt.savefig(outfile)
plt.close()
return
def test_science(self):
set_matplotlib_pdf_backend()
import matplotlib.pyplot as plt
test_dir = test_subdir_create("qa_test_science")
log_file = os.path.join(test_dir, "log.txt")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
# For this test, we will use just 2 science target classes, in order to verify
# we get approximately the correct distribution
sdist = [(3000, 1, 0.25, "QSO"), (2000, 1, 0.75, "ELG")]
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
0,
density=self.density_science,
science_frac=sdist)
log_msg = "Simulated {} science targets\n".format(nscience)
tgs = Targets()
load_target_file(tgs, input_mtl)
# Read hardware properties
fp, exclude, state = sim_focalplane(rundate=test_assign_date)
hw = load_hardware(focalplane=(fp, exclude, state))
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
# Precompute target positions
tile_targetids, tile_x, tile_y = targets_in_tiles(hw, tgs, tiles)
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tiles, tile_targetids, tile_x, tile_y)
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Pass empty map of STUCK positioners that land on good sky
stucksky = {}
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail, stucksky)
# First-pass assignment of science targets
asgn.assign_unused(TARGET_TYPE_SCIENCE)
# Redistribute
asgn.redistribute_science()
write_assignment_fits(tiles, asgn, out_dir=test_dir, all_targets=True)
tile_ids = list(tiles.id)
merge_results([input_mtl],
list(),
tile_ids,
result_dir=test_dir,
copy_fba=False)
# FIXME: In order to use the qa_targets function, we need to know the
# starting requested number of observations (NUMOBS_INIT). Then we can use
# that value for each target and compare to the number actually assigned.
# However, the NUMOBS_INIT column was removed from the merged TARGET table.
# If we are ever able to reach consensus on restoring that column, then we
# can re-enable these tests below.
#
# qa_targets(
# hw,
# tiles,
# result_dir=test_dir,
# result_prefix="fiberassign-"
# )
#
# # Load the target catalog so that we have access to the target properties
#
# fd = fitsio.FITS(input_mtl, "r")
# scidata = np.array(np.sort(fd[1].read(), order="TARGETID"))
# fd.close()
# del fd
#
# # How many possible positioner assignments did we have?
# nassign = 5000 * len(tile_ids)
#
# possible = dict()
# achieved = dict()
#
# namepat = re.compile(r".*/qa_target_count_(.*)_init-(.*)\.fits")
# for qafile in glob.glob("{}/qa_target_count_*.fits".format(test_dir)):
# namemat = namepat.match(qafile)
# name = namemat.group(1)
# obs = int(namemat.group(2))
# if obs == 0:
# continue
# fd = fitsio.FITS(qafile, "r")
# fdata = fd["COUNTS"].read()
# # Sort by target ID so we can select easily
# fdata = np.sort(fdata, order="TARGETID")
# tgid = np.array(fdata["TARGETID"])
# counts = np.array(fdata["NUMOBS_DONE"])
# avail = np.array(fdata["NUMOBS_AVAIL"])
# del fdata
# fd.close()
#
# # Select target properties. BOTH TARGET LISTS MUST BE SORTED.
# rows = np.where(np.isin(scidata["TARGETID"], tgid, assume_unique=True))[0]
#
# ra = np.array(scidata["RA"][rows])
# dec = np.array(scidata["DEC"][rows])
# dtarget = np.array(scidata["DESI_TARGET"][rows])
# init = np.array(scidata["NUMOBS_INIT"][rows])
#
# requested = obs * np.ones_like(avail)
#
# under = np.where(avail < requested)[0]
# over = np.where(avail > requested)[0]
#
# limavail = np.array(avail)
# limavail[over] = obs
#
# deficit = np.zeros(len(limavail), dtype=np.int)
#
# deficit[:] = limavail - counts
# deficit[avail == 0] = 0
#
# possible[name] = np.sum(limavail)
# achieved[name] = np.sum(counts)
#
# log_msg += "{}-{}:\n".format(name, obs)
#
# pindx = np.where(deficit > 0)[0]
# poor_tgid = tgid[pindx]
# poor_dtarget = dtarget[pindx]
# log_msg += " Deficit > 0: {}\n".format(len(poor_tgid))
# poor_ra = ra[pindx]
# poor_dec = dec[pindx]
# poor_deficit = deficit[pindx]
#
# # Plot Target availability
# # Commented out by default, since in the case of high target density
# # needed for maximizing assignments, there are far more targets than
# # the number of available fiber placements.
#
# # marksize = 4 * np.ones_like(deficit)
# #
# # fig = plt.figure(figsize=(12, 12))
# # ax = fig.add_subplot(1, 1, 1)
# # ax.scatter(ra, dec, s=2, c="black", marker="o")
# # for pt, pr, pd, pdef in zip(poor_tgid, poor_ra, poor_dec, poor_deficit):
# # ploc = plt.Circle(
# # (pr, pd), radius=(0.05*pdef), fc="none", ec="red"
# # )
# # ax.add_artist(ploc)
# # ax.set_xlabel("RA", fontsize="large")
# # ax.set_ylabel("DEC", fontsize="large")
# # ax.set_title(
# # "Target \"{}\": (min(avail, requested) - counts) > 0".format(
# # name, obs
# # )
# # )
# # #ax.legend(handles=lg, framealpha=1.0, loc="upper right")
# # plt.savefig(os.path.join(test_dir, "{}-{}_deficit.pdf".format(name, obs)), dpi=300, format="pdf")
#
# log_msg += \
# "Assigned {} tiles for total of {} possible target observations\n".format(
# len(tile_ids), nassign
# )
# ach = 0
# for nm in possible.keys():
# ach += achieved[nm]
# log_msg += \
# " type {} had {} possible target obs and achieved {}\n".format(
# nm, possible[nm], achieved[nm]
# )
# frac = 100.0 * ach / nassign
# log_msg += \
# " {} / {} = {:0.2f}% of fibers were assigned\n".format(
# ach, nassign, frac
# )
# for nm in possible.keys():
# log_msg += \
# " type {} had {:0.2f}% of achieved observations\n".format(
# nm, achieved[nm] / ach
# )
# with open(log_file, "w") as f:
# f.write(log_msg)
#
# self.assertGreaterEqual(frac, 99.0)
# Test if qa-fiberassign script runs without crashing
script = os.path.join(self.binDir, "qa-fiberassign")
if os.path.exists(script):
fafiles = glob.glob(f"{test_dir}/fiberassign-*.fits")
cmd = "{} --targets {}".format(script, " ".join(fafiles))
err = subprocess.call(cmd.split())
self.assertEqual(err, 0, f"FAILED ({err}): {cmd}")
else:
print(f"ERROR: didn't find {script}")
def test_fieldrot(self):
test_dir = test_subdir_create("assign_test_fieldrot")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
input_suppsky = os.path.join(test_dir, "suppsky.fits")
tgoff = 0
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
tgoff,
density=self.density_science)
tgoff += nscience
nstd = sim_targets(input_std,
TARGET_TYPE_STANDARD,
tgoff,
density=self.density_standards)
tgoff += nstd
nsky = sim_targets(input_sky,
TARGET_TYPE_SKY,
tgoff,
density=self.density_sky)
tgoff += nsky
nsuppsky = sim_targets(input_suppsky,
TARGET_TYPE_SUPPSKY,
tgoff,
density=self.density_suppsky)
# Simulate the tiles
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
# petal mapping
rotator = petal_rotation(1, reverse=False)
rots = [0, 36]
tile_ids = None
for rt in rots:
odir = "theta_{:02d}".format(rt)
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
load_target_file(tgs, input_suppsky)
# Create a hierarchical triangle mesh lookup of the targets
# positions
tree = TargetTree(tgs, 0.01)
# Manually override the field rotation
tiles = load_tiles(tiles_file=tfile, obstheta=float(rt))
if tile_ids is None:
tile_ids = list(tiles.id)
# Simulate a fake focalplane
fp, exclude, state = sim_focalplane(rundate=test_assign_date,
fakepos=True)
# Load the focalplane
hw = load_hardware(focalplane=(fp, exclude, state))
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Pass empty map of STUCK positioners that land on good sky
stucksky = {}
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail, stucksky)
# First-pass assignment of science targets
asgn.assign_unused(TARGET_TYPE_SCIENCE)
out = os.path.join(test_dir, odir)
write_assignment_fits(tiles, asgn, out_dir=out, all_targets=True)
ppet = 6
if odir == "theta_36":
ppet = rotator[6]
plot_tiles(hw,
tiles,
result_dir=out,
plot_dir=out,
real_shapes=True,
petals=[ppet],
serial=True)
# Explicitly free everything
del asgn
del favail
del tgsavail
del hw
del tiles
del tree
del tgs
# For each tile, compare the assignment output and verify that they
# agree with a one-petal rotation.
# NOTE: The comparison below will NOT pass, since we are still
# Sorting by highest priority available target and then (in case
# of a tie) by fiber ID. See line 333 of assign.cpp. Re-enable this
# test after that is changed to sort by location in case of a tie.
# for tl in tile_ids:
# orig_path = os.path.join(
# test_dir, "theta_00", "fiberassign_{:06d}.fits".format(tl)
# )
# orig_header, orig_data, _, _, _ = \
# read_assignment_fits_tile((tl, orig_path))
# rot_path = os.path.join(
# test_dir, "theta_36", "fiberassign_{:06d}.fits".format(tl)
# )
# rot_header, rot_data, _, _, _ = \
# read_assignment_fits_tile((tl, rot_path))
# comppath = os.path.join(
# test_dir, "comp_00-36_{:06d}.txt".format(tl)
# )
# with open(comppath, "w") as fc:
# for dev, petal, tg in zip(
# orig_data["DEVICE_LOC"], orig_data["PETAL_LOC"],
# orig_data["TARGETID"]
# ):
# for newdev, newpetal, newtg in zip(
# rot_data["DEVICE_LOC"], rot_data["PETAL_LOC"],
# rot_data["TARGETID"]
# ):
# rpet = rotator[newpetal]
# if (newdev == dev) and (rpet == petal):
# fc.write(
# "{}, {} = {} : {}, {} = {}\n"
# .format(petal, dev, tg, rpet, newdev, newtg)
# )
# # self.assertEqual(newtg, tg)
return
def test_io(self):
np.random.seed(123456789)
test_dir = test_subdir_create("assign_test_io")
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
input_suppsky = os.path.join(test_dir, "suppsky.fits")
tgoff = 0
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
tgoff,
density=self.density_science)
tgoff += nscience
nstd = sim_targets(input_std,
TARGET_TYPE_STANDARD,
tgoff,
density=self.density_standards)
tgoff += nstd
nsky = sim_targets(input_sky,
TARGET_TYPE_SKY,
tgoff,
density=self.density_sky)
tgoff += nsky
nsuppsky = sim_targets(input_suppsky,
TARGET_TYPE_SUPPSKY,
tgoff,
density=self.density_suppsky)
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
load_target_file(tgs, input_suppsky)
# Create a hierarchical triangle mesh lookup of the targets positions
tree = TargetTree(tgs, 0.01)
# Compute the targets available to each fiber for each tile.
fp, exclude, state = sim_focalplane(rundate=test_assign_date)
hw = load_hardware(focalplane=(fp, exclude, state))
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Free the tree
del tree
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Pass empty map of STUCK positioners that land on good sky
stucksky = {}
# First pass assignment
asgn = Assignment(tgs, tgsavail, favail, stucksky)
asgn.assign_unused(TARGET_TYPE_SCIENCE)
# Write out, merge, read back in and verify
write_assignment_ascii(tiles,
asgn,
out_dir=test_dir,
out_prefix="test_io_ascii_")
write_assignment_fits(tiles,
asgn,
out_dir=test_dir,
out_prefix="basic_",
all_targets=False)
write_assignment_fits(tiles,
asgn,
out_dir=test_dir,
out_prefix="full_",
all_targets=True)
plotpetals = [0]
# plotpetals = None
plot_tiles(hw,
tiles,
result_dir=test_dir,
result_prefix="basic_",
plot_dir=test_dir,
plot_prefix="basic_",
result_split_dir=False,
petals=plotpetals,
serial=True)
plot_tiles(hw,
tiles,
result_dir=test_dir,
result_prefix="full_",
plot_dir=test_dir,
plot_prefix="full_",
result_split_dir=False,
petals=plotpetals,
serial=True)
target_files = [input_mtl, input_sky, input_std]
tile_ids = list(tiles.id)
merge_results(target_files,
list(),
tile_ids,
result_dir=test_dir,
result_prefix="basic_",
out_dir=test_dir,
out_prefix="basic_tile-",
copy_fba=False)
merge_results(target_files,
list(),
tile_ids,
result_dir=test_dir,
result_prefix="full_",
out_dir=test_dir,
out_prefix="full_tile-",
copy_fba=False)
# Here we test reading with the standard reading function
for tid in tile_ids:
tdata = asgn.tile_location_target(tid)
avail = tgsavail.tile_data(tid)
# Check basic format
infile = os.path.join(test_dir,
"basic_tile-{:06d}.fits".format(tid))
inhead, fiber_data, targets_data, avail_data, gfa_targets = \
read_assignment_fits_tile((tid, infile))
for lid, tgid, tgra, tgdec in zip(fiber_data["LOCATION"],
fiber_data["TARGETID"],
fiber_data["TARGET_RA"],
fiber_data["TARGET_DEC"]):
if tgid >= 0:
self.assertEqual(tgid, tdata[lid])
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
# Check full format
infile = os.path.join(test_dir,
"full_tile-{:06d}.fits".format(tid))
inhead, fiber_data, targets_data, avail_data, gfa_targets = \
read_assignment_fits_tile((tid, infile))
for lid, tgid, tgra, tgdec in zip(fiber_data["LOCATION"],
fiber_data["TARGETID"],
fiber_data["TARGET_RA"],
fiber_data["TARGET_DEC"]):
if tgid >= 0:
self.assertEqual(tgid, tdata[lid])
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
# Now read the files directly with fitsio and verify against the input
# target data.
for tid in tile_ids:
tdata = asgn.tile_location_target(tid)
avail = tgsavail.tile_data(tid)
# Check basic format
infile = os.path.join(test_dir,
"basic_tile-{:06d}.fits".format(tid))
fdata = fitsio.FITS(infile, "r")
fassign = fdata["FIBERASSIGN"].read()
ftargets = fdata["TARGETS"].read()
for lid, tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip(
fassign["LOCATION"], fassign["TARGETID"],
fassign["TARGET_RA"], fassign["TARGET_DEC"],
fassign["SUBPRIORITY"], fassign["PRIORITY"],
fassign["OBSCONDITIONS"]):
if tgid >= 0:
self.assertEqual(tgid, tdata[lid])
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
self.assertEqual(tgsub, props.subpriority)
self.assertEqual(tgprior, props.priority)
self.assertEqual(tgobs, props.obscond)
for tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip(
ftargets["TARGETID"], ftargets["RA"], ftargets["DEC"],
ftargets["SUBPRIORITY"], ftargets["PRIORITY"],
ftargets["OBSCONDITIONS"]):
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
self.assertEqual(tgsub, props.subpriority)
self.assertEqual(tgprior, props.priority)
self.assertEqual(tgobs, props.obscond)
# Check full format
infile = os.path.join(test_dir,
"full_tile-{:06d}.fits".format(tid))
fdata = fitsio.FITS(infile, "r")
fassign = fdata["FIBERASSIGN"].read()
ftargets = fdata["TARGETS"].read()
for lid, tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip(
fassign["LOCATION"], fassign["TARGETID"],
fassign["TARGET_RA"], fassign["TARGET_DEC"],
fassign["SUBPRIORITY"], fassign["PRIORITY"],
fassign["OBSCONDITIONS"]):
if tgid >= 0:
self.assertEqual(tgid, tdata[lid])
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
self.assertEqual(tgsub, props.subpriority)
self.assertEqual(tgprior, props.priority)
self.assertEqual(tgobs, props.obscond)
for tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip(
ftargets["TARGETID"], ftargets["RA"], ftargets["DEC"],
ftargets["SUBPRIORITY"], ftargets["PRIORITY"],
ftargets["OBSCONDITIONS"]):
props = tgs.get(tgid)
self.assertEqual(tgra, props.ra)
self.assertEqual(tgdec, props.dec)
self.assertEqual(tgsub, props.subpriority)
self.assertEqual(tgprior, props.priority)
self.assertEqual(tgobs, props.obscond)
plot_tiles(hw,
tiles,
result_dir=test_dir,
result_prefix="basic_tile-",
plot_dir=test_dir,
plot_prefix="basic_tile-",
result_split_dir=False,
petals=plotpetals,
serial=True)
plot_tiles(hw,
tiles,
result_dir=test_dir,
result_prefix="full_tile-",
plot_dir=test_dir,
plot_prefix="full_tile-",
result_split_dir=False,
petals=plotpetals,
serial=True)
return
def test_full(self):
test_dir = test_subdir_create("assign_test_full")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
nscience = sim_targets(input_mtl, TARGET_TYPE_SCIENCE, 0)
nstd = sim_targets(input_std, TARGET_TYPE_STANDARD, nscience)
nsky = sim_targets(input_sky, TARGET_TYPE_SKY, (nscience + nstd))
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
# Create a hierarchical triangle mesh lookup of the targets positions
tree = TargetTree(tgs, 0.01)
# Read hardware properties
fstatus = os.path.join(test_dir, "fiberstatus.ecsv")
sim_status(fstatus)
hw = load_hardware(status_file=fstatus)
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Free the tree
del tree
# Compute the fibers on all tiles available for each target
favail = FibersAvailable(tgsavail)
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail)
# First-pass assignment of science targets
asgn.assign_unused(TARGET_TYPE_SCIENCE)
# Redistribute science targets
asgn.redistribute_science()
# Assign standards, 10 per petal
asgn.assign_unused(TARGET_TYPE_STANDARD, 10)
asgn.assign_force(TARGET_TYPE_STANDARD, 10)
# Assign sky to unused fibers, up to 40 per petal
asgn.assign_unused(TARGET_TYPE_SKY, 40)
asgn.assign_force(TARGET_TYPE_SKY, 40)
# If there are any unassigned fibers, try to place them somewhere.
asgn.assign_unused(TARGET_TYPE_SCIENCE)
asgn.assign_unused(TARGET_TYPE_SKY)
write_assignment_fits(tiles, asgn, out_dir=test_dir, all_targets=True)
plot_tiles(hw,
tiles,
result_dir=test_dir,
plot_dir=test_dir,
petals=[0],
serial=True)
qa_tiles(hw, tiles, result_dir=test_dir)
qadata = None
with open(os.path.join(test_dir, "qa.json"), "r") as f:
qadata = json.load(f)
for tile, props in qadata.items():
self.assertEqual(4495, props["assign_science"])
self.assertEqual(100, props["assign_std"])
self.assertEqual(400, props["assign_sky"])
plot_qa(qadata,
os.path.join(test_dir, "qa"),
outformat="pdf",
labels=True)
return
def test_full(self, do_stucksky=False):
test_dir = test_subdir_create("assign_test_full")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
input_suppsky = os.path.join(test_dir, "suppsky.fits")
tgoff = 0
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
tgoff,
density=self.density_science)
tgoff += nscience
nstd = sim_targets(input_std,
TARGET_TYPE_STANDARD,
tgoff,
density=self.density_standards)
tgoff += nstd
nsky = sim_targets(input_sky,
TARGET_TYPE_SKY,
tgoff,
density=self.density_sky)
tgoff += nsky
nsuppsky = sim_targets(input_suppsky,
TARGET_TYPE_SUPPSKY,
tgoff,
density=self.density_suppsky)
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
load_target_file(tgs, input_suppsky)
# Create a hierarchical triangle mesh lookup of the targets positions
tree = TargetTree(tgs, 0.01)
# Read hardware properties
fp, exclude, state = sim_focalplane(rundate=test_assign_date)
hw = load_hardware(focalplane=(fp, exclude, state))
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
if do_stucksky:
sim_stuck_sky(test_dir, hw, tiles)
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Free the tree
del tree
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Pass empty map of STUCK positioners that land on good sky
stucksky = None
if do_stucksky:
stucksky = stuck_on_sky(hw, tiles)
if stucksky is None:
# (the pybind code doesn't like None when a dict is expected...)
stucksky = {}
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail, stucksky)
run(asgn)
write_assignment_fits(tiles,
asgn,
out_dir=test_dir,
all_targets=True,
stucksky=stucksky)
plotpetals = [0]
#plotpetals = None
plot_tiles(hw,
tiles,
result_dir=test_dir,
plot_dir=test_dir,
result_prefix="fba-",
real_shapes=True,
petals=plotpetals,
serial=True)
qa_tiles(hw, tiles, result_dir=test_dir)
qadata = None
with open(os.path.join(test_dir, "qa.json"), "r") as f:
qadata = json.load(f)
for tile, props in qadata.items():
self.assertTrue(props["assign_science"] >= 4485)
self.assertEqual(100, props["assign_std"])
if do_stucksky:
# We get 3 stuck positioners landing on good sky!
self.assertTrue(
(props["assign_sky"] + props["assign_suppsky"]) >= 397)
else:
self.assertTrue(
(props["assign_sky"] + props["assign_suppsky"]) >= 400)
plot_qa(qadata,
os.path.join(test_dir, "qa"),
outformat="pdf",
labels=True)
return
def test_science(self):
set_matplotlib_pdf_backend()
import matplotlib.pyplot as plt
test_dir = test_subdir_create("qa_test_science")
log_file = os.path.join(test_dir, "log.txt")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
# For this test, we will use just 2 science target classes, in order to verify
# we get approximately the correct distribution
sdist = [(3000, 1, 0.25, "QSO"), (2000, 1, 0.75, "ELG")]
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
0,
density=self.density_science,
science_frac=sdist)
log_msg = "Simulated {} science targets\n".format(nscience)
tgs = Targets()
load_target_file(tgs, input_mtl)
# Create a hierarchical triangle mesh lookup of the targets positions
tree = TargetTree(tgs, 0.01)
# Read hardware properties
fp, exclude, state = sim_focalplane(rundate=test_assign_date)
hw = load_hardware(focalplane=(fp, exclude, state))
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Free the tree
del tree
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail)
# First-pass assignment of science targets
asgn.assign_unused(TARGET_TYPE_SCIENCE)
# Redistribute
asgn.redistribute_science()
write_assignment_fits(tiles, asgn, out_dir=test_dir, all_targets=True)
tile_ids = list(tiles.id)
merge_results([input_mtl],
list(),
tile_ids,
result_dir=test_dir,
copy_fba=False)
# if "TRAVIS" not in os.environ:
# plot_tiles(
# hw,
# tiles,
# result_dir=test_dir,
# plot_dir=test_dir,
# real_shapes=True,
# serial=True
# )
qa_targets(hw,
tiles,
result_dir=test_dir,
result_prefix="fiberassign-")
# Load the target catalog so that we have access to the target properties
fd = fitsio.FITS(input_mtl, "r")
scidata = np.array(np.sort(fd[1].read(), order="TARGETID"))
fd.close()
del fd
# How many possible positioner assignments did we have?
nassign = 5000 * len(tile_ids)
possible = dict()
achieved = dict()
namepat = re.compile(r".*/qa_target_count_(.*)_init-(.*)\.fits")
for qafile in glob.glob("{}/qa_target_count_*.fits".format(test_dir)):
namemat = namepat.match(qafile)
name = namemat.group(1)
obs = int(namemat.group(2))
if obs == 0:
continue
fd = fitsio.FITS(qafile, "r")
fdata = fd["COUNTS"].read()
# Sort by target ID so we can select easily
fdata = np.sort(fdata, order="TARGETID")
tgid = np.array(fdata["TARGETID"])
counts = np.array(fdata["NUMOBS_DONE"])
avail = np.array(fdata["NUMOBS_AVAIL"])
del fdata
fd.close()
# Select target properties. BOTH TARGET LISTS MUST BE SORTED.
rows = np.where(
np.isin(scidata["TARGETID"], tgid, assume_unique=True))[0]
ra = np.array(scidata["RA"][rows])
dec = np.array(scidata["DEC"][rows])
dtarget = np.array(scidata["DESI_TARGET"][rows])
init = np.array(scidata["NUMOBS_MORE"][rows])
requested = obs * np.ones_like(avail)
under = np.where(avail < requested)[0]
over = np.where(avail > requested)[0]
limavail = np.array(avail)
limavail[over] = obs
deficit = np.zeros(len(limavail), dtype=np.int)
deficit[:] = limavail - counts
deficit[avail == 0] = 0
possible[name] = np.sum(limavail)
achieved[name] = np.sum(counts)
log_msg += "{}-{}:\n".format(name, obs)
pindx = np.where(deficit > 0)[0]
poor_tgid = tgid[pindx]
poor_dtarget = dtarget[pindx]
log_msg += " Deficit > 0: {}\n".format(len(poor_tgid))
poor_ra = ra[pindx]
poor_dec = dec[pindx]
poor_deficit = deficit[pindx]
# Plot Target availability
# Commented out by default, since in the case of high target density
# needed for maximizing assignments, there are far more targets than
# the number of available fiber placements.
# marksize = 4 * np.ones_like(deficit)
#
# fig = plt.figure(figsize=(12, 12))
# ax = fig.add_subplot(1, 1, 1)
# ax.scatter(ra, dec, s=2, c="black", marker="o")
# for pt, pr, pd, pdef in zip(poor_tgid, poor_ra, poor_dec, poor_deficit):
# ploc = plt.Circle(
# (pr, pd), radius=(0.05*pdef), fc="none", ec="red"
# )
# ax.add_artist(ploc)
# ax.set_xlabel("RA", fontsize="large")
# ax.set_ylabel("DEC", fontsize="large")
# ax.set_title(
# "Target \"{}\": (min(avail, requested) - counts) > 0".format(
# name, obs
# )
# )
# #ax.legend(handles=lg, framealpha=1.0, loc="upper right")
# plt.savefig(os.path.join(test_dir, "{}-{}_deficit.pdf".format(name, obs)), dpi=300, format="pdf")
log_msg += \
"Assigned {} tiles for total of {} possible target observations\n".format(
len(tile_ids), nassign
)
ach = 0
for nm in possible.keys():
ach += achieved[nm]
log_msg += \
" type {} had {} possible target obs and achieved {}\n".format(
nm, possible[nm], achieved[nm]
)
frac = 100.0 * ach / nassign
log_msg += \
" {} / {} = {:0.2f}% of fibers were assigned\n".format(
ach, nassign, frac
)
for nm in possible.keys():
log_msg += \
" type {} had {:0.2f}% of achieved observations\n".format(
nm, achieved[nm] / ach
)
with open(log_file, "w") as f:
f.write(log_msg)
self.assertGreaterEqual(frac, 99.0)
#- Test if qa-fiberassign script runs without crashing
bindir = os.path.join(os.path.dirname(fiberassign.__file__), '..',
'..', 'bin')
script = os.path.join(os.path.abspath(bindir), 'qa-fiberassign')
if os.path.exists(script):
fafiles = glob.glob(f"{test_dir}/fiberassign-*.fits")
cmd = "{} --targets {}".format(script, " ".join(fafiles))
err = subprocess.call(cmd.split())
self.assertEqual(err, 0, f"FAILED ({err}): {cmd}")
else:
print(f"ERROR: didn't find {script}")
def test_thetaphi_range(self):
# Function to test that all positioners can reach a circle of
# targets at fixed distance from their center.
def check_reachable(hrdw, radius, increments, log_fail=True):
centers = hw.loc_pos_curved_mm
theta_arms = hw.loc_theta_arm
phi_arms = hw.loc_phi_arm
theta_mins = hw.loc_theta_min
theta_maxs = hw.loc_theta_max
theta_offsets = hw.loc_theta_offset
phi_mins = hw.loc_phi_min
phi_maxs = hw.loc_phi_max
phi_offsets = hw.loc_phi_offset
n_failed = 0
for loc in hw.locations:
center = centers[loc]
theta_arm = theta_arms[loc]
phi_arm = phi_arms[loc]
theta_min = theta_mins[loc]
theta_max = theta_maxs[loc]
theta_offset = theta_offsets[loc]
phi_min = phi_mins[loc]
phi_max = phi_maxs[loc]
phi_offset = phi_offsets[loc]
ang = np.arange(increments) / (2 * np.pi)
test_x = radius * np.cos(ang) + center[0]
test_y = radius * np.sin(ang) + center[1]
for xy in zip(test_x, test_y):
result = hrdw.xy_to_thetaphi(center, xy, theta_arm,
phi_arm, theta_offset,
phi_offset, theta_min,
phi_min, theta_max, phi_max)
if result[0] is None or result[1] is None:
if log_fail:
print("loc {} at ({}, {}) cannot reach ({}, {})".
format(loc, center[0], center[1], xy[0],
xy[1]),
flush=True)
n_failed += 1
break
else:
if not log_fail:
# log success instead
print(
"loc {} at ({}, {}) to ({}, {}) with ({}, {})".
format(loc, center[0], center[1], xy[0], xy[1],
result[0] * 180.0 / np.pi,
result[1] * 180.0 / np.pi),
flush=True)
return n_failed
# Test nominal focalplane
hw = load_hardware(rundate=test_assign_date)
failed = check_reachable(hw, 3.0, 100)
if (failed > 0):
print("{} positioners failed to reach ring at 3mm from center".
format(failed),
flush=True)
self.assertTrue(False)
# Now we are going to artificially restrict the phi angle range and test that
# we cannot access the outer areas of the patrol radius.
runtime = datetime.strptime(test_assign_date, "%Y-%m-%dT%H:%M:%S")
fp, exclude, state, tmstr = dmio.load_focalplane(runtime)
# make a copy so that we aren't modifying the desimodel cache
fp = fp.copy()
limit_radius = 2.0
open_limit = 2.0 * np.arcsin(0.5 * limit_radius / 3.0)
phi_limit_min = (np.pi - open_limit) * 180.0 / np.pi
new_min = phi_limit_min - np.array(fp["OFFSET_P"])
fp["MIN_P"][:] = new_min
hw = load_hardware(focalplane=(fp, exclude, state))
failed = check_reachable(hw, 3.0, 100, log_fail=False)
if (failed != len(hw.locations)):
print(
"{} positioners reached 3mm from center, despite restricted phi"
.format(len(hw.locations) - failed),
flush=True)
self.assertTrue(False)
return
def test_thetaphi_xy(self):
# Test round trip consistency.
def check_positioner(hrdw, radius, increments, log_fail=True):
centers = hw.loc_pos_curved_mm
theta_arms = hw.loc_theta_arm
phi_arms = hw.loc_phi_arm
theta_mins = hw.loc_theta_min
theta_maxs = hw.loc_theta_max
theta_offsets = hw.loc_theta_offset
phi_mins = hw.loc_phi_min
phi_maxs = hw.loc_phi_max
phi_offsets = hw.loc_phi_offset
n_failed = 0
for loc in hw.locations:
center = centers[loc]
theta_arm = theta_arms[loc]
phi_arm = phi_arms[loc]
theta_min = theta_mins[loc]
theta_max = theta_maxs[loc]
theta_offset = theta_offsets[loc]
phi_min = phi_mins[loc]
phi_max = phi_maxs[loc]
phi_offset = phi_offsets[loc]
ang = np.arange(increments) / (2 * np.pi)
test_x = radius * np.cos(ang) + center[0]
test_y = radius * np.sin(ang) + center[1]
for xy in zip(test_x, test_y):
thetaphi = hrdw.xy_to_thetaphi(center, xy, theta_arm,
phi_arm, theta_offset,
phi_offset, theta_min,
phi_min, theta_max, phi_max)
if thetaphi[0] is None or thetaphi[1] is None:
if log_fail:
print("loc {} at ({}, {}) cannot reach ({}, {})".
format(loc, center[0], center[1], xy[0],
xy[1]),
flush=True)
n_failed += 1
break
else:
if not log_fail:
# log success instead
print(
"loc {} at ({}, {}) to ({}, {}) with ({}, {})".
format(loc, center[0], center[1], xy[0], xy[1],
thetaphi[0] * 180.0 / np.pi,
thetaphi[1] * 180.0 / np.pi),
flush=True)
result = hrdw.thetaphi_to_xy(center, thetaphi[0],
thetaphi[1], theta_arm,
phi_arm, theta_offset,
phi_offset, theta_min,
phi_min, theta_max, phi_max)
if result[0] is None or result[1] is None:
if log_fail:
print("loc {} at ({}, {}) invalid angles ({}, {})".
format(loc, center[0], center[1],
thetaphi[0] * 180.0 / np.pi,
thetaphi[1] * 180.0 / np.pi),
flush=True)
n_failed += 1
break
else:
if not log_fail:
# log success instead
print(
"loc {} at ({}, {}) angles ({}, {}) to ({}, {})"
.format(loc, center[0], center[1],
thetaphi[0] * 180.0 / np.pi,
thetaphi[1] * 180.0 / np.pi, result[0],
result[1]),
flush=True)
if not np.allclose([xy[0], xy[1]], [result[0], result[1]]):
print(
"loc {} at ({}, {}) failed roundtrip ({}, {}) != ({}, {})"
.format(loc, center[0], center[1], xy[0], xy[1],
result[0], result[1]),
flush=True)
n_failed += 1
return n_failed
# Test nominal focalplane
hw = load_hardware(rundate=test_assign_date)
failed = check_positioner(hw, 3.0, 100)
if (failed > 0):
print("{} positioners failed X/Y roundtrip at 3mm from center".
format(failed),
flush=True)
self.assertTrue(False)
return
def _load_and_plotpos(self, time, dir, suffix, simple=False):
hw = load_hardware(rundate=time)
locs = hw.locations
center_mm = hw.loc_pos_curved_mm
theta_arm = hw.loc_theta_arm
phi_arm = hw.loc_phi_arm
theta_offset = hw.loc_theta_offset
theta_min = hw.loc_theta_min
theta_max = hw.loc_theta_max
phi_offset = hw.loc_phi_offset
phi_min = hw.loc_phi_min
phi_max = hw.loc_phi_max
fig = plt.figure(figsize=(8, 8))
ax = fig.add_subplot(1, 1, 1)
ax.set_aspect("equal")
# Compute the font size to use for detector labels
figdpi = 75
fontpix = 0.2 * figdpi
fontpt = int(0.75 * fontpix)
# angle increments
nincr = 8
configincr = 2.0 * np.pi / nincr
# Plot the first fiber in a variety of positions
lid = locs[0]
patrol_mm = theta_arm[lid] + phi_arm[lid]
center = center_mm[lid]
# Plot data range in mm
width = 1.2 * (2.0 * patrol_mm)
height = 1.2 * (2.0 * patrol_mm)
linewidth = 0.1
# phipos = [np.pi/2.0]
# phicol = ["r"]
phipos = [0.0, np.pi/2.0]
phicol = ["r", "b"]
for configindx, (angphi, col) in enumerate(zip(phipos, phicol)):
shptheta = Shape()
shpphi = Shape()
theta = theta_offset[lid] + theta_min[lid]
phi = phi_offset[lid] + phi_min[lid] + angphi
failed = hw.loc_position_thetaphi(
lid, theta, phi, shptheta, shpphi
)
if failed:
print(
"Failed to move positioner {} to theta = {}, phi = {}"
.format(lid, theta, phi)
)
else:
if simple:
plot_positioner_simple(
ax, patrol_mm, lid, center, theta, theta_arm[lid], phi,
phi_arm[lid], color="black", linewidth=linewidth
)
else:
plot_positioner(
ax, patrol_mm, lid, center, shptheta, shpphi,
color="black", linewidth=linewidth
)
for inc in range(1, nincr):
ang = inc * configincr
effrad = 0.5 * patrol_mm * np.sin(angphi)
theta = theta_offset[lid] + theta_min[lid] + ang
xoff = effrad * np.cos(theta) + center[0]
yoff = effrad * np.sin(theta) + center[1]
failed = hw.loc_position_thetaphi(
lid, theta, phi, shptheta, shpphi
)
if failed:
print(
"Failed to move positioner {} to theta = {}, phi = {}"
.format(lid, theta, phi)
)
else:
if simple:
plot_positioner_simple(
ax, patrol_mm, lid, center, theta, theta_arm[lid],
phi, phi_arm[lid], color=col, linewidth=linewidth
)
else:
plot_positioner(
ax, patrol_mm, lid, center, shptheta, shpphi,
color=col, linewidth=linewidth
)
xend = xoff
yend = yoff
ax.text(xend, yend, "{}".format(inc),
color='k', fontsize=fontpt,
horizontalalignment='center',
verticalalignment='center',
bbox=dict(fc='w', ec='none', pad=1, alpha=1.0))
pxcent = center[0]
pycent = center[1]
half_width = 0.5 * width
half_height = 0.5 * height
ax.set_xlabel("Millimeters", fontsize="large")
ax.set_ylabel("Millimeters", fontsize="large")
ax.set_xlim([pxcent-half_width, pxcent+half_width])
ax.set_ylim([pycent-half_height, pycent+half_height])
outfile = os.path.join(dir, "test_plotpos_{}.pdf".format(suffix))
plt.savefig(outfile, dpi=300, format="pdf")
plt.close()
def test_full(self):
test_dir = test_subdir_create("assign_test_full")
np.random.seed(123456789)
input_mtl = os.path.join(test_dir, "mtl.fits")
input_std = os.path.join(test_dir, "standards.fits")
input_sky = os.path.join(test_dir, "sky.fits")
input_suppsky = os.path.join(test_dir, "suppsky.fits")
tgoff = 0
nscience = sim_targets(input_mtl,
TARGET_TYPE_SCIENCE,
tgoff,
density=self.density_science)
tgoff += nscience
nstd = sim_targets(input_std,
TARGET_TYPE_STANDARD,
tgoff,
density=self.density_standards)
tgoff += nstd
nsky = sim_targets(input_sky,
TARGET_TYPE_SKY,
tgoff,
density=self.density_sky)
tgoff += nsky
nsuppsky = sim_targets(input_suppsky,
TARGET_TYPE_SUPPSKY,
tgoff,
density=self.density_suppsky)
tgs = Targets()
load_target_file(tgs, input_mtl)
load_target_file(tgs, input_std)
load_target_file(tgs, input_sky)
load_target_file(tgs, input_suppsky)
# Create a hierarchical triangle mesh lookup of the targets positions
tree = TargetTree(tgs, 0.01)
# Read hardware properties
fp, exclude, state = sim_focalplane(rundate=test_assign_date)
hw = load_hardware(focalplane=(fp, exclude, state))
tfile = os.path.join(test_dir, "footprint.fits")
sim_tiles(tfile)
tiles = load_tiles(tiles_file=tfile)
# Compute the targets available to each fiber for each tile.
tgsavail = TargetsAvailable(hw, tgs, tiles, tree)
# Free the tree
del tree
# Compute the fibers on all tiles available for each target
favail = LocationsAvailable(tgsavail)
# Create assignment object
asgn = Assignment(tgs, tgsavail, favail)
# First-pass assignment of science targets
asgn.assign_unused(TARGET_TYPE_SCIENCE)
# Redistribute science targets
asgn.redistribute_science()
# Assign standards, 10 per petal
asgn.assign_unused(TARGET_TYPE_STANDARD, 10)
asgn.assign_force(TARGET_TYPE_STANDARD, 10)
# Assign sky to unused fibers, up to 40 per petal
asgn.assign_unused(TARGET_TYPE_SKY, 40)
asgn.assign_force(TARGET_TYPE_SKY, 40)
# Use supplemental sky to meet our requirements
asgn.assign_unused(TARGET_TYPE_SUPPSKY, 40)
asgn.assign_force(TARGET_TYPE_SUPPSKY, 40)
# If there are any unassigned fibers, try to place them somewhere.
asgn.assign_unused(TARGET_TYPE_SCIENCE)
asgn.assign_unused(TARGET_TYPE_SKY)
asgn.assign_unused(TARGET_TYPE_SUPPSKY)
write_assignment_fits(tiles, asgn, out_dir=test_dir, all_targets=True)
plotpetals = [0]
#plotpetals = None
plot_tiles(hw,
tiles,
result_dir=test_dir,
plot_dir=test_dir,
result_prefix="fba-",
real_shapes=True,
petals=plotpetals,
serial=True)
qa_tiles(hw, tiles, result_dir=test_dir)
qadata = None
with open(os.path.join(test_dir, "qa.json"), "r") as f:
qadata = json.load(f)
for tile, props in qadata.items():
self.assertTrue(props["assign_science"] >= 4485)
self.assertEqual(100, props["assign_std"])
self.assertTrue(
(props["assign_sky"] + props["assign_suppsky"]) >= 400)
plot_qa(qadata,
os.path.join(test_dir, "qa"),
outformat="pdf",
labels=True)
return
