def test_available(self): test_dir = test_subdir_create("targets_test_available") input_mtl = os.path.join(test_dir, "mtl.fits") input_std = os.path.join(test_dir, "standards.fits") input_sky = os.path.join(test_dir, "sky.fits") nscience = sim_targets(input_mtl, TARGET_TYPE_SCIENCE, 0) nstd = sim_targets(input_std, TARGET_TYPE_STANDARD, nscience) nsky = sim_targets(input_sky, TARGET_TYPE_SKY, (nscience + nstd)) tgs = Targets() load_target_file(tgs, input_mtl) load_target_file(tgs, input_std) load_target_file(tgs, input_sky) print(tgs) # Create a hierarchical triangle mesh lookup of the targets positions tree = TargetTree(tgs, 0.01) # Compute the targets available to each fiber for each tile. hw = load_hardware() tfile = os.path.join(test_dir, "footprint.fits") sim_tiles(tfile) tiles = load_tiles(tiles_file=tfile) tgsavail = TargetsAvailable(hw, tgs, tiles, tree) # Free the tree del tree # Compute the fibers on all tiles available for each target favail = FibersAvailable(tgsavail) return
def test_read(self): hw = load_hardware(rundate=test_assign_date) print(hw) locs = hw.locations cs5 = hw.loc_pos_cs5_mm curved = hw.loc_pos_curved_mm # for p in locs: # print("{}: curved = [{}, {}] cs5 = [{}, {}]".format( # p, curved[p][0], curved[p][1], cs5[p][0], cs5[p][1] # )) return
def mkfa(targetf,tileff,rd,fad,srun,nrun,DESIMODEL): ''' make the fiberassign files needed targetf is the target file (e.g., an mtl file) tilef is the root string for the tile files produced for each epoch rd is the output directory fad is the directory for the data fiberassign files srun is the initial epoch nrun is the number of epochs DESIMODEL is the directory for where to find the focal plane model for running these ''' os.environ['DESIMODEL'] = DESIMODEL #targetf = e2eout +program+'/randoms_mtl_cuttod.fits' #above file, cut to ~e2e area with significant padding #use fiberassign tools to read in randoms to be assigned tgs = Targets() load_target_file(tgs,targetf) print('loaded target file '+targetf) tree = TargetTree(tgs, 0.01) for run in range(srun,srun+nrun): #make the tile file for this run #e2e.mke2etiles(run,program=program) tilef = tileff+str(run)+'.fits' #+str(run)+'.fits' randir = rd +str(run) #+str(run) if os.path.isdir(randir): ofls = glob.glob(randir+'/*') for fl in ofls: os.remove(fl) #remove the old files if we are rerunning else: os.mkdir(randir) fafls = glob.glob(fad+str(run)+'/fiberassign/fiberassign*') #+str(run)+'/fiberassign/fiberassign*') hd = fitsio.read_header(fafls[0]) dt = hd['FA_RUN'] hw = load_hardware(rundate=dt) tiles = load_tiles(tiles_file=tilef) tgsavail = TargetsAvailable(hw, tgs, tiles, tree) favail = LocationsAvailable(tgsavail) #del tree asgn = Assignment(tgs, tgsavail, favail) asgn.assign_unused(TARGET_TYPE_SCIENCE) write_assignment_fits(tiles, asgn, out_dir=randir, all_targets=True) print('wrote assignment files to '+randir)
def test_plotfp(self): test_dir = test_subdir_create("vis_test_plotfp") time = test_assign_date hw = load_hardware(rundate=time) suffix = "{}_simple".format(time) self._load_and_plotfp(hw, test_dir, suffix, simple=True) suffix = "{}".format(time) self._load_and_plotfp(hw, test_dir, suffix, simple=False) # time = "2012-12-12T00:00:00" # hw = load_hardware(rundate=time) # suffix = "{}_simple".format(time) # self._load_and_plotfp(hw, test_dir, suffix, simple=True) # suffix = "{}".format(time) # self._load_and_plotfp(hw, test_dir, suffix, simple=False) return
def test_plot_fakefp(self): test_dir = test_subdir_create("vis_test_fakefp") time = test_assign_date # Simulate a fake focalplane fp, exclude, state = sim_focalplane(rundate=test_assign_date, fakepos=True) # Load the focalplane hw = load_hardware(focalplane=(fp, exclude, state)) suffix = "{}".format(time) self._load_and_plotfp(hw, test_dir, suffix, simple=False) time = "2012-12-12T00:00:00" suffix = "{}".format(time) self._load_and_plotfp(hw, test_dir, suffix, simple=False) return
def test_available(self): test_dir = test_subdir_create("targets_test_available") input_mtl = os.path.join(test_dir, "mtl.fits") input_std = os.path.join(test_dir, "standards.fits") input_sky = os.path.join(test_dir, "sky.fits") input_suppsky = os.path.join(test_dir, "suppsky.fits") tgoff = 0 nscience = sim_targets(input_mtl, TARGET_TYPE_SCIENCE, tgoff) tgoff += nscience nstd = sim_targets(input_std, TARGET_TYPE_STANDARD, tgoff) tgoff += nstd nsky = sim_targets(input_sky, TARGET_TYPE_SKY, tgoff) tgoff += nsky nsuppsky = sim_targets(input_suppsky, TARGET_TYPE_SUPPSKY, tgoff) tgs = Targets() load_target_file(tgs, input_mtl) load_target_file(tgs, input_std) load_target_file(tgs, input_sky) load_target_file(tgs, input_suppsky) print(tgs) # Test access ids = tgs.ids() tt = tgs.get(ids[0]) tt.ra += 1.0e-5 tt.dec += 1.0e-5 tt.subpriority = 0.99 # Create a hierarchical triangle mesh lookup of the targets positions tree = TargetTree(tgs, 0.01) # Compute the targets available to each fiber for each tile. hw = load_hardware() tfile = os.path.join(test_dir, "footprint.fits") sim_tiles(tfile) tiles = load_tiles(tiles_file=tfile) tgsavail = TargetsAvailable(hw, tgs, tiles, tree) # Free the tree del tree # Compute the fibers on all tiles available for each target favail = LocationsAvailable(tgsavail) return
def test_collision_thetaphi(self): hw = load_hardware() fiber_id = hw.fiber_id ntheta = 10 nphi = 10 thetaincr = 2 * np.pi / ntheta phiincr = np.pi / nphi tm = Timer() tm.start() for thetarot in range(ntheta): for phirot in range(nphi): theta = [thetarot * thetaincr for x in fiber_id] phi = [phirot * phiincr for x in fiber_id] result = hw.check_collisions_thetaphi(fiber_id, theta, phi, 0) tm.stop() tm.report("check_collisions_thetaphi 100 configurations") return
def test_collision_thetaphi(self): hw = load_hardware(rundate=test_assign_date) locs = hw.locations ntheta = 10 nphi = 10 thetaincr = 2 * np.pi / ntheta phiincr = np.pi / nphi tm = Timer() tm.start() for thetarot in range(ntheta): for phirot in range(nphi): theta = [thetarot * thetaincr for x in locs] phi = [phirot * phiincr for x in locs] result = hw.check_collisions_thetaphi(locs, theta, phi, 0) tm.stop() tm.report("check_collisions_thetaphi 100 configurations") return
def test_collision_xy(self): hw = load_hardware(rundate=test_assign_date) center_mm = hw.loc_pos_curved_mm locs = hw.locations nrot = 100 rotrad = 0.5 rotincr = 2 * np.pi / nrot tm = Timer() tm.start() for rot in range(nrot): xoff = rotrad * np.cos(rot * rotincr) yoff = rotrad * np.sin(rot * rotincr) xy = [(center_mm[p][0] + xoff, center_mm[p][1] + yoff) for p in locs] result = hw.check_collisions_xy(locs, xy, 0) tm.stop() tm.report("check_collisions_xy 100 configurations") return
def test_collision_xy(self): hw = load_hardware() center_mm = hw.fiber_pos_xy_mm fiber_id = hw.fiber_id nrot = 100 rotrad = 0.5 rotincr = 2 * np.pi / nrot tm = Timer() tm.start() for rot in range(nrot): xoff = rotrad * np.cos(rot * rotincr) yoff = rotrad * np.sin(rot * rotincr) xy = [(center_mm[p][0] + xoff, center_mm[p][1] + yoff) for p in fiber_id] result = hw.check_collisions_xy(fiber_id, xy, 0) tm.stop() tm.report("check_collisions_xy 100 configurations") return
def test_read(self): test_dir = test_subdir_create("tiles_test_read") print('test_dir', test_dir) hw = load_hardware() tfile = os.path.join(test_dir, "footprint.fits") sfile = os.path.join(test_dir, "footprint_keep.txt") sim_tiles(tfile, selectfile=sfile) stiles = list() with open(sfile, "r") as f: for line in f: # Try to convert the first column to an integer. try: stiles.append(int(line.split()[0])) except ValueError: pass tls = load_tiles(tiles_file=tfile, select=stiles) print(tls) indx = 0 for st in stiles: self.assertEqual(tls.order[st], indx) indx += 1 #- Syntax / coverage tests tls = load_tiles(tiles_file=tfile, select=stiles, obstime='2020-10-20') self.assertEqual(tls.obstime[0], Time('2020-10-20T00:00:00').isot) tls = load_tiles(tiles_file=tfile, select=stiles, obstime='2020-10-20T10:20:30') self.assertEqual(tls.obstime[0], Time('2020-10-20T10:20:30').isot) #- including obsdate in tile file tiles = Table.read(tfile) tiles['OBSDATE'] = '2020-11-22' # creates full column tiles['OBSDATE'][1] = '2020-11-23' # override second entry tfile = os.path.join(test_dir, "footprint-obsdate.fits") tiles.write(tfile) tls = load_tiles(tiles_file=tfile, select=stiles) self.assertEqual(tls.obstime[0], Time('2020-11-22').isot) self.assertEqual(tls.obstime[1], Time('2020-11-23').isot) return
def getfatiles(targetf, tilef, dirout='', dt='2020-03-10T00:00:00'): ''' will write out fiberassignment files for each tile with the FASSIGN, FTARGETS, FAVAIL HDUS these are what are required to determine the geometry of what fiberassign thinks could have been observed and also match to actual observations (though FASSIGN is not really necessary) targetf is file with all targets to be run through tilef lists the tiles to "assign" dirout is the directory where this all gets written out !make sure this is unique for every different target! ''' tgs = Targets() load_target_file(tgs, targetf) print('loaded target file ' + targetf) tree = TargetTree(tgs, 0.01) hw = load_hardware(rundate=dt) tiles = load_tiles(tiles_file=tilef) tgsavail = TargetsAvailable(hw, tgs, tiles, tree) favail = LocationsAvailable(tgsavail) del tree asgn = Assignment(tgs, tgsavail, favail) asgn.assign_unused(TARGET_TYPE_SCIENCE) write_assignment_fits(tiles, asgn, out_dir=dirout, all_targets=True) print('wrote assignment files to ' + dirout)
tilesfile = args.tiles outdir = args.outdir label = args.label #rank_group = args.rank_group # python --cpu-bind=verbose parallel_pip_bianchi_general_testing.py --mtl /global/project/projectdirs/desi/users/arroyoc/pip_scripts/parallel_pip_bianchi/mtlz-patch-thesis-version.fits --stdstar-dark /global/project/projectdirs/desi/users/jguy/mocks/darksky-v1.0.1-v3/targets-0/standards-dark.fits --stdstar-bright /global/project/projectdirs/desi/users/jguy/mocks/darksky-v1.0.1-v3/targets-0/standards-bright.fits --sky /global/project/projectdirs/desi/users/jguy/mocks/darksky-v1.0.1-v3/targets-0/sky.fits --tiles /global/project/projectdirs/desi/users/arroyoc/pip_scripts/parallel_pip_bianchi/test-tiles.fits --rank-group 0 # -------------------------------------------------------------------------------------------------- # READING AND LOADING THE RELEVANT DATA (FIRST PART) # # In this part of the code the main objects to be used are defined and the data to make the # assignation process is loaded. # -------------------------------------------------------------------------------------------------- # Read hardware properties. hw = load_hardware() # Read the tiles to be used and define the number of tiles. Path obtained from command line # argument. if (tilesfile is None): nominal_tiles = load_tiles() else: nominal_tiles = load_tiles(tiles_file=tilesfile) ntiles = len(nominal_tiles.id) # Container for the Mersenne Twister pseudo-random number generator. Since every MPI rank is seeded # with a different number, size number of different subpriorities are generated. #seed = 62*rank_group + rank seed = rank random_generator = RandomState(seed=seed)
def main(): os.environ['DESIMODEL'] = '/global/homes/d/dstn/desimodel-data' global hw global stuck_x global stuck_y global stuck_loc global starkd hw = load_hardware() # From fiberassign/stucksky.py: find X,Y positions of stuck positioners. # (grab the hw dictionaries once -- these are python wrappers over C++ so not simple accessors) state = hw.state devtype = hw.loc_device_type stuck_loc = [ loc for loc in hw.locations if (((state[loc] & (FIBER_STATE_STUCK | FIBER_STATE_BROKEN) ) == FIBER_STATE_STUCK) and (devtype[loc] == 'POS')) ] print(len(stuck_loc), 'stuck positioners') theta_pos = hw.loc_theta_pos theta_off = hw.loc_theta_offset phi_pos = hw.loc_phi_pos phi_off = hw.loc_phi_offset stuck_theta = [theta_pos[loc] + theta_off[loc] for loc in stuck_loc] stuck_phi = [phi_pos[loc] + phi_off[loc] for loc in stuck_loc] curved_mm = hw.loc_pos_curved_mm theta_arm = hw.loc_theta_arm phi_arm = hw.loc_phi_arm theta_min = hw.loc_theta_min theta_max = hw.loc_theta_max phi_min = hw.loc_phi_min phi_max = hw.loc_phi_max # Convert positioner angle orientations to curved focal surface X / Y (not CS5) # Note: we could add some methods to the python bindings to vectorize this or make it less clunky... stuck_x = np.zeros(len(stuck_loc)) stuck_y = np.zeros(len(stuck_loc)) for iloc, (loc, theta, phi) in enumerate(zip(stuck_loc, stuck_theta, stuck_phi)): loc_x, loc_y = hw.thetaphi_to_xy(curved_mm[loc], theta, phi, theta_arm[loc], phi_arm[loc], theta_off[loc], phi_off[loc], theta_min[loc], phi_min[loc], theta_max[loc], phi_max[loc], True) stuck_x[iloc] = loc_x stuck_y[iloc] = loc_y tiles = Table.read( '/global/cfs/cdirs/desi/target/surveyops/ops/tiles-main.ecsv') print(len(tiles), 'tiles') # Deduplicate tiles with same RA,Dec center tilera = tiles['RA'] tiledec = tiles['DEC'] tileid = tiles['TILEID'] rdtile = {} tilemap = {} for tid, r, d in zip(tileid, tilera, tiledec): key = r, d if key in rdtile: # already seen a tile with this RA,Dec; point to it tilemap[tid] = rdtile[key] else: rdtile[key] = tid del rdtile tnow = datetime.now() tile_obstime = tnow.isoformat(timespec='seconds') mjd = Time(tnow).mjd stars = fits_table( '/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/masking/gaia-mask-dr9.fits.gz' ) print(len(stars), 'stars for masking') print('Moving to MJD', mjd) ra, dec = radec_at_mjd(stars.ra, stars.dec, stars.ref_epoch.astype(float), stars.pmra, stars.pmdec, stars.parallax, mjd) assert (np.all(np.isfinite(ra))) assert (np.all(np.isfinite(dec))) stars.ra = ra stars.dec = dec print('Building kd-tree...') starkd = tree_build_radec(stars.ra, stars.dec) match_radius = deg2dist(30. / 3600.) stuck_loc = np.array(stuck_loc) allresults = {} mp = multiproc(32) print('Building arg lists...') args = [] for tid, tile_ra, tile_dec, tile_obsha in zip(tileid, tilera, tiledec, tiles['DESIGNHA']): # skip duplicate tiles if tid in tilemap: continue # "fieldrot" tile_theta = field_rotation_angle(tile_ra, tile_dec, mjd) args.append((tid, tile_ra, tile_dec, tile_obstime, tile_theta, tile_obsha, match_radius)) print('Matching', len(args), 'unique tile RA,Decs in parallel...') res = mp.map(_match_tile, args) print('Organizing results...') T = fits_table() T.tileid = [] T.loc = [] T.petal = [] T.device = [] T.fiber = [] T.pos_ra = [] T.pos_dec = [] T.star_ra = [] T.star_dec = [] T.dist_arcsec = [] T.mask_mag = [] loc_to_petal = hw.loc_petal loc_to_device = hw.loc_device loc_to_fiber = hw.loc_fiber for vals in res: if vals is None: continue tileid, I, pos_ra, pos_dec, pos_loc, dists = vals T.tileid.extend([tileid] * len(I)) T.loc.extend(pos_loc) for loc in pos_loc: T.petal.append(loc_to_petal[loc]) T.device.append(loc_to_device[loc]) T.fiber.append(loc_to_fiber[loc]) T.pos_ra.extend(pos_ra) T.pos_dec.extend(pos_dec) T.star_ra.extend(stars.ra[I]) T.star_dec.extend(stars.dec[I]) T.dist_arcsec.extend(dists) T.mask_mag.extend(stars.mask_mag[I]) T.to_np_arrays() T.writeto('stuck-on-stars.fits')
tgs = Targets() load_target_file(tgs, targetf) print('loaded target file ' + targetf) tree = TargetTree(tgs, 0.01) for run in range(srun, srun + nrun): #make the tile file for this run mke2etiles(run, program=program) tilef = e2eout + 'e2etiles_run' + str(run) + '.fits' randir = e2eout + program + '/randoms/' + str(run) if os.path.isdir(randir): pass else: os.mkdir(randir) fafls = glob.glob(e2ein + 'run/quicksurvey/' + program + '/' + str(run) + '/fiberassign/fiberassign*') hd = fitsio.read_header(fafls[0]) dt = hd['FA_RUN'] hw = load_hardware(rundate=dt) tiles = load_tiles(tiles_file=tilef) tgsavail = TargetsAvailable(hw, tgs, tiles, tree) favail = LocationsAvailable(tgsavail) #del tree asgn = Assignment(tgs, tgsavail, favail) asgn.assign_unused(TARGET_TYPE_SCIENCE) write_assignment_fits(tiles, asgn, out_dir=randir, all_targets=True) print('wrote assignment files to ' + randir)
def test_read(self): hw = load_hardware() print(hw) return
def main(): log = Logger.get() mpi_procs = MPI.COMM_WORLD.size mpi_rank = MPI.COMM_WORLD.rank parser = argparse.ArgumentParser() parser.add_argument( "--survey_log", type=str, required=False, help="Eventually we would pass in a file containing the log" " of when each fiber assignment was run and for which tiles, " "along with the options that were used.") parser.add_argument( "--sky", type=str, required=False, help="Input file with sky or supp_sky targets. " "These target files are assumed to be constant and not " "tracked by the MTL ledger.") parser.add_argument( "--mtl", type=str, required=True, help="The MTL ledger. This is still a work in progress and" " I am not sure what the interface will be, but given the " "fiber assignment dates in the survey log, we should be able" " to get the MTL state at that time. For now, this option" " is just one or more target files.") parser.add_argument("--footprint", type=str, required=False, default=None, help="Optional FITS file defining the footprint. If" " not specified, the default footprint from desimodel" " is used.") parser.add_argument("--tiles", type=str, required=False, default=None, help="Optional text file containing a subset of the" " tile IDs to use in the footprint, one ID per line." " Default uses all tiles in the footprint.") parser.add_argument("--out", type=str, required=False, default=None, help="Output directory.") parser.add_argument("--realizations", type=int, required=False, default=10, help="Number of realizations.") args = parser.parse_args() if args.sky is None: args.sky = list() # Set output directory if args.out is None: args.out = "." # Read tiles we are using tileselect = None if args.tiles is not None: tileselect = list() with open(args.tiles, "r") as f: for line in f: # Try to convert the first column to an integer. try: tileselect.append(int(line.split()[0])) except ValueError: pass tiles = load_tiles( tiles_file=args.footprint, select=tileselect, ) # Create empty target list tgs = Targets() # Append each input target file. These target files must all be of the # same survey type, and will set the Targets object to be of that survey. print(args.mtl) print(args.sky) #for tgfile in args.targets: # load_target_file(tgs, tgfile) load_target_file(tgs, args.mtl) # Just the science target IDs tg_science = tgs.ids() tg_science2indx = {y: x for x, y in enumerate(tg_science)} # Now load the sky target files. survey = tgs.survey() #for tgfile in args.sky: # load_target_file(tgs, tgfile) load_target_file(tgs, args.sky) # Divide up realizations among the processes. n_realization = args.realizations realizations = np.arange(n_realization, dtype=np.int32) my_realizations = np.array_split(realizations, mpi_procs)[mpi_rank] # Bitarray for all targets and realizations #tgarray = bitarray(len(tg_science) * n_realization) #tgarray.setall(False) tgarray = np.zeros(len(tg_science) * n_realization, dtype='bool') # Target tree tree = TargetTree(tgs) hw = load_hardware() for realization in my_realizations: # Set the seed based on the realization, so that the result is reproducible # regardless of which process is working on the realization. np.random.seed(realization) # Comment out the next block to avoid randomizing subpriority # ---- # Randomize science target subpriority for this realization new_subpriority = np.random.random_sample(size=len(tg_science)) for indx, tgid in enumerate(tg_science): tg = tgs.get(tgid) tg.subpriority = new_subpriority[indx] # Comment out the next block to avoid dithering tiles # ---- # Dither tiles centers by the same # Compute available targets / locations tgsavail = TargetsAvailable(hw, tgs, tiles, tree) favail = LocationsAvailable(tgsavail) asgn = Assignment(tgs, tgsavail, favail) # Replay the survey log for each time fiber assignment was run. For now, this # is just doing the tiles all at once. for assign_event in range(1): # In the future, load MTL updates to the obs remaining for each target here # Read hardware properties- in the future, pass in the assignment run date # to this function. hw = load_hardware() # Run assignment for this event. run(asgn) # Update bit arrays for assigned science targets for tile_id in tiles.id: #(): adata = asgn.tile_location_target(tile_id) for loc, tgid in adata.items(): try: idx = tg_science2indx[tgid] tgarray[idx * n_realization + realization] = True except KeyError: # Not a science target pass # Reduce bitarrays to root process. The bitarray type conforms to the # buffer protocol. tgall = None #if mpi_rank == 0: # tgall = bitarray(tgarray) # tgall.setall(False) MPI.COMM_WORLD.Reduce(tgarray, tgall, op=MPI.BOR, root=0) # Write it out if mpi_rank == 0: #pass print(len(tgall))
def test_plotpos(self): test_dir = test_subdir_create("vis_test_plotpos") hw = load_hardware() patrol_mm = hw.patrol_mm fiber_id = hw.fiber_id center_mm = hw.fiber_pos_xy_mm # Plot data range in mm width = 1.2 * (2.0 * patrol_mm) height = 1.2 * (2.0 * patrol_mm) # Plot size in inches xfigsize = 8 yfigsize = 8 figdpi = 75 # Compute the font size to use for detector labels fontpix = 0.2 * figdpi fontpt = int(0.75 * fontpix) # Plot the first fiber in a variety of positions fid = fiber_id[0] nincr = 8 configincr = 2.0 * np.pi / nincr center = center_mm[fid] for configindx, (configrad, col) in \ enumerate(zip([0.5*patrol_mm, patrol_mm], ["r", "b"])): fig = plt.figure(figsize=(xfigsize, yfigsize), dpi=figdpi) ax = fig.add_subplot(1, 1, 1) ax.set_aspect("equal") cb, fh = hw.fiber_position(fid, center_mm[fid]) plot_positioner(ax, patrol_mm, fid, center, cb, fh, color="k") for inc in range(nincr): ang = inc * configincr xoff = configrad * np.cos(ang) + center_mm[fid][0] yoff = configrad * np.sin(ang) + center_mm[fid][1] cb, fh = hw.fiber_position(fid, (xoff, yoff)) plot_positioner(ax, patrol_mm, fid, center, cb, fh, color=col) xend = xoff yend = yoff ax.plot([center_mm[fid][0], xend], [center_mm[fid][1], yend], color="k", linewidth="0.5") ax.text(xend, yend, "{}".format(inc), color='k', fontsize=fontpt, horizontalalignment='center', verticalalignment='center', bbox=dict(fc='w', ec='none', pad=1, alpha=1.0)) pxcent = center_mm[fid][0] pycent = center_mm[fid][1] half_width = 0.5 * width half_height = 0.5 * height ax.set_xlabel("Millimeters", fontsize="large") ax.set_ylabel("Millimeters", fontsize="large") ax.set_xlim([pxcent - half_width, pxcent + half_width]) ax.set_ylim([pycent - half_height, pycent + half_height]) outfile = os.path.join(test_dir, "test_plotpos_{}.png".format(configindx)) plt.savefig(outfile) plt.close() return
def test_science(self): set_matplotlib_pdf_backend() import matplotlib.pyplot as plt test_dir = test_subdir_create("qa_test_science") log_file = os.path.join(test_dir, "log.txt") np.random.seed(123456789) input_mtl = os.path.join(test_dir, "mtl.fits") # For this test, we will use just 2 science target classes, in order to verify # we get approximately the correct distribution sdist = [(3000, 1, 0.25, "QSO"), (2000, 1, 0.75, "ELG")] nscience = sim_targets(input_mtl, TARGET_TYPE_SCIENCE, 0, density=self.density_science, science_frac=sdist) log_msg = "Simulated {} science targets\n".format(nscience) tgs = Targets() load_target_file(tgs, input_mtl) # Read hardware properties fp, exclude, state = sim_focalplane(rundate=test_assign_date) hw = load_hardware(focalplane=(fp, exclude, state)) tfile = os.path.join(test_dir, "footprint.fits") sim_tiles(tfile) tiles = load_tiles(tiles_file=tfile) # Precompute target positions tile_targetids, tile_x, tile_y = targets_in_tiles(hw, tgs, tiles) # Compute the targets available to each fiber for each tile. tgsavail = TargetsAvailable(hw, tiles, tile_targetids, tile_x, tile_y) # Compute the fibers on all tiles available for each target favail = LocationsAvailable(tgsavail) # Pass empty map of STUCK positioners that land on good sky stucksky = {} # Create assignment object asgn = Assignment(tgs, tgsavail, favail, stucksky) # First-pass assignment of science targets asgn.assign_unused(TARGET_TYPE_SCIENCE) # Redistribute asgn.redistribute_science() write_assignment_fits(tiles, asgn, out_dir=test_dir, all_targets=True) tile_ids = list(tiles.id) merge_results([input_mtl], list(), tile_ids, result_dir=test_dir, copy_fba=False) # FIXME: In order to use the qa_targets function, we need to know the # starting requested number of observations (NUMOBS_INIT). Then we can use # that value for each target and compare to the number actually assigned. # However, the NUMOBS_INIT column was removed from the merged TARGET table. # If we are ever able to reach consensus on restoring that column, then we # can re-enable these tests below. # # qa_targets( # hw, # tiles, # result_dir=test_dir, # result_prefix="fiberassign-" # ) # # # Load the target catalog so that we have access to the target properties # # fd = fitsio.FITS(input_mtl, "r") # scidata = np.array(np.sort(fd[1].read(), order="TARGETID")) # fd.close() # del fd # # # How many possible positioner assignments did we have? # nassign = 5000 * len(tile_ids) # # possible = dict() # achieved = dict() # # namepat = re.compile(r".*/qa_target_count_(.*)_init-(.*)\.fits") # for qafile in glob.glob("{}/qa_target_count_*.fits".format(test_dir)): # namemat = namepat.match(qafile) # name = namemat.group(1) # obs = int(namemat.group(2)) # if obs == 0: # continue # fd = fitsio.FITS(qafile, "r") # fdata = fd["COUNTS"].read() # # Sort by target ID so we can select easily # fdata = np.sort(fdata, order="TARGETID") # tgid = np.array(fdata["TARGETID"]) # counts = np.array(fdata["NUMOBS_DONE"]) # avail = np.array(fdata["NUMOBS_AVAIL"]) # del fdata # fd.close() # # # Select target properties. BOTH TARGET LISTS MUST BE SORTED. # rows = np.where(np.isin(scidata["TARGETID"], tgid, assume_unique=True))[0] # # ra = np.array(scidata["RA"][rows]) # dec = np.array(scidata["DEC"][rows]) # dtarget = np.array(scidata["DESI_TARGET"][rows]) # init = np.array(scidata["NUMOBS_INIT"][rows]) # # requested = obs * np.ones_like(avail) # # under = np.where(avail < requested)[0] # over = np.where(avail > requested)[0] # # limavail = np.array(avail) # limavail[over] = obs # # deficit = np.zeros(len(limavail), dtype=np.int) # # deficit[:] = limavail - counts # deficit[avail == 0] = 0 # # possible[name] = np.sum(limavail) # achieved[name] = np.sum(counts) # # log_msg += "{}-{}:\n".format(name, obs) # # pindx = np.where(deficit > 0)[0] # poor_tgid = tgid[pindx] # poor_dtarget = dtarget[pindx] # log_msg += " Deficit > 0: {}\n".format(len(poor_tgid)) # poor_ra = ra[pindx] # poor_dec = dec[pindx] # poor_deficit = deficit[pindx] # # # Plot Target availability # # Commented out by default, since in the case of high target density # # needed for maximizing assignments, there are far more targets than # # the number of available fiber placements. # # # marksize = 4 * np.ones_like(deficit) # # # # fig = plt.figure(figsize=(12, 12)) # # ax = fig.add_subplot(1, 1, 1) # # ax.scatter(ra, dec, s=2, c="black", marker="o") # # for pt, pr, pd, pdef in zip(poor_tgid, poor_ra, poor_dec, poor_deficit): # # ploc = plt.Circle( # # (pr, pd), radius=(0.05*pdef), fc="none", ec="red" # # ) # # ax.add_artist(ploc) # # ax.set_xlabel("RA", fontsize="large") # # ax.set_ylabel("DEC", fontsize="large") # # ax.set_title( # # "Target \"{}\": (min(avail, requested) - counts) > 0".format( # # name, obs # # ) # # ) # # #ax.legend(handles=lg, framealpha=1.0, loc="upper right") # # plt.savefig(os.path.join(test_dir, "{}-{}_deficit.pdf".format(name, obs)), dpi=300, format="pdf") # # log_msg += \ # "Assigned {} tiles for total of {} possible target observations\n".format( # len(tile_ids), nassign # ) # ach = 0 # for nm in possible.keys(): # ach += achieved[nm] # log_msg += \ # " type {} had {} possible target obs and achieved {}\n".format( # nm, possible[nm], achieved[nm] # ) # frac = 100.0 * ach / nassign # log_msg += \ # " {} / {} = {:0.2f}% of fibers were assigned\n".format( # ach, nassign, frac # ) # for nm in possible.keys(): # log_msg += \ # " type {} had {:0.2f}% of achieved observations\n".format( # nm, achieved[nm] / ach # ) # with open(log_file, "w") as f: # f.write(log_msg) # # self.assertGreaterEqual(frac, 99.0) # Test if qa-fiberassign script runs without crashing script = os.path.join(self.binDir, "qa-fiberassign") if os.path.exists(script): fafiles = glob.glob(f"{test_dir}/fiberassign-*.fits") cmd = "{} --targets {}".format(script, " ".join(fafiles)) err = subprocess.call(cmd.split()) self.assertEqual(err, 0, f"FAILED ({err}): {cmd}") else: print(f"ERROR: didn't find {script}")
def test_fieldrot(self): test_dir = test_subdir_create("assign_test_fieldrot") np.random.seed(123456789) input_mtl = os.path.join(test_dir, "mtl.fits") input_std = os.path.join(test_dir, "standards.fits") input_sky = os.path.join(test_dir, "sky.fits") input_suppsky = os.path.join(test_dir, "suppsky.fits") tgoff = 0 nscience = sim_targets(input_mtl, TARGET_TYPE_SCIENCE, tgoff, density=self.density_science) tgoff += nscience nstd = sim_targets(input_std, TARGET_TYPE_STANDARD, tgoff, density=self.density_standards) tgoff += nstd nsky = sim_targets(input_sky, TARGET_TYPE_SKY, tgoff, density=self.density_sky) tgoff += nsky nsuppsky = sim_targets(input_suppsky, TARGET_TYPE_SUPPSKY, tgoff, density=self.density_suppsky) # Simulate the tiles tfile = os.path.join(test_dir, "footprint.fits") sim_tiles(tfile) # petal mapping rotator = petal_rotation(1, reverse=False) rots = [0, 36] tile_ids = None for rt in rots: odir = "theta_{:02d}".format(rt) tgs = Targets() load_target_file(tgs, input_mtl) load_target_file(tgs, input_std) load_target_file(tgs, input_sky) load_target_file(tgs, input_suppsky) # Create a hierarchical triangle mesh lookup of the targets # positions tree = TargetTree(tgs, 0.01) # Manually override the field rotation tiles = load_tiles(tiles_file=tfile, obstheta=float(rt)) if tile_ids is None: tile_ids = list(tiles.id) # Simulate a fake focalplane fp, exclude, state = sim_focalplane(rundate=test_assign_date, fakepos=True) # Load the focalplane hw = load_hardware(focalplane=(fp, exclude, state)) # Compute the targets available to each fiber for each tile. tgsavail = TargetsAvailable(hw, tgs, tiles, tree) # Compute the fibers on all tiles available for each target favail = LocationsAvailable(tgsavail) # Pass empty map of STUCK positioners that land on good sky stucksky = {} # Create assignment object asgn = Assignment(tgs, tgsavail, favail, stucksky) # First-pass assignment of science targets asgn.assign_unused(TARGET_TYPE_SCIENCE) out = os.path.join(test_dir, odir) write_assignment_fits(tiles, asgn, out_dir=out, all_targets=True) ppet = 6 if odir == "theta_36": ppet = rotator[6] plot_tiles(hw, tiles, result_dir=out, plot_dir=out, real_shapes=True, petals=[ppet], serial=True) # Explicitly free everything del asgn del favail del tgsavail del hw del tiles del tree del tgs # For each tile, compare the assignment output and verify that they # agree with a one-petal rotation. # NOTE: The comparison below will NOT pass, since we are still # Sorting by highest priority available target and then (in case # of a tie) by fiber ID. See line 333 of assign.cpp. Re-enable this # test after that is changed to sort by location in case of a tie. # for tl in tile_ids: # orig_path = os.path.join( # test_dir, "theta_00", "fiberassign_{:06d}.fits".format(tl) # ) # orig_header, orig_data, _, _, _ = \ # read_assignment_fits_tile((tl, orig_path)) # rot_path = os.path.join( # test_dir, "theta_36", "fiberassign_{:06d}.fits".format(tl) # ) # rot_header, rot_data, _, _, _ = \ # read_assignment_fits_tile((tl, rot_path)) # comppath = os.path.join( # test_dir, "comp_00-36_{:06d}.txt".format(tl) # ) # with open(comppath, "w") as fc: # for dev, petal, tg in zip( # orig_data["DEVICE_LOC"], orig_data["PETAL_LOC"], # orig_data["TARGETID"] # ): # for newdev, newpetal, newtg in zip( # rot_data["DEVICE_LOC"], rot_data["PETAL_LOC"], # rot_data["TARGETID"] # ): # rpet = rotator[newpetal] # if (newdev == dev) and (rpet == petal): # fc.write( # "{}, {} = {} : {}, {} = {}\n" # .format(petal, dev, tg, rpet, newdev, newtg) # ) # # self.assertEqual(newtg, tg) return
def test_io(self): np.random.seed(123456789) test_dir = test_subdir_create("assign_test_io") input_mtl = os.path.join(test_dir, "mtl.fits") input_std = os.path.join(test_dir, "standards.fits") input_sky = os.path.join(test_dir, "sky.fits") input_suppsky = os.path.join(test_dir, "suppsky.fits") tgoff = 0 nscience = sim_targets(input_mtl, TARGET_TYPE_SCIENCE, tgoff, density=self.density_science) tgoff += nscience nstd = sim_targets(input_std, TARGET_TYPE_STANDARD, tgoff, density=self.density_standards) tgoff += nstd nsky = sim_targets(input_sky, TARGET_TYPE_SKY, tgoff, density=self.density_sky) tgoff += nsky nsuppsky = sim_targets(input_suppsky, TARGET_TYPE_SUPPSKY, tgoff, density=self.density_suppsky) tgs = Targets() load_target_file(tgs, input_mtl) load_target_file(tgs, input_std) load_target_file(tgs, input_sky) load_target_file(tgs, input_suppsky) # Create a hierarchical triangle mesh lookup of the targets positions tree = TargetTree(tgs, 0.01) # Compute the targets available to each fiber for each tile. fp, exclude, state = sim_focalplane(rundate=test_assign_date) hw = load_hardware(focalplane=(fp, exclude, state)) tfile = os.path.join(test_dir, "footprint.fits") sim_tiles(tfile) tiles = load_tiles(tiles_file=tfile) tgsavail = TargetsAvailable(hw, tgs, tiles, tree) # Free the tree del tree # Compute the fibers on all tiles available for each target favail = LocationsAvailable(tgsavail) # Pass empty map of STUCK positioners that land on good sky stucksky = {} # First pass assignment asgn = Assignment(tgs, tgsavail, favail, stucksky) asgn.assign_unused(TARGET_TYPE_SCIENCE) # Write out, merge, read back in and verify write_assignment_ascii(tiles, asgn, out_dir=test_dir, out_prefix="test_io_ascii_") write_assignment_fits(tiles, asgn, out_dir=test_dir, out_prefix="basic_", all_targets=False) write_assignment_fits(tiles, asgn, out_dir=test_dir, out_prefix="full_", all_targets=True) plotpetals = [0] # plotpetals = None plot_tiles(hw, tiles, result_dir=test_dir, result_prefix="basic_", plot_dir=test_dir, plot_prefix="basic_", result_split_dir=False, petals=plotpetals, serial=True) plot_tiles(hw, tiles, result_dir=test_dir, result_prefix="full_", plot_dir=test_dir, plot_prefix="full_", result_split_dir=False, petals=plotpetals, serial=True) target_files = [input_mtl, input_sky, input_std] tile_ids = list(tiles.id) merge_results(target_files, list(), tile_ids, result_dir=test_dir, result_prefix="basic_", out_dir=test_dir, out_prefix="basic_tile-", copy_fba=False) merge_results(target_files, list(), tile_ids, result_dir=test_dir, result_prefix="full_", out_dir=test_dir, out_prefix="full_tile-", copy_fba=False) # Here we test reading with the standard reading function for tid in tile_ids: tdata = asgn.tile_location_target(tid) avail = tgsavail.tile_data(tid) # Check basic format infile = os.path.join(test_dir, "basic_tile-{:06d}.fits".format(tid)) inhead, fiber_data, targets_data, avail_data, gfa_targets = \ read_assignment_fits_tile((tid, infile)) for lid, tgid, tgra, tgdec in zip(fiber_data["LOCATION"], fiber_data["TARGETID"], fiber_data["TARGET_RA"], fiber_data["TARGET_DEC"]): if tgid >= 0: self.assertEqual(tgid, tdata[lid]) props = tgs.get(tgid) self.assertEqual(tgra, props.ra) self.assertEqual(tgdec, props.dec) # Check full format infile = os.path.join(test_dir, "full_tile-{:06d}.fits".format(tid)) inhead, fiber_data, targets_data, avail_data, gfa_targets = \ read_assignment_fits_tile((tid, infile)) for lid, tgid, tgra, tgdec in zip(fiber_data["LOCATION"], fiber_data["TARGETID"], fiber_data["TARGET_RA"], fiber_data["TARGET_DEC"]): if tgid >= 0: self.assertEqual(tgid, tdata[lid]) props = tgs.get(tgid) self.assertEqual(tgra, props.ra) self.assertEqual(tgdec, props.dec) # Now read the files directly with fitsio and verify against the input # target data. for tid in tile_ids: tdata = asgn.tile_location_target(tid) avail = tgsavail.tile_data(tid) # Check basic format infile = os.path.join(test_dir, "basic_tile-{:06d}.fits".format(tid)) fdata = fitsio.FITS(infile, "r") fassign = fdata["FIBERASSIGN"].read() ftargets = fdata["TARGETS"].read() for lid, tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip( fassign["LOCATION"], fassign["TARGETID"], fassign["TARGET_RA"], fassign["TARGET_DEC"], fassign["SUBPRIORITY"], fassign["PRIORITY"], fassign["OBSCONDITIONS"]): if tgid >= 0: self.assertEqual(tgid, tdata[lid]) props = tgs.get(tgid) self.assertEqual(tgra, props.ra) self.assertEqual(tgdec, props.dec) self.assertEqual(tgsub, props.subpriority) self.assertEqual(tgprior, props.priority) self.assertEqual(tgobs, props.obscond) for tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip( ftargets["TARGETID"], ftargets["RA"], ftargets["DEC"], ftargets["SUBPRIORITY"], ftargets["PRIORITY"], ftargets["OBSCONDITIONS"]): props = tgs.get(tgid) self.assertEqual(tgra, props.ra) self.assertEqual(tgdec, props.dec) self.assertEqual(tgsub, props.subpriority) self.assertEqual(tgprior, props.priority) self.assertEqual(tgobs, props.obscond) # Check full format infile = os.path.join(test_dir, "full_tile-{:06d}.fits".format(tid)) fdata = fitsio.FITS(infile, "r") fassign = fdata["FIBERASSIGN"].read() ftargets = fdata["TARGETS"].read() for lid, tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip( fassign["LOCATION"], fassign["TARGETID"], fassign["TARGET_RA"], fassign["TARGET_DEC"], fassign["SUBPRIORITY"], fassign["PRIORITY"], fassign["OBSCONDITIONS"]): if tgid >= 0: self.assertEqual(tgid, tdata[lid]) props = tgs.get(tgid) self.assertEqual(tgra, props.ra) self.assertEqual(tgdec, props.dec) self.assertEqual(tgsub, props.subpriority) self.assertEqual(tgprior, props.priority) self.assertEqual(tgobs, props.obscond) for tgid, tgra, tgdec, tgsub, tgprior, tgobs in zip( ftargets["TARGETID"], ftargets["RA"], ftargets["DEC"], ftargets["SUBPRIORITY"], ftargets["PRIORITY"], ftargets["OBSCONDITIONS"]): props = tgs.get(tgid) self.assertEqual(tgra, props.ra) self.assertEqual(tgdec, props.dec) self.assertEqual(tgsub, props.subpriority) self.assertEqual(tgprior, props.priority) self.assertEqual(tgobs, props.obscond) plot_tiles(hw, tiles, result_dir=test_dir, result_prefix="basic_tile-", plot_dir=test_dir, plot_prefix="basic_tile-", result_split_dir=False, petals=plotpetals, serial=True) plot_tiles(hw, tiles, result_dir=test_dir, result_prefix="full_tile-", plot_dir=test_dir, plot_prefix="full_tile-", result_split_dir=False, petals=plotpetals, serial=True) return
def test_full(self): test_dir = test_subdir_create("assign_test_full") np.random.seed(123456789) input_mtl = os.path.join(test_dir, "mtl.fits") input_std = os.path.join(test_dir, "standards.fits") input_sky = os.path.join(test_dir, "sky.fits") nscience = sim_targets(input_mtl, TARGET_TYPE_SCIENCE, 0) nstd = sim_targets(input_std, TARGET_TYPE_STANDARD, nscience) nsky = sim_targets(input_sky, TARGET_TYPE_SKY, (nscience + nstd)) tgs = Targets() load_target_file(tgs, input_mtl) load_target_file(tgs, input_std) load_target_file(tgs, input_sky) # Create a hierarchical triangle mesh lookup of the targets positions tree = TargetTree(tgs, 0.01) # Read hardware properties fstatus = os.path.join(test_dir, "fiberstatus.ecsv") sim_status(fstatus) hw = load_hardware(status_file=fstatus) tfile = os.path.join(test_dir, "footprint.fits") sim_tiles(tfile) tiles = load_tiles(tiles_file=tfile) # Compute the targets available to each fiber for each tile. tgsavail = TargetsAvailable(hw, tgs, tiles, tree) # Free the tree del tree # Compute the fibers on all tiles available for each target favail = FibersAvailable(tgsavail) # Create assignment object asgn = Assignment(tgs, tgsavail, favail) # First-pass assignment of science targets asgn.assign_unused(TARGET_TYPE_SCIENCE) # Redistribute science targets asgn.redistribute_science() # Assign standards, 10 per petal asgn.assign_unused(TARGET_TYPE_STANDARD, 10) asgn.assign_force(TARGET_TYPE_STANDARD, 10) # Assign sky to unused fibers, up to 40 per petal asgn.assign_unused(TARGET_TYPE_SKY, 40) asgn.assign_force(TARGET_TYPE_SKY, 40) # If there are any unassigned fibers, try to place them somewhere. asgn.assign_unused(TARGET_TYPE_SCIENCE) asgn.assign_unused(TARGET_TYPE_SKY) write_assignment_fits(tiles, asgn, out_dir=test_dir, all_targets=True) plot_tiles(hw, tiles, result_dir=test_dir, plot_dir=test_dir, petals=[0], serial=True) qa_tiles(hw, tiles, result_dir=test_dir) qadata = None with open(os.path.join(test_dir, "qa.json"), "r") as f: qadata = json.load(f) for tile, props in qadata.items(): self.assertEqual(4495, props["assign_science"]) self.assertEqual(100, props["assign_std"]) self.assertEqual(400, props["assign_sky"]) plot_qa(qadata, os.path.join(test_dir, "qa"), outformat="pdf", labels=True) return
def test_full(self, do_stucksky=False): test_dir = test_subdir_create("assign_test_full") np.random.seed(123456789) input_mtl = os.path.join(test_dir, "mtl.fits") input_std = os.path.join(test_dir, "standards.fits") input_sky = os.path.join(test_dir, "sky.fits") input_suppsky = os.path.join(test_dir, "suppsky.fits") tgoff = 0 nscience = sim_targets(input_mtl, TARGET_TYPE_SCIENCE, tgoff, density=self.density_science) tgoff += nscience nstd = sim_targets(input_std, TARGET_TYPE_STANDARD, tgoff, density=self.density_standards) tgoff += nstd nsky = sim_targets(input_sky, TARGET_TYPE_SKY, tgoff, density=self.density_sky) tgoff += nsky nsuppsky = sim_targets(input_suppsky, TARGET_TYPE_SUPPSKY, tgoff, density=self.density_suppsky) tgs = Targets() load_target_file(tgs, input_mtl) load_target_file(tgs, input_std) load_target_file(tgs, input_sky) load_target_file(tgs, input_suppsky) # Create a hierarchical triangle mesh lookup of the targets positions tree = TargetTree(tgs, 0.01) # Read hardware properties fp, exclude, state = sim_focalplane(rundate=test_assign_date) hw = load_hardware(focalplane=(fp, exclude, state)) tfile = os.path.join(test_dir, "footprint.fits") sim_tiles(tfile) tiles = load_tiles(tiles_file=tfile) if do_stucksky: sim_stuck_sky(test_dir, hw, tiles) # Compute the targets available to each fiber for each tile. tgsavail = TargetsAvailable(hw, tgs, tiles, tree) # Free the tree del tree # Compute the fibers on all tiles available for each target favail = LocationsAvailable(tgsavail) # Pass empty map of STUCK positioners that land on good sky stucksky = None if do_stucksky: stucksky = stuck_on_sky(hw, tiles) if stucksky is None: # (the pybind code doesn't like None when a dict is expected...) stucksky = {} # Create assignment object asgn = Assignment(tgs, tgsavail, favail, stucksky) run(asgn) write_assignment_fits(tiles, asgn, out_dir=test_dir, all_targets=True, stucksky=stucksky) plotpetals = [0] #plotpetals = None plot_tiles(hw, tiles, result_dir=test_dir, plot_dir=test_dir, result_prefix="fba-", real_shapes=True, petals=plotpetals, serial=True) qa_tiles(hw, tiles, result_dir=test_dir) qadata = None with open(os.path.join(test_dir, "qa.json"), "r") as f: qadata = json.load(f) for tile, props in qadata.items(): self.assertTrue(props["assign_science"] >= 4485) self.assertEqual(100, props["assign_std"]) if do_stucksky: # We get 3 stuck positioners landing on good sky! self.assertTrue( (props["assign_sky"] + props["assign_suppsky"]) >= 397) else: self.assertTrue( (props["assign_sky"] + props["assign_suppsky"]) >= 400) plot_qa(qadata, os.path.join(test_dir, "qa"), outformat="pdf", labels=True) return
def test_science(self): set_matplotlib_pdf_backend() import matplotlib.pyplot as plt test_dir = test_subdir_create("qa_test_science") log_file = os.path.join(test_dir, "log.txt") np.random.seed(123456789) input_mtl = os.path.join(test_dir, "mtl.fits") # For this test, we will use just 2 science target classes, in order to verify # we get approximately the correct distribution sdist = [(3000, 1, 0.25, "QSO"), (2000, 1, 0.75, "ELG")] nscience = sim_targets(input_mtl, TARGET_TYPE_SCIENCE, 0, density=self.density_science, science_frac=sdist) log_msg = "Simulated {} science targets\n".format(nscience) tgs = Targets() load_target_file(tgs, input_mtl) # Create a hierarchical triangle mesh lookup of the targets positions tree = TargetTree(tgs, 0.01) # Read hardware properties fp, exclude, state = sim_focalplane(rundate=test_assign_date) hw = load_hardware(focalplane=(fp, exclude, state)) tfile = os.path.join(test_dir, "footprint.fits") sim_tiles(tfile) tiles = load_tiles(tiles_file=tfile) # Compute the targets available to each fiber for each tile. tgsavail = TargetsAvailable(hw, tgs, tiles, tree) # Free the tree del tree # Compute the fibers on all tiles available for each target favail = LocationsAvailable(tgsavail) # Create assignment object asgn = Assignment(tgs, tgsavail, favail) # First-pass assignment of science targets asgn.assign_unused(TARGET_TYPE_SCIENCE) # Redistribute asgn.redistribute_science() write_assignment_fits(tiles, asgn, out_dir=test_dir, all_targets=True) tile_ids = list(tiles.id) merge_results([input_mtl], list(), tile_ids, result_dir=test_dir, copy_fba=False) # if "TRAVIS" not in os.environ: # plot_tiles( # hw, # tiles, # result_dir=test_dir, # plot_dir=test_dir, # real_shapes=True, # serial=True # ) qa_targets(hw, tiles, result_dir=test_dir, result_prefix="fiberassign-") # Load the target catalog so that we have access to the target properties fd = fitsio.FITS(input_mtl, "r") scidata = np.array(np.sort(fd[1].read(), order="TARGETID")) fd.close() del fd # How many possible positioner assignments did we have? nassign = 5000 * len(tile_ids) possible = dict() achieved = dict() namepat = re.compile(r".*/qa_target_count_(.*)_init-(.*)\.fits") for qafile in glob.glob("{}/qa_target_count_*.fits".format(test_dir)): namemat = namepat.match(qafile) name = namemat.group(1) obs = int(namemat.group(2)) if obs == 0: continue fd = fitsio.FITS(qafile, "r") fdata = fd["COUNTS"].read() # Sort by target ID so we can select easily fdata = np.sort(fdata, order="TARGETID") tgid = np.array(fdata["TARGETID"]) counts = np.array(fdata["NUMOBS_DONE"]) avail = np.array(fdata["NUMOBS_AVAIL"]) del fdata fd.close() # Select target properties. BOTH TARGET LISTS MUST BE SORTED. rows = np.where( np.isin(scidata["TARGETID"], tgid, assume_unique=True))[0] ra = np.array(scidata["RA"][rows]) dec = np.array(scidata["DEC"][rows]) dtarget = np.array(scidata["DESI_TARGET"][rows]) init = np.array(scidata["NUMOBS_MORE"][rows]) requested = obs * np.ones_like(avail) under = np.where(avail < requested)[0] over = np.where(avail > requested)[0] limavail = np.array(avail) limavail[over] = obs deficit = np.zeros(len(limavail), dtype=np.int) deficit[:] = limavail - counts deficit[avail == 0] = 0 possible[name] = np.sum(limavail) achieved[name] = np.sum(counts) log_msg += "{}-{}:\n".format(name, obs) pindx = np.where(deficit > 0)[0] poor_tgid = tgid[pindx] poor_dtarget = dtarget[pindx] log_msg += " Deficit > 0: {}\n".format(len(poor_tgid)) poor_ra = ra[pindx] poor_dec = dec[pindx] poor_deficit = deficit[pindx] # Plot Target availability # Commented out by default, since in the case of high target density # needed for maximizing assignments, there are far more targets than # the number of available fiber placements. # marksize = 4 * np.ones_like(deficit) # # fig = plt.figure(figsize=(12, 12)) # ax = fig.add_subplot(1, 1, 1) # ax.scatter(ra, dec, s=2, c="black", marker="o") # for pt, pr, pd, pdef in zip(poor_tgid, poor_ra, poor_dec, poor_deficit): # ploc = plt.Circle( # (pr, pd), radius=(0.05*pdef), fc="none", ec="red" # ) # ax.add_artist(ploc) # ax.set_xlabel("RA", fontsize="large") # ax.set_ylabel("DEC", fontsize="large") # ax.set_title( # "Target \"{}\": (min(avail, requested) - counts) > 0".format( # name, obs # ) # ) # #ax.legend(handles=lg, framealpha=1.0, loc="upper right") # plt.savefig(os.path.join(test_dir, "{}-{}_deficit.pdf".format(name, obs)), dpi=300, format="pdf") log_msg += \ "Assigned {} tiles for total of {} possible target observations\n".format( len(tile_ids), nassign ) ach = 0 for nm in possible.keys(): ach += achieved[nm] log_msg += \ " type {} had {} possible target obs and achieved {}\n".format( nm, possible[nm], achieved[nm] ) frac = 100.0 * ach / nassign log_msg += \ " {} / {} = {:0.2f}% of fibers were assigned\n".format( ach, nassign, frac ) for nm in possible.keys(): log_msg += \ " type {} had {:0.2f}% of achieved observations\n".format( nm, achieved[nm] / ach ) with open(log_file, "w") as f: f.write(log_msg) self.assertGreaterEqual(frac, 99.0) #- Test if qa-fiberassign script runs without crashing bindir = os.path.join(os.path.dirname(fiberassign.__file__), '..', '..', 'bin') script = os.path.join(os.path.abspath(bindir), 'qa-fiberassign') if os.path.exists(script): fafiles = glob.glob(f"{test_dir}/fiberassign-*.fits") cmd = "{} --targets {}".format(script, " ".join(fafiles)) err = subprocess.call(cmd.split()) self.assertEqual(err, 0, f"FAILED ({err}): {cmd}") else: print(f"ERROR: didn't find {script}")
def test_thetaphi_range(self): # Function to test that all positioners can reach a circle of # targets at fixed distance from their center. def check_reachable(hrdw, radius, increments, log_fail=True): centers = hw.loc_pos_curved_mm theta_arms = hw.loc_theta_arm phi_arms = hw.loc_phi_arm theta_mins = hw.loc_theta_min theta_maxs = hw.loc_theta_max theta_offsets = hw.loc_theta_offset phi_mins = hw.loc_phi_min phi_maxs = hw.loc_phi_max phi_offsets = hw.loc_phi_offset n_failed = 0 for loc in hw.locations: center = centers[loc] theta_arm = theta_arms[loc] phi_arm = phi_arms[loc] theta_min = theta_mins[loc] theta_max = theta_maxs[loc] theta_offset = theta_offsets[loc] phi_min = phi_mins[loc] phi_max = phi_maxs[loc] phi_offset = phi_offsets[loc] ang = np.arange(increments) / (2 * np.pi) test_x = radius * np.cos(ang) + center[0] test_y = radius * np.sin(ang) + center[1] for xy in zip(test_x, test_y): result = hrdw.xy_to_thetaphi(center, xy, theta_arm, phi_arm, theta_offset, phi_offset, theta_min, phi_min, theta_max, phi_max) if result[0] is None or result[1] is None: if log_fail: print("loc {} at ({}, {}) cannot reach ({}, {})". format(loc, center[0], center[1], xy[0], xy[1]), flush=True) n_failed += 1 break else: if not log_fail: # log success instead print( "loc {} at ({}, {}) to ({}, {}) with ({}, {})". format(loc, center[0], center[1], xy[0], xy[1], result[0] * 180.0 / np.pi, result[1] * 180.0 / np.pi), flush=True) return n_failed # Test nominal focalplane hw = load_hardware(rundate=test_assign_date) failed = check_reachable(hw, 3.0, 100) if (failed > 0): print("{} positioners failed to reach ring at 3mm from center". format(failed), flush=True) self.assertTrue(False) # Now we are going to artificially restrict the phi angle range and test that # we cannot access the outer areas of the patrol radius. runtime = datetime.strptime(test_assign_date, "%Y-%m-%dT%H:%M:%S") fp, exclude, state, tmstr = dmio.load_focalplane(runtime) # make a copy so that we aren't modifying the desimodel cache fp = fp.copy() limit_radius = 2.0 open_limit = 2.0 * np.arcsin(0.5 * limit_radius / 3.0) phi_limit_min = (np.pi - open_limit) * 180.0 / np.pi new_min = phi_limit_min - np.array(fp["OFFSET_P"]) fp["MIN_P"][:] = new_min hw = load_hardware(focalplane=(fp, exclude, state)) failed = check_reachable(hw, 3.0, 100, log_fail=False) if (failed != len(hw.locations)): print( "{} positioners reached 3mm from center, despite restricted phi" .format(len(hw.locations) - failed), flush=True) self.assertTrue(False) return
def test_thetaphi_xy(self): # Test round trip consistency. def check_positioner(hrdw, radius, increments, log_fail=True): centers = hw.loc_pos_curved_mm theta_arms = hw.loc_theta_arm phi_arms = hw.loc_phi_arm theta_mins = hw.loc_theta_min theta_maxs = hw.loc_theta_max theta_offsets = hw.loc_theta_offset phi_mins = hw.loc_phi_min phi_maxs = hw.loc_phi_max phi_offsets = hw.loc_phi_offset n_failed = 0 for loc in hw.locations: center = centers[loc] theta_arm = theta_arms[loc] phi_arm = phi_arms[loc] theta_min = theta_mins[loc] theta_max = theta_maxs[loc] theta_offset = theta_offsets[loc] phi_min = phi_mins[loc] phi_max = phi_maxs[loc] phi_offset = phi_offsets[loc] ang = np.arange(increments) / (2 * np.pi) test_x = radius * np.cos(ang) + center[0] test_y = radius * np.sin(ang) + center[1] for xy in zip(test_x, test_y): thetaphi = hrdw.xy_to_thetaphi(center, xy, theta_arm, phi_arm, theta_offset, phi_offset, theta_min, phi_min, theta_max, phi_max) if thetaphi[0] is None or thetaphi[1] is None: if log_fail: print("loc {} at ({}, {}) cannot reach ({}, {})". format(loc, center[0], center[1], xy[0], xy[1]), flush=True) n_failed += 1 break else: if not log_fail: # log success instead print( "loc {} at ({}, {}) to ({}, {}) with ({}, {})". format(loc, center[0], center[1], xy[0], xy[1], thetaphi[0] * 180.0 / np.pi, thetaphi[1] * 180.0 / np.pi), flush=True) result = hrdw.thetaphi_to_xy(center, thetaphi[0], thetaphi[1], theta_arm, phi_arm, theta_offset, phi_offset, theta_min, phi_min, theta_max, phi_max) if result[0] is None or result[1] is None: if log_fail: print("loc {} at ({}, {}) invalid angles ({}, {})". format(loc, center[0], center[1], thetaphi[0] * 180.0 / np.pi, thetaphi[1] * 180.0 / np.pi), flush=True) n_failed += 1 break else: if not log_fail: # log success instead print( "loc {} at ({}, {}) angles ({}, {}) to ({}, {})" .format(loc, center[0], center[1], thetaphi[0] * 180.0 / np.pi, thetaphi[1] * 180.0 / np.pi, result[0], result[1]), flush=True) if not np.allclose([xy[0], xy[1]], [result[0], result[1]]): print( "loc {} at ({}, {}) failed roundtrip ({}, {}) != ({}, {})" .format(loc, center[0], center[1], xy[0], xy[1], result[0], result[1]), flush=True) n_failed += 1 return n_failed # Test nominal focalplane hw = load_hardware(rundate=test_assign_date) failed = check_positioner(hw, 3.0, 100) if (failed > 0): print("{} positioners failed X/Y roundtrip at 3mm from center". format(failed), flush=True) self.assertTrue(False) return
def _load_and_plotpos(self, time, dir, suffix, simple=False): hw = load_hardware(rundate=time) locs = hw.locations center_mm = hw.loc_pos_curved_mm theta_arm = hw.loc_theta_arm phi_arm = hw.loc_phi_arm theta_offset = hw.loc_theta_offset theta_min = hw.loc_theta_min theta_max = hw.loc_theta_max phi_offset = hw.loc_phi_offset phi_min = hw.loc_phi_min phi_max = hw.loc_phi_max fig = plt.figure(figsize=(8, 8)) ax = fig.add_subplot(1, 1, 1) ax.set_aspect("equal") # Compute the font size to use for detector labels figdpi = 75 fontpix = 0.2 * figdpi fontpt = int(0.75 * fontpix) # angle increments nincr = 8 configincr = 2.0 * np.pi / nincr # Plot the first fiber in a variety of positions lid = locs[0] patrol_mm = theta_arm[lid] + phi_arm[lid] center = center_mm[lid] # Plot data range in mm width = 1.2 * (2.0 * patrol_mm) height = 1.2 * (2.0 * patrol_mm) linewidth = 0.1 # phipos = [np.pi/2.0] # phicol = ["r"] phipos = [0.0, np.pi/2.0] phicol = ["r", "b"] for configindx, (angphi, col) in enumerate(zip(phipos, phicol)): shptheta = Shape() shpphi = Shape() theta = theta_offset[lid] + theta_min[lid] phi = phi_offset[lid] + phi_min[lid] + angphi failed = hw.loc_position_thetaphi( lid, theta, phi, shptheta, shpphi ) if failed: print( "Failed to move positioner {} to theta = {}, phi = {}" .format(lid, theta, phi) ) else: if simple: plot_positioner_simple( ax, patrol_mm, lid, center, theta, theta_arm[lid], phi, phi_arm[lid], color="black", linewidth=linewidth ) else: plot_positioner( ax, patrol_mm, lid, center, shptheta, shpphi, color="black", linewidth=linewidth ) for inc in range(1, nincr): ang = inc * configincr effrad = 0.5 * patrol_mm * np.sin(angphi) theta = theta_offset[lid] + theta_min[lid] + ang xoff = effrad * np.cos(theta) + center[0] yoff = effrad * np.sin(theta) + center[1] failed = hw.loc_position_thetaphi( lid, theta, phi, shptheta, shpphi ) if failed: print( "Failed to move positioner {} to theta = {}, phi = {}" .format(lid, theta, phi) ) else: if simple: plot_positioner_simple( ax, patrol_mm, lid, center, theta, theta_arm[lid], phi, phi_arm[lid], color=col, linewidth=linewidth ) else: plot_positioner( ax, patrol_mm, lid, center, shptheta, shpphi, color=col, linewidth=linewidth ) xend = xoff yend = yoff ax.text(xend, yend, "{}".format(inc), color='k', fontsize=fontpt, horizontalalignment='center', verticalalignment='center', bbox=dict(fc='w', ec='none', pad=1, alpha=1.0)) pxcent = center[0] pycent = center[1] half_width = 0.5 * width half_height = 0.5 * height ax.set_xlabel("Millimeters", fontsize="large") ax.set_ylabel("Millimeters", fontsize="large") ax.set_xlim([pxcent-half_width, pxcent+half_width]) ax.set_ylim([pycent-half_height, pycent+half_height]) outfile = os.path.join(dir, "test_plotpos_{}.pdf".format(suffix)) plt.savefig(outfile, dpi=300, format="pdf") plt.close()
def test_full(self): test_dir = test_subdir_create("assign_test_full") np.random.seed(123456789) input_mtl = os.path.join(test_dir, "mtl.fits") input_std = os.path.join(test_dir, "standards.fits") input_sky = os.path.join(test_dir, "sky.fits") input_suppsky = os.path.join(test_dir, "suppsky.fits") tgoff = 0 nscience = sim_targets(input_mtl, TARGET_TYPE_SCIENCE, tgoff, density=self.density_science) tgoff += nscience nstd = sim_targets(input_std, TARGET_TYPE_STANDARD, tgoff, density=self.density_standards) tgoff += nstd nsky = sim_targets(input_sky, TARGET_TYPE_SKY, tgoff, density=self.density_sky) tgoff += nsky nsuppsky = sim_targets(input_suppsky, TARGET_TYPE_SUPPSKY, tgoff, density=self.density_suppsky) tgs = Targets() load_target_file(tgs, input_mtl) load_target_file(tgs, input_std) load_target_file(tgs, input_sky) load_target_file(tgs, input_suppsky) # Create a hierarchical triangle mesh lookup of the targets positions tree = TargetTree(tgs, 0.01) # Read hardware properties fp, exclude, state = sim_focalplane(rundate=test_assign_date) hw = load_hardware(focalplane=(fp, exclude, state)) tfile = os.path.join(test_dir, "footprint.fits") sim_tiles(tfile) tiles = load_tiles(tiles_file=tfile) # Compute the targets available to each fiber for each tile. tgsavail = TargetsAvailable(hw, tgs, tiles, tree) # Free the tree del tree # Compute the fibers on all tiles available for each target favail = LocationsAvailable(tgsavail) # Create assignment object asgn = Assignment(tgs, tgsavail, favail) # First-pass assignment of science targets asgn.assign_unused(TARGET_TYPE_SCIENCE) # Redistribute science targets asgn.redistribute_science() # Assign standards, 10 per petal asgn.assign_unused(TARGET_TYPE_STANDARD, 10) asgn.assign_force(TARGET_TYPE_STANDARD, 10) # Assign sky to unused fibers, up to 40 per petal asgn.assign_unused(TARGET_TYPE_SKY, 40) asgn.assign_force(TARGET_TYPE_SKY, 40) # Use supplemental sky to meet our requirements asgn.assign_unused(TARGET_TYPE_SUPPSKY, 40) asgn.assign_force(TARGET_TYPE_SUPPSKY, 40) # If there are any unassigned fibers, try to place them somewhere. asgn.assign_unused(TARGET_TYPE_SCIENCE) asgn.assign_unused(TARGET_TYPE_SKY) asgn.assign_unused(TARGET_TYPE_SUPPSKY) write_assignment_fits(tiles, asgn, out_dir=test_dir, all_targets=True) plotpetals = [0] #plotpetals = None plot_tiles(hw, tiles, result_dir=test_dir, plot_dir=test_dir, result_prefix="fba-", real_shapes=True, petals=plotpetals, serial=True) qa_tiles(hw, tiles, result_dir=test_dir) qadata = None with open(os.path.join(test_dir, "qa.json"), "r") as f: qadata = json.load(f) for tile, props in qadata.items(): self.assertTrue(props["assign_science"] >= 4485) self.assertEqual(100, props["assign_std"]) self.assertTrue( (props["assign_sky"] + props["assign_suppsky"]) >= 400) plot_qa(qadata, os.path.join(test_dir, "qa"), outformat="pdf", labels=True) return