def get_next_expid(n=None): """ Return the next exposure ID to use from {proddir}/etc/next_expid.txt and update the exposure ID in that file. Use file locking to prevent multiple readers from getting the same ID or accidentally clobbering each other while writing. Optional Input: n : integer, number of contiguous expids to return as a list. If None, return a scalar. Note that n=1 returns a list of length 1. BUGS: * if etc/next_expid.txt doesn't exist, initial file creation is probably not threadsafe. * File locking mechanism doesn't work on NERSC Edison, to turned off for now. """ #- Full path to next_expid.txt file filename = io.simdir() + '/etc/next_expid.txt' if not os.path.exists(io.simdir() + '/etc/'): os.makedirs(io.simdir() + '/etc/') #- Create file if needed; is this threadsafe? Probably not. if not os.path.exists(filename): fw = open(filename, 'w') fw.write("0\n") fw.close() #- Open the file, but get exclusive lock before reading f0 = open(filename) ### fcntl.flock(f0, fcntl.LOCK_EX) expid = int(f0.readline()) #- Write update expid to the file fw = open(filename, 'w') if n is None: fw.write(str(expid + 1) + '\n') else: fw.write(str(expid + n) + '\n') fw.close() #- Release the file lock ### fcntl.flock(f0, fcntl.LOCK_UN) f0.close() if n is None: return expid else: return range(expid, expid + n)
def get_next_expid(n=None): """ Return the next exposure ID to use from {proddir}/etc/next_expid.txt and update the exposure ID in that file. Use file locking to prevent multiple readers from getting the same ID or accidentally clobbering each other while writing. Optional Input: n : integer, number of contiguous expids to return as a list. If None, return a scalar. Note that n=1 returns a list of length 1. BUGS: * if etc/next_expid.txt doesn't exist, initial file creation is probably not threadsafe. * File locking mechanism doesn't work on NERSC Edison, to turned off for now. """ #- Full path to next_expid.txt file filename = io.simdir()+'/etc/next_expid.txt' if not os.path.exists(io.simdir()+'/etc/'): os.makedirs(io.simdir()+'/etc/') #- Create file if needed; is this threadsafe? Probably not. if not os.path.exists(filename): fw = open(filename, 'w') fw.write("0\n") fw.close() #- Open the file, but get exclusive lock before reading f0 = open(filename) ### fcntl.flock(f0, fcntl.LOCK_EX) expid = int(f0.readline()) #- Write update expid to the file fw = open(filename, 'w') if n is None: fw.write(str(expid+1)+'\n') else: fw.write(str(expid+n)+'\n') fw.close() #- Release the file lock ### fcntl.flock(f0, fcntl.LOCK_UN) f0.close() if n is None: return expid else: return range(expid, expid+n)
def update_obslog(obstype='science', program='DARK', expid=None, dateobs=None, tileid=-1, ra=None, dec=None): """ Update obslog with a new exposure obstype : 'arc', 'flat', 'bias', 'test', 'science', ... program : 'DARK', 'GRAY', 'BRIGHT', 'CALIB' expid : integer exposure ID, default from get_next_expid() dateobs : time.struct_time tuple; default time.localtime() tileid : integer TileID, default -1, i.e. not a DESI tile ra, dec : float (ra, dec) coordinates, default tile ra,dec or (0,0) returns tuple (expid, dateobs) TODO: normalize obstype vs. program; see desisim issue #97 """ #- Connect to sqlite database file and create DB if needed dbdir = io.simdir() + '/etc' if not os.path.exists(dbdir): os.makedirs(dbdir) dbfile = dbdir+'/obslog.sqlite' with sqlite3.connect(dbfile, isolation_level="EXCLUSIVE") as db: db.execute("""\ CREATE TABLE IF NOT EXISTS obslog ( expid INTEGER PRIMARY KEY, dateobs DATETIME, -- seconds since Unix Epoch (1970) night TEXT, -- YEARMMDD obstype TEXT DEFAULT "science", program TEXT DEFAULT "DARK", tileid INTEGER DEFAULT -1, ra REAL DEFAULT 0.0, dec REAL DEFAULT 0.0 ) """) #- Fill in defaults if expid is None: expid = get_next_expid() if dateobs is None: dateobs = time.localtime() if ra is None: assert (dec is None) if tileid < 0: ra, dec = (0.0, 0.0) else: ra, dec = io.get_tile_radec(tileid) night = get_night(utc=dateobs) insert = """\ INSERT OR REPLACE INTO obslog(expid,dateobs,night,obstype,program,tileid,ra,dec) VALUES (?,?,?,?,?,?,?,?) """ db.execute(insert, (int(expid), time.mktime(dateobs), str(night), str(obstype.upper()), str(program.upper()), int(tileid), float(ra), float(dec))) db.commit() return expid, dateobs
def get_next_tileid(program='DARK'): """ Return tileid of next tile to observe Args: program (optional): dark, gray, or bright Note: Simultaneous calls will return the same tileid; it does *not* reserve the tileid. """ program = program.upper() if program not in ('DARK', 'GRAY', 'GREY', 'BRIGHT', 'ELG', 'LRG', 'QSO', 'LYA', 'BGS', 'MWS'): return -1 #- Read DESI tiling and trim to just tiles in DESI footprint tiles = table.Table(desimodel.io.load_tiles()) #- HACK: update tilelist to include PROGRAM, etc. if 'PROGRAM' not in tiles.colnames: log.error('You are using an out-of-date desi-tiles.fits file from desimodel') log.error('please update your copy of desimodel/data') log.warning('proceeding anyway with a workaround for now...') tiles['PASS'] -= min(tiles['PASS']) #- standardize to starting at 0 not 1 brighttiles = tiles[tiles['PASS'] <= 2].copy() brighttiles['TILEID'] += 50000 brighttiles['PASS'] += 5 tiles = table.vstack([tiles, brighttiles]) program_col = table.Column(name='PROGRAM', length=len(tiles), dtype=(str, 6)) tiles.add_column(program_col) tiles['PROGRAM'][tiles['PASS'] <= 3] = 'DARK' tiles['PROGRAM'][tiles['PASS'] == 4] = 'GRAY' tiles['PROGRAM'][tiles['PASS'] >= 5] = 'BRIGHT' else: tiles['PROGRAM'] = np.char.strip(tiles['PROGRAM']) #- If obslog doesn't exist yet, start at tile 0 dbfile = io.simdir()+'/etc/obslog.sqlite' if not os.path.exists(dbfile): obstiles = set() else: #- Read obslog to get tiles that have already been observed db = sqlite3.connect(dbfile) result = db.execute('SELECT tileid FROM obslog WHERE program="{}"'.format(program)) obstiles = set( [row[0] for row in result] ) db.close() #- Just pick the next tile in sequential order program_tiles = tiles['TILEID'][tiles['PROGRAM'] == program] nexttile = int(min(set(program_tiles) - obstiles)) log.debug('{} tiles in program {}'.format(len(program_tiles), program)) log.debug('{} observed tiles'.format(len(obstiles))) return nexttile
def simulate_frame(night, expid, camera, ccdshape=None, **kwargs): """ Simulate a single frame, including I/O Args: night: YEARMMDD string expid: integer exposure ID camera: b0, r1, .. z9 Options: ccdshape = (npix_y, npix_x) primarily used to limit memory while testing Additional keyword args are passed to pixsim.simulate() Reads: $DESI_SPECTRO_SIM/$PIXPROD/{night}/simspec-{expid}.fits Writes: $DESI_SPECTRO_SIM/$PIXPROD/{night}/simpix-{camera}-{expid}.fits $DESI_SPECTRO_SIM/$PIXPROD/{night}/desi-{expid}.fits $DESI_SPECTRO_SIM/$PIXPROD/{night}/pix-{camera}-{expid}.fits For a lower-level pixel simulation interface that doesn't perform I/O, see pixsim.simulate() """ #- night, expid, camera -> input file names simspecfile = io.findfile('simspec', night=night, expid=expid) #- Read inputs psf = desimodel.io.load_psf(camera[0]) simspec = io.read_simspec(simspecfile) #- Trim effective CCD size; mainly to limit memory for testing if ccdshape is not None: psf.npix_y, psf.npix_x = ccdshape if 'cosmics' in kwargs: shape = (psf.npix_y, psf.npix_x) kwargs['cosmics'] = io.read_cosmics(kwargs['cosmics'], expid, shape=shape) image, rawpix, truepix = simulate(camera, simspec, psf, **kwargs) #- Outputs; force "real" data files into simulation directory simpixfile = io.findfile('simpix', night=night, expid=expid, camera=camera) io.write_simpix(simpixfile, truepix, camera=camera, meta=image.meta) simdir = io.simdir(night=night) rawfile = desispec.io.findfile('desi', night=night, expid=expid) rawfile = os.path.join(simdir, os.path.basename(rawfile)) desispec.io.write_raw(rawfile, rawpix, image.meta, camera=camera) pixfile = desispec.io.findfile('pix', night=night, expid=expid, camera=camera) pixfile = os.path.join(simdir, os.path.basename(pixfile)) desispec.io.write_image(pixfile, image)
def update_obslog(obstype='science', expid=None, dateobs=None, tileid=-1, ra=None, dec=None): """ Update obslog with a new exposure obstype : 'science', 'arc', 'flat', 'bias', 'dark', or 'test' expid : integer exposure ID, default from get_next_expid() dateobs : time.struct_time tuple; default time.localtime() tileid : integer TileID, default -1, i.e. not a DESI tile ra, dec : float (ra, dec) coordinates, default tile ra,dec or (0,0) returns tuple (expid, dateobs) """ #- Connect to sqlite database file and create DB if needed dbdir = io.simdir() + '/etc' if not os.path.exists(dbdir): os.makedirs(dbdir) dbfile = dbdir+'/obslog.sqlite' db = sqlite3.connect(dbfile) db.execute("""\ CREATE TABLE IF NOT EXISTS obslog ( expid INTEGER PRIMARY KEY, dateobs DATETIME, -- seconds since Unix Epoch (1970) night TEXT, -- YEARMMDD obstype TEXT DEFAULT "science", tileid INTEGER DEFAULT -1, ra REAL DEFAULT 0.0, dec REAL DEFAULT 0.0 ) """) #- Fill in defaults if expid is None: expid = get_next_expid() if dateobs is None: dateobs = time.localtime() if ra is None: assert (dec is None) if tileid < 0: ra, dec = (0.0, 0.0) else: ra, dec = io.get_tile_radec(tileid) night = get_night(utc=dateobs) insert = """\ INSERT OR REPLACE INTO obslog(expid,dateobs,night,obstype,tileid,ra,dec) VALUES (?,?,?,?,?,?,?) """ db.execute(insert, (expid, time.mktime(dateobs), night, obstype, tileid, ra, dec)) db.commit() return expid, dateobs
def get_next_tileid(): """ Return tileid of next tile to observe Note: simultaneous calls will return the same tileid; it does *not* reserve the tileid """ #- Read DESI tiling and trim to just tiles in DESI footprint tiles = desimodel.io.load_tiles() #- If obslog doesn't exist yet, start at tile 0 dbfile = io.simdir() + '/etc/obslog.sqlite' if not os.path.exists(dbfile): obstiles = set() else: #- Read obslog to get tiles that have already been observed db = sqlite3.connect(dbfile) result = db.execute('SELECT tileid FROM obslog') obstiles = set([row[0] for row in result]) db.close() #- Just pick the next tile in sequential order nexttile = int(min(set(tiles['TILEID']) - obstiles)) return nexttile
def get_next_tileid(): """ Return tileid of next tile to observe Note: simultaneous calls will return the same tileid; it does *not* reserve the tileid """ #- Read DESI tiling and trim to just tiles in DESI footprint tiles = desimodel.io.load_tiles() #- If obslog doesn't exist yet, start at tile 0 dbfile = io.simdir()+'/etc/obslog.sqlite' if not os.path.exists(dbfile): obstiles = set() else: #- Read obslog to get tiles that have already been observed db = sqlite3.connect(dbfile) result = db.execute('SELECT tileid FROM obslog') obstiles = set( [row[0] for row in result] ) db.close() #- Just pick the next tile in sequential order nexttile = int(min(set(tiles['TILEID']) - obstiles)) return nexttile
def update_obslog(obstype='science', expid=None, dateobs=None, tileid=-1, ra=None, dec=None): """ Update obslog with a new exposure obstype : 'science', 'arc', 'flat', 'bias', 'dark', or 'test' expid : integer exposure ID, default from get_next_expid() dateobs : time.struct_time tuple; default time.localtime() tileid : integer TileID, default -1, i.e. not a DESI tile ra, dec : float (ra, dec) coordinates, default tile ra,dec or (0,0) returns tuple (expid, dateobs) """ #- Connect to sqlite database file and create DB if needed dbdir = io.simdir() + '/etc' if not os.path.exists(dbdir): os.makedirs(dbdir) dbfile = dbdir + '/obslog.sqlite' db = sqlite3.connect(dbfile) db.execute("""\ CREATE TABLE IF NOT EXISTS obslog ( expid INTEGER PRIMARY KEY, dateobs DATETIME, -- seconds since Unix Epoch (1970) night TEXT, -- YEARMMDD obstype TEXT DEFAULT "science", tileid INTEGER DEFAULT -1, ra REAL DEFAULT 0.0, dec REAL DEFAULT 0.0 ) """) #- Fill in defaults if expid is None: expid = get_next_expid() if dateobs is None: dateobs = time.localtime() if ra is None: assert (dec is None) if tileid < 0: ra, dec = (0.0, 0.0) else: ra, dec = io.get_tile_radec(tileid) night = get_night(utc=dateobs) insert = """\ INSERT OR REPLACE INTO obslog(expid,dateobs,night,obstype,tileid,ra,dec) VALUES (?,?,?,?,?,?,?) """ db.execute(insert, (expid, time.mktime(dateobs), night, obstype, tileid, ra, dec)) db.commit() return expid, dateobs
def simulate(night, expid, camera, nspec=None, verbose=False, ncpu=None, trimxy=None): """ Run pixel-level simulation of input spectra Args: night : YEARMMDD string expid : integer exposure id camera : e.g. b0, r1, z9 nspec (optional) : number of spectra to simulate verbose (optional) : if True, print status messages ncpu (optional) : number of CPU cores to use Reads: $DESI_SPECTRO_SIM/$PIXPROD/{night}/simspec-{expid}.fits Writes: $DESI_SPECTRO_SIM/$PIXPROD/{night}/simpix-{camera}-{expid}.fits $DESI_SPECTRO_SIM/$PIXPROD/{night}/pix-{camera}-{expid}.fits """ simdir = io.simdir(night) simfile = "{}/simspec-{:08d}.fits".format(simdir, expid) if verbose: print "Reading input files" channel = camera[0].upper() ispec = int(camera[1]) assert channel in "BRZ" assert 0 <= ispec < 10 # - Load DESI parameters params = desimodel.io.load_desiparams() nfibers = params["spectro"]["nfibers"] # - Check that this camera has simulated spectra hdr = fits.getheader(simfile, "PHOT_" + channel) nspec_in = hdr["NAXIS2"] if ispec * nfibers >= nspec_in: print "ERROR: camera {} not in the {} spectra in {}/{}".format( camera, nspec_in, night, os.path.basename(simfile) ) return # - Load input photon data phot = fits.getdata(simfile, "PHOT_" + channel) try: phot += fits.getdata(simfile, "SKYPHOT_" + channel) except KeyError: pass # - arcs and flats don't have SKYPHOT nwave = phot.shape[1] wave = hdr["CRVAL1"] + np.arange(nwave) * hdr["CDELT1"] # - Load PSF psf = desimodel.io.load_psf(channel) # - Trim to just the spectra for this spectrograph if nspec is None: ii = slice(nfibers * ispec, nfibers * (ispec + 1)) phot = phot[ii] else: ii = slice(nfibers * ispec, nfibers * ispec + nspec) phot = phot[ii] # - check if simulation has less than 500 input spectra if phot.shape[0] < nspec: nspec = phot.shape[0] # - Project to image and append that to file if verbose: print "Projecting photons onto CCD" img = parallel_project(psf, wave, phot, ncpu=ncpu) if trimxy: xmin, xmax, ymin, ymax = psf.xyrange((0, nspec), wave) img = img[0:ymax, 0:xmax] # img = img[ymin:ymax, xmin:xmax] # hdr['CRVAL1'] = xmin+1 # hdr['CRVAL2'] = ymin+1 # - Add noise and write output files tmp = "/".join(simfile.split("/")[-3:]) # - last 3 elements of path hdr["SIMFILE"] = (tmp, "Input simulation file") pixfile = io.write_simpix(img, camera, "science", night, expid, header=hdr) if verbose: print "Wrote " + pixfile
def simulate(night, expid, camera, nspec=None, verbose=False, ncpu=None, trimxy=None): """ Run pixel-level simulation of input spectra Args: night : YEARMMDD string expid : integer exposure id camera : e.g. b0, r1, z9 nspec (optional) : number of spectra to simulate verbose (optional) : if True, print status messages ncpu (optional) : number of CPU cores to use Reads: $DESI_SPECTRO_SIM/$PIXPROD/{night}/simspec-{expid}.fits Writes: $DESI_SPECTRO_SIM/$PIXPROD/{night}/simpix-{camera}-{expid}.fits $DESI_SPECTRO_SIM/$PIXPROD/{night}/pix-{camera}-{expid}.fits """ simdir = io.simdir(night) simfile = '{}/simspec-{:08d}.fits'.format(simdir, expid) if verbose: print "Reading input files" channel = camera[0].upper() ispec = int(camera[1]) assert channel in 'BRZ' assert 0 <= ispec < 10 #- Load DESI parameters params = desimodel.io.load_desiparams() nfibers = params['spectro']['nfibers'] #- Check that this camera has simulated spectra fx = fits.open(simfile) hdr = fx['PHOT_'+channel].header nspec_in = hdr['NAXIS2'] if ispec*nfibers >= nspec_in: print "ERROR: camera {} not in the {} spectra in {}/{}".format( camera, nspec_in, night, os.path.basename(simfile)) return #- Load input photon data phot = fx['PHOT_'+channel].data wave = fx['WAVE_'+channel].data if 'SKYPHOT_'+channel in fx: phot += fx['SKYPHOT_'+channel].data #- Load PSF psf = desimodel.io.load_psf(channel) #- Trim to just the spectra for this spectrograph if nspec is None: ii = slice(nfibers*ispec, nfibers*(ispec+1)) phot = phot[ii] else: ii = slice(nfibers*ispec, nfibers*ispec + nspec) phot = phot[ii] #- check if simulation has less than 500 input spectra if phot.shape[0] < nspec: nspec = phot.shape[0] #- Project to image and append that to file if verbose: print "Projecting photons onto CCD" img = parallel_project(psf, wave, phot, ncpu=ncpu) if trimxy: xmin, xmax, ymin, ymax = psf.xyrange((0,nspec), wave) img = img[0:ymax, 0:xmax] # img = img[ymin:ymax, xmin:xmax] # hdr['CRVAL1'] = xmin+1 # hdr['CRVAL2'] = ymin+1 #- Prepare header hdr = fx[0].header tmp = '/'.join(simfile.split('/')[-3:]) #- last 3 elements of path hdr['SIMFILE'] = (tmp, 'Input simulation file') #- Strip unnecessary keywords for key in ('EXTNAME', 'LOGLAM', 'AIRORVAC', 'CRVAL1', 'CDELT1'): if key in hdr: del hdr[key] #- Add noise and write output files pixfile = io.write_simpix(img, camera, night, expid, header=hdr) fx.close() if verbose: print "Wrote "+pixfile