def _write_fibermap(self): """Write a fake fiberflat""" fibermap = io.empty_fibermap(self.nspec) for i in range(0, self.nspec, 3): fibermap['OBJTYPE'][i] = 'SKY' io.write_fibermap(self.fibermapfile, fibermap)
def test_main(self): """ Test the main program. """ # generate the frame data wave, flux, ivar, mask = _get_data() nspec, nwave = flux.shape #- Setup data for a Resolution matrix sigma = 4.0 ndiag = 11 xx = np.linspace(-(ndiag - 1) / 2.0, +(ndiag - 1) / 2.0, ndiag) Rdata = np.zeros((nspec, ndiag, nwave)) kernel = np.exp(-xx**2 / (2 * sigma)) kernel /= sum(kernel) for i in range(nspec): for j in range(nwave): Rdata[i, :, j] = kernel #- Convolve the data with the resolution matrix convflux = np.empty_like(flux) for i in range(nspec): convflux[i] = Resolution(Rdata[i]).dot(flux[i]) # create a fake fibermap fibermap = io.empty_fibermap(nspec, nwave) for i in range(0, nspec): fibermap['OBJTYPE'][i] = 'FAKE' io.write_fibermap(self.testfibermap, fibermap) #- write out the frame frame = Frame(wave, convflux, ivar, mask, Rdata, spectrograph=0, fibermap=fibermap, meta=dict(FLAVOR='flat')) write_frame(self.testframe, frame, fibermap=fibermap) # set program arguments argstr = ['--infile', self.testframe, '--outfile', self.testflat] # run it args = ffscript.parse(options=argstr) ffscript.main(args)
def test_main(self): """ Test the main program. """ # generate the frame data wave, flux, ivar, mask = _get_data() nspec, nwave = flux.shape #- Setup data for a Resolution matrix sigma = 4.0 ndiag = 11 xx = np.linspace(-(ndiag-1)/2.0, +(ndiag-1)/2.0, ndiag) Rdata = np.zeros( (nspec, ndiag, nwave) ) kernel = np.exp(-xx**2/(2*sigma)) kernel /= sum(kernel) for i in range(nspec): for j in range(nwave): Rdata[i,:,j] = kernel #- Convolve the data with the resolution matrix convflux = np.empty_like(flux) for i in range(nspec): convflux[i] = Resolution(Rdata[i]).dot(flux[i]) # create a fake fibermap fibermap = io.empty_fibermap(nspec, nwave) for i in range(0, nspec): fibermap['OBJTYPE'][i] = 'FAKE' io.write_fibermap(self.testfibermap, fibermap) #- write out the frame frame = Frame(wave, convflux, ivar, mask, Rdata, spectrograph=0, fibermap=fibermap) write_frame(self.testframe, frame, fibermap=fibermap) # set program arguments argstr = [ '--infile', self.testframe, '--outfile', self.testflat ] # run it args = ffscript.parse(options=argstr) ffscript.main(args)
def setUp(self): self.proddir = 'test-prod-' + uuid4().hex self.testraw = 'test-' + uuid4().hex os.mkdir(self.testraw) self.night = time.strftime('%Y%m%d', time.localtime(time.time() - 12 * 3600)) self.nightdir = os.path.join(self.testraw, self.night) os.mkdir(self.nightdir) for expid in [0, 1]: fibermap = io.fibermap.empty_fibermap(10) for key in fibermap.dtype.names: column = fibermap[key] fibermap[key] = np.random.random(column.shape).astype( column.dtype) hdr = {'flavor': 'flat'} fmfile = os.path.join(self.nightdir, "fibermap-{:08d}.fits".format(expid)) io.write_fibermap(fmfile, fibermap, header=hdr) for expid in [2, 3]: fibermap = io.fibermap.empty_fibermap(10) for key in fibermap.dtype.names: column = fibermap[key] fibermap[key] = np.random.random(column.shape).astype( column.dtype) hdr = {'flavor': 'arc'} fmfile = os.path.join(self.nightdir, "fibermap-{:08d}.fits".format(expid)) io.write_fibermap(fmfile, fibermap, header=hdr) for expid in [4, 5]: fibermap = io.fibermap.empty_fibermap(10) for key in fibermap.dtype.names: column = fibermap[key] fibermap[key] = np.random.random(column.shape).astype( column.dtype) hdr = {'flavor': 'science'} fmfile = os.path.join(self.nightdir, "fibermap-{:08d}.fits".format(expid)) io.write_fibermap(fmfile, fibermap, header=hdr) for expid in range(6): for band in ['b', 'r', 'z']: for spec in range(10): cam = "{}{}".format(band, spec) pixfile = os.path.join( self.nightdir, "pix-{}-{:08d}.fits".format(cam, expid)) with open(pixfile, 'w') as p: p.write("")
def setUp(self): self.proddir = 'test-prod-'+uuid4().hex self.testraw = 'test-'+uuid4().hex os.mkdir(self.testraw) self.night = time.strftime('%Y%m%d', time.localtime(time.time()-12*3600)) self.nightdir = os.path.join(self.testraw, self.night) os.mkdir(self.nightdir) for expid in [0, 1]: fibermap = io.fibermap.empty_fibermap(10) for key in fibermap.dtype.names: column = fibermap[key] fibermap[key] = np.random.random(column.shape).astype(column.dtype) hdr = {'flavor': 'flat'} fmfile = os.path.join(self.nightdir, "fibermap-{:08d}.fits".format(expid)) io.write_fibermap(fmfile, fibermap, header=hdr) for expid in [2, 3]: fibermap = io.fibermap.empty_fibermap(10) for key in fibermap.dtype.names: column = fibermap[key] fibermap[key] = np.random.random(column.shape).astype(column.dtype) hdr = {'flavor': 'arc'} fmfile = os.path.join(self.nightdir, "fibermap-{:08d}.fits".format(expid)) io.write_fibermap(fmfile, fibermap, header=hdr) for expid in [4, 5]: fibermap = io.fibermap.empty_fibermap(10) for key in fibermap.dtype.names: column = fibermap[key] fibermap[key] = np.random.random(column.shape).astype(column.dtype) hdr = {'flavor': 'science'} fmfile = os.path.join(self.nightdir, "fibermap-{:08d}.fits".format(expid)) io.write_fibermap(fmfile, fibermap, header=hdr) for expid in range(6): for band in ['b', 'r', 'z']: for spec in range(10): cam = "{}{}".format(band, spec) pixfile = os.path.join(self.nightdir, "pix-{}-{:08d}.fits".format(cam, expid)) with open(pixfile, 'w') as p: p.write("")
def _write_fibermap(self): """Write a fake fibermap""" fibermap = self._get_fibermap() io.write_fibermap(self.fibermapfile, fibermap)
def setUp(self): self.uid = uuid4().hex self.testraw = "test_raw-{}".format(self.uid) os.mkdir(self.testraw) self.testprod = "test_redux-{}".format(self.uid) os.mkdir(self.testprod) self.night = time.strftime('%Y%m%d', time.localtime(time.time()-12*3600)) self.rawnightdir = os.path.join(self.testraw, self.night) os.mkdir(self.rawnightdir) self.cal2d = os.path.join(self.testprod, 'calib2d') if not os.path.isdir(self.cal2d): os.makedirs(self.cal2d) self.calpsf = os.path.join(self.cal2d, 'psf') if not os.path.isdir(self.calpsf): os.makedirs(self.calpsf) self.calpsfnight = os.path.join(self.calpsf, self.night) if not os.path.isdir(self.calpsfnight): os.makedirs(self.calpsfnight) self.expdir = os.path.join(self.testprod, 'exposures') if not os.path.isdir(self.expdir): os.makedirs(self.expdir) self.expnight = os.path.join(self.expdir, self.night) if not os.path.isdir(self.expnight): os.makedirs(self.expnight) self.brkdir = os.path.join(self.testprod, 'bricks') if not os.path.isdir(self.brkdir): os.makedirs(self.brkdir) self.logdir = os.path.join(self.testprod, 'logs') if not os.path.isdir(self.logdir): os.makedirs(self.logdir) self.faildir = os.path.join(self.testprod, 'failed') if not os.path.isdir(self.faildir): os.makedirs(self.faildir) self.scriptdir = os.path.join(self.testprod, 'scripts') if not os.path.isdir(self.scriptdir): os.makedirs(self.scriptdir) for expid in [0, 1]: fibermap = io.fibermap.empty_fibermap(10) for key in fibermap.dtype.names: column = fibermap[key] fibermap[key] = np.random.random(column.shape).astype(column.dtype) hdr = {'flavor': 'flat'} fmfile = os.path.join(self.rawnightdir, "fibermap-{:08d}.fits".format(expid)) io.write_fibermap(fmfile, fibermap, header=hdr) for expid in [2, 3]: fibermap = io.fibermap.empty_fibermap(10) for key in fibermap.dtype.names: column = fibermap[key] fibermap[key] = np.random.random(column.shape).astype(column.dtype) hdr = {'flavor': 'arc'} fmfile = os.path.join(self.rawnightdir, "fibermap-{:08d}.fits".format(expid)) io.write_fibermap(fmfile, fibermap, header=hdr) for expid in [4, 5]: fibermap = io.fibermap.empty_fibermap(10) for key in fibermap.dtype.names: column = fibermap[key] fibermap[key] = np.random.random(column.shape).astype(column.dtype) hdr = {'flavor': 'science'} fmfile = os.path.join(self.rawnightdir, "fibermap-{:08d}.fits".format(expid)) io.write_fibermap(fmfile, fibermap, header=hdr) for expid in range(6): for band in ['b', 'r', 'z']: for spec in range(10): cam = "{}{}".format(band, spec) pixfile = os.path.join(self.rawnightdir, "pix-{}-{:08d}.fits".format(cam, expid)) with open(pixfile, 'w') as p: p.write("") self.grph, expcount, bricks = graph_night(self.testraw, self.night) graph_write(os.path.join(self.testraw, "{}_graph.yml".format(self.night)), self.grph)
def setUp(self): #- Cleanup in case a previous failed test left something behind if os.path.isdir(self.testraw): shutil.rmtree(self.testraw) if os.path.isdir(self.testprod): shutil.rmtree(self.testprod) #- Now create a fresh copy to work from os.mkdir(self.testraw) os.mkdir(self.testprod) self.night = time.strftime('%Y%m%d', time.localtime(time.time() - 12 * 3600)) self.rawnightdir = os.path.join(self.testraw, self.night) os.mkdir(self.rawnightdir) self.cal2d = os.path.join(self.testprod, 'calib2d') if not os.path.isdir(self.cal2d): os.makedirs(self.cal2d) self.calpsf = os.path.join(self.cal2d, 'psf') if not os.path.isdir(self.calpsf): os.makedirs(self.calpsf) self.calpsfnight = os.path.join(self.calpsf, self.night) if not os.path.isdir(self.calpsfnight): os.makedirs(self.calpsfnight) self.expdir = os.path.join(self.testprod, 'exposures') if not os.path.isdir(self.expdir): os.makedirs(self.expdir) self.expnight = os.path.join(self.expdir, self.night) if not os.path.isdir(self.expnight): os.makedirs(self.expnight) self.brkdir = os.path.join(self.testprod, 'bricks') if not os.path.isdir(self.brkdir): os.makedirs(self.brkdir) self.logdir = os.path.join(self.testprod, 'logs') if not os.path.isdir(self.logdir): os.makedirs(self.logdir) self.faildir = os.path.join(self.testprod, 'failed') if not os.path.isdir(self.faildir): os.makedirs(self.faildir) self.scriptdir = os.path.join(self.testprod, 'scripts') if not os.path.isdir(self.scriptdir): os.makedirs(self.scriptdir) for expid in [0, 1]: fibermap = io.fibermap.empty_fibermap(10) for key in fibermap.dtype.names: column = fibermap[key] fibermap[key] = np.random.random(column.shape).astype( column.dtype) hdr = {'flavor': 'flat'} fmfile = os.path.join(self.rawnightdir, "fibermap-{:08d}.fits".format(expid)) io.write_fibermap(fmfile, fibermap, header=hdr) for expid in [2, 3]: fibermap = io.fibermap.empty_fibermap(10) for key in fibermap.dtype.names: column = fibermap[key] fibermap[key] = np.random.random(column.shape).astype( column.dtype) hdr = {'flavor': 'arc'} fmfile = os.path.join(self.rawnightdir, "fibermap-{:08d}.fits".format(expid)) io.write_fibermap(fmfile, fibermap, header=hdr) for expid in [4, 5]: fibermap = io.fibermap.empty_fibermap(10) for key in fibermap.dtype.names: column = fibermap[key] fibermap[key] = np.random.random(column.shape).astype( column.dtype) hdr = {'flavor': 'science'} fmfile = os.path.join(self.rawnightdir, "fibermap-{:08d}.fits".format(expid)) io.write_fibermap(fmfile, fibermap, header=hdr) for expid in range(6): for band in ['b', 'r', 'z']: for spec in range(10): cam = "{}{}".format(band, spec) pixfile = os.path.join( self.rawnightdir, "pix-{}-{:08d}.fits".format(cam, expid)) with open(pixfile, 'w') as p: p.write("")
def fake_raw(): dirhash = uuid.uuid4() night = fake_night() rawdir = "test_pipe_raw_{}".format(dirhash) if os.path.exists(rawdir): shutil.rmtree(rawdir) os.makedirs(rawdir) nightdir = os.path.join(rawdir, night) # set up one spectrograph (500 fibers) nspec = 500 # arc expid = "00000000" tileid = "0" flavor = "arc" telera = "0.0" teledec = "0.0" fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid)) hdr = dict( NIGHT=(night, 'Night of observation YEARMMDD'), EXPID=(expid, 'DESI exposure ID'), TILEID=(tileid, 'DESI tile ID'), FLAVOR=(flavor, 'Flavor [arc, flat, science, ...]'), TELRA=(telera, 'Telescope pointing RA [degrees]'), TELDEC=(teledec, 'Telescope pointing dec [degrees]'), ) fibermap = io.empty_fibermap(nspec) fibermap['OBJTYPE'] = 'ARC' io.write_fibermap(fmfile, fibermap, header=hdr) for cam in ["r0", "b0", "z0"]: pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid)) pix = np.random.normal(0, 3.0, size=(10, 10)) ivar = np.ones_like(pix) / 3.0**2 mask = np.zeros(pix.shape, dtype=np.uint32) img = dimg.Image(pix, ivar, mask, camera=cam) io.write_image(pfile, img) # flat expid = "00000001" tileid = "1" flavor = "flat" telera = "0.0" teledec = "0.0" fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid)) hdr = dict( NIGHT=(night, 'Night of observation YEARMMDD'), EXPID=(expid, 'DESI exposure ID'), TILEID=(tileid, 'DESI tile ID'), FLAVOR=(flavor, 'Flavor [arc, flat, science, ...]'), TELRA=(telera, 'Telescope pointing RA [degrees]'), TELDEC=(teledec, 'Telescope pointing dec [degrees]'), ) fibermap = io.empty_fibermap(nspec) fibermap['OBJTYPE'] = 'FLAT' io.write_fibermap(fmfile, fibermap, header=hdr) for cam in ["r0", "b0", "z0"]: pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid)) pix = np.random.normal(0, 3.0, size=(10, 10)) ivar = np.ones_like(pix) / 3.0**2 mask = np.zeros(pix.shape, dtype=np.uint32) img = dimg.Image(pix, ivar, mask, camera=cam) io.write_image(pfile, img) # science expid = "00000002" tileid = "2" flavor = "dark" telera = "0.0" teledec = "0.0" fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid)) hdr = dict( NIGHT=(night, 'Night of observation YEARMMDD'), EXPID=(expid, 'DESI exposure ID'), TILEID=(tileid, 'DESI tile ID'), FLAVOR=(flavor, 'Flavor [arc, flat, science, ...]'), TELRA=(telera, 'Telescope pointing RA [degrees]'), TELDEC=(teledec, 'Telescope pointing dec [degrees]'), ) fibermap = io.empty_fibermap(nspec) fibermap['OBJTYPE'] = 'ELG' fibermap['FIBER'] = np.arange(nspec, dtype='i4') fibermap['TARGETID'] = np.random.randint(sys.maxsize, size=nspec) fibermap['BRICKNAME'] = ['3412p195' for x in range(nspec)] io.write_fibermap(fmfile, fibermap, header=hdr) for cam in ["r0", "b0", "z0"]: pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid)) pix = np.random.normal(0, 3.0, size=(10, 10)) ivar = np.ones_like(pix) / 3.0**2 mask = np.zeros(pix.shape, dtype=np.uint32) img = dimg.Image(pix, ivar, mask, camera=cam) io.write_image(pfile, img) return rawdir
def fake_raw(): dirhash = uuid.uuid4() night = fake_night() rawdir = "test_pipe_raw_{}".format(dirhash) if os.path.exists(rawdir): shutil.rmtree(rawdir) os.makedirs(rawdir) nightdir = os.path.join(rawdir, night) # set up one spectrograph (500 fibers) nspec = 500 # arc expid = "00000000" tileid = "0" flavor = "arc" telera = "0.0" teledec = "0.0" fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid)) hdr = dict( NIGHT = (night, 'Night of observation YEARMMDD'), EXPID = (expid, 'DESI exposure ID'), TILEID = (tileid, 'DESI tile ID'), FLAVOR = (flavor, 'Flavor [arc, flat, science, ...]'), TELRA = (telera, 'Telescope pointing RA [degrees]'), TELDEC = (teledec, 'Telescope pointing dec [degrees]'), ) fibermap = io.empty_fibermap(nspec) fibermap['OBJTYPE'] = 'ARC' io.write_fibermap(fmfile, fibermap, header=hdr) for cam in ["r0", "b0", "z0"]: pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid)) pix = np.random.normal(0, 3.0, size=(10,10)) ivar = np.ones_like(pix) / 3.0**2 mask = np.zeros(pix.shape, dtype=np.uint32) img = dimg.Image(pix, ivar, mask, camera=cam) io.write_image(pfile, img) # flat expid = "00000001" tileid = "1" flavor = "flat" telera = "0.0" teledec = "0.0" fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid)) hdr = dict( NIGHT = (night, 'Night of observation YEARMMDD'), EXPID = (expid, 'DESI exposure ID'), TILEID = (tileid, 'DESI tile ID'), FLAVOR = (flavor, 'Flavor [arc, flat, science, ...]'), TELRA = (telera, 'Telescope pointing RA [degrees]'), TELDEC = (teledec, 'Telescope pointing dec [degrees]'), ) fibermap = io.empty_fibermap(nspec) fibermap['OBJTYPE'] = 'FLAT' io.write_fibermap(fmfile, fibermap, header=hdr) for cam in ["r0", "b0", "z0"]: pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid)) pix = np.random.normal(0, 3.0, size=(10,10)) ivar = np.ones_like(pix) / 3.0**2 mask = np.zeros(pix.shape, dtype=np.uint32) img = dimg.Image(pix, ivar, mask, camera=cam) io.write_image(pfile, img) # science expid = "00000002" tileid = "2" flavor = "dark" telera = "0.0" teledec = "0.0" fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid)) hdr = dict( NIGHT = (night, 'Night of observation YEARMMDD'), EXPID = (expid, 'DESI exposure ID'), TILEID = (tileid, 'DESI tile ID'), FLAVOR = (flavor, 'Flavor [arc, flat, science, ...]'), TELRA = (telera, 'Telescope pointing RA [degrees]'), TELDEC = (teledec, 'Telescope pointing dec [degrees]'), ) fibermap = io.empty_fibermap(nspec) fibermap['OBJTYPE'] = 'ELG' fibermap['FIBER'] = np.arange(nspec, dtype='i4') fibermap['TARGETID'] = np.random.randint(sys.maxsize, size=nspec) fibermap['BRICKNAME'] = [ '3412p195' for x in range(nspec) ] io.write_fibermap(fmfile, fibermap, header=hdr) for cam in ["r0", "b0", "z0"]: pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid)) pix = np.random.normal(0, 3.0, size=(10,10)) ivar = np.ones_like(pix) / 3.0**2 mask = np.zeros(pix.shape, dtype=np.uint32) img = dimg.Image(pix, ivar, mask, camera=cam) io.write_image(pfile, img) return rawdir