def _write_fibermap(self):
"""Write a fake fiberflat"""
fibermap = io.empty_fibermap(self.nspec)
for i in range(0, self.nspec, 3):
fibermap['OBJTYPE'][i] = 'SKY'
io.write_fibermap(self.fibermapfile, fibermap)
def test_main(self):
"""
Test the main program.
"""
# generate the frame data
wave, flux, ivar, mask = _get_data()
nspec, nwave = flux.shape
#- Setup data for a Resolution matrix
sigma = 4.0
ndiag = 11
xx = np.linspace(-(ndiag - 1) / 2.0, +(ndiag - 1) / 2.0, ndiag)
Rdata = np.zeros((nspec, ndiag, nwave))
kernel = np.exp(-xx**2 / (2 * sigma))
kernel /= sum(kernel)
for i in range(nspec):
for j in range(nwave):
Rdata[i, :, j] = kernel
#- Convolve the data with the resolution matrix
convflux = np.empty_like(flux)
for i in range(nspec):
convflux[i] = Resolution(Rdata[i]).dot(flux[i])
# create a fake fibermap
fibermap = io.empty_fibermap(nspec, nwave)
for i in range(0, nspec):
fibermap['OBJTYPE'][i] = 'FAKE'
io.write_fibermap(self.testfibermap, fibermap)
#- write out the frame
frame = Frame(wave,
convflux,
ivar,
mask,
Rdata,
spectrograph=0,
fibermap=fibermap,
meta=dict(FLAVOR='flat'))
write_frame(self.testframe, frame, fibermap=fibermap)
# set program arguments
argstr = ['--infile', self.testframe, '--outfile', self.testflat]
# run it
args = ffscript.parse(options=argstr)
ffscript.main(args)
def test_main(self):
"""
Test the main program.
"""
# generate the frame data
wave, flux, ivar, mask = _get_data()
nspec, nwave = flux.shape
#- Setup data for a Resolution matrix
sigma = 4.0
ndiag = 11
xx = np.linspace(-(ndiag-1)/2.0, +(ndiag-1)/2.0, ndiag)
Rdata = np.zeros( (nspec, ndiag, nwave) )
kernel = np.exp(-xx**2/(2*sigma))
kernel /= sum(kernel)
for i in range(nspec):
for j in range(nwave):
Rdata[i,:,j] = kernel
#- Convolve the data with the resolution matrix
convflux = np.empty_like(flux)
for i in range(nspec):
convflux[i] = Resolution(Rdata[i]).dot(flux[i])
# create a fake fibermap
fibermap = io.empty_fibermap(nspec, nwave)
for i in range(0, nspec):
fibermap['OBJTYPE'][i] = 'FAKE'
io.write_fibermap(self.testfibermap, fibermap)
#- write out the frame
frame = Frame(wave, convflux, ivar, mask, Rdata, spectrograph=0, fibermap=fibermap)
write_frame(self.testframe, frame, fibermap=fibermap)
# set program arguments
argstr = [
'--infile', self.testframe,
'--outfile', self.testflat
]
# run it
args = ffscript.parse(options=argstr)
ffscript.main(args)
def setUp(self):
self.proddir = 'test-prod-' + uuid4().hex
self.testraw = 'test-' + uuid4().hex
os.mkdir(self.testraw)
self.night = time.strftime('%Y%m%d',
time.localtime(time.time() - 12 * 3600))
self.nightdir = os.path.join(self.testraw, self.night)
os.mkdir(self.nightdir)
for expid in [0, 1]:
fibermap = io.fibermap.empty_fibermap(10)
for key in fibermap.dtype.names:
column = fibermap[key]
fibermap[key] = np.random.random(column.shape).astype(
column.dtype)
hdr = {'flavor': 'flat'}
fmfile = os.path.join(self.nightdir,
"fibermap-{:08d}.fits".format(expid))
io.write_fibermap(fmfile, fibermap, header=hdr)
for expid in [2, 3]:
fibermap = io.fibermap.empty_fibermap(10)
for key in fibermap.dtype.names:
column = fibermap[key]
fibermap[key] = np.random.random(column.shape).astype(
column.dtype)
hdr = {'flavor': 'arc'}
fmfile = os.path.join(self.nightdir,
"fibermap-{:08d}.fits".format(expid))
io.write_fibermap(fmfile, fibermap, header=hdr)
for expid in [4, 5]:
fibermap = io.fibermap.empty_fibermap(10)
for key in fibermap.dtype.names:
column = fibermap[key]
fibermap[key] = np.random.random(column.shape).astype(
column.dtype)
hdr = {'flavor': 'science'}
fmfile = os.path.join(self.nightdir,
"fibermap-{:08d}.fits".format(expid))
io.write_fibermap(fmfile, fibermap, header=hdr)
for expid in range(6):
for band in ['b', 'r', 'z']:
for spec in range(10):
cam = "{}{}".format(band, spec)
pixfile = os.path.join(
self.nightdir, "pix-{}-{:08d}.fits".format(cam, expid))
with open(pixfile, 'w') as p:
p.write("")
def setUp(self):
self.proddir = 'test-prod-'+uuid4().hex
self.testraw = 'test-'+uuid4().hex
os.mkdir(self.testraw)
self.night = time.strftime('%Y%m%d', time.localtime(time.time()-12*3600))
self.nightdir = os.path.join(self.testraw, self.night)
os.mkdir(self.nightdir)
for expid in [0, 1]:
fibermap = io.fibermap.empty_fibermap(10)
for key in fibermap.dtype.names:
column = fibermap[key]
fibermap[key] = np.random.random(column.shape).astype(column.dtype)
hdr = {'flavor': 'flat'}
fmfile = os.path.join(self.nightdir, "fibermap-{:08d}.fits".format(expid))
io.write_fibermap(fmfile, fibermap, header=hdr)
for expid in [2, 3]:
fibermap = io.fibermap.empty_fibermap(10)
for key in fibermap.dtype.names:
column = fibermap[key]
fibermap[key] = np.random.random(column.shape).astype(column.dtype)
hdr = {'flavor': 'arc'}
fmfile = os.path.join(self.nightdir, "fibermap-{:08d}.fits".format(expid))
io.write_fibermap(fmfile, fibermap, header=hdr)
for expid in [4, 5]:
fibermap = io.fibermap.empty_fibermap(10)
for key in fibermap.dtype.names:
column = fibermap[key]
fibermap[key] = np.random.random(column.shape).astype(column.dtype)
hdr = {'flavor': 'science'}
fmfile = os.path.join(self.nightdir, "fibermap-{:08d}.fits".format(expid))
io.write_fibermap(fmfile, fibermap, header=hdr)
for expid in range(6):
for band in ['b', 'r', 'z']:
for spec in range(10):
cam = "{}{}".format(band, spec)
pixfile = os.path.join(self.nightdir, "pix-{}-{:08d}.fits".format(cam, expid))
with open(pixfile, 'w') as p:
p.write("")
def _write_fibermap(self):
"""Write a fake fibermap"""
fibermap = self._get_fibermap()
io.write_fibermap(self.fibermapfile, fibermap)
def setUp(self):
self.uid = uuid4().hex
self.testraw = "test_raw-{}".format(self.uid)
os.mkdir(self.testraw)
self.testprod = "test_redux-{}".format(self.uid)
os.mkdir(self.testprod)
self.night = time.strftime('%Y%m%d', time.localtime(time.time()-12*3600))
self.rawnightdir = os.path.join(self.testraw, self.night)
os.mkdir(self.rawnightdir)
self.cal2d = os.path.join(self.testprod, 'calib2d')
if not os.path.isdir(self.cal2d):
os.makedirs(self.cal2d)
self.calpsf = os.path.join(self.cal2d, 'psf')
if not os.path.isdir(self.calpsf):
os.makedirs(self.calpsf)
self.calpsfnight = os.path.join(self.calpsf, self.night)
if not os.path.isdir(self.calpsfnight):
os.makedirs(self.calpsfnight)
self.expdir = os.path.join(self.testprod, 'exposures')
if not os.path.isdir(self.expdir):
os.makedirs(self.expdir)
self.expnight = os.path.join(self.expdir, self.night)
if not os.path.isdir(self.expnight):
os.makedirs(self.expnight)
self.brkdir = os.path.join(self.testprod, 'bricks')
if not os.path.isdir(self.brkdir):
os.makedirs(self.brkdir)
self.logdir = os.path.join(self.testprod, 'logs')
if not os.path.isdir(self.logdir):
os.makedirs(self.logdir)
self.faildir = os.path.join(self.testprod, 'failed')
if not os.path.isdir(self.faildir):
os.makedirs(self.faildir)
self.scriptdir = os.path.join(self.testprod, 'scripts')
if not os.path.isdir(self.scriptdir):
os.makedirs(self.scriptdir)
for expid in [0, 1]:
fibermap = io.fibermap.empty_fibermap(10)
for key in fibermap.dtype.names:
column = fibermap[key]
fibermap[key] = np.random.random(column.shape).astype(column.dtype)
hdr = {'flavor': 'flat'}
fmfile = os.path.join(self.rawnightdir, "fibermap-{:08d}.fits".format(expid))
io.write_fibermap(fmfile, fibermap, header=hdr)
for expid in [2, 3]:
fibermap = io.fibermap.empty_fibermap(10)
for key in fibermap.dtype.names:
column = fibermap[key]
fibermap[key] = np.random.random(column.shape).astype(column.dtype)
hdr = {'flavor': 'arc'}
fmfile = os.path.join(self.rawnightdir, "fibermap-{:08d}.fits".format(expid))
io.write_fibermap(fmfile, fibermap, header=hdr)
for expid in [4, 5]:
fibermap = io.fibermap.empty_fibermap(10)
for key in fibermap.dtype.names:
column = fibermap[key]
fibermap[key] = np.random.random(column.shape).astype(column.dtype)
hdr = {'flavor': 'science'}
fmfile = os.path.join(self.rawnightdir, "fibermap-{:08d}.fits".format(expid))
io.write_fibermap(fmfile, fibermap, header=hdr)
for expid in range(6):
for band in ['b', 'r', 'z']:
for spec in range(10):
cam = "{}{}".format(band, spec)
pixfile = os.path.join(self.rawnightdir, "pix-{}-{:08d}.fits".format(cam, expid))
with open(pixfile, 'w') as p:
p.write("")
self.grph, expcount, bricks = graph_night(self.testraw, self.night)
graph_write(os.path.join(self.testraw, "{}_graph.yml".format(self.night)), self.grph)
def setUp(self):
#- Cleanup in case a previous failed test left something behind
if os.path.isdir(self.testraw):
shutil.rmtree(self.testraw)
if os.path.isdir(self.testprod):
shutil.rmtree(self.testprod)
#- Now create a fresh copy to work from
os.mkdir(self.testraw)
os.mkdir(self.testprod)
self.night = time.strftime('%Y%m%d',
time.localtime(time.time() - 12 * 3600))
self.rawnightdir = os.path.join(self.testraw, self.night)
os.mkdir(self.rawnightdir)
self.cal2d = os.path.join(self.testprod, 'calib2d')
if not os.path.isdir(self.cal2d):
os.makedirs(self.cal2d)
self.calpsf = os.path.join(self.cal2d, 'psf')
if not os.path.isdir(self.calpsf):
os.makedirs(self.calpsf)
self.calpsfnight = os.path.join(self.calpsf, self.night)
if not os.path.isdir(self.calpsfnight):
os.makedirs(self.calpsfnight)
self.expdir = os.path.join(self.testprod, 'exposures')
if not os.path.isdir(self.expdir):
os.makedirs(self.expdir)
self.expnight = os.path.join(self.expdir, self.night)
if not os.path.isdir(self.expnight):
os.makedirs(self.expnight)
self.brkdir = os.path.join(self.testprod, 'bricks')
if not os.path.isdir(self.brkdir):
os.makedirs(self.brkdir)
self.logdir = os.path.join(self.testprod, 'logs')
if not os.path.isdir(self.logdir):
os.makedirs(self.logdir)
self.faildir = os.path.join(self.testprod, 'failed')
if not os.path.isdir(self.faildir):
os.makedirs(self.faildir)
self.scriptdir = os.path.join(self.testprod, 'scripts')
if not os.path.isdir(self.scriptdir):
os.makedirs(self.scriptdir)
for expid in [0, 1]:
fibermap = io.fibermap.empty_fibermap(10)
for key in fibermap.dtype.names:
column = fibermap[key]
fibermap[key] = np.random.random(column.shape).astype(
column.dtype)
hdr = {'flavor': 'flat'}
fmfile = os.path.join(self.rawnightdir,
"fibermap-{:08d}.fits".format(expid))
io.write_fibermap(fmfile, fibermap, header=hdr)
for expid in [2, 3]:
fibermap = io.fibermap.empty_fibermap(10)
for key in fibermap.dtype.names:
column = fibermap[key]
fibermap[key] = np.random.random(column.shape).astype(
column.dtype)
hdr = {'flavor': 'arc'}
fmfile = os.path.join(self.rawnightdir,
"fibermap-{:08d}.fits".format(expid))
io.write_fibermap(fmfile, fibermap, header=hdr)
for expid in [4, 5]:
fibermap = io.fibermap.empty_fibermap(10)
for key in fibermap.dtype.names:
column = fibermap[key]
fibermap[key] = np.random.random(column.shape).astype(
column.dtype)
hdr = {'flavor': 'science'}
fmfile = os.path.join(self.rawnightdir,
"fibermap-{:08d}.fits".format(expid))
io.write_fibermap(fmfile, fibermap, header=hdr)
for expid in range(6):
for band in ['b', 'r', 'z']:
for spec in range(10):
cam = "{}{}".format(band, spec)
pixfile = os.path.join(
self.rawnightdir,
"pix-{}-{:08d}.fits".format(cam, expid))
with open(pixfile, 'w') as p:
p.write("")
def fake_raw():
dirhash = uuid.uuid4()
night = fake_night()
rawdir = "test_pipe_raw_{}".format(dirhash)
if os.path.exists(rawdir):
shutil.rmtree(rawdir)
os.makedirs(rawdir)
nightdir = os.path.join(rawdir, night)
# set up one spectrograph (500 fibers)
nspec = 500
# arc
expid = "00000000"
tileid = "0"
flavor = "arc"
telera = "0.0"
teledec = "0.0"
fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid))
hdr = dict(
NIGHT=(night, 'Night of observation YEARMMDD'),
EXPID=(expid, 'DESI exposure ID'),
TILEID=(tileid, 'DESI tile ID'),
FLAVOR=(flavor, 'Flavor [arc, flat, science, ...]'),
TELRA=(telera, 'Telescope pointing RA [degrees]'),
TELDEC=(teledec, 'Telescope pointing dec [degrees]'),
)
fibermap = io.empty_fibermap(nspec)
fibermap['OBJTYPE'] = 'ARC'
io.write_fibermap(fmfile, fibermap, header=hdr)
for cam in ["r0", "b0", "z0"]:
pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid))
pix = np.random.normal(0, 3.0, size=(10, 10))
ivar = np.ones_like(pix) / 3.0**2
mask = np.zeros(pix.shape, dtype=np.uint32)
img = dimg.Image(pix, ivar, mask, camera=cam)
io.write_image(pfile, img)
# flat
expid = "00000001"
tileid = "1"
flavor = "flat"
telera = "0.0"
teledec = "0.0"
fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid))
hdr = dict(
NIGHT=(night, 'Night of observation YEARMMDD'),
EXPID=(expid, 'DESI exposure ID'),
TILEID=(tileid, 'DESI tile ID'),
FLAVOR=(flavor, 'Flavor [arc, flat, science, ...]'),
TELRA=(telera, 'Telescope pointing RA [degrees]'),
TELDEC=(teledec, 'Telescope pointing dec [degrees]'),
)
fibermap = io.empty_fibermap(nspec)
fibermap['OBJTYPE'] = 'FLAT'
io.write_fibermap(fmfile, fibermap, header=hdr)
for cam in ["r0", "b0", "z0"]:
pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid))
pix = np.random.normal(0, 3.0, size=(10, 10))
ivar = np.ones_like(pix) / 3.0**2
mask = np.zeros(pix.shape, dtype=np.uint32)
img = dimg.Image(pix, ivar, mask, camera=cam)
io.write_image(pfile, img)
# science
expid = "00000002"
tileid = "2"
flavor = "dark"
telera = "0.0"
teledec = "0.0"
fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid))
hdr = dict(
NIGHT=(night, 'Night of observation YEARMMDD'),
EXPID=(expid, 'DESI exposure ID'),
TILEID=(tileid, 'DESI tile ID'),
FLAVOR=(flavor, 'Flavor [arc, flat, science, ...]'),
TELRA=(telera, 'Telescope pointing RA [degrees]'),
TELDEC=(teledec, 'Telescope pointing dec [degrees]'),
)
fibermap = io.empty_fibermap(nspec)
fibermap['OBJTYPE'] = 'ELG'
fibermap['FIBER'] = np.arange(nspec, dtype='i4')
fibermap['TARGETID'] = np.random.randint(sys.maxsize, size=nspec)
fibermap['BRICKNAME'] = ['3412p195' for x in range(nspec)]
io.write_fibermap(fmfile, fibermap, header=hdr)
for cam in ["r0", "b0", "z0"]:
pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid))
pix = np.random.normal(0, 3.0, size=(10, 10))
ivar = np.ones_like(pix) / 3.0**2
mask = np.zeros(pix.shape, dtype=np.uint32)
img = dimg.Image(pix, ivar, mask, camera=cam)
io.write_image(pfile, img)
return rawdir
def fake_raw():
dirhash = uuid.uuid4()
night = fake_night()
rawdir = "test_pipe_raw_{}".format(dirhash)
if os.path.exists(rawdir):
shutil.rmtree(rawdir)
os.makedirs(rawdir)
nightdir = os.path.join(rawdir, night)
# set up one spectrograph (500 fibers)
nspec = 500
# arc
expid = "00000000"
tileid = "0"
flavor = "arc"
telera = "0.0"
teledec = "0.0"
fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid))
hdr = dict(
NIGHT = (night, 'Night of observation YEARMMDD'),
EXPID = (expid, 'DESI exposure ID'),
TILEID = (tileid, 'DESI tile ID'),
FLAVOR = (flavor, 'Flavor [arc, flat, science, ...]'),
TELRA = (telera, 'Telescope pointing RA [degrees]'),
TELDEC = (teledec, 'Telescope pointing dec [degrees]'),
)
fibermap = io.empty_fibermap(nspec)
fibermap['OBJTYPE'] = 'ARC'
io.write_fibermap(fmfile, fibermap, header=hdr)
for cam in ["r0", "b0", "z0"]:
pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid))
pix = np.random.normal(0, 3.0, size=(10,10))
ivar = np.ones_like(pix) / 3.0**2
mask = np.zeros(pix.shape, dtype=np.uint32)
img = dimg.Image(pix, ivar, mask, camera=cam)
io.write_image(pfile, img)
# flat
expid = "00000001"
tileid = "1"
flavor = "flat"
telera = "0.0"
teledec = "0.0"
fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid))
hdr = dict(
NIGHT = (night, 'Night of observation YEARMMDD'),
EXPID = (expid, 'DESI exposure ID'),
TILEID = (tileid, 'DESI tile ID'),
FLAVOR = (flavor, 'Flavor [arc, flat, science, ...]'),
TELRA = (telera, 'Telescope pointing RA [degrees]'),
TELDEC = (teledec, 'Telescope pointing dec [degrees]'),
)
fibermap = io.empty_fibermap(nspec)
fibermap['OBJTYPE'] = 'FLAT'
io.write_fibermap(fmfile, fibermap, header=hdr)
for cam in ["r0", "b0", "z0"]:
pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid))
pix = np.random.normal(0, 3.0, size=(10,10))
ivar = np.ones_like(pix) / 3.0**2
mask = np.zeros(pix.shape, dtype=np.uint32)
img = dimg.Image(pix, ivar, mask, camera=cam)
io.write_image(pfile, img)
# science
expid = "00000002"
tileid = "2"
flavor = "dark"
telera = "0.0"
teledec = "0.0"
fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid))
hdr = dict(
NIGHT = (night, 'Night of observation YEARMMDD'),
EXPID = (expid, 'DESI exposure ID'),
TILEID = (tileid, 'DESI tile ID'),
FLAVOR = (flavor, 'Flavor [arc, flat, science, ...]'),
TELRA = (telera, 'Telescope pointing RA [degrees]'),
TELDEC = (teledec, 'Telescope pointing dec [degrees]'),
)
fibermap = io.empty_fibermap(nspec)
fibermap['OBJTYPE'] = 'ELG'
fibermap['FIBER'] = np.arange(nspec, dtype='i4')
fibermap['TARGETID'] = np.random.randint(sys.maxsize, size=nspec)
fibermap['BRICKNAME'] = [ '3412p195' for x in range(nspec) ]
io.write_fibermap(fmfile, fibermap, header=hdr)
for cam in ["r0", "b0", "z0"]:
pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid))
pix = np.random.normal(0, 3.0, size=(10,10))
ivar = np.ones_like(pix) / 3.0**2
mask = np.zeros(pix.shape, dtype=np.uint32)
img = dimg.Image(pix, ivar, mask, camera=cam)
io.write_image(pfile, img)
return rawdir
