def run_pa(self,
input_raw,
camera,
bias=False,
pixflat=False,
mask=False,
dump=False,
dumpfile=None):
import desispec.preproc
rawimage = input_raw[camera.upper()].data
header = input_raw[camera.upper()].header
if 'INHERIT' in header and header['INHERIT']:
h0 = input_raw[0].header
for key in h0.keys():
if key not in header:
header[key] = h0[key]
img = desispec.preproc.preproc(rawimage,
header,
bias=bias,
pixflat=pixflat,
mask=mask)
if dump and dumpfile is not None:
from desispec import io
night = img.meta['NIGHT']
expid = img.meta['EXPID']
io.write_image(dumpfile, img)
log.info("Wrote intermediate file %s after %s" %
(dumpfile, self.name))
return img
def run_pa(self,input_raw,camera,bias=False,pixflat=False,mask=False,dumpfile=None):
import desispec.preproc
rawimage=input_raw[camera.upper()].data
header=input_raw[camera.upper()].header
primary_header=input_raw[0].header
if 'INHERIT' in header and header['INHERIT']:
h0 = input_raw[0].header
for key in h0:
if key not in header:
header[key] = h0[key]
#- WARNING!!!This is a hack for QL to run on old raw images for QLF to be working on old set of data
if "PROGRAM" not in header:
log.info("WARNING!!! Temporary hack for QL to add header key PROGRAM. Only to facilitate QLF to work on their dataset. Remove this after some time and run with new data set")
header["PROGRAM"]= 'dark'
if header["FLAVOR"] not in [None,'bias','arc','flat','science']:
header["FLAVOR"] = 'science'
img = desispec.preproc.preproc(rawimage,header,primary_header,bias=bias,pixflat=pixflat,mask=mask)
if dumpfile is not None:
from desispec import io
night = img.meta['NIGHT']
expid = img.meta['EXPID']
io.write_image(dumpfile, img)
log.info("Wrote intermediate file %s after %s"%(dumpfile,self.name))
return img
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
if args.cameras is None:
args.cameras = [c + str(i) for c in 'brz' for i in range(10)]
else:
args.cameras = args.cameras.split(',')
if (args.bias is not None) or (args.pixflat is not None) or (args.mask
is not None):
if len(args.cameras) > 1:
raise ValueError(
'must use only one camera with --bias, --pixflat, --mask options'
)
if (args.pixfile is not None) and len(args.cameras) > 1:
raise ValueError('must use only one camera with --pixfile option')
if args.outdir is None:
args.outdir = os.getcwd()
for camera in args.cameras:
try:
img = io.read_raw(args.infile,
camera,
bias=args.bias,
pixflat=args.pixflat,
mask=args.mask)
except IOError:
log.error('Camera {} not in {}'.format(camera, args.infile))
continue
if args.pixfile is None:
night = img.meta['NIGHT']
expid = img.meta['EXPID']
pixfile = io.findfile('pix',
night=night,
expid=expid,
camera=camera,
outdir=args.outdir)
else:
pixfile = args.pixfile
io.write_image(pixfile, img)
def run_pa(self,input_raw,camera,bias=False,pixflat=False,mask=False,dump=False,dumpfile=None):
import desispec.preproc
rawimage=input_raw[camera.upper()].data
header=input_raw[camera.upper()].header
if 'INHERIT' in header and header['INHERIT']:
h0 = input_raw[0].header
for key in h0.keys():
if key not in header:
header[key] = h0[key]
img = desispec.preproc.preproc(rawimage,header,bias=bias,pixflat=pixflat,mask=mask)
if dump and dumpfile is not None:
from desispec import io
night = img.meta['NIGHT']
expid = img.meta['EXPID']
io.write_image(dumpfile, img)
log.info("Wrote intermediate file %s after %s"%(dumpfile,self.name))
return img
def run(self):
file_dir = '/global/cscratch1/sd/zhangkai/desi/test/'
if use_mpi:
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
test_file = file_dir + 'test' + str(rank).zfill(2) + '.fits.fz'
test_file_output = file_dir + 'test' + str(rank).zfill(
2) + '_out.fits.fz'
##### Read ######
print('Read ' + test_file)
rawdata = read_raw(test_file, camera='R1')
h = fitsio.read_header(test_file, 1)
##### Write ######
print('Write ' + test_file_output)
write_image(test_file_output,
rawdata) #write_raw(test_file_output,rawdata,None)
else:
pass
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
if args.cameras is None:
args.cameras = [c+str(i) for c in 'brz' for i in range(10)]
else:
args.cameras = args.cameras.split(',')
if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None):
if len(args.cameras) > 1:
raise ValueError('must use only one camera with --bias, --pixflat, --mask options')
if (args.pixfile is not None) and len(args.cameras) > 1:
raise ValueError('must use only one camera with --pixfile option')
if args.outdir is None:
args.outdir = os.getcwd()
for camera in args.cameras:
try:
img = io.read_raw(args.infile, camera,
bias=args.bias, pixflat=args.pixflat, mask=args.mask)
except IOError:
log.error('Camera {} not in {}'.format(camera, args.infile))
continue
if args.pixfile is None:
night = img.meta['NIGHT']
expid = img.meta['EXPID']
pixfile = io.findfile('pix', night=night, expid=expid, camera=camera,
outdir=args.outdir)
else:
pixfile = args.pixfile
io.write_image(pixfile, img)
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
bias=True
if args.bias : bias=args.bias
if args.nobias : bias=False
dark=True
if args.dark : dark=args.dark
if args.nodark : dark=False
pixflat=True
if args.pixflat : pixflat=args.pixflat
if args.nopixflat : pixflat=False
mask=True
if args.mask : mask=args.mask
if args.nomask : mask=False
if args.cameras is None:
args.cameras = [c+str(i) for c in 'brz' for i in range(10)]
else:
args.cameras = args.cameras.split(',')
if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None) or (args.dark is not None):
if len(args.cameras) > 1:
raise ValueError('must use only one camera with --bias, --dark, --pixflat, --mask options')
if (args.pixfile is not None):
log.warning('--pixfile is deprecated; please use --outfile instead')
if args.outfile is None:
args.outfile = args.pixfile
else:
log.critical("Set --outfile not --pixfile and certainly not both")
sys.exit(1)
if (args.outfile is not None) and len(args.cameras) > 1:
raise ValueError('must use only one camera with --outfile option')
if args.outdir is None:
args.outdir = os.getcwd()
log.warning('--outdir not specified; using {}'.format(args.outdir))
ccd_calibration_filename = None
if args.no_ccd_calib_filename :
ccd_calibration_filename = False
elif args.ccd_calib_filename is not None :
ccd_calibration_filename = args.ccd_calib_filename
for camera in args.cameras:
try:
img = io.read_raw(args.infile, camera,
bias=bias, dark=dark, pixflat=pixflat, mask=mask, bkgsub=args.bkgsub,
nocosmic=args.nocosmic,
cosmics_nsig=args.cosmics_nsig,
cosmics_cfudge=args.cosmics_cfudge,
cosmics_c2fudge=args.cosmics_c2fudge,
ccd_calibration_filename=ccd_calibration_filename,
nocrosstalk=args.nocrosstalk,
nogain=args.nogain,
fill_header=args.fill_header
)
except IOError:
log.error('Error while reading or preprocessing camera {} in {}'.format(camera, args.infile))
continue
if(args.zero_masked) :
img.pix *= (img.mask==0)
if args.outfile is None:
night = img.meta['NIGHT']
expid = img.meta['EXPID']
outfile = io.findfile('preproc', night=night, expid=expid, camera=camera,
outdir=args.outdir)
else:
outfile = args.outfile
io.write_image(outfile, img)
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
bias=True
if args.bias : bias=args.bias
if args.nobias : bias=False
dark=True
if args.dark : dark=args.dark
if args.nodark : dark=False
pixflat=True
if args.pixflat : pixflat=args.pixflat
if args.nopixflat : pixflat=False
mask=True
if args.mask : mask=args.mask
if args.nomask : mask=False
if args.cameras is None:
args.cameras = [c+str(i) for c in 'brz' for i in range(10)]
else:
args.cameras = args.cameras.split(',')
if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None) or (args.dark is not None):
if len(args.cameras) > 1:
raise ValueError('must use only one camera with --bias, --dark, --pixflat, --mask options')
if (args.outfile is not None) and len(args.cameras) > 1:
raise ValueError('must use only one camera with --outfile option')
if args.outdir is None:
args.outdir = os.getcwd()
log.warning('--outdir not specified; using {}'.format(args.outdir))
ccd_calibration_filename = None
if args.no_ccd_calib_filename :
ccd_calibration_filename = False
elif args.ccd_calib_filename is not None :
ccd_calibration_filename = args.ccd_calib_filename
if args.fibermap and not os.path.exists(fibermap):
raise ValueError('--fibermap {} not found'.format(args.fibermap))
if args.fibermap is None:
datadir, infile = os.path.split(os.path.abspath(args.infile))
fibermapfile = infile.replace('desi-', 'fibermap-').replace('.fits.fz', '.fits')
args.fibermap = os.path.join(datadir, fibermapfile)
if args.nofibermap:
fibermap = None
elif os.path.exists(args.fibermap):
fibermap = io.read_fibermap(args.fibermap)
else:
log.warning('fibermap file not found; creating blank fibermap')
fibermap = io.empty_fibermap(5000)
for camera in args.cameras:
try:
img = io.read_raw(args.infile, camera,
bias=bias, dark=dark, pixflat=pixflat, mask=mask, bkgsub=args.bkgsub,
nocosmic=args.nocosmic,
cosmics_nsig=args.cosmics_nsig,
cosmics_cfudge=args.cosmics_cfudge,
cosmics_c2fudge=args.cosmics_c2fudge,
ccd_calibration_filename=ccd_calibration_filename,
nocrosstalk=args.nocrosstalk,
nogain=args.nogain,
nodarktrail=args.nodarktrail,
fill_header=args.fill_header,
)
except IOError:
log.error('Error while reading or preprocessing camera {} in {}'.format(camera, args.infile))
continue
if(args.zero_masked) :
img.pix *= (img.mask==0)
if args.outfile is None:
night = img.meta['NIGHT']
expid = img.meta['EXPID']
outfile = io.findfile('preproc', night=night, expid=expid, camera=camera,
outdir=args.outdir)
else:
outfile = args.outfile
if fibermap:
petal_loc = int(img.camera[1])
ii = (fibermap['PETAL_LOC'] == petal_loc)
img.fibermap = fibermap[ii]
io.write_image(outfile, img)
log.info("Wrote {}".format(outfile))
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
bias = True
if args.bias: bias = args.bias
if args.nobias: bias = False
dark = True
if args.dark: dark = args.dark
if args.nodark: dark = False
pixflat = True
if args.pixflat: pixflat = args.pixflat
if args.nopixflat: pixflat = False
mask = True
if args.mask: mask = args.mask
if args.nomask: mask = False
if args.cameras is None:
args.cameras = [c + str(i) for c in 'brz' for i in range(10)]
else:
args.cameras = args.cameras.split(',')
if (args.bias is not None) or (args.pixflat is not None) or (
args.mask is not None) or (args.dark is not None):
if len(args.cameras) > 1:
raise ValueError(
'must use only one camera with --bias, --dark, --pixflat, --mask options'
)
if (args.pixfile is not None):
log.warning('--pixfile is deprecated; please use --outfile instead')
if args.outfile is None:
args.outfile = args.pixfile
else:
log.critical("Set --outfile not --pixfile and certainly not both")
sys.exit(1)
if (args.outfile is not None) and len(args.cameras) > 1:
raise ValueError('must use only one camera with --outfile option')
if args.outdir is None:
args.outdir = os.getcwd()
log.warning('--outdir not specified; using {}'.format(args.outdir))
ccd_calibration_filename = None
if args.no_ccd_calib_filename:
ccd_calibration_filename = False
elif args.ccd_calib_filename is not None:
ccd_calibration_filename = args.ccd_calib_filename
for camera in args.cameras:
try:
img = io.read_raw(
args.infile,
camera,
bias=bias,
dark=dark,
pixflat=pixflat,
mask=mask,
bkgsub=args.bkgsub,
nocosmic=args.nocosmic,
cosmics_nsig=args.cosmics_nsig,
cosmics_cfudge=args.cosmics_cfudge,
cosmics_c2fudge=args.cosmics_c2fudge,
ccd_calibration_filename=ccd_calibration_filename,
nocrosstalk=args.nocrosstalk,
nogain=args.nogain,
fill_header=args.fill_header)
except IOError:
log.error(
'Error while reading or preprocessing camera {} in {}'.format(
camera, args.infile))
continue
if (args.zero_masked):
img.pix *= (img.mask == 0)
if args.outfile is None:
night = img.meta['NIGHT']
expid = img.meta['EXPID']
outfile = io.findfile('preproc',
night=night,
expid=expid,
camera=camera,
outdir=args.outdir)
else:
outfile = args.outfile
io.write_image(outfile, img)
def preproc_file(infile,
camera,
outfile=None,
outdir=None,
fibermap=None,
zero_masked=False,
**preproc_opts):
"""
Preprocess a single camera from a single input file
Args:
infile : input raw data file
camera : camera, e.g. 'b0', 'r1', 'z9'
Options:
outfile: output preprocessed image file to write
outdir: output directory; derive filename from infile NIGHT and EXPID
fibermap: fibermap filename to include in output
zero_masked (bool): set masked pixels to 0
preproc_opts: dictionary to pass to preproc
Returns error code (1=error, 0=success) but will not raise exception if
there is an I/O or preprocessing failure (allows other parallel procs to
proceed).
Note: either `outfile` or `outdir` must be provided
"""
try:
img = io.read_raw(infile,
camera,
fibermapfile=fibermap,
**preproc_opts)
except IOError as e:
#- print error and return error code, but don't raise exception so
#- that this won't block other cameras for multiprocessing
log.error(
'Error while reading or preprocessing camera {} in {}'.format(
camera, infile))
log.error(e)
return 1
if zero_masked:
log.info("Setting masked pixels values to zero")
img.pix *= (img.mask == 0)
if outfile is None:
night = img.meta['NIGHT']
if not 'EXPID' in img.meta.keys():
if 'EXPNUM' in img.meta.keys():
img.meta['EXPID'] = img.meta['EXPNUM']
else:
mess = "no EXPID nor EXPNUM in img.meta, cannot create output filename"
log.error(mess)
raise KeyError(mess)
expid = img.meta['EXPID']
outfile = io.findfile('preproc',
night=night,
expid=expid,
camera=camera,
outdir=outdir)
io.write_image(outfile, img)
log.info("Wrote {}".format(outfile))
return 0
def main(args=None):
log = get_logger()
if args is None:
args = parse()
first_preproc = None
images = []
ivars = []
masks = []
log.info("read inputs ...")
for filename in args.infile:
log.info(" read {}".format(filename))
tmp = read_image(filename)
if first_preproc is None:
first_preproc = tmp
images.append(tmp.pix.ravel())
# make sure we don't include data with ivar=0
mask = tmp.mask + (tmp.ivar == 0).astype(int)
masks.append(mask.ravel())
ivars.append((tmp.ivar * (tmp.mask == 0)).ravel())
images = np.array(images)
masks = np.array(masks)
ivars = np.array(ivars)
smask = np.sum(masks, axis=0)
log.info("compute masked median image ...")
medimage = masked_median(images, masks)
log.info("use masked median image to discard outlier ...")
good = ((images - medimage)**2 * (ivars > 0) /
((medimage > 0) * (medimage * args.fracerr)**2 + 1. /
(ivars + (ivars == 0)))) < args.nsig**3
ivars *= good.astype(float)
if args.weighted:
log.info("compute weighted mean ...")
sw = np.sum(ivars, axis=0)
swf = np.sum(ivars * images, axis=0)
meanimage = swf / (sw + (sw == 0))
meanivar = sw
else:
log.info("compute unweighted mean ...")
s1 = np.sum((ivars > 0).astype(int), axis=0)
sf = np.sum((ivars > 0) * images, axis=0)
meanimage = sf / (s1 + (s1 == 0))
meanvar = np.sum(
(ivars > 0) / (ivars + (ivars == 0))) / (s1 + (s1 == 0))**2
meanivar = (meanvar > 0) / (meanvar + (meanvar == 0))
log.info("write nimages.fits ...")
fitsio.write("nimages.fits",
s1.reshape(first_preproc.pix.shape),
clobber=True)
log.info("compute mask ...")
meanmask = masks[0]
for mask in masks[1:]:
meanmask &= mask
log.info("write image ...")
preproc = first_preproc
shape = preproc.pix.shape
preproc.pix = meanimage.reshape(shape)
preproc.ivar = meanivar.reshape(shape)
preproc.mask = meanmask.reshape(shape)
write_image(args.outfile, preproc)
log.info("wrote {}".format(args.outfile))
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
t0 = time.time()
log = get_logger()
# guess if it is a preprocessed or a raw image
hdulist = fits.open(args.image)
is_input_preprocessed = ("IMAGE" in hdulist) & ("IVAR" in hdulist)
primary_header = hdulist[0].header
hdulist.close()
if is_input_preprocessed:
image = read_image(args.image)
else:
if args.camera is None:
print(
"ERROR: Need to specify camera to open a raw fits image (with all cameras in different fits HDUs)"
)
print(
"Try adding the option '--camera xx', with xx in {brz}{0-9}, like r7, or type 'desi_qproc --help' for more options"
)
sys.exit(12)
image = read_raw(args.image, args.camera, fill_header=[
1,
])
if args.auto:
log.debug("AUTOMATIC MODE")
try:
night = image.meta['NIGHT']
if not 'EXPID' in image.meta:
if 'EXPNUM' in image.meta:
log.warning('using EXPNUM {} for EXPID'.format(
image.meta['EXPNUM']))
image.meta['EXPID'] = image.meta['EXPNUM']
expid = image.meta['EXPID']
except KeyError as e:
log.error(
"Need at least NIGHT and EXPID (or EXPNUM) to run in auto mode. Retry without the --auto option."
)
log.error(str(e))
sys.exit(12)
indir = os.path.dirname(args.image)
if args.fibermap is None:
filename = '{}/fibermap-{:08d}.fits'.format(indir, expid)
if os.path.isfile(filename):
log.debug("auto-mode: found a fibermap, {}, using it!".format(
filename))
args.fibermap = filename
if args.output_preproc is None:
if not is_input_preprocessed:
args.output_preproc = '{}/preproc-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera.lower(), expid)
log.debug("auto-mode: will write preproc in " +
args.output_preproc)
else:
log.debug(
"auto-mode: will not write preproc because input is a preprocessed image"
)
if args.auto_output_dir != '.':
if not os.path.isdir(args.auto_output_dir):
log.debug("auto-mode: creating directory " +
args.auto_output_dir)
os.makedirs(args.auto_output_dir)
if args.output_preproc is not None:
write_image(args.output_preproc, image)
cfinder = None
if args.psf is None:
if cfinder is None:
cfinder = CalibFinder([image.meta, primary_header])
args.psf = cfinder.findfile("PSF")
log.info(" Using PSF {}".format(args.psf))
tset = read_xytraceset(args.psf)
# add fibermap
if args.fibermap:
if os.path.isfile(args.fibermap):
fibermap = read_fibermap(args.fibermap)
else:
log.error("no fibermap file {}".format(args.fibermap))
fibermap = None
else:
fibermap = None
if "OBSTYPE" in image.meta:
obstype = image.meta["OBSTYPE"].upper()
image.meta["OBSTYPE"] = obstype # make sure it's upper case
qframe = None
else:
log.warning("No OBSTYPE keyword, trying to guess ...")
qframe = qproc_boxcar_extraction(tset,
image,
width=args.width,
fibermap=fibermap)
obstype = check_qframe_flavor(
qframe, input_flavor=image.meta["FLAVOR"]).upper()
image.meta["OBSTYPE"] = obstype
log.info("OBSTYPE = '{}'".format(obstype))
if args.auto:
# now set the things to do
if obstype == "SKY" or obstype == "TWILIGHT" or obstype == "SCIENCE":
args.shift_psf = True
args.output_psf = '{}/psf-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.apply_fiberflat = True
args.skysub = True
args.output_skyframe = '{}/qsky-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.fluxcalib = True
args.outframe = '{}/qcframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
elif obstype == "ARC" or obstype == "TESTARC":
args.shift_psf = True
args.output_psf = '{}/psf-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.compute_lsf_sigma = True
elif obstype == "FLAT" or obstype == "TESTFLAT":
args.shift_psf = True
args.output_psf = '{}/psf-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.compute_fiberflat = '{}/qfiberflat-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
if args.shift_psf:
# using the trace shift script
if args.auto:
options = option_list({
"psf":
args.psf,
"image":
"dummy",
"outpsf":
"dummy",
"continuum": ((obstype == "FLAT") | (obstype == "TESTFLAT")),
"sky": ((obstype == "SCIENCE") | (obstype == "SKY"))
})
else:
options = option_list({
"psf": args.psf,
"image": "dummy",
"outpsf": "dummy"
})
tmp_args = trace_shifts_script.parse(options=options)
tset = trace_shifts_script.fit_trace_shifts(image=image, args=tmp_args)
qframe = qproc_boxcar_extraction(tset,
image,
width=args.width,
fibermap=fibermap)
if tset.meta is not None:
# add traceshift info in the qframe, this will be saved in the qframe header
if qframe.meta is None:
qframe.meta = dict()
for k in tset.meta.keys():
qframe.meta[k] = tset.meta[k]
if args.output_rawframe is not None:
write_qframe(args.output_rawframe, qframe)
log.info("wrote raw extracted frame in {}".format(
args.output_rawframe))
if args.compute_lsf_sigma:
tset = process_arc(qframe, tset, linelist=None, npoly=2, nbins=2)
if args.output_psf is not None:
for k in qframe.meta:
if k not in tset.meta:
tset.meta[k] = qframe.meta[k]
write_xytraceset(args.output_psf, tset)
if args.compute_fiberflat is not None:
fiberflat = qproc_compute_fiberflat(qframe)
#write_qframe(args.compute_fiberflat,qflat)
write_fiberflat(args.compute_fiberflat, fiberflat, header=qframe.meta)
log.info("wrote fiberflat in {}".format(args.compute_fiberflat))
if args.apply_fiberflat or args.input_fiberflat:
if args.input_fiberflat is None:
if cfinder is None:
cfinder = CalibFinder([image.meta, primary_header])
try:
args.input_fiberflat = cfinder.findfile("FIBERFLAT")
except KeyError as e:
log.error("no FIBERFLAT for this spectro config")
sys.exit(12)
log.info("applying fiber flat {}".format(args.input_fiberflat))
flat = read_fiberflat(args.input_fiberflat)
qproc_apply_fiberflat(qframe, flat)
if args.skysub:
log.info("sky subtraction")
if args.output_skyframe is not None:
skyflux = qproc_sky_subtraction(qframe, return_skymodel=True)
sqframe = QFrame(qframe.wave, skyflux, np.ones(skyflux.shape))
write_qframe(args.output_skyframe, sqframe)
log.info("wrote sky model in {}".format(args.output_skyframe))
else:
qproc_sky_subtraction(qframe)
if args.fluxcalib:
if cfinder is None:
cfinder = CalibFinder([image.meta, primary_header])
# check for flux calib
if cfinder.haskey("FLUXCALIB"):
fluxcalib_filename = cfinder.findfile("FLUXCALIB")
fluxcalib = read_average_flux_calibration(fluxcalib_filename)
log.info("read average calib in {}".format(fluxcalib_filename))
seeing = qframe.meta["SEEING"]
airmass = qframe.meta["AIRMASS"]
exptime = qframe.meta["EXPTIME"]
exposure_calib = fluxcalib.value(seeing=seeing, airmass=airmass)
for q in range(qframe.nspec):
fiber_calib = np.interp(qframe.wave[q], fluxcalib.wave,
exposure_calib) * exptime
inv_calib = (fiber_calib > 0) / (fiber_calib +
(fiber_calib == 0))
qframe.flux[q] *= inv_calib
qframe.ivar[q] *= fiber_calib**2 * (fiber_calib > 0)
# add keyword in header giving the calibration factor applied at a reference wavelength
band = qframe.meta["CAMERA"].upper()[0]
if band == "B":
refwave = 4500
elif band == "R":
refwave = 6500
else:
refwave = 8500
calvalue = np.interp(refwave, fluxcalib.wave,
exposure_calib) * exptime
qframe.meta["CALWAVE"] = refwave
qframe.meta["CALVALUE"] = calvalue
else:
log.error(
"Cannot calibrate fluxes because no FLUXCALIB keywork in calibration files"
)
fibers = parse_fibers(args.fibers)
if fibers is None:
fibers = qframe.flux.shape[0]
else:
ii = np.arange(qframe.fibers.size)[np.in1d(qframe.fibers, fibers)]
if ii.size == 0:
log.error("no such fibers in frame,")
log.error("fibers are in range [{}:{}]".format(
qframe.fibers[0], qframe.fibers[-1] + 1))
sys.exit(12)
qframe = qframe[ii]
if args.outframe is not None:
write_qframe(args.outframe, qframe)
log.info("wrote {}".format(args.outframe))
t1 = time.time()
log.info("all done in {:3.1f} sec".format(t1 - t0))
if args.plot:
log.info("plotting {} spectra".format(qframe.wave.shape[0]))
import matplotlib.pyplot as plt
fig = plt.figure()
for i in range(qframe.wave.shape[0]):
j = (qframe.ivar[i] > 0)
plt.plot(qframe.wave[i, j], qframe.flux[i, j])
plt.grid()
plt.xlabel("wavelength")
plt.ylabel("flux")
plt.show()
def fake_raw():
dirhash = uuid.uuid4()
night = fake_night()
rawdir = "test_pipe_raw_{}".format(dirhash)
if os.path.exists(rawdir):
shutil.rmtree(rawdir)
os.makedirs(rawdir)
nightdir = os.path.join(rawdir, night)
# set up one spectrograph (500 fibers)
nspec = 500
# arc
expid = "00000000"
tileid = "0"
flavor = "arc"
telera = "0.0"
teledec = "0.0"
fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid))
hdr = dict(
NIGHT=(night, 'Night of observation YEARMMDD'),
EXPID=(expid, 'DESI exposure ID'),
TILEID=(tileid, 'DESI tile ID'),
FLAVOR=(flavor, 'Flavor [arc, flat, science, ...]'),
TELRA=(telera, 'Telescope pointing RA [degrees]'),
TELDEC=(teledec, 'Telescope pointing dec [degrees]'),
)
fibermap = io.empty_fibermap(nspec)
fibermap['OBJTYPE'] = 'ARC'
io.write_fibermap(fmfile, fibermap, header=hdr)
for cam in ["r0", "b0", "z0"]:
pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid))
pix = np.random.normal(0, 3.0, size=(10, 10))
ivar = np.ones_like(pix) / 3.0**2
mask = np.zeros(pix.shape, dtype=np.uint32)
img = dimg.Image(pix, ivar, mask, camera=cam)
io.write_image(pfile, img)
# flat
expid = "00000001"
tileid = "1"
flavor = "flat"
telera = "0.0"
teledec = "0.0"
fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid))
hdr = dict(
NIGHT=(night, 'Night of observation YEARMMDD'),
EXPID=(expid, 'DESI exposure ID'),
TILEID=(tileid, 'DESI tile ID'),
FLAVOR=(flavor, 'Flavor [arc, flat, science, ...]'),
TELRA=(telera, 'Telescope pointing RA [degrees]'),
TELDEC=(teledec, 'Telescope pointing dec [degrees]'),
)
fibermap = io.empty_fibermap(nspec)
fibermap['OBJTYPE'] = 'FLAT'
io.write_fibermap(fmfile, fibermap, header=hdr)
for cam in ["r0", "b0", "z0"]:
pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid))
pix = np.random.normal(0, 3.0, size=(10, 10))
ivar = np.ones_like(pix) / 3.0**2
mask = np.zeros(pix.shape, dtype=np.uint32)
img = dimg.Image(pix, ivar, mask, camera=cam)
io.write_image(pfile, img)
# science
expid = "00000002"
tileid = "2"
flavor = "dark"
telera = "0.0"
teledec = "0.0"
fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid))
hdr = dict(
NIGHT=(night, 'Night of observation YEARMMDD'),
EXPID=(expid, 'DESI exposure ID'),
TILEID=(tileid, 'DESI tile ID'),
FLAVOR=(flavor, 'Flavor [arc, flat, science, ...]'),
TELRA=(telera, 'Telescope pointing RA [degrees]'),
TELDEC=(teledec, 'Telescope pointing dec [degrees]'),
)
fibermap = io.empty_fibermap(nspec)
fibermap['OBJTYPE'] = 'ELG'
fibermap['FIBER'] = np.arange(nspec, dtype='i4')
fibermap['TARGETID'] = np.random.randint(sys.maxsize, size=nspec)
fibermap['BRICKNAME'] = ['3412p195' for x in range(nspec)]
io.write_fibermap(fmfile, fibermap, header=hdr)
for cam in ["r0", "b0", "z0"]:
pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid))
pix = np.random.normal(0, 3.0, size=(10, 10))
ivar = np.ones_like(pix) / 3.0**2
mask = np.zeros(pix.shape, dtype=np.uint32)
img = dimg.Image(pix, ivar, mask, camera=cam)
io.write_image(pfile, img)
return rawdir
def fake_raw():
dirhash = uuid.uuid4()
night = fake_night()
rawdir = "test_pipe_raw_{}".format(dirhash)
if os.path.exists(rawdir):
shutil.rmtree(rawdir)
os.makedirs(rawdir)
nightdir = os.path.join(rawdir, night)
# set up one spectrograph (500 fibers)
nspec = 500
# arc
expid = "00000000"
tileid = "0"
flavor = "arc"
telera = "0.0"
teledec = "0.0"
fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid))
hdr = dict(
NIGHT = (night, 'Night of observation YEARMMDD'),
EXPID = (expid, 'DESI exposure ID'),
TILEID = (tileid, 'DESI tile ID'),
FLAVOR = (flavor, 'Flavor [arc, flat, science, ...]'),
TELRA = (telera, 'Telescope pointing RA [degrees]'),
TELDEC = (teledec, 'Telescope pointing dec [degrees]'),
)
fibermap = io.empty_fibermap(nspec)
fibermap['OBJTYPE'] = 'ARC'
io.write_fibermap(fmfile, fibermap, header=hdr)
for cam in ["r0", "b0", "z0"]:
pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid))
pix = np.random.normal(0, 3.0, size=(10,10))
ivar = np.ones_like(pix) / 3.0**2
mask = np.zeros(pix.shape, dtype=np.uint32)
img = dimg.Image(pix, ivar, mask, camera=cam)
io.write_image(pfile, img)
# flat
expid = "00000001"
tileid = "1"
flavor = "flat"
telera = "0.0"
teledec = "0.0"
fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid))
hdr = dict(
NIGHT = (night, 'Night of observation YEARMMDD'),
EXPID = (expid, 'DESI exposure ID'),
TILEID = (tileid, 'DESI tile ID'),
FLAVOR = (flavor, 'Flavor [arc, flat, science, ...]'),
TELRA = (telera, 'Telescope pointing RA [degrees]'),
TELDEC = (teledec, 'Telescope pointing dec [degrees]'),
)
fibermap = io.empty_fibermap(nspec)
fibermap['OBJTYPE'] = 'FLAT'
io.write_fibermap(fmfile, fibermap, header=hdr)
for cam in ["r0", "b0", "z0"]:
pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid))
pix = np.random.normal(0, 3.0, size=(10,10))
ivar = np.ones_like(pix) / 3.0**2
mask = np.zeros(pix.shape, dtype=np.uint32)
img = dimg.Image(pix, ivar, mask, camera=cam)
io.write_image(pfile, img)
# science
expid = "00000002"
tileid = "2"
flavor = "dark"
telera = "0.0"
teledec = "0.0"
fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid))
hdr = dict(
NIGHT = (night, 'Night of observation YEARMMDD'),
EXPID = (expid, 'DESI exposure ID'),
TILEID = (tileid, 'DESI tile ID'),
FLAVOR = (flavor, 'Flavor [arc, flat, science, ...]'),
TELRA = (telera, 'Telescope pointing RA [degrees]'),
TELDEC = (teledec, 'Telescope pointing dec [degrees]'),
)
fibermap = io.empty_fibermap(nspec)
fibermap['OBJTYPE'] = 'ELG'
fibermap['FIBER'] = np.arange(nspec, dtype='i4')
fibermap['TARGETID'] = np.random.randint(sys.maxsize, size=nspec)
fibermap['BRICKNAME'] = [ '3412p195' for x in range(nspec) ]
io.write_fibermap(fmfile, fibermap, header=hdr)
for cam in ["r0", "b0", "z0"]:
pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid))
pix = np.random.normal(0, 3.0, size=(10,10))
ivar = np.ones_like(pix) / 3.0**2
mask = np.zeros(pix.shape, dtype=np.uint32)
img = dimg.Image(pix, ivar, mask, camera=cam)
io.write_image(pfile, img)
return rawdir
from astropy.io import fits
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import os
from os import listdir
import pdb
from matplotlib.backends.backend_pdf import PdfPages
from scipy.signal import butter, lfilter, freqz
from scipy.signal import savgol_filter
from desispec import io
"""
img = io.read_raw(args.infile, camera,
fibermapfile=args.fibermap,
bias=bias, dark=dark, pixflat=pixflat, mask=mask, bkgsub=args.bkgsub,
nocosmic=args.nocosmic,
cosmics_nsig=args.cosmics_nsig,
cosmics_cfudge=args.cosmics_cfudge,
cosmics_c2fudge=args.cosmics_c2fudge,
ccd_calibration_filename=ccd_calibration_filename,
nocrosstalk=args.nocrosstalk,
nogain=args.nogain,
use_savgol=args.use_savgol,
nodarktrail=args.nodarktrail,
fill_header=args.fill_header,
remove_scattered_light=args.scattered_light
)
io.write_image(outfile, img)
#- Actually write or update the file
if os.path.exists(filename):