def run_pa(self, input_raw, camera, bias=False, pixflat=False, mask=False, dump=False, dumpfile=None): import desispec.preproc rawimage = input_raw[camera.upper()].data header = input_raw[camera.upper()].header if 'INHERIT' in header and header['INHERIT']: h0 = input_raw[0].header for key in h0.keys(): if key not in header: header[key] = h0[key] img = desispec.preproc.preproc(rawimage, header, bias=bias, pixflat=pixflat, mask=mask) if dump and dumpfile is not None: from desispec import io night = img.meta['NIGHT'] expid = img.meta['EXPID'] io.write_image(dumpfile, img) log.info("Wrote intermediate file %s after %s" % (dumpfile, self.name)) return img
def run_pa(self,input_raw,camera,bias=False,pixflat=False,mask=False,dumpfile=None): import desispec.preproc rawimage=input_raw[camera.upper()].data header=input_raw[camera.upper()].header primary_header=input_raw[0].header if 'INHERIT' in header and header['INHERIT']: h0 = input_raw[0].header for key in h0: if key not in header: header[key] = h0[key] #- WARNING!!!This is a hack for QL to run on old raw images for QLF to be working on old set of data if "PROGRAM" not in header: log.info("WARNING!!! Temporary hack for QL to add header key PROGRAM. Only to facilitate QLF to work on their dataset. Remove this after some time and run with new data set") header["PROGRAM"]= 'dark' if header["FLAVOR"] not in [None,'bias','arc','flat','science']: header["FLAVOR"] = 'science' img = desispec.preproc.preproc(rawimage,header,primary_header,bias=bias,pixflat=pixflat,mask=mask) if dumpfile is not None: from desispec import io night = img.meta['NIGHT'] expid = img.meta['EXPID'] io.write_image(dumpfile, img) log.info("Wrote intermediate file %s after %s"%(dumpfile,self.name)) return img
def main(args=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) if args.cameras is None: args.cameras = [c + str(i) for c in 'brz' for i in range(10)] else: args.cameras = args.cameras.split(',') if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None): if len(args.cameras) > 1: raise ValueError( 'must use only one camera with --bias, --pixflat, --mask options' ) if (args.pixfile is not None) and len(args.cameras) > 1: raise ValueError('must use only one camera with --pixfile option') if args.outdir is None: args.outdir = os.getcwd() for camera in args.cameras: try: img = io.read_raw(args.infile, camera, bias=args.bias, pixflat=args.pixflat, mask=args.mask) except IOError: log.error('Camera {} not in {}'.format(camera, args.infile)) continue if args.pixfile is None: night = img.meta['NIGHT'] expid = img.meta['EXPID'] pixfile = io.findfile('pix', night=night, expid=expid, camera=camera, outdir=args.outdir) else: pixfile = args.pixfile io.write_image(pixfile, img)
def run_pa(self,input_raw,camera,bias=False,pixflat=False,mask=False,dump=False,dumpfile=None): import desispec.preproc rawimage=input_raw[camera.upper()].data header=input_raw[camera.upper()].header if 'INHERIT' in header and header['INHERIT']: h0 = input_raw[0].header for key in h0.keys(): if key not in header: header[key] = h0[key] img = desispec.preproc.preproc(rawimage,header,bias=bias,pixflat=pixflat,mask=mask) if dump and dumpfile is not None: from desispec import io night = img.meta['NIGHT'] expid = img.meta['EXPID'] io.write_image(dumpfile, img) log.info("Wrote intermediate file %s after %s"%(dumpfile,self.name)) return img
def run(self): file_dir = '/global/cscratch1/sd/zhangkai/desi/test/' if use_mpi: comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() test_file = file_dir + 'test' + str(rank).zfill(2) + '.fits.fz' test_file_output = file_dir + 'test' + str(rank).zfill( 2) + '_out.fits.fz' ##### Read ###### print('Read ' + test_file) rawdata = read_raw(test_file, camera='R1') h = fitsio.read_header(test_file, 1) ##### Write ###### print('Write ' + test_file_output) write_image(test_file_output, rawdata) #write_raw(test_file_output,rawdata,None) else: pass
def main(args=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) if args.cameras is None: args.cameras = [c+str(i) for c in 'brz' for i in range(10)] else: args.cameras = args.cameras.split(',') if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None): if len(args.cameras) > 1: raise ValueError('must use only one camera with --bias, --pixflat, --mask options') if (args.pixfile is not None) and len(args.cameras) > 1: raise ValueError('must use only one camera with --pixfile option') if args.outdir is None: args.outdir = os.getcwd() for camera in args.cameras: try: img = io.read_raw(args.infile, camera, bias=args.bias, pixflat=args.pixflat, mask=args.mask) except IOError: log.error('Camera {} not in {}'.format(camera, args.infile)) continue if args.pixfile is None: night = img.meta['NIGHT'] expid = img.meta['EXPID'] pixfile = io.findfile('pix', night=night, expid=expid, camera=camera, outdir=args.outdir) else: pixfile = args.pixfile io.write_image(pixfile, img)
def main(args=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) bias=True if args.bias : bias=args.bias if args.nobias : bias=False dark=True if args.dark : dark=args.dark if args.nodark : dark=False pixflat=True if args.pixflat : pixflat=args.pixflat if args.nopixflat : pixflat=False mask=True if args.mask : mask=args.mask if args.nomask : mask=False if args.cameras is None: args.cameras = [c+str(i) for c in 'brz' for i in range(10)] else: args.cameras = args.cameras.split(',') if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None) or (args.dark is not None): if len(args.cameras) > 1: raise ValueError('must use only one camera with --bias, --dark, --pixflat, --mask options') if (args.pixfile is not None): log.warning('--pixfile is deprecated; please use --outfile instead') if args.outfile is None: args.outfile = args.pixfile else: log.critical("Set --outfile not --pixfile and certainly not both") sys.exit(1) if (args.outfile is not None) and len(args.cameras) > 1: raise ValueError('must use only one camera with --outfile option') if args.outdir is None: args.outdir = os.getcwd() log.warning('--outdir not specified; using {}'.format(args.outdir)) ccd_calibration_filename = None if args.no_ccd_calib_filename : ccd_calibration_filename = False elif args.ccd_calib_filename is not None : ccd_calibration_filename = args.ccd_calib_filename for camera in args.cameras: try: img = io.read_raw(args.infile, camera, bias=bias, dark=dark, pixflat=pixflat, mask=mask, bkgsub=args.bkgsub, nocosmic=args.nocosmic, cosmics_nsig=args.cosmics_nsig, cosmics_cfudge=args.cosmics_cfudge, cosmics_c2fudge=args.cosmics_c2fudge, ccd_calibration_filename=ccd_calibration_filename, nocrosstalk=args.nocrosstalk, nogain=args.nogain, fill_header=args.fill_header ) except IOError: log.error('Error while reading or preprocessing camera {} in {}'.format(camera, args.infile)) continue if(args.zero_masked) : img.pix *= (img.mask==0) if args.outfile is None: night = img.meta['NIGHT'] expid = img.meta['EXPID'] outfile = io.findfile('preproc', night=night, expid=expid, camera=camera, outdir=args.outdir) else: outfile = args.outfile io.write_image(outfile, img)
def main(args=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) bias=True if args.bias : bias=args.bias if args.nobias : bias=False dark=True if args.dark : dark=args.dark if args.nodark : dark=False pixflat=True if args.pixflat : pixflat=args.pixflat if args.nopixflat : pixflat=False mask=True if args.mask : mask=args.mask if args.nomask : mask=False if args.cameras is None: args.cameras = [c+str(i) for c in 'brz' for i in range(10)] else: args.cameras = args.cameras.split(',') if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None) or (args.dark is not None): if len(args.cameras) > 1: raise ValueError('must use only one camera with --bias, --dark, --pixflat, --mask options') if (args.outfile is not None) and len(args.cameras) > 1: raise ValueError('must use only one camera with --outfile option') if args.outdir is None: args.outdir = os.getcwd() log.warning('--outdir not specified; using {}'.format(args.outdir)) ccd_calibration_filename = None if args.no_ccd_calib_filename : ccd_calibration_filename = False elif args.ccd_calib_filename is not None : ccd_calibration_filename = args.ccd_calib_filename if args.fibermap and not os.path.exists(fibermap): raise ValueError('--fibermap {} not found'.format(args.fibermap)) if args.fibermap is None: datadir, infile = os.path.split(os.path.abspath(args.infile)) fibermapfile = infile.replace('desi-', 'fibermap-').replace('.fits.fz', '.fits') args.fibermap = os.path.join(datadir, fibermapfile) if args.nofibermap: fibermap = None elif os.path.exists(args.fibermap): fibermap = io.read_fibermap(args.fibermap) else: log.warning('fibermap file not found; creating blank fibermap') fibermap = io.empty_fibermap(5000) for camera in args.cameras: try: img = io.read_raw(args.infile, camera, bias=bias, dark=dark, pixflat=pixflat, mask=mask, bkgsub=args.bkgsub, nocosmic=args.nocosmic, cosmics_nsig=args.cosmics_nsig, cosmics_cfudge=args.cosmics_cfudge, cosmics_c2fudge=args.cosmics_c2fudge, ccd_calibration_filename=ccd_calibration_filename, nocrosstalk=args.nocrosstalk, nogain=args.nogain, nodarktrail=args.nodarktrail, fill_header=args.fill_header, ) except IOError: log.error('Error while reading or preprocessing camera {} in {}'.format(camera, args.infile)) continue if(args.zero_masked) : img.pix *= (img.mask==0) if args.outfile is None: night = img.meta['NIGHT'] expid = img.meta['EXPID'] outfile = io.findfile('preproc', night=night, expid=expid, camera=camera, outdir=args.outdir) else: outfile = args.outfile if fibermap: petal_loc = int(img.camera[1]) ii = (fibermap['PETAL_LOC'] == petal_loc) img.fibermap = fibermap[ii] io.write_image(outfile, img) log.info("Wrote {}".format(outfile))
def main(args=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) bias = True if args.bias: bias = args.bias if args.nobias: bias = False dark = True if args.dark: dark = args.dark if args.nodark: dark = False pixflat = True if args.pixflat: pixflat = args.pixflat if args.nopixflat: pixflat = False mask = True if args.mask: mask = args.mask if args.nomask: mask = False if args.cameras is None: args.cameras = [c + str(i) for c in 'brz' for i in range(10)] else: args.cameras = args.cameras.split(',') if (args.bias is not None) or (args.pixflat is not None) or ( args.mask is not None) or (args.dark is not None): if len(args.cameras) > 1: raise ValueError( 'must use only one camera with --bias, --dark, --pixflat, --mask options' ) if (args.pixfile is not None): log.warning('--pixfile is deprecated; please use --outfile instead') if args.outfile is None: args.outfile = args.pixfile else: log.critical("Set --outfile not --pixfile and certainly not both") sys.exit(1) if (args.outfile is not None) and len(args.cameras) > 1: raise ValueError('must use only one camera with --outfile option') if args.outdir is None: args.outdir = os.getcwd() log.warning('--outdir not specified; using {}'.format(args.outdir)) ccd_calibration_filename = None if args.no_ccd_calib_filename: ccd_calibration_filename = False elif args.ccd_calib_filename is not None: ccd_calibration_filename = args.ccd_calib_filename for camera in args.cameras: try: img = io.read_raw( args.infile, camera, bias=bias, dark=dark, pixflat=pixflat, mask=mask, bkgsub=args.bkgsub, nocosmic=args.nocosmic, cosmics_nsig=args.cosmics_nsig, cosmics_cfudge=args.cosmics_cfudge, cosmics_c2fudge=args.cosmics_c2fudge, ccd_calibration_filename=ccd_calibration_filename, nocrosstalk=args.nocrosstalk, nogain=args.nogain, fill_header=args.fill_header) except IOError: log.error( 'Error while reading or preprocessing camera {} in {}'.format( camera, args.infile)) continue if (args.zero_masked): img.pix *= (img.mask == 0) if args.outfile is None: night = img.meta['NIGHT'] expid = img.meta['EXPID'] outfile = io.findfile('preproc', night=night, expid=expid, camera=camera, outdir=args.outdir) else: outfile = args.outfile io.write_image(outfile, img)
def preproc_file(infile, camera, outfile=None, outdir=None, fibermap=None, zero_masked=False, **preproc_opts): """ Preprocess a single camera from a single input file Args: infile : input raw data file camera : camera, e.g. 'b0', 'r1', 'z9' Options: outfile: output preprocessed image file to write outdir: output directory; derive filename from infile NIGHT and EXPID fibermap: fibermap filename to include in output zero_masked (bool): set masked pixels to 0 preproc_opts: dictionary to pass to preproc Returns error code (1=error, 0=success) but will not raise exception if there is an I/O or preprocessing failure (allows other parallel procs to proceed). Note: either `outfile` or `outdir` must be provided """ try: img = io.read_raw(infile, camera, fibermapfile=fibermap, **preproc_opts) except IOError as e: #- print error and return error code, but don't raise exception so #- that this won't block other cameras for multiprocessing log.error( 'Error while reading or preprocessing camera {} in {}'.format( camera, infile)) log.error(e) return 1 if zero_masked: log.info("Setting masked pixels values to zero") img.pix *= (img.mask == 0) if outfile is None: night = img.meta['NIGHT'] if not 'EXPID' in img.meta.keys(): if 'EXPNUM' in img.meta.keys(): img.meta['EXPID'] = img.meta['EXPNUM'] else: mess = "no EXPID nor EXPNUM in img.meta, cannot create output filename" log.error(mess) raise KeyError(mess) expid = img.meta['EXPID'] outfile = io.findfile('preproc', night=night, expid=expid, camera=camera, outdir=outdir) io.write_image(outfile, img) log.info("Wrote {}".format(outfile)) return 0
def main(args=None): log = get_logger() if args is None: args = parse() first_preproc = None images = [] ivars = [] masks = [] log.info("read inputs ...") for filename in args.infile: log.info(" read {}".format(filename)) tmp = read_image(filename) if first_preproc is None: first_preproc = tmp images.append(tmp.pix.ravel()) # make sure we don't include data with ivar=0 mask = tmp.mask + (tmp.ivar == 0).astype(int) masks.append(mask.ravel()) ivars.append((tmp.ivar * (tmp.mask == 0)).ravel()) images = np.array(images) masks = np.array(masks) ivars = np.array(ivars) smask = np.sum(masks, axis=0) log.info("compute masked median image ...") medimage = masked_median(images, masks) log.info("use masked median image to discard outlier ...") good = ((images - medimage)**2 * (ivars > 0) / ((medimage > 0) * (medimage * args.fracerr)**2 + 1. / (ivars + (ivars == 0)))) < args.nsig**3 ivars *= good.astype(float) if args.weighted: log.info("compute weighted mean ...") sw = np.sum(ivars, axis=0) swf = np.sum(ivars * images, axis=0) meanimage = swf / (sw + (sw == 0)) meanivar = sw else: log.info("compute unweighted mean ...") s1 = np.sum((ivars > 0).astype(int), axis=0) sf = np.sum((ivars > 0) * images, axis=0) meanimage = sf / (s1 + (s1 == 0)) meanvar = np.sum( (ivars > 0) / (ivars + (ivars == 0))) / (s1 + (s1 == 0))**2 meanivar = (meanvar > 0) / (meanvar + (meanvar == 0)) log.info("write nimages.fits ...") fitsio.write("nimages.fits", s1.reshape(first_preproc.pix.shape), clobber=True) log.info("compute mask ...") meanmask = masks[0] for mask in masks[1:]: meanmask &= mask log.info("write image ...") preproc = first_preproc shape = preproc.pix.shape preproc.pix = meanimage.reshape(shape) preproc.ivar = meanivar.reshape(shape) preproc.mask = meanmask.reshape(shape) write_image(args.outfile, preproc) log.info("wrote {}".format(args.outfile))
def main(args=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) t0 = time.time() log = get_logger() # guess if it is a preprocessed or a raw image hdulist = fits.open(args.image) is_input_preprocessed = ("IMAGE" in hdulist) & ("IVAR" in hdulist) primary_header = hdulist[0].header hdulist.close() if is_input_preprocessed: image = read_image(args.image) else: if args.camera is None: print( "ERROR: Need to specify camera to open a raw fits image (with all cameras in different fits HDUs)" ) print( "Try adding the option '--camera xx', with xx in {brz}{0-9}, like r7, or type 'desi_qproc --help' for more options" ) sys.exit(12) image = read_raw(args.image, args.camera, fill_header=[ 1, ]) if args.auto: log.debug("AUTOMATIC MODE") try: night = image.meta['NIGHT'] if not 'EXPID' in image.meta: if 'EXPNUM' in image.meta: log.warning('using EXPNUM {} for EXPID'.format( image.meta['EXPNUM'])) image.meta['EXPID'] = image.meta['EXPNUM'] expid = image.meta['EXPID'] except KeyError as e: log.error( "Need at least NIGHT and EXPID (or EXPNUM) to run in auto mode. Retry without the --auto option." ) log.error(str(e)) sys.exit(12) indir = os.path.dirname(args.image) if args.fibermap is None: filename = '{}/fibermap-{:08d}.fits'.format(indir, expid) if os.path.isfile(filename): log.debug("auto-mode: found a fibermap, {}, using it!".format( filename)) args.fibermap = filename if args.output_preproc is None: if not is_input_preprocessed: args.output_preproc = '{}/preproc-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera.lower(), expid) log.debug("auto-mode: will write preproc in " + args.output_preproc) else: log.debug( "auto-mode: will not write preproc because input is a preprocessed image" ) if args.auto_output_dir != '.': if not os.path.isdir(args.auto_output_dir): log.debug("auto-mode: creating directory " + args.auto_output_dir) os.makedirs(args.auto_output_dir) if args.output_preproc is not None: write_image(args.output_preproc, image) cfinder = None if args.psf is None: if cfinder is None: cfinder = CalibFinder([image.meta, primary_header]) args.psf = cfinder.findfile("PSF") log.info(" Using PSF {}".format(args.psf)) tset = read_xytraceset(args.psf) # add fibermap if args.fibermap: if os.path.isfile(args.fibermap): fibermap = read_fibermap(args.fibermap) else: log.error("no fibermap file {}".format(args.fibermap)) fibermap = None else: fibermap = None if "OBSTYPE" in image.meta: obstype = image.meta["OBSTYPE"].upper() image.meta["OBSTYPE"] = obstype # make sure it's upper case qframe = None else: log.warning("No OBSTYPE keyword, trying to guess ...") qframe = qproc_boxcar_extraction(tset, image, width=args.width, fibermap=fibermap) obstype = check_qframe_flavor( qframe, input_flavor=image.meta["FLAVOR"]).upper() image.meta["OBSTYPE"] = obstype log.info("OBSTYPE = '{}'".format(obstype)) if args.auto: # now set the things to do if obstype == "SKY" or obstype == "TWILIGHT" or obstype == "SCIENCE": args.shift_psf = True args.output_psf = '{}/psf-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.apply_fiberflat = True args.skysub = True args.output_skyframe = '{}/qsky-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.fluxcalib = True args.outframe = '{}/qcframe-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) elif obstype == "ARC" or obstype == "TESTARC": args.shift_psf = True args.output_psf = '{}/psf-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.compute_lsf_sigma = True elif obstype == "FLAT" or obstype == "TESTFLAT": args.shift_psf = True args.output_psf = '{}/psf-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.compute_fiberflat = '{}/qfiberflat-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) if args.shift_psf: # using the trace shift script if args.auto: options = option_list({ "psf": args.psf, "image": "dummy", "outpsf": "dummy", "continuum": ((obstype == "FLAT") | (obstype == "TESTFLAT")), "sky": ((obstype == "SCIENCE") | (obstype == "SKY")) }) else: options = option_list({ "psf": args.psf, "image": "dummy", "outpsf": "dummy" }) tmp_args = trace_shifts_script.parse(options=options) tset = trace_shifts_script.fit_trace_shifts(image=image, args=tmp_args) qframe = qproc_boxcar_extraction(tset, image, width=args.width, fibermap=fibermap) if tset.meta is not None: # add traceshift info in the qframe, this will be saved in the qframe header if qframe.meta is None: qframe.meta = dict() for k in tset.meta.keys(): qframe.meta[k] = tset.meta[k] if args.output_rawframe is not None: write_qframe(args.output_rawframe, qframe) log.info("wrote raw extracted frame in {}".format( args.output_rawframe)) if args.compute_lsf_sigma: tset = process_arc(qframe, tset, linelist=None, npoly=2, nbins=2) if args.output_psf is not None: for k in qframe.meta: if k not in tset.meta: tset.meta[k] = qframe.meta[k] write_xytraceset(args.output_psf, tset) if args.compute_fiberflat is not None: fiberflat = qproc_compute_fiberflat(qframe) #write_qframe(args.compute_fiberflat,qflat) write_fiberflat(args.compute_fiberflat, fiberflat, header=qframe.meta) log.info("wrote fiberflat in {}".format(args.compute_fiberflat)) if args.apply_fiberflat or args.input_fiberflat: if args.input_fiberflat is None: if cfinder is None: cfinder = CalibFinder([image.meta, primary_header]) try: args.input_fiberflat = cfinder.findfile("FIBERFLAT") except KeyError as e: log.error("no FIBERFLAT for this spectro config") sys.exit(12) log.info("applying fiber flat {}".format(args.input_fiberflat)) flat = read_fiberflat(args.input_fiberflat) qproc_apply_fiberflat(qframe, flat) if args.skysub: log.info("sky subtraction") if args.output_skyframe is not None: skyflux = qproc_sky_subtraction(qframe, return_skymodel=True) sqframe = QFrame(qframe.wave, skyflux, np.ones(skyflux.shape)) write_qframe(args.output_skyframe, sqframe) log.info("wrote sky model in {}".format(args.output_skyframe)) else: qproc_sky_subtraction(qframe) if args.fluxcalib: if cfinder is None: cfinder = CalibFinder([image.meta, primary_header]) # check for flux calib if cfinder.haskey("FLUXCALIB"): fluxcalib_filename = cfinder.findfile("FLUXCALIB") fluxcalib = read_average_flux_calibration(fluxcalib_filename) log.info("read average calib in {}".format(fluxcalib_filename)) seeing = qframe.meta["SEEING"] airmass = qframe.meta["AIRMASS"] exptime = qframe.meta["EXPTIME"] exposure_calib = fluxcalib.value(seeing=seeing, airmass=airmass) for q in range(qframe.nspec): fiber_calib = np.interp(qframe.wave[q], fluxcalib.wave, exposure_calib) * exptime inv_calib = (fiber_calib > 0) / (fiber_calib + (fiber_calib == 0)) qframe.flux[q] *= inv_calib qframe.ivar[q] *= fiber_calib**2 * (fiber_calib > 0) # add keyword in header giving the calibration factor applied at a reference wavelength band = qframe.meta["CAMERA"].upper()[0] if band == "B": refwave = 4500 elif band == "R": refwave = 6500 else: refwave = 8500 calvalue = np.interp(refwave, fluxcalib.wave, exposure_calib) * exptime qframe.meta["CALWAVE"] = refwave qframe.meta["CALVALUE"] = calvalue else: log.error( "Cannot calibrate fluxes because no FLUXCALIB keywork in calibration files" ) fibers = parse_fibers(args.fibers) if fibers is None: fibers = qframe.flux.shape[0] else: ii = np.arange(qframe.fibers.size)[np.in1d(qframe.fibers, fibers)] if ii.size == 0: log.error("no such fibers in frame,") log.error("fibers are in range [{}:{}]".format( qframe.fibers[0], qframe.fibers[-1] + 1)) sys.exit(12) qframe = qframe[ii] if args.outframe is not None: write_qframe(args.outframe, qframe) log.info("wrote {}".format(args.outframe)) t1 = time.time() log.info("all done in {:3.1f} sec".format(t1 - t0)) if args.plot: log.info("plotting {} spectra".format(qframe.wave.shape[0])) import matplotlib.pyplot as plt fig = plt.figure() for i in range(qframe.wave.shape[0]): j = (qframe.ivar[i] > 0) plt.plot(qframe.wave[i, j], qframe.flux[i, j]) plt.grid() plt.xlabel("wavelength") plt.ylabel("flux") plt.show()
def fake_raw(): dirhash = uuid.uuid4() night = fake_night() rawdir = "test_pipe_raw_{}".format(dirhash) if os.path.exists(rawdir): shutil.rmtree(rawdir) os.makedirs(rawdir) nightdir = os.path.join(rawdir, night) # set up one spectrograph (500 fibers) nspec = 500 # arc expid = "00000000" tileid = "0" flavor = "arc" telera = "0.0" teledec = "0.0" fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid)) hdr = dict( NIGHT=(night, 'Night of observation YEARMMDD'), EXPID=(expid, 'DESI exposure ID'), TILEID=(tileid, 'DESI tile ID'), FLAVOR=(flavor, 'Flavor [arc, flat, science, ...]'), TELRA=(telera, 'Telescope pointing RA [degrees]'), TELDEC=(teledec, 'Telescope pointing dec [degrees]'), ) fibermap = io.empty_fibermap(nspec) fibermap['OBJTYPE'] = 'ARC' io.write_fibermap(fmfile, fibermap, header=hdr) for cam in ["r0", "b0", "z0"]: pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid)) pix = np.random.normal(0, 3.0, size=(10, 10)) ivar = np.ones_like(pix) / 3.0**2 mask = np.zeros(pix.shape, dtype=np.uint32) img = dimg.Image(pix, ivar, mask, camera=cam) io.write_image(pfile, img) # flat expid = "00000001" tileid = "1" flavor = "flat" telera = "0.0" teledec = "0.0" fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid)) hdr = dict( NIGHT=(night, 'Night of observation YEARMMDD'), EXPID=(expid, 'DESI exposure ID'), TILEID=(tileid, 'DESI tile ID'), FLAVOR=(flavor, 'Flavor [arc, flat, science, ...]'), TELRA=(telera, 'Telescope pointing RA [degrees]'), TELDEC=(teledec, 'Telescope pointing dec [degrees]'), ) fibermap = io.empty_fibermap(nspec) fibermap['OBJTYPE'] = 'FLAT' io.write_fibermap(fmfile, fibermap, header=hdr) for cam in ["r0", "b0", "z0"]: pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid)) pix = np.random.normal(0, 3.0, size=(10, 10)) ivar = np.ones_like(pix) / 3.0**2 mask = np.zeros(pix.shape, dtype=np.uint32) img = dimg.Image(pix, ivar, mask, camera=cam) io.write_image(pfile, img) # science expid = "00000002" tileid = "2" flavor = "dark" telera = "0.0" teledec = "0.0" fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid)) hdr = dict( NIGHT=(night, 'Night of observation YEARMMDD'), EXPID=(expid, 'DESI exposure ID'), TILEID=(tileid, 'DESI tile ID'), FLAVOR=(flavor, 'Flavor [arc, flat, science, ...]'), TELRA=(telera, 'Telescope pointing RA [degrees]'), TELDEC=(teledec, 'Telescope pointing dec [degrees]'), ) fibermap = io.empty_fibermap(nspec) fibermap['OBJTYPE'] = 'ELG' fibermap['FIBER'] = np.arange(nspec, dtype='i4') fibermap['TARGETID'] = np.random.randint(sys.maxsize, size=nspec) fibermap['BRICKNAME'] = ['3412p195' for x in range(nspec)] io.write_fibermap(fmfile, fibermap, header=hdr) for cam in ["r0", "b0", "z0"]: pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid)) pix = np.random.normal(0, 3.0, size=(10, 10)) ivar = np.ones_like(pix) / 3.0**2 mask = np.zeros(pix.shape, dtype=np.uint32) img = dimg.Image(pix, ivar, mask, camera=cam) io.write_image(pfile, img) return rawdir
def fake_raw(): dirhash = uuid.uuid4() night = fake_night() rawdir = "test_pipe_raw_{}".format(dirhash) if os.path.exists(rawdir): shutil.rmtree(rawdir) os.makedirs(rawdir) nightdir = os.path.join(rawdir, night) # set up one spectrograph (500 fibers) nspec = 500 # arc expid = "00000000" tileid = "0" flavor = "arc" telera = "0.0" teledec = "0.0" fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid)) hdr = dict( NIGHT = (night, 'Night of observation YEARMMDD'), EXPID = (expid, 'DESI exposure ID'), TILEID = (tileid, 'DESI tile ID'), FLAVOR = (flavor, 'Flavor [arc, flat, science, ...]'), TELRA = (telera, 'Telescope pointing RA [degrees]'), TELDEC = (teledec, 'Telescope pointing dec [degrees]'), ) fibermap = io.empty_fibermap(nspec) fibermap['OBJTYPE'] = 'ARC' io.write_fibermap(fmfile, fibermap, header=hdr) for cam in ["r0", "b0", "z0"]: pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid)) pix = np.random.normal(0, 3.0, size=(10,10)) ivar = np.ones_like(pix) / 3.0**2 mask = np.zeros(pix.shape, dtype=np.uint32) img = dimg.Image(pix, ivar, mask, camera=cam) io.write_image(pfile, img) # flat expid = "00000001" tileid = "1" flavor = "flat" telera = "0.0" teledec = "0.0" fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid)) hdr = dict( NIGHT = (night, 'Night of observation YEARMMDD'), EXPID = (expid, 'DESI exposure ID'), TILEID = (tileid, 'DESI tile ID'), FLAVOR = (flavor, 'Flavor [arc, flat, science, ...]'), TELRA = (telera, 'Telescope pointing RA [degrees]'), TELDEC = (teledec, 'Telescope pointing dec [degrees]'), ) fibermap = io.empty_fibermap(nspec) fibermap['OBJTYPE'] = 'FLAT' io.write_fibermap(fmfile, fibermap, header=hdr) for cam in ["r0", "b0", "z0"]: pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid)) pix = np.random.normal(0, 3.0, size=(10,10)) ivar = np.ones_like(pix) / 3.0**2 mask = np.zeros(pix.shape, dtype=np.uint32) img = dimg.Image(pix, ivar, mask, camera=cam) io.write_image(pfile, img) # science expid = "00000002" tileid = "2" flavor = "dark" telera = "0.0" teledec = "0.0" fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid)) hdr = dict( NIGHT = (night, 'Night of observation YEARMMDD'), EXPID = (expid, 'DESI exposure ID'), TILEID = (tileid, 'DESI tile ID'), FLAVOR = (flavor, 'Flavor [arc, flat, science, ...]'), TELRA = (telera, 'Telescope pointing RA [degrees]'), TELDEC = (teledec, 'Telescope pointing dec [degrees]'), ) fibermap = io.empty_fibermap(nspec) fibermap['OBJTYPE'] = 'ELG' fibermap['FIBER'] = np.arange(nspec, dtype='i4') fibermap['TARGETID'] = np.random.randint(sys.maxsize, size=nspec) fibermap['BRICKNAME'] = [ '3412p195' for x in range(nspec) ] io.write_fibermap(fmfile, fibermap, header=hdr) for cam in ["r0", "b0", "z0"]: pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid)) pix = np.random.normal(0, 3.0, size=(10,10)) ivar = np.ones_like(pix) / 3.0**2 mask = np.zeros(pix.shape, dtype=np.uint32) img = dimg.Image(pix, ivar, mask, camera=cam) io.write_image(pfile, img) return rawdir
from astropy.io import fits import matplotlib.pyplot as plt import numpy as np import pandas as pd import os from os import listdir import pdb from matplotlib.backends.backend_pdf import PdfPages from scipy.signal import butter, lfilter, freqz from scipy.signal import savgol_filter from desispec import io """ img = io.read_raw(args.infile, camera, fibermapfile=args.fibermap, bias=bias, dark=dark, pixflat=pixflat, mask=mask, bkgsub=args.bkgsub, nocosmic=args.nocosmic, cosmics_nsig=args.cosmics_nsig, cosmics_cfudge=args.cosmics_cfudge, cosmics_c2fudge=args.cosmics_c2fudge, ccd_calibration_filename=ccd_calibration_filename, nocrosstalk=args.nocrosstalk, nogain=args.nogain, use_savgol=args.use_savgol, nodarktrail=args.nodarktrail, fill_header=args.fill_header, remove_scattered_light=args.scattered_light ) io.write_image(outfile, img) #- Actually write or update the file if os.path.exists(filename):