Пример #1
0
 def test_init(self):
     """Cleanup test files if they exist.
     """
     
     pheader={"DATE-OBS":'2018-11-30T12:42:10.442593-05:00',"DOSVER":'SIM'}
     header={"DETECTOR":'SIM',"CAMERA":'b0      ',"FEEVER":'SIM'}
     cfinder = CalibFinder([pheader,header])
     print(cfinder.value("DETECTOR"))
     if cfinder.haskey("BIAS") :
         print(cfinder.findfile("BIAS"))
Пример #2
0
    def test_init(self):
        """Cleanup test files if they exist.
        """

        pheader = {
            "DATE-OBS": '2018-11-30T12:42:10.442593-05:00',
            "DOSVER": 'SIM'
        }
        header = {"DETECTOR": 'SIM', "CAMERA": 'b0      ', "FEEVER": 'SIM'}
        cfinder = CalibFinder([pheader, header])
        print(cfinder.value("DETECTOR"))
        if cfinder.haskey("BIAS"):
            print(cfinder.findfile("BIAS"))
Пример #3
0
    def run(self, indir):
        '''TODO: document'''

        log = desiutil.log.get_logger()

        results = list()

        infiles = glob.glob(os.path.join(indir, 'qframe-*.fits'))
        if len(infiles) == 0:
            log.error("no qframe in {}".format(indir))
            return None

        for filename in infiles:
            qframe = read_qframe(filename)
            night = int(qframe.meta['NIGHT'])
            expid = int(qframe.meta['EXPID'])
            cam = qframe.meta['CAMERA'][0].upper()
            spectro = int(qframe.meta['CAMERA'][1])

            try:
                cfinder = CalibFinder([qframe.meta])
            except:
                log.error(
                    "failed to find calib for qframe {}".format(filename))
                continue
            if not cfinder.haskey("FIBERFLAT"):
                log.warning(
                    "no known fiberflat for qframe {}".format(filename))
                continue
            fflat = read_fiberflat(cfinder.findfile("FIBERFLAT"))
            tmp = np.median(fflat.fiberflat, axis=1)
            reference_fflat = tmp / np.median(tmp)

            tmp = np.median(qframe.flux, axis=1)
            this_fflat = tmp / np.median(tmp)

            for f, fiber in enumerate(qframe.fibermap["FIBER"]):
                results.append(
                    collections.OrderedDict(NIGHT=night,
                                            EXPID=expid,
                                            SPECTRO=spectro,
                                            CAM=cam,
                                            FIBER=fiber,
                                            FIBERFLAT=this_fflat[f],
                                            REF_FIBERFLAT=reference_fflat[f]))

        if len(results) == 0:
            return None
        return Table(results, names=results[0].keys())
Пример #4
0
def subtract_overscan(fx,camera):
    import desispec.preproc
    from desispec.calibfinder import parse_date_obs, CalibFinder
    rawimage = fx[camera.upper()].data
    rawimage = rawimage.astype(np.float64)
    header = fx[camera.upper()].header
    hdu=0
    primary_header= fx[hdu].header
    ccd_calibration_filename=None
    log=desispec.preproc.get_logger()

    cfinder = CalibFinder([header, primary_header], yaml_file=ccd_calibration_filename)
    amp_ids = desispec.preproc.get_amp_ids(header)
    #######################
    use_overscan_row = False
    overscan_per_row=False
    #######################
    # Subtract overscan 
    for amp in amp_ids:
        # Grab the sections
        ov_col = desispec.preproc.parse_sec_keyword(header['BIASSEC'+amp])
        if 'ORSEC'+amp in header.keys():
            ov_row = desispec.preproc.parse_sec_keyword(header['ORSEC'+amp])
        elif use_overscan_row:
            log.error('No ORSEC{} keyword; not using overscan_row'.format(amp))
            use_overscan_row = False

        # Generate the overscan images
        raw_overscan_col = rawimage[ov_col].copy()

        if use_overscan_row:
            raw_overscan_row = rawimage[ov_row].copy()
            overscan_row = np.zeros_like(raw_overscan_row)

            # Remove overscan_col from overscan_row
            raw_overscan_squared = rawimage[ov_row[0], ov_col[1]].copy()
            for row in range(raw_overscan_row.shape[0]):
                o,r = _overscan(raw_overscan_squared[row])
                overscan_row[row] = raw_overscan_row[row] - o

        # Now remove the overscan_col
        nrows=raw_overscan_col.shape[0]
        log.info("nrows in overscan=%d"%nrows)
        overscan_col = np.zeros(nrows)
        rdnoise  = np.zeros(nrows)
        if (cfinder and cfinder.haskey('OVERSCAN'+amp) and cfinder.value("OVERSCAN"+amp).upper()=="PER_ROW") or overscan_per_row:
            log.info("Subtracting overscan per row for amplifier %s of camera %s"%(amp,camera))
            for j in range(nrows) :
                if np.isnan(np.sum(overscan_col[j])) :
                    log.warning("NaN values in row %d of overscan of amplifier %s of camera %s"%(j,amp,camera))
                    continue
                o,r =  _overscan(raw_overscan_col[j])
                #log.info("%d %f %f"%(j,o,r))
                overscan_col[j]=o
                rdnoise[j]=r
        else :
            log.info("Subtracting average overscan for amplifier %s of camera %s"%(amp,camera))
            o,r =  desispec.preproc._overscan(raw_overscan_col)
            overscan_col += o
            rdnoise  += r

        #- subtract overscan from data region and apply gain
        jj = desispec.preproc.parse_sec_keyword(header['DATASEC'+amp])
        kk = desispec.preproc.parse_sec_keyword(header['CCDSEC'+amp])

        data = rawimage[jj].copy()
        # Subtract columns
        for k in range(nrows):
            data[k] -= overscan_col[k]
        # And now the rows
        if use_overscan_row:
            # Savgol?
            if use_savgol:
                log.info("Using savgol")
                collapse_oscan_row = np.zeros(overscan_row.shape[1])
                for col in range(overscan_row.shape[1]):
                    o, _ = _overscan(overscan_row[:,col])
                    collapse_oscan_row[col] = o
                oscan_row = _savgol_clipped(collapse_oscan_row, niter=0)
                oimg_row = np.outer(np.ones(data.shape[0]), oscan_row)
                data -= oimg_row
            else:
                o,r = _overscan(overscan_row)
                data -= o
        rawimage[jj]=data
    return rawimage
Пример #5
0
def preproc(rawimage,
            header,
            primary_header,
            bias=True,
            dark=True,
            pixflat=True,
            mask=True,
            bkgsub=False,
            nocosmic=False,
            cosmics_nsig=6,
            cosmics_cfudge=3.,
            cosmics_c2fudge=0.5,
            ccd_calibration_filename=None,
            nocrosstalk=False,
            nogain=False,
            overscan_per_row=False,
            use_overscan_row=False,
            use_savgol=None,
            nodarktrail=False,
            remove_scattered_light=False,
            psf_filename=None,
            bias_img=None):
    '''
    preprocess image using metadata in header

    image = ((rawimage-bias-overscan)*gain)/pixflat

    Args:
        rawimage : 2D numpy array directly from raw data file
        header : dict-like metadata, e.g. from FITS header, with keywords
            CAMERA, BIASSECx, DATASECx, CCDSECx
            where x = A, B, C, D for each of the 4 amplifiers
            (also supports old naming convention 1, 2, 3, 4).
        primary_header: dict-like metadata fit keywords EXPTIME, DOSVER
            DATE-OBS is also required if bias, pixflat, or mask=True

    Optional bias, pixflat, and mask can each be:
        False: don't apply that step
        True: use default calibration data for that night
        ndarray: use that array
        filename (str or unicode): read HDU 0 and use that

    Optional overscan features:
        overscan_per_row : bool,  Subtract the overscan_col values
            row by row from the data.
        use_overscan_row : bool,  Subtract off the overscan_row
            from the data (default: False).  Requires ORSEC in
            the Header
        use_savgol : bool,  Specify whether to use Savitsky-Golay filter for
            the overscan.   (default: False).  Requires use_overscan_row=True
            to have any effect.

    Optional background subtraction with median filtering if bkgsub=True

    Optional disabling of cosmic ray rejection if nocosmic=True
    Optional disabling of dark trail correction if nodarktrail=True

    Optional bias image (testing only) may be provided by bias_img=

    Optional tuning of cosmic ray rejection parameters:
        cosmics_nsig: number of sigma above background required
        cosmics_cfudge: number of sigma inconsistent with PSF required
        cosmics_c2fudge:  fudge factor applied to PSF

    Optional fit and subtraction of scattered light

    Returns Image object with member variables:
        pix : 2D preprocessed image in units of electrons per pixel
        ivar : 2D inverse variance of image
        mask : 2D mask of image (0=good)
        readnoise : 2D per-pixel readnoise of image
        meta : metadata dictionary
        TODO: define what keywords are included

    preprocessing includes the following steps:
        - bias image subtraction
        - overscan subtraction (from BIASSEC* keyword defined regions)
        - readnoise estimation (from BIASSEC* keyword defined regions)
        - gain correction (from GAIN* keywords)
        - pixel flat correction
        - cosmic ray masking
        - propagation of input known bad pixel mask
        - inverse variance estimation

    Notes:

    The bias image is subtracted before any other calculation to remove any
    non-uniformities in the overscan regions prior to calculating overscan
    levels and readnoise.

    The readnoise is an image not just one number per amp, because the pixflat
    image also affects the interpreted readnoise.

    The inverse variance is estimated from the readnoise and the image itself,
    and thus is biased.
    '''
    log = get_logger()

    header = header.copy()

    cfinder = None

    if ccd_calibration_filename is not False:
        cfinder = CalibFinder([header, primary_header],
                              yaml_file=ccd_calibration_filename)

    #- TODO: Check for required keywords first

    #- Subtract bias image
    camera = header['CAMERA'].lower()

    #- convert rawimage to float64 : this is the output format of read_image
    rawimage = rawimage.astype(np.float64)

    # Savgol
    if cfinder and cfinder.haskey("USE_ORSEC"):
        use_overscan_row = cfinder.value("USE_ORSEC")
    if cfinder and cfinder.haskey("SAVGOL"):
        use_savgol = cfinder.value("SAVGOL")

    # Set bias image, as desired
    if bias_img is None:
        bias = get_calibration_image(cfinder, "BIAS", bias)
    else:
        bias = bias_img

    if bias is not False:  #- it's an array
        if bias.shape == rawimage.shape:
            log.info("subtracting bias")
            rawimage = rawimage - bias
        else:
            raise ValueError('shape mismatch bias {} != rawimage {}'.format(
                bias.shape, rawimage.shape))

    #- Check if this file uses amp names 1,2,3,4 (old) or A,B,C,D (new)
    amp_ids = get_amp_ids(header)

    #- Double check that we have the necessary keywords
    missing_keywords = list()
    for prefix in ['CCDSEC', 'BIASSEC']:
        for amp in amp_ids:
            key = prefix + amp
            if not key in header:
                log.error('No {} keyword in header'.format(key))
                missing_keywords.append(key)

    if len(missing_keywords) > 0:
        raise KeyError("Missing keywords {}".format(
            ' '.join(missing_keywords)))

    #- Output arrays
    ny = 0
    nx = 0
    for amp in amp_ids:
        yy, xx = parse_sec_keyword(header['CCDSEC%s' % amp])
        ny = max(ny, yy.stop)
        nx = max(nx, xx.stop)
    image = np.zeros((ny, nx))

    readnoise = np.zeros_like(image)

    #- Load mask
    mask = get_calibration_image(cfinder, "MASK", mask)

    if mask is False:
        mask = np.zeros(image.shape, dtype=np.int32)
    else:
        if mask.shape != image.shape:
            raise ValueError('shape mismatch mask {} != image {}'.format(
                mask.shape, image.shape))

    #- Load dark
    dark = get_calibration_image(cfinder, "DARK", dark)

    if dark is not False:
        if dark.shape != image.shape:
            log.error('shape mismatch dark {} != image {}'.format(
                dark.shape, image.shape))
            raise ValueError('shape mismatch dark {} != image {}'.format(
                dark.shape, image.shape))

        if cfinder and cfinder.haskey("EXPTIMEKEY"):
            exptime_key = cfinder.value("EXPTIMEKEY")
            log.info("Using exposure time keyword %s for dark normalization" %
                     exptime_key)
        else:
            exptime_key = "EXPTIME"
        exptime = primary_header[exptime_key]

        log.info("Multiplying dark by exptime %f" % (exptime))
        dark *= exptime

    for amp in amp_ids:
        # Grab the sections
        ov_col = parse_sec_keyword(header['BIASSEC' + amp])
        if 'ORSEC' + amp in header.keys():
            ov_row = parse_sec_keyword(header['ORSEC' + amp])
        elif use_overscan_row:
            log.error('No ORSEC{} keyword; not using overscan_row'.format(amp))
            use_overscan_row = False

        if nogain:
            gain = 1.
        else:
            #- Initial teststand data may be missing GAIN* keywords; don't crash
            if 'GAIN' + amp in header:
                gain = header['GAIN' + amp]  #- gain = electrons / ADU
            else:
                if cfinder and cfinder.haskey('GAIN' + amp):
                    gain = float(cfinder.value('GAIN' + amp))
                    log.info('Using GAIN{}={} from calibration data'.format(
                        amp, gain))
                else:
                    gain = 1.0
                    log.warning(
                        'Missing keyword GAIN{} in header and nothing in calib data; using {}'
                        .format(amp, gain))

        #- Add saturation level
        if 'SATURLEV' + amp in header:
            saturlev = header['SATURLEV' + amp]  # in electrons
        else:
            if cfinder and cfinder.haskey('SATURLEV' + amp):
                saturlev = float(cfinder.value('SATURLEV' + amp))
                log.info('Using SATURLEV{}={} from calibration data'.format(
                    amp, saturlev))
            else:
                saturlev = 200000
                log.warning(
                    'Missing keyword SATURLEV{} in header and nothing in calib data; using 200000'
                    .format(amp, saturlev))

        # Generate the overscan images
        raw_overscan_col = rawimage[ov_col].copy()

        if use_overscan_row:
            raw_overscan_row = rawimage[ov_row].copy()
            overscan_row = np.zeros_like(raw_overscan_row)

            # Remove overscan_col from overscan_row
            raw_overscan_squared = rawimage[ov_row[0], ov_col[1]].copy()
            for row in range(raw_overscan_row.shape[0]):
                o, r = _overscan(raw_overscan_squared[row])
                overscan_row[row] = raw_overscan_row[row] - o

        # Now remove the overscan_col
        nrows = raw_overscan_col.shape[0]
        log.info("nrows in overscan=%d" % nrows)
        overscan_col = np.zeros(nrows)
        rdnoise = np.zeros(nrows)
        if (cfinder and cfinder.haskey('OVERSCAN' + amp)
                and cfinder.value("OVERSCAN" + amp).upper()
                == "PER_ROW") or overscan_per_row:
            log.info(
                "Subtracting overscan per row for amplifier %s of camera %s" %
                (amp, camera))
            for j in range(nrows):
                if np.isnan(np.sum(overscan_col[j])):
                    log.warning(
                        "NaN values in row %d of overscan of amplifier %s of camera %s"
                        % (j, amp, camera))
                    continue
                o, r = _overscan(raw_overscan_col[j])
                #log.info("%d %f %f"%(j,o,r))
                overscan_col[j] = o
                rdnoise[j] = r
        else:
            log.info(
                "Subtracting average overscan for amplifier %s of camera %s" %
                (amp, camera))
            o, r = _overscan(raw_overscan_col)
            overscan_col += o
            rdnoise += r

        rdnoise *= gain
        median_rdnoise = np.median(rdnoise)
        median_overscan = np.median(overscan_col)
        log.info("Median rdnoise and overscan= %f %f" %
                 (median_rdnoise, median_overscan))

        kk = parse_sec_keyword(header['CCDSEC' + amp])
        for j in range(nrows):
            readnoise[kk][j] = rdnoise[j]

        header['OVERSCN' + amp] = (median_overscan, 'ADUs (gain not applied)')
        if gain != 1:
            rdnoise_message = 'electrons (gain is applied)'
            gain_message = 'e/ADU (gain applied to image)'
        else:
            rdnoise_message = 'ADUs (gain not applied)'
            gain_message = 'gain not applied to image'
        header['OBSRDN' + amp] = (median_rdnoise, rdnoise_message)
        header['GAIN' + amp] = (gain, gain_message)

        #- Warn/error if measured readnoise is very different from expected if exists
        if 'RDNOISE' + amp in header:
            expected_readnoise = header['RDNOISE' + amp]
            if median_rdnoise < 0.5 * expected_readnoise:
                log.error(
                    'Amp {} measured readnoise {:.2f} < 0.5 * expected readnoise {:.2f}'
                    .format(amp, median_rdnoise, expected_readnoise))
            elif median_rdnoise < 0.9 * expected_readnoise:
                log.warning(
                    'Amp {} measured readnoise {:.2f} < 0.9 * expected readnoise {:.2f}'
                    .format(amp, median_rdnoise, expected_readnoise))
            elif median_rdnoise > 2.0 * expected_readnoise:
                log.error(
                    'Amp {} measured readnoise {:.2f} > 2 * expected readnoise {:.2f}'
                    .format(amp, median_rdnoise, expected_readnoise))
            elif median_rdnoise > 1.2 * expected_readnoise:
                log.warning(
                    'Amp {} measured readnoise {:.2f} > 1.2 * expected readnoise {:.2f}'
                    .format(amp, median_rdnoise, expected_readnoise))
        #else:
        #    log.warning('Expected readnoise keyword {} missing'.format('RDNOISE'+amp))

        log.info("Measured readnoise for AMP %s = %f" % (amp, median_rdnoise))

        #- subtract overscan from data region and apply gain
        jj = parse_sec_keyword(header['DATASEC' + amp])

        data = rawimage[jj].copy()
        # Subtract columns
        for k in range(nrows):
            data[k] -= overscan_col[k]
        # And now the rows
        if use_overscan_row:
            # Savgol?
            if use_savgol:
                log.info("Using savgol")
                collapse_oscan_row = np.zeros(overscan_row.shape[1])
                for col in range(overscan_row.shape[1]):
                    o, _ = _overscan(overscan_row[:, col])
                    collapse_oscan_row[col] = o
                oscan_row = _savgol_clipped(collapse_oscan_row, niter=0)
                oimg_row = np.outer(np.ones(data.shape[0]), oscan_row)
                data -= oimg_row
            else:
                o, r = _overscan(overscan_row)
                data -= o

        #- apply saturlev (defined in ADU), prior to multiplication by gain
        saturated = (rawimage[jj] >= saturlev)
        mask[kk][saturated] |= ccdmask.SATURATED

        #- ADC to electrons
        image[kk] = data * gain

    if not nocrosstalk:
        #- apply cross-talk

        # the ccd looks like :
        # C D
        # A B
        # for cross talk, we need a symmetric 4x4 flip_matrix
        # of coordinates ABCD giving flip of both axis
        # when computing crosstalk of
        #    A   B   C   D
        #
        # A  AA  AB  AC  AD
        # B  BA  BB  BC  BD
        # C  CA  CB  CC  CD
        # D  DA  DB  DC  BB
        # orientation_matrix_defines change of orientation
        #
        fip_axis_0 = np.array([[1, 1, -1, -1], [1, 1, -1, -1], [-1, -1, 1, 1],
                               [-1, -1, 1, 1]])
        fip_axis_1 = np.array([[1, -1, 1, -1], [-1, 1, -1, 1], [1, -1, 1, -1],
                               [-1, 1, -1, 1]])

        for a1 in range(len(amp_ids)):
            amp1 = amp_ids[a1]
            ii1 = parse_sec_keyword(header['CCDSEC' + amp1])
            a1flux = image[ii1]
            #a1mask=mask[ii1]

            for a2 in range(len(amp_ids)):
                if a1 == a2:
                    continue
                amp2 = amp_ids[a2]
                if cfinder is None: continue
                if not cfinder.haskey("CROSSTALK%s%s" % (amp1, amp2)): continue
                crosstalk = cfinder.value("CROSSTALK%s%s" % (amp1, amp2))
                if crosstalk == 0.: continue
                log.info("Correct for crosstalk=%f from AMP %s into %s" %
                         (crosstalk, amp1, amp2))
                a12flux = crosstalk * a1flux.copy()
                #a12mask=a1mask.copy()
                if fip_axis_0[a1, a2] == -1:
                    a12flux = a12flux[::-1]
                    #a12mask=a12mask[::-1]
                if fip_axis_1[a1, a2] == -1:
                    a12flux = a12flux[:, ::-1]
                    #a12mask=a12mask[:,::-1]
                ii2 = parse_sec_keyword(header['CCDSEC' + amp2])
                image[ii2] -= a12flux
                # mask[ii2]  |= a12mask (not sure we really need to propagate the mask)

    #- Poisson noise variance (prior to dark subtraction and prior to pixel flat field)
    #- This is biasing, but that's what we have for now
    poisson_var = image.clip(0)

    #- subtract dark after multiplication by gain
    if dark is not False:
        log.info("subtracting dark for amp %s" % amp)
        image -= dark

    #- Correct for dark trails if any
    if not nodarktrail and cfinder is not None:
        for amp in amp_ids:
            if cfinder.haskey("DARKTRAILAMP%s" % amp):
                amplitude = cfinder.value("DARKTRAILAMP%s" % amp)
                width = cfinder.value("DARKTRAILWIDTH%s" % amp)
                ii = _parse_sec_keyword(header["CCDSEC" + amp])
                log.info(
                    "Removing dark trails for amplifier %s with width=%3.1f and amplitude=%5.4f"
                    % (amp, width, amplitude))
                correct_dark_trail(image,
                                   ii,
                                   left=((amp == "B") | (amp == "D")),
                                   width=width,
                                   amplitude=amplitude)

    #- Divide by pixflat image
    pixflat = get_calibration_image(cfinder, "PIXFLAT", pixflat)
    if pixflat is not False:
        if pixflat.shape != image.shape:
            raise ValueError('shape mismatch pixflat {} != image {}'.format(
                pixflat.shape, image.shape))

        almost_zero = 0.001

        if np.all(pixflat > almost_zero):
            image /= pixflat
            readnoise /= pixflat
            poisson_var /= pixflat**2
        else:
            good = (pixflat > almost_zero)
            image[good] /= pixflat[good]
            readnoise[good] /= pixflat[good]
            poisson_var[good] /= pixflat[good]**2
            mask[~good] |= ccdmask.PIXFLATZERO

        lowpixflat = (0 < pixflat) & (pixflat < 0.1)
        if np.any(lowpixflat):
            mask[lowpixflat] |= ccdmask.PIXFLATLOW

    #- Inverse variance, estimated directly from the data (BEWARE: biased!)
    var = poisson_var + readnoise**2
    ivar = np.zeros(var.shape)
    ivar[var > 0] = 1.0 / var[var > 0]

    #- Ridiculously high readnoise is bad
    mask[readnoise > 100] |= ccdmask.BADREADNOISE

    if bkgsub:
        bkg = _background(image, header)
        image -= bkg

    img = Image(image,
                ivar=ivar,
                mask=mask,
                meta=header,
                readnoise=readnoise,
                camera=camera)

    #- update img.mask to mask cosmic rays

    if not nocosmic:
        cosmics.reject_cosmic_rays(img,
                                   nsig=cosmics_nsig,
                                   cfudge=cosmics_cfudge,
                                   c2fudge=cosmics_c2fudge)

    if remove_scattered_light:
        if psf_filename is None:
            psf_filename = cfinder.findfile("PSF")
        xyset = read_xytraceset(psf_filename)
        img.pix -= model_scattered_light(img, xyset)

    return img
Пример #6
0
def main(args=None):

    if args is None:
        args = parse()
    elif isinstance(args, (list, tuple)):
        args = parse(args)

    t0 = time.time()
    log = get_logger()

    # guess if it is a preprocessed or a raw image
    hdulist = fits.open(args.image)
    is_input_preprocessed = ("IMAGE" in hdulist) & ("IVAR" in hdulist)
    primary_header = hdulist[0].header
    hdulist.close()

    if is_input_preprocessed:
        image = read_image(args.image)
    else:
        if args.camera is None:
            print(
                "ERROR: Need to specify camera to open a raw fits image (with all cameras in different fits HDUs)"
            )
            print(
                "Try adding the option '--camera xx', with xx in {brz}{0-9}, like r7,  or type 'desi_qproc --help' for more options"
            )
            sys.exit(12)
        image = read_raw(args.image, args.camera, fill_header=[
            1,
        ])

    if args.auto:
        log.debug("AUTOMATIC MODE")
        try:
            night = image.meta['NIGHT']
            if not 'EXPID' in image.meta:
                if 'EXPNUM' in image.meta:
                    log.warning('using EXPNUM {} for EXPID'.format(
                        image.meta['EXPNUM']))
                    image.meta['EXPID'] = image.meta['EXPNUM']
            expid = image.meta['EXPID']
        except KeyError as e:
            log.error(
                "Need at least NIGHT and EXPID (or EXPNUM) to run in auto mode. Retry without the --auto option."
            )
            log.error(str(e))
            sys.exit(12)

        indir = os.path.dirname(args.image)
        if args.fibermap is None:
            filename = '{}/fibermap-{:08d}.fits'.format(indir, expid)
            if os.path.isfile(filename):
                log.debug("auto-mode: found a fibermap, {}, using it!".format(
                    filename))
                args.fibermap = filename
        if args.output_preproc is None:
            if not is_input_preprocessed:
                args.output_preproc = '{}/preproc-{}-{:08d}.fits'.format(
                    args.auto_output_dir, args.camera.lower(), expid)
                log.debug("auto-mode: will write preproc in " +
                          args.output_preproc)
            else:
                log.debug(
                    "auto-mode: will not write preproc because input is a preprocessed image"
                )

        if args.auto_output_dir != '.':
            if not os.path.isdir(args.auto_output_dir):
                log.debug("auto-mode: creating directory " +
                          args.auto_output_dir)
                os.makedirs(args.auto_output_dir)

    if args.output_preproc is not None:
        write_image(args.output_preproc, image)

    cfinder = None

    if args.psf is None:
        if cfinder is None:
            cfinder = CalibFinder([image.meta, primary_header])
        args.psf = cfinder.findfile("PSF")
        log.info(" Using PSF {}".format(args.psf))

    tset = read_xytraceset(args.psf)

    # add fibermap
    if args.fibermap:
        if os.path.isfile(args.fibermap):
            fibermap = read_fibermap(args.fibermap)
        else:
            log.error("no fibermap file {}".format(args.fibermap))
            fibermap = None
    else:
        fibermap = None

    if "OBSTYPE" in image.meta:
        obstype = image.meta["OBSTYPE"].upper()
        image.meta["OBSTYPE"] = obstype  # make sure it's upper case
        qframe = None
    else:
        log.warning("No OBSTYPE keyword, trying to guess ...")
        qframe = qproc_boxcar_extraction(tset,
                                         image,
                                         width=args.width,
                                         fibermap=fibermap)
        obstype = check_qframe_flavor(
            qframe, input_flavor=image.meta["FLAVOR"]).upper()
        image.meta["OBSTYPE"] = obstype

    log.info("OBSTYPE = '{}'".format(obstype))

    if args.auto:

        # now set the things to do
        if obstype == "SKY" or obstype == "TWILIGHT" or obstype == "SCIENCE":

            args.shift_psf = True
            args.output_psf = '{}/psf-{}-{:08d}.fits'.format(
                args.auto_output_dir, args.camera, expid)
            args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format(
                args.auto_output_dir, args.camera, expid)
            args.apply_fiberflat = True
            args.skysub = True
            args.output_skyframe = '{}/qsky-{}-{:08d}.fits'.format(
                args.auto_output_dir, args.camera, expid)
            args.fluxcalib = True
            args.outframe = '{}/qcframe-{}-{:08d}.fits'.format(
                args.auto_output_dir, args.camera, expid)

        elif obstype == "ARC" or obstype == "TESTARC":

            args.shift_psf = True
            args.output_psf = '{}/psf-{}-{:08d}.fits'.format(
                args.auto_output_dir, args.camera, expid)
            args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format(
                args.auto_output_dir, args.camera, expid)
            args.compute_lsf_sigma = True

        elif obstype == "FLAT" or obstype == "TESTFLAT":
            args.shift_psf = True
            args.output_psf = '{}/psf-{}-{:08d}.fits'.format(
                args.auto_output_dir, args.camera, expid)
            args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format(
                args.auto_output_dir, args.camera, expid)
            args.compute_fiberflat = '{}/qfiberflat-{}-{:08d}.fits'.format(
                args.auto_output_dir, args.camera, expid)

    if args.shift_psf:

        # using the trace shift script
        if args.auto:
            options = option_list({
                "psf":
                args.psf,
                "image":
                "dummy",
                "outpsf":
                "dummy",
                "continuum": ((obstype == "FLAT") | (obstype == "TESTFLAT")),
                "sky": ((obstype == "SCIENCE") | (obstype == "SKY"))
            })
        else:
            options = option_list({
                "psf": args.psf,
                "image": "dummy",
                "outpsf": "dummy"
            })
        tmp_args = trace_shifts_script.parse(options=options)
        tset = trace_shifts_script.fit_trace_shifts(image=image, args=tmp_args)

    qframe = qproc_boxcar_extraction(tset,
                                     image,
                                     width=args.width,
                                     fibermap=fibermap)

    if tset.meta is not None:
        # add traceshift info in the qframe, this will be saved in the qframe header
        if qframe.meta is None:
            qframe.meta = dict()
        for k in tset.meta.keys():
            qframe.meta[k] = tset.meta[k]

    if args.output_rawframe is not None:
        write_qframe(args.output_rawframe, qframe)
        log.info("wrote raw extracted frame in {}".format(
            args.output_rawframe))

    if args.compute_lsf_sigma:
        tset = process_arc(qframe, tset, linelist=None, npoly=2, nbins=2)

    if args.output_psf is not None:
        for k in qframe.meta:
            if k not in tset.meta:
                tset.meta[k] = qframe.meta[k]
        write_xytraceset(args.output_psf, tset)

    if args.compute_fiberflat is not None:
        fiberflat = qproc_compute_fiberflat(qframe)
        #write_qframe(args.compute_fiberflat,qflat)
        write_fiberflat(args.compute_fiberflat, fiberflat, header=qframe.meta)
        log.info("wrote fiberflat in {}".format(args.compute_fiberflat))

    if args.apply_fiberflat or args.input_fiberflat:

        if args.input_fiberflat is None:
            if cfinder is None:
                cfinder = CalibFinder([image.meta, primary_header])
            try:
                args.input_fiberflat = cfinder.findfile("FIBERFLAT")
            except KeyError as e:
                log.error("no FIBERFLAT for this spectro config")
                sys.exit(12)
        log.info("applying fiber flat {}".format(args.input_fiberflat))
        flat = read_fiberflat(args.input_fiberflat)
        qproc_apply_fiberflat(qframe, flat)

    if args.skysub:
        log.info("sky subtraction")
        if args.output_skyframe is not None:
            skyflux = qproc_sky_subtraction(qframe, return_skymodel=True)
            sqframe = QFrame(qframe.wave, skyflux, np.ones(skyflux.shape))
            write_qframe(args.output_skyframe, sqframe)
            log.info("wrote sky model in {}".format(args.output_skyframe))
        else:
            qproc_sky_subtraction(qframe)

    if args.fluxcalib:
        if cfinder is None:
            cfinder = CalibFinder([image.meta, primary_header])
        # check for flux calib
        if cfinder.haskey("FLUXCALIB"):
            fluxcalib_filename = cfinder.findfile("FLUXCALIB")
            fluxcalib = read_average_flux_calibration(fluxcalib_filename)
            log.info("read average calib in {}".format(fluxcalib_filename))
            seeing = qframe.meta["SEEING"]
            airmass = qframe.meta["AIRMASS"]
            exptime = qframe.meta["EXPTIME"]
            exposure_calib = fluxcalib.value(seeing=seeing, airmass=airmass)
            for q in range(qframe.nspec):
                fiber_calib = np.interp(qframe.wave[q], fluxcalib.wave,
                                        exposure_calib) * exptime
                inv_calib = (fiber_calib > 0) / (fiber_calib +
                                                 (fiber_calib == 0))
                qframe.flux[q] *= inv_calib
                qframe.ivar[q] *= fiber_calib**2 * (fiber_calib > 0)

            # add keyword in header giving the calibration factor applied at a reference wavelength
            band = qframe.meta["CAMERA"].upper()[0]
            if band == "B":
                refwave = 4500
            elif band == "R":
                refwave = 6500
            else:
                refwave = 8500
            calvalue = np.interp(refwave, fluxcalib.wave,
                                 exposure_calib) * exptime
            qframe.meta["CALWAVE"] = refwave
            qframe.meta["CALVALUE"] = calvalue
        else:
            log.error(
                "Cannot calibrate fluxes because no FLUXCALIB keywork in calibration files"
            )

    fibers = parse_fibers(args.fibers)
    if fibers is None:
        fibers = qframe.flux.shape[0]
    else:
        ii = np.arange(qframe.fibers.size)[np.in1d(qframe.fibers, fibers)]
        if ii.size == 0:
            log.error("no such fibers in frame,")
            log.error("fibers are in range [{}:{}]".format(
                qframe.fibers[0], qframe.fibers[-1] + 1))
            sys.exit(12)
        qframe = qframe[ii]

    if args.outframe is not None:
        write_qframe(args.outframe, qframe)
        log.info("wrote {}".format(args.outframe))

    t1 = time.time()
    log.info("all done in {:3.1f} sec".format(t1 - t0))

    if args.plot:
        log.info("plotting {} spectra".format(qframe.wave.shape[0]))

        import matplotlib.pyplot as plt
        fig = plt.figure()
        for i in range(qframe.wave.shape[0]):
            j = (qframe.ivar[i] > 0)
            plt.plot(qframe.wave[i, j], qframe.flux[i, j])
        plt.grid()
        plt.xlabel("wavelength")
        plt.ylabel("flux")
        plt.show()
Пример #7
0
 if "MJD-OBS" in pheader :
     mjdobs = pheader["MJD-OBS"]
 else :
     mjdobs = 0.
 if "DATE-OBS" in pheader :
     try : 
         dateobs = Time(pheader["DATE-OBS"]).mjd
     except :
         dateobs = 0.
 else :
     dateobs = 0.
 img=img.astype(float)
 sub = None
 if not args.nobias :
     filename=args.bias
     if filename is None and cfinder is not None and cfinder.haskey("BIAS") :
         filename=cfinder.findfile("BIAS")
     if filename is not None :
         print("subtracting bias",filename)
         bias=fitsio.read(filename)
         sub = img - bias
 if sub is None :
     sub = img # we don't do bias subtraction
        
 rms_scale = 1.
 if args.gradient :
     tmp = sub[:,1:] - sub[:,:-1]
     sub[:,1:] = tmp
     sub[:,0]  = 0
     rms_scale = 1./np.sqrt(2.)
 i=0
Пример #8
0
def main(args=None, comm=None):
    if args is None:
        args = parse()
    # elif isinstance(args, (list, tuple)):
    #     args = parse(args)

    log = get_logger()

    start_mpi_connect = time.time()
    if comm is not None:
        #- Use the provided comm to determine rank and size
        rank = comm.rank
        size = comm.size
    else:
        #- Check MPI flags and determine the comm, rank, and size given the arguments
        comm, rank, size = assign_mpi(do_mpi=args.mpi,
                                      do_batch=args.batch,
                                      log=log)
    stop_mpi_connect = time.time()

    #- Start timer; only print log messages from rank 0 (others are silent)
    timer = desiutil.timer.Timer(silent=(rank > 0))

    #- Fill in timing information for steps before we had the timer created
    if args.starttime is not None:
        timer.start('startup', starttime=args.starttime)
        timer.stop('startup', stoptime=start_imports)

    timer.start('imports', starttime=start_imports)
    timer.stop('imports', stoptime=stop_imports)

    timer.start('mpi_connect', starttime=start_mpi_connect)
    timer.stop('mpi_connect', stoptime=stop_mpi_connect)

    #- Freeze IERS after parsing args so that it doesn't bother if only --help
    timer.start('freeze_iers')
    desiutil.iers.freeze_iers()
    timer.stop('freeze_iers')

    #- Preflight checks
    timer.start('preflight')
    if rank > 0:
        #- Let rank 0 fetch these, and then broadcast
        args, hdr, camhdr = None, None, None
    else:
        args, hdr, camhdr = update_args_with_headers(args)

    ## Make sure badamps is formatted properly
    if comm is not None and rank == 0 and args.badamps is not None:
        args.badamps = validate_badamps(args.badamps)

    if comm is not None:
        args = comm.bcast(args, root=0)
        hdr = comm.bcast(hdr, root=0)
        camhdr = comm.bcast(camhdr, root=0)

    known_obstype = [
        'SCIENCE', 'ARC', 'FLAT', 'ZERO', 'DARK', 'TESTARC', 'TESTFLAT',
        'PIXFLAT', 'SKY', 'TWILIGHT', 'OTHER'
    ]
    if args.obstype not in known_obstype:
        raise RuntimeError('obstype {} not in {}'.format(
            args.obstype, known_obstype))

    timer.stop('preflight')

    #-------------------------------------------------------------------------
    #- Create and submit a batch job if requested

    if args.batch:
        #exp_str = '{:08d}'.format(args.expid)
        jobdesc = args.obstype.lower()
        if args.obstype == 'SCIENCE':
            # if not doing pre-stdstar fitting or stdstar fitting and if there is
            # no flag stopping flux calibration, set job to poststdstar
            if args.noprestdstarfit and args.nostdstarfit and (
                    not args.nofluxcalib):
                jobdesc = 'poststdstar'
            # elif told not to do std or post stdstar but the flag for prestdstar isn't set,
            # then perform prestdstar
            elif (not args.noprestdstarfit
                  ) and args.nostdstarfit and args.nofluxcalib:
                jobdesc = 'prestdstar'
            #elif (not args.noprestdstarfit) and (not args.nostdstarfit) and (not args.nofluxcalib):
            #    jobdesc = 'science'
        scriptfile = create_desi_proc_batch_script(night=args.night, exp=args.expid, cameras=args.cameras,\
                                                jobdesc=jobdesc, queue=args.queue, runtime=args.runtime,\
                                                batch_opts=args.batch_opts, timingfile=args.timingfile,
                                                system_name=args.system_name)
        err = 0
        if not args.nosubmit:
            err = subprocess.call(['sbatch', scriptfile])
        sys.exit(err)

    #-------------------------------------------------------------------------
    #- Proceeding with running

    #- What are we going to do?
    if rank == 0:
        log.info('----------')
        log.info('Input {}'.format(args.input))
        log.info('Night {} expid {}'.format(args.night, args.expid))
        log.info('Obstype {}'.format(args.obstype))
        log.info('Cameras {}'.format(args.cameras))
        log.info('Output root {}'.format(desispec.io.specprod_root()))
        log.info('----------')

    #- Create output directories if needed
    if rank == 0:
        preprocdir = os.path.dirname(
            findfile('preproc', args.night, args.expid, 'b0'))
        expdir = os.path.dirname(
            findfile('frame', args.night, args.expid, 'b0'))
        os.makedirs(preprocdir, exist_ok=True)
        os.makedirs(expdir, exist_ok=True)

    #- Wait for rank 0 to make directories before proceeding
    if comm is not None:
        comm.barrier()

    #-------------------------------------------------------------------------
    #- Preproc
    #- All obstypes get preprocessed

    timer.start('fibermap')

    #- Assemble fibermap for science exposures
    fibermap = None
    fibermap_ok = None
    if rank == 0 and args.obstype == 'SCIENCE':
        fibermap = findfile('fibermap', args.night, args.expid)
        if not os.path.exists(fibermap):
            tmp = findfile('preproc', args.night, args.expid, 'b0')
            preprocdir = os.path.dirname(tmp)
            fibermap = os.path.join(preprocdir, os.path.basename(fibermap))

            log.info('Creating fibermap {}'.format(fibermap))
            cmd = 'assemble_fibermap -n {} -e {} -o {}'.format(
                args.night, args.expid, fibermap)
            if args.badamps is not None:
                cmd += ' --badamps={}'.format(args.badamps)
            runcmd(cmd, inputs=[], outputs=[fibermap])

        fibermap_ok = os.path.exists(fibermap)

        #- Some commissioning files didn't have coords* files that caused assemble_fibermap to fail
        #- these are well known failures with no other solution, so for those, just force creation
        #- of a fibermap with null coordinate information
        if not fibermap_ok and int(args.night) < 20200310:
            log.info(
                "Since night is before 20200310, trying to force fibermap creation without coords file"
            )
            cmd += ' --force'
            runcmd(cmd, inputs=[], outputs=[fibermap])
            fibermap_ok = os.path.exists(fibermap)

    #- If assemble_fibermap failed and obstype is SCIENCE, exit now
    if comm is not None:
        fibermap_ok = comm.bcast(fibermap_ok, root=0)

    if args.obstype == 'SCIENCE' and not fibermap_ok:
        sys.stdout.flush()
        if rank == 0:
            log.critical(
                'assemble_fibermap failed for science exposure; exiting now')

        sys.exit(13)

    #- Wait for rank 0 to make fibermap if needed
    if comm is not None:
        fibermap = comm.bcast(fibermap, root=0)

    timer.stop('fibermap')

    if not (args.obstype in ['SCIENCE'] and args.noprestdstarfit):
        timer.start('preproc')
        for i in range(rank, len(args.cameras), size):
            camera = args.cameras[i]
            outfile = findfile('preproc', args.night, args.expid, camera)
            outdir = os.path.dirname(outfile)
            cmd = "desi_preproc -i {} -o {} --outdir {} --cameras {}".format(
                args.input, outfile, outdir, camera)
            if args.scattered_light:
                cmd += " --scattered-light"
            if fibermap is not None:
                cmd += " --fibermap {}".format(fibermap)
            if not args.obstype in ['ARC']:  # never model variance for arcs
                if not args.no_model_pixel_variance:
                    cmd += " --model-variance"
            runcmd(cmd, inputs=[args.input], outputs=[outfile])

        timer.stop('preproc')
        if comm is not None:
            comm.barrier()

    #-------------------------------------------------------------------------
    #- Get input PSFs
    timer.start('findpsf')
    input_psf = dict()
    if rank == 0:
        for camera in args.cameras:
            if args.psf is not None:
                input_psf[camera] = args.psf
            elif args.calibnight is not None:
                # look for a psfnight psf for this calib night
                psfnightfile = findfile('psfnight', args.calibnight,
                                        args.expid, camera)
                if not os.path.isfile(psfnightfile):
                    log.error("no {}".format(psfnightfile))
                    raise IOError("no {}".format(psfnightfile))
                input_psf[camera] = psfnightfile
            else:
                # look for a psfnight psf
                psfnightfile = findfile('psfnight', args.night, args.expid,
                                        camera)
                if os.path.isfile(psfnightfile):
                    input_psf[camera] = psfnightfile
                elif args.most_recent_calib:
                    nightfile = find_most_recent(args.night,
                                                 file_type='psfnight')
                    if nightfile is None:
                        input_psf[camera] = findcalibfile(
                            [hdr, camhdr[camera]], 'PSF')
                    else:
                        input_psf[camera] = nightfile
                else:
                    input_psf[camera] = findcalibfile([hdr, camhdr[camera]],
                                                      'PSF')
            log.info("Will use input PSF : {}".format(input_psf[camera]))

    if comm is not None:
        input_psf = comm.bcast(input_psf, root=0)

    timer.stop('findpsf')

    #-------------------------------------------------------------------------
    #- Traceshift

    if ( args.obstype in ['FLAT', 'TESTFLAT', 'SKY', 'TWILIGHT']     )   or \
    ( args.obstype in ['SCIENCE'] and (not args.noprestdstarfit) ):

        timer.start('traceshift')

        if rank == 0 and args.traceshift:
            log.info('Starting traceshift at {}'.format(time.asctime()))

        for i in range(rank, len(args.cameras), size):
            camera = args.cameras[i]
            preprocfile = findfile('preproc', args.night, args.expid, camera)
            inpsf = input_psf[camera]
            outpsf = findfile('psf', args.night, args.expid, camera)
            if not os.path.isfile(outpsf):
                if args.traceshift:
                    cmd = "desi_compute_trace_shifts"
                    cmd += " -i {}".format(preprocfile)
                    cmd += " --psf {}".format(inpsf)
                    cmd += " --outpsf {}".format(outpsf)
                    cmd += " --degxx 2 --degxy 0"
                    if args.obstype in ['FLAT', 'TESTFLAT', 'TWILIGHT']:
                        cmd += " --continuum"
                    else:
                        cmd += " --degyx 2 --degyy 0"
                    if args.obstype in ['SCIENCE', 'SKY']:
                        cmd += ' --sky'
                else:
                    cmd = "ln -s {} {}".format(inpsf, outpsf)
                runcmd(cmd, inputs=[preprocfile, inpsf], outputs=[outpsf])
            else:
                log.info("PSF {} exists".format(outpsf))

        timer.stop('traceshift')
        if comm is not None:
            comm.barrier()

    #-------------------------------------------------------------------------
    #- PSF
    #- MPI parallelize this step

    if args.obstype in ['ARC', 'TESTARC']:

        timer.start('arc_traceshift')

        if rank == 0:
            log.info('Starting traceshift before specex PSF fit at {}'.format(
                time.asctime()))

        for i in range(rank, len(args.cameras), size):
            camera = args.cameras[i]
            preprocfile = findfile('preproc', args.night, args.expid, camera)
            inpsf = input_psf[camera]
            outpsf = findfile('psf', args.night, args.expid, camera)
            outpsf = replace_prefix(outpsf, "psf", "shifted-input-psf")
            if not os.path.isfile(outpsf):
                cmd = "desi_compute_trace_shifts"
                cmd += " -i {}".format(preprocfile)
                cmd += " --psf {}".format(inpsf)
                cmd += " --outpsf {}".format(outpsf)
                cmd += " --degxx 0 --degxy 0 --degyx 0 --degyy 0"
                cmd += ' --arc-lamps'
                runcmd(cmd, inputs=[preprocfile, inpsf], outputs=[outpsf])
            else:
                log.info("PSF {} exists".format(outpsf))

        timer.stop('arc_traceshift')
        if comm is not None:
            comm.barrier()

        timer.start('psf')

        if rank == 0:
            log.info('Starting specex PSF fitting at {}'.format(
                time.asctime()))

        if rank > 0:
            cmds = inputs = outputs = None
        else:
            cmds = dict()
            inputs = dict()
            outputs = dict()
            for camera in args.cameras:
                preprocfile = findfile('preproc', args.night, args.expid,
                                       camera)
                tmpname = findfile('psf', args.night, args.expid, camera)
                inpsf = replace_prefix(tmpname, "psf", "shifted-input-psf")
                outpsf = replace_prefix(tmpname, "psf", "fit-psf")

                log.info("now run specex psf fit")

                cmd = 'desi_compute_psf'
                cmd += ' --input-image {}'.format(preprocfile)
                cmd += ' --input-psf {}'.format(inpsf)
                cmd += ' --output-psf {}'.format(outpsf)

                # look for fiber blacklist
                cfinder = CalibFinder([hdr, camhdr[camera]])
                blacklistkey = "FIBERBLACKLIST"
                if not cfinder.haskey(blacklistkey) and cfinder.haskey(
                        "BROKENFIBERS"):
                    log.warning(
                        "BROKENFIBERS yaml keyword deprecated, please use FIBERBLACKLIST"
                    )
                    blacklistkey = "BROKENFIBERS"

                if cfinder.haskey(blacklistkey):
                    blacklist = cfinder.value(blacklistkey)
                    cmd += ' --broken-fibers {}'.format(blacklist)
                    if rank == 0:
                        log.warning('broken fibers: {}'.format(blacklist))

                if not os.path.exists(outpsf):
                    cmds[camera] = cmd
                    inputs[camera] = [preprocfile, inpsf]
                    outputs[camera] = [
                        outpsf,
                    ]

        if comm is not None:
            cmds = comm.bcast(cmds, root=0)
            inputs = comm.bcast(inputs, root=0)
            outputs = comm.bcast(outputs, root=0)
            #- split communicator by 20 (number of bundles)
            group_size = 20
            if (rank == 0) and (size % group_size != 0):
                log.warning(
                    'MPI size={} should be evenly divisible by {}'.format(
                        size, group_size))

            group = rank // group_size
            num_groups = (size + group_size - 1) // group_size
            comm_group = comm.Split(color=group)

            if rank == 0:
                log.info(
                    f'Fitting PSFs with {num_groups} sub-communicators of size {group_size}'
                )

            for i in range(group, len(args.cameras), num_groups):
                camera = args.cameras[i]
                if camera in cmds:
                    cmdargs = cmds[camera].split()[1:]
                    cmdargs = desispec.scripts.specex.parse(cmdargs)
                    if comm_group.rank == 0:
                        print('RUNNING: {}'.format(cmds[camera]))
                        t0 = time.time()
                        timestamp = time.asctime()
                        log.info(
                            f'MPI group {group} ranks {rank}-{rank+group_size-1} fitting PSF for {camera} at {timestamp}'
                        )
                    try:
                        desispec.scripts.specex.main(cmdargs, comm=comm_group)
                    except Exception as e:
                        if comm_group.rank == 0:
                            log.error(
                                f'FAILED: MPI group {group} ranks {rank}-{rank+group_size-1} camera {camera}'
                            )
                            log.error('FAILED: {}'.format(cmds[camera]))
                            log.error(e)

                    if comm_group.rank == 0:
                        specex_time = time.time() - t0
                        log.info(
                            f'specex fit for {camera} took {specex_time:.1f} seconds'
                        )

            comm.barrier()

        else:
            log.warning(
                'fitting PSFs without MPI parallelism; this will be SLOW')
            for camera in args.cameras:
                if camera in cmds:
                    runcmd(cmds[camera],
                           inputs=inputs[camera],
                           outputs=outputs[camera])

        if comm is not None:
            comm.barrier()

        # loop on all cameras and interpolate bad fibers
        for camera in args.cameras[rank::size]:
            t0 = time.time()
            log.info(f'Rank {rank} interpolating {camera} PSF over bad fibers')
            # look for fiber blacklist
            cfinder = CalibFinder([hdr, camhdr[camera]])
            blacklistkey = "FIBERBLACKLIST"
            if not cfinder.haskey(blacklistkey) and cfinder.haskey(
                    "BROKENFIBERS"):
                log.warning(
                    "BROKENFIBERS yaml keyword deprecated, please use FIBERBLACKLIST"
                )
                blacklistkey = "BROKENFIBERS"

            if cfinder.haskey(blacklistkey):
                fiberblacklist = cfinder.value(blacklistkey)
                tmpname = findfile('psf', args.night, args.expid, camera)
                inpsf = replace_prefix(tmpname, "psf", "fit-psf")
                outpsf = replace_prefix(tmpname, "psf",
                                        "fit-psf-fixed-blacklisted")
                if os.path.isfile(inpsf) and not os.path.isfile(outpsf):
                    cmd = 'desi_interpolate_fiber_psf'
                    cmd += ' --infile {}'.format(inpsf)
                    cmd += ' --outfile {}'.format(outpsf)
                    cmd += ' --fibers {}'.format(fiberblacklist)
                    log.info(
                        'For camera {} interpolating PSF for broken fibers: {}'
                        .format(camera, fiberblacklist))
                    runcmd(cmd, inputs=[inpsf], outputs=[outpsf])
                    if os.path.isfile(outpsf):
                        os.rename(
                            inpsf,
                            inpsf.replace("fit-psf",
                                          "fit-psf-before-blacklisted-fix"))
                        subprocess.call('cp {} {}'.format(outpsf, inpsf),
                                        shell=True)

            dt = time.time() - t0
            log.info(
                f'Rank {rank} {camera} PSF interpolation took {dt:.1f} sec')

        timer.stop('psf')

    #-------------------------------------------------------------------------
    #- Merge PSF of night if applicable

    #if args.obstype in ['ARC']:
    if False:
        if rank == 0:
            for camera in args.cameras:
                psfnightfile = findfile('psfnight', args.night, args.expid,
                                        camera)
                if not os.path.isfile(
                        psfnightfile
                ):  # we still don't have a psf night, see if we can compute it ...
                    psfs = glob.glob(
                        findfile('psf', args.night, args.expid,
                                 camera).replace("psf", "fit-psf").replace(
                                     str(args.expid), "*"))
                    log.info(
                        "Number of PSF for night={} camera={} = {}".format(
                            args.night, camera, len(psfs)))
                    if len(psfs) > 4:  # lets do it!
                        log.info("Computing psfnight ...")
                        dirname = os.path.dirname(psfnightfile)
                        if not os.path.isdir(dirname):
                            os.makedirs(dirname)
                        desispec.scripts.specex.mean_psf(psfs, psfnightfile)
                if os.path.isfile(psfnightfile):  # now use this one
                    input_psf[camera] = psfnightfile

    #-------------------------------------------------------------------------
    #- Extract
    #- This is MPI parallel so handle a bit differently

    # maybe add ARC and TESTARC too
    if ( args.obstype in ['FLAT', 'TESTFLAT', 'SKY', 'TWILIGHT']     )   or \
    ( args.obstype in ['SCIENCE'] and (not args.noprestdstarfit) ):

        timer.start('extract')
        if rank == 0:
            log.info('Starting extractions at {}'.format(time.asctime()))

        if rank > 0:
            cmds = inputs = outputs = None
        else:
            cmds = dict()
            inputs = dict()
            outputs = dict()
            for camera in args.cameras:
                cmd = 'desi_extract_spectra'

                #- Based on data from SM1-SM8, looking at central and edge fibers
                #- with in mind overlapping arc lamps lines
                if camera.startswith('b'):
                    cmd += ' -w 3600.0,5800.0,0.8'
                elif camera.startswith('r'):
                    cmd += ' -w 5760.0,7620.0,0.8'
                elif camera.startswith('z'):
                    cmd += ' -w 7520.0,9824.0,0.8'

                preprocfile = findfile('preproc', args.night, args.expid,
                                       camera)
                psffile = findfile('psf', args.night, args.expid, camera)
                framefile = findfile('frame', args.night, args.expid, camera)
                cmd += ' -i {}'.format(preprocfile)
                cmd += ' -p {}'.format(psffile)
                cmd += ' -o {}'.format(framefile)
                cmd += ' --psferr 0.1'

                if args.obstype == 'SCIENCE' or args.obstype == 'SKY':
                    if rank == 0:
                        log.info('Include barycentric correction')
                    cmd += ' --barycentric-correction'

                if not os.path.exists(framefile):
                    cmds[camera] = cmd
                    inputs[camera] = [preprocfile, psffile]
                    outputs[camera] = [
                        framefile,
                    ]

        #- TODO: refactor/combine this with PSF comm splitting logic
        if comm is not None:
            cmds = comm.bcast(cmds, root=0)
            inputs = comm.bcast(inputs, root=0)
            outputs = comm.bcast(outputs, root=0)

            #- split communicator by 20 (number of bundles)
            extract_size = 20
            if (rank == 0) and (size % extract_size != 0):
                log.warning(
                    'MPI size={} should be evenly divisible by {}'.format(
                        size, extract_size))

            extract_group = rank // extract_size
            num_extract_groups = (size + extract_size - 1) // extract_size
            comm_extract = comm.Split(color=extract_group)

            for i in range(extract_group, len(args.cameras),
                           num_extract_groups):
                camera = args.cameras[i]
                if camera in cmds:
                    cmdargs = cmds[camera].split()[1:]
                    extract_args = desispec.scripts.extract.parse(cmdargs)
                    if comm_extract.rank == 0:
                        print('RUNNING: {}'.format(cmds[camera]))

                    desispec.scripts.extract.main_mpi(extract_args,
                                                      comm=comm_extract)

            comm.barrier()

        else:
            log.warning(
                'running extractions without MPI parallelism; this will be SLOW'
            )
            for camera in args.cameras:
                if camera in cmds:
                    runcmd(cmds[camera],
                           inputs=inputs[camera],
                           outputs=outputs[camera])

        timer.stop('extract')
        if comm is not None:
            comm.barrier()

    #-------------------------------------------------------------------------
    #- Fiberflat

    if args.obstype in ['FLAT', 'TESTFLAT']:
        timer.start('fiberflat')
        if rank == 0:
            log.info('Starting fiberflats at {}'.format(time.asctime()))

        for i in range(rank, len(args.cameras), size):
            camera = args.cameras[i]
            framefile = findfile('frame', args.night, args.expid, camera)
            fiberflatfile = findfile('fiberflat', args.night, args.expid,
                                     camera)
            cmd = "desi_compute_fiberflat"
            cmd += " -i {}".format(framefile)
            cmd += " -o {}".format(fiberflatfile)
            runcmd(cmd, inputs=[
                framefile,
            ], outputs=[
                fiberflatfile,
            ])

        timer.stop('fiberflat')
        if comm is not None:
            comm.barrier()

    #-------------------------------------------------------------------------
    #- Average and auto-calib fiberflats of night if applicable

    #if args.obstype in ['FLAT']:
    if False:
        if rank == 0:
            fiberflatnightfile = findfile('fiberflatnight', args.night,
                                          args.expid, args.cameras[0])
            fiberflatdirname = os.path.dirname(fiberflatnightfile)
            if not os.path.isfile(fiberflatnightfile) and len(
                    args.cameras
            ) >= 6:  # we still don't have them, see if we can compute them, but need at least 2 spectros ...
                flats = glob.glob(
                    findfile('fiberflat', args.night, args.expid,
                             "b0").replace(str(args.expid),
                                           "*").replace("b0", "*"))
                log.info("Number of fiberflat for night {} = {}".format(
                    args.night, len(flats)))
                if len(flats) >= 3 * 4 * len(
                        args.cameras
                ):  # lets do it! (3 exposures x 4 lamps x N cameras)
                    log.info(
                        "Computing fiberflatnight per lamp and camera ...")
                    tmpdir = os.path.join(fiberflatdirname, "tmp")
                    if not os.path.isdir(tmpdir):
                        os.makedirs(tmpdir)

                    log.info(
                        "First average measurements per camera and per lamp")
                    average_flats = dict()
                    for camera in args.cameras:
                        # list of flats for this camera
                        flats_for_this_camera = []
                        for flat in flats:
                            if flat.find(camera) >= 0:
                                flats_for_this_camera.append(flat)
                        #log.info("For camera {} , flats = {}".format(camera,flats_for_this_camera))
                        #sys.exit(12)

                        # average per lamp (and camera)
                        average_flats[camera] = list()
                        for lampbox in range(4):
                            ofile = os.path.join(
                                tmpdir,
                                "fiberflatnight-camera-{}-lamp-{}.fits".format(
                                    camera, lampbox))
                            if not os.path.isfile(ofile):
                                log.info(
                                    "Average flat for camera {} and lamp box #{}"
                                    .format(camera, lampbox))
                                pg = "CALIB DESI-CALIB-0{} LEDs only".format(
                                    lampbox)

                                cmd = "desi_average_fiberflat --program '{}' --outfile {} -i ".format(
                                    pg, ofile)
                                for flat in flats_for_this_camera:
                                    cmd += " {} ".format(flat)
                                runcmd(cmd,
                                       inputs=flats_for_this_camera,
                                       outputs=[
                                           ofile,
                                       ])
                                if os.path.isfile(ofile):
                                    average_flats[camera].append(ofile)
                            else:
                                log.info("Will use existing {}".format(ofile))
                                average_flats[camera].append(ofile)

                    log.info(
                        "Auto-calibration across lamps and spectro  per camera arm (b,r,z)"
                    )
                    for camera_arm in ["b", "r", "z"]:
                        cameras_for_this_arm = []
                        flats_for_this_arm = []
                        for camera in args.cameras:
                            if camera[0].lower() == camera_arm:
                                cameras_for_this_arm.append(camera)
                                if camera in average_flats:
                                    for flat in average_flats[camera]:
                                        flats_for_this_arm.append(flat)
                        cmd = "desi_autocalib_fiberflat --night {} --arm {} -i ".format(
                            args.night, camera_arm)
                        for flat in flats_for_this_arm:
                            cmd += " {} ".format(flat)
                        runcmd(cmd, inputs=flats_for_this_arm, outputs=[])
                    log.info("Done with fiber flats per night")

        if comm is not None:
            comm.barrier()

    #-------------------------------------------------------------------------
    #- Get input fiberflat
    if args.obstype in ['SCIENCE', 'SKY'] and (not args.nofiberflat):
        timer.start('find_fiberflat')
        input_fiberflat = dict()
        if rank == 0:
            for camera in args.cameras:
                if args.fiberflat is not None:
                    input_fiberflat[camera] = args.fiberflat
                elif args.calibnight is not None:
                    # look for a fiberflatnight for this calib night
                    fiberflatnightfile = findfile('fiberflatnight',
                                                  args.calibnight, args.expid,
                                                  camera)
                    if not os.path.isfile(fiberflatnightfile):
                        log.error("no {}".format(fiberflatnightfile))
                        raise IOError("no {}".format(fiberflatnightfile))
                    input_fiberflat[camera] = fiberflatnightfile
                else:
                    # look for a fiberflatnight fiberflat
                    fiberflatnightfile = findfile('fiberflatnight', args.night,
                                                  args.expid, camera)
                    if os.path.isfile(fiberflatnightfile):
                        input_fiberflat[camera] = fiberflatnightfile
                    elif args.most_recent_calib:
                        nightfile = find_most_recent(
                            args.night, file_type='fiberflatnight')
                        if nightfile is None:
                            input_fiberflat[camera] = findcalibfile(
                                [hdr, camhdr[camera]], 'FIBERFLAT')
                        else:
                            input_fiberflat[camera] = nightfile
                    else:
                        input_fiberflat[camera] = findcalibfile(
                            [hdr, camhdr[camera]], 'FIBERFLAT')
                log.info("Will use input FIBERFLAT: {}".format(
                    input_fiberflat[camera]))

        if comm is not None:
            input_fiberflat = comm.bcast(input_fiberflat, root=0)

        timer.stop('find_fiberflat')

    #-------------------------------------------------------------------------
    #- Apply fiberflat and write fframe file

    if args.obstype in ['SCIENCE', 'SKY'] and args.fframe and \
    ( not args.nofiberflat ) and (not args.noprestdstarfit):
        timer.start('apply_fiberflat')
        if rank == 0:
            log.info('Applying fiberflat at {}'.format(time.asctime()))

        for i in range(rank, len(args.cameras), size):
            camera = args.cameras[i]
            fframefile = findfile('fframe', args.night, args.expid, camera)
            if not os.path.exists(fframefile):
                framefile = findfile('frame', args.night, args.expid, camera)
                fr = desispec.io.read_frame(framefile)
                flatfilename = input_fiberflat[camera]
                if flatfilename is not None:
                    ff = desispec.io.read_fiberflat(flatfilename)
                    fr.meta['FIBERFLT'] = desispec.io.shorten_filename(
                        flatfilename)
                    apply_fiberflat(fr, ff)

                    fframefile = findfile('fframe', args.night, args.expid,
                                          camera)
                    desispec.io.write_frame(fframefile, fr)
                else:
                    log.warning(
                        "Missing fiberflat for camera {}".format(camera))

        timer.stop('apply_fiberflat')
        if comm is not None:
            comm.barrier()

    #-------------------------------------------------------------------------
    #- Select random sky fibers (inplace update of frame file)
    #- TODO: move this to a function somewhere
    #- TODO: this assigns different sky fibers to each frame of same spectrograph

    if (args.obstype in [
            'SKY', 'SCIENCE'
    ]) and (not args.noskysub) and (not args.noprestdstarfit):
        timer.start('picksky')
        if rank == 0:
            log.info('Picking sky fibers at {}'.format(time.asctime()))

        for i in range(rank, len(args.cameras), size):
            camera = args.cameras[i]
            framefile = findfile('frame', args.night, args.expid, camera)
            orig_frame = desispec.io.read_frame(framefile)

            #- Make a copy so that we can apply fiberflat
            fr = deepcopy(orig_frame)

            if np.any(fr.fibermap['OBJTYPE'] == 'SKY'):
                log.info('{} sky fibers already set; skipping'.format(
                    os.path.basename(framefile)))
                continue

            #- Apply fiberflat then select random fibers below a flux cut
            flatfilename = input_fiberflat[camera]
            if flatfilename is None:
                log.error("No fiberflat for {}".format(camera))
                continue
            ff = desispec.io.read_fiberflat(flatfilename)
            apply_fiberflat(fr, ff)
            sumflux = np.sum(fr.flux, axis=1)
            fluxcut = np.percentile(sumflux, 30)
            iisky = np.where(sumflux < fluxcut)[0]
            iisky = np.random.choice(iisky, size=100, replace=False)

            #- Update fibermap or original frame and write out
            orig_frame.fibermap['OBJTYPE'][iisky] = 'SKY'
            orig_frame.fibermap['DESI_TARGET'][iisky] |= desi_mask.SKY

            desispec.io.write_frame(framefile, orig_frame)

        timer.stop('picksky')
        if comm is not None:
            comm.barrier()

    #-------------------------------------------------------------------------
    #- Sky subtraction
    if args.obstype in [
            'SCIENCE', 'SKY'
    ] and (not args.noskysub) and (not args.noprestdstarfit):
        timer.start('skysub')
        if rank == 0:
            log.info('Starting sky subtraction at {}'.format(time.asctime()))

        for i in range(rank, len(args.cameras), size):
            camera = args.cameras[i]
            framefile = findfile('frame', args.night, args.expid, camera)
            hdr = fitsio.read_header(framefile, 'FLUX')
            fiberflatfile = input_fiberflat[camera]
            if fiberflatfile is None:
                log.error("No fiberflat for {}".format(camera))
                continue
            skyfile = findfile('sky', args.night, args.expid, camera)

            cmd = "desi_compute_sky"
            cmd += " -i {}".format(framefile)
            cmd += " --fiberflat {}".format(fiberflatfile)
            cmd += " --o {}".format(skyfile)
            if args.no_extra_variance:
                cmd += " --no-extra-variance"
            if not args.no_sky_wavelength_adjustment:
                cmd += " --adjust-wavelength"
            if not args.no_sky_lsf_adjustment: cmd += " --adjust-lsf"

            runcmd(cmd, inputs=[framefile, fiberflatfile], outputs=[
                skyfile,
            ])

            #- sframe = flatfielded sky-subtracted but not flux calibrated frame
            #- Note: this re-reads and re-does steps previously done for picking
            #- sky fibers; desi_proc is about human efficiency,
            #- not I/O or CPU efficiency...
            sframefile = desispec.io.findfile('sframe', args.night, args.expid,
                                              camera)
            if not os.path.exists(sframefile):
                frame = desispec.io.read_frame(framefile)
                fiberflat = desispec.io.read_fiberflat(fiberflatfile)
                sky = desispec.io.read_sky(skyfile)
                apply_fiberflat(frame, fiberflat)
                subtract_sky(frame, sky, apply_throughput_correction=True)
                frame.meta['IN_SKY'] = shorten_filename(skyfile)
                frame.meta['FIBERFLT'] = shorten_filename(fiberflatfile)
                desispec.io.write_frame(sframefile, frame)

        timer.stop('skysub')
        if comm is not None:
            comm.barrier()

    #-------------------------------------------------------------------------
    #- Standard Star Fitting

    if args.obstype in ['SCIENCE',] and \
            (not args.noskysub ) and \
            (not args.nostdstarfit) :

        timer.start('stdstarfit')
        if rank == 0:
            log.info('Starting flux calibration at {}'.format(time.asctime()))

        #- Group inputs by spectrograph
        framefiles = dict()
        skyfiles = dict()
        fiberflatfiles = dict()
        night, expid = args.night, args.expid  #- shorter
        for camera in args.cameras:
            sp = int(camera[1])
            if sp not in framefiles:
                framefiles[sp] = list()
                skyfiles[sp] = list()
                fiberflatfiles[sp] = list()

            framefiles[sp].append(findfile('frame', night, expid, camera))
            skyfiles[sp].append(findfile('sky', night, expid, camera))
            fiberflatfiles[sp].append(input_fiberflat[camera])

        #- Hardcoded stdstar model version
        starmodels = os.path.join(os.getenv('DESI_BASIS_TEMPLATES'),
                                  'stdstar_templates_v2.2.fits')

        #- Fit stdstars per spectrograph (not per-camera)
        spectro_nums = sorted(framefiles.keys())
        ## for sp in spectro_nums[rank::size]:
        for i in range(rank, len(spectro_nums), size):
            sp = spectro_nums[i]

            stdfile = findfile('stdstars', night, expid, spectrograph=sp)
            cmd = "desi_fit_stdstars"
            cmd += " --frames {}".format(' '.join(framefiles[sp]))
            cmd += " --skymodels {}".format(' '.join(skyfiles[sp]))
            cmd += " --fiberflats {}".format(' '.join(fiberflatfiles[sp]))
            cmd += " --starmodels {}".format(starmodels)
            cmd += " --outfile {}".format(stdfile)
            cmd += " --delta-color 0.1"
            if args.maxstdstars is not None:
                cmd += " --maxstdstars {}".format(args.maxstdstars)

            inputs = framefiles[sp] + skyfiles[sp] + fiberflatfiles[sp]
            runcmd(cmd, inputs=inputs, outputs=[stdfile])

        timer.stop('stdstarfit')
        if comm is not None:
            comm.barrier()

    # -------------------------------------------------------------------------
    # - Flux calibration

    if args.obstype in ['SCIENCE'] and \
                (not args.noskysub) and \
                (not args.nofluxcalib):
        timer.start('fluxcalib')

        night, expid = args.night, args.expid  #- shorter
        #- Compute flux calibration vectors per camera
        for camera in args.cameras[rank::size]:
            framefile = findfile('frame', night, expid, camera)
            skyfile = findfile('sky', night, expid, camera)
            spectrograph = int(camera[1])
            stdfile = findfile('stdstars',
                               night,
                               expid,
                               spectrograph=spectrograph)
            calibfile = findfile('fluxcalib', night, expid, camera)

            fiberflatfile = input_fiberflat[camera]

            cmd = "desi_compute_fluxcalibration"
            cmd += " --infile {}".format(framefile)
            cmd += " --sky {}".format(skyfile)
            cmd += " --fiberflat {}".format(fiberflatfile)
            cmd += " --models {}".format(stdfile)
            cmd += " --outfile {}".format(calibfile)
            cmd += " --delta-color-cut 0.1"

            inputs = [framefile, skyfile, fiberflatfile, stdfile]
            runcmd(cmd, inputs=inputs, outputs=[
                calibfile,
            ])

        timer.stop('fluxcalib')
        if comm is not None:
            comm.barrier()

    #-------------------------------------------------------------------------
    #- Applying flux calibration

    if args.obstype in [
            'SCIENCE',
    ] and (not args.noskysub) and (not args.nofluxcalib):

        night, expid = args.night, args.expid  #- shorter

        timer.start('applycalib')
        if rank == 0:
            log.info('Starting cframe file creation at {}'.format(
                time.asctime()))

        for camera in args.cameras[rank::size]:
            framefile = findfile('frame', night, expid, camera)
            skyfile = findfile('sky', night, expid, camera)
            spectrograph = int(camera[1])
            stdfile = findfile('stdstars',
                               night,
                               expid,
                               spectrograph=spectrograph)
            calibfile = findfile('fluxcalib', night, expid, camera)
            cframefile = findfile('cframe', night, expid, camera)

            fiberflatfile = input_fiberflat[camera]

            cmd = "desi_process_exposure"
            cmd += " --infile {}".format(framefile)
            cmd += " --fiberflat {}".format(fiberflatfile)
            cmd += " --sky {}".format(skyfile)
            cmd += " --calib {}".format(calibfile)
            cmd += " --outfile {}".format(cframefile)
            cmd += " --cosmics-nsig 6"
            if args.no_xtalk:
                cmd += " --no-xtalk"

            inputs = [framefile, fiberflatfile, skyfile, calibfile]
            runcmd(cmd, inputs=inputs, outputs=[
                cframefile,
            ])

        if comm is not None:
            comm.barrier()

        timer.stop('applycalib')

    #-------------------------------------------------------------------------
    #- Wrap up

    # if rank == 0:
    #     report = timer.report()
    #     log.info('Rank 0 timing report:\n' + report)

    if comm is not None:
        timers = comm.gather(timer, root=0)
    else:
        timers = [
            timer,
        ]

    if rank == 0:
        stats = desiutil.timer.compute_stats(timers)
        log.info('Timing summary statistics:\n' + json.dumps(stats, indent=2))

        if args.timingfile:
            if os.path.exists(args.timingfile):
                with open(args.timingfile) as fx:
                    previous_stats = json.load(fx)

                #- augment previous_stats with new entries, but don't overwrite old
                for name in stats:
                    if name not in previous_stats:
                        previous_stats[name] = stats[name]

                stats = previous_stats

            tmpfile = args.timingfile + '.tmp'
            with open(tmpfile, 'w') as fx:
                json.dump(stats, fx, indent=2)
            os.rename(tmpfile, args.timingfile)

    if rank == 0:
        log.info('All done at {}'.format(time.asctime()))
Пример #9
0
def cal_rdnoise(hdul, camera, ccd_calibration_filename, use_overscan_row,
                overscan_per_row, use_active):
    if True:
        hdul_this = hdul[camera]
        header = hdul_this.header
        rawimage = hdul_this.data
        # works!  preproc.preproc(hdul['B0'].data,hdul['B0'].header,h1)
        amp_ids = preproc.get_amp_ids(header)

        #- Output arrays
        ny = 0
        nx = 0
        for amp in amp_ids:
            yy, xx = parse_sec_keyword(header['CCDSEC%s' % amp])
            ny = max(ny, yy.stop)
            nx = max(nx, xx.stop)
        image = np.zeros((ny, nx))

        readnoise = np.zeros_like(image)
        nogain = False

        cfinder = None

        if ccd_calibration_filename is not False:
            cfinder = CalibFinder([header, primary_header],
                                  yaml_file=ccd_calibration_filename)

        for amp in amp_ids:
            # Grab the sections
            ov_col = parse_sec_keyword(header['BIASSEC' + amp])
            ov_row = parse_sec_keyword(header['ORSEC' + amp])

            if use_active:
                if amp == 'A' or amp == 'D':
                    ov_col2 = np.s_[ov_col[1].start:ov_col[1].stop, 0:100]
                else:
                    ov_col2 = np.s_[ov_col[1].start:ov_col[1].stop, 0:100]
            import pdb
            pdb.set_trace()

            if 'ORSEC' + amp in header.keys():
                ov_row = parse_sec_keyword(header['ORSEC' + amp])
            elif use_overscan_row:
                log.error(
                    'No ORSEC{} keyword; not using overscan_row'.format(amp))
                use_overscan_row = False

            if nogain:
                gain = 1.
            else:
                #- Initial teststand data may be missing GAIN* keywords; don't crash
                if 'GAIN' + amp in header:
                    gain = header['GAIN' + amp]  #- gain = electrons / ADU
                else:
                    if cfinder and cfinder.haskey('GAIN' + amp):
                        gain = float(cfinder.value('GAIN' + amp))
                        log.info(
                            'Using GAIN{}={} from calibration data'.format(
                                amp, gain))
                    else:
                        gain = 1.0
                        log.warning(
                            'Missing keyword GAIN{} in header and nothing in calib data; using {}'
                            .format(amp, gain))

            #- Add saturation level
            if 'SATURLEV' + amp in header:
                saturlev = header['SATURLEV' + amp]  # in electrons
            else:
                if cfinder and cfinder.haskey('SATURLEV' + amp):
                    saturlev = float(cfinder.value('SATURLEV' + amp))
                    log.info(
                        'Using SATURLEV{}={} from calibration data'.format(
                            amp, saturlev))
                else:
                    saturlev = 200000
                    log.warning(
                        'Missing keyword SATURLEV{} in header and nothing in calib data; using 200000'
                        .format(amp, saturlev))

            # Generate the overscan images
            raw_overscan_col = rawimage[ov_col].copy()  # 2064*64

            if use_overscan_row:
                raw_overscan_row = rawimage[ov_row].copy()  # 32*2057
                overscan_row = np.zeros_like(raw_overscan_row)

                # Remove overscan_col from overscan_row
                raw_overscan_squared = rawimage[ov_row[0],
                                                ov_col[1]].copy()  # 32*64
                for row in range(raw_overscan_row.shape[0]):
                    o, r = _overscan(raw_overscan_squared[row])
                    overscan_row[row] = raw_overscan_row[row] - o

            # Now remove the overscan_col
            nrows = raw_overscan_col.shape[0]
            log.info("nrows in overscan=%d" % nrows)
            overscan_col = np.zeros(nrows)
            rdnoise = np.zeros(nrows)
            if (cfinder and cfinder.haskey('OVERSCAN' + amp)
                    and cfinder.value("OVERSCAN" + amp).upper()
                    == "PER_ROW") or overscan_per_row:
                log.info(
                    "Subtracting overscan per row for amplifier %s of camera %s"
                    % (amp, camera))
                for j in range(nrows):
                    if np.isnan(np.sum(overscan_col[j])):
                        log.warning(
                            "NaN values in row %d of overscan of amplifier %s of camera %s"
                            % (j, amp, camera))
                        continue
                    o, r = _overscan(raw_overscan_col[j])
                    #log.info("%d %f %f"%(j,o,r))
                    overscan_col[j] = o
                    rdnoise[j] = r
            else:
                log.info(
                    "Subtracting average overscan for amplifier %s of camera %s"
                    % (amp, camera))
                o, r = _overscan(raw_overscan_col)
                overscan_col += o
                rdnoise += r

            rdnoise *= gain
            median_rdnoise = np.median(rdnoise)
            median_overscan = np.median(overscan_col)
    return rdnoise, median_rdnoise
Пример #10
0
def compute_bias_file(rawfiles, outfile, camera, explistfile=None):
    """
    Compute a bias file from input ZERO rawfiles

    Args:
        rawfiles: list of input raw file names
        outfile (str): output filename
        camera (str): camera, e.g. b0, r1, z9

    Options:
        explistfile: filename with text list of NIGHT EXPID to use

    Notes: explistfile is only used if rawfiles=None; it should have
    one NIGHT EXPID entry per line.
    """
    log = get_logger()

    if explistfile is not None:
        if rawfiles is not None:
            msg = "specify rawfiles or explistfile, but not both"
            log.error(msg)
            raise ValueError(msg)

        rawfiles = list()
        with open(explistfile, 'r') as fx:
            for line in fx:
                line = line.strip()
                if line.startswith('#') or len(line) < 2:
                    continue
                night, expid = map(int, line.split())
                filename = io.findfile('raw', night, expid)
                if not os.path.exists(filename):
                    msg = f'Missing {filename}'
                    log.critical(msg)
                    raise RuntimeError(msg)

                rawfiles.append(filename)

    log.info("read images ...")
    images = []
    shape = None
    first_image_header = None
    for filename in rawfiles:
        log.info("reading %s" % filename)
        fitsfile = pyfits.open(filename)

        primary_header = fitsfile[0].header
        image_header = fitsfile[camera].header

        if first_image_header is None:
            first_image_header = image_header

        flavor = image_header['FLAVOR'].upper()
        if flavor != 'ZERO':
            message = f'Input {filename} flavor {flavor} != ZERO'
            log.error(message)
            raise ValueError(message)

        # subtract overscan region
        cfinder = CalibFinder([image_header, primary_header])

        image = fitsfile[camera].data.astype("float64")

        if cfinder and cfinder.haskey("AMPLIFIERS"):
            amp_ids = list(cfinder.value("AMPLIFIERS"))
        else:
            amp_ids = ['A', 'B', 'C', 'D']

        n0 = image.shape[0] // 2
        n1 = image.shape[1] // 2

        for a, amp in enumerate(amp_ids):
            ii = parse_sec_keyword(image_header['BIASSEC' + amp])
            overscan_image = image[ii].copy()
            overscan, rdnoise = _overscan(overscan_image)
            log.info("amp {} overscan = {}".format(amp, overscan))
            if ii[0].start < n0 and ii[1].start < n1:
                image[:n0, :n1] -= overscan
            elif ii[0].start < n0 and ii[1].start >= n1:
                image[:n0, n1:] -= overscan
            elif ii[0].start >= n0 and ii[1].start < n1:
                image[n0:, :n1] -= overscan
            elif ii[0].start >= n0 and ii[1].start >= n1:
                image[n0:, n1:] -= overscan

        if shape is None:
            shape = image.shape
        images.append(image.ravel())

        fitsfile.close()

    images = np.array(images)
    print(images.shape)

    # compute a mask
    log.info("compute median image ...")
    medimage = np.median(images, axis=0)  #.reshape(shape)
    log.info("compute mask ...")
    ares = np.abs(images - medimage)
    nsig = 4.
    mask = (ares < nsig * 1.4826 * np.median(ares, axis=0))
    # average (not median)
    log.info("compute average ...")
    meanimage = np.sum(images * mask, axis=0) / np.sum(mask, axis=0)
    meanimage = meanimage.reshape(shape)

    log.info("write result in %s ..." % outfile)
    hdus = pyfits.HDUList([pyfits.PrimaryHDU(meanimage.astype('float32'))])

    # copy some keywords
    for key in [
            "TELESCOP", "INSTRUME", "SPECGRPH", "SPECID", "DETECTOR", "CAMERA",
            "CCDNAME", "CCDPREP", "CCDSIZE", "CCDTEMP", "CPUTEMP", "CASETEMP",
            "CCDTMING", "CCDCFG", "SETTINGS", "VESSEL", "FEEVER", "FEEBOX",
            "PRESECA", "PRRSECA", "DATASECA", "TRIMSECA", "BIASSECA", "ORSECA",
            "CCDSECA", "DETSECA", "AMPSECA", "PRESECB", "PRRSECB", "DATASECB",
            "TRIMSECB", "BIASSECB", "ORSECB", "CCDSECB", "DETSECB", "AMPSECB",
            "PRESECC", "PRRSECC", "DATASECC", "TRIMSECC", "BIASSECC", "ORSECC",
            "CCDSECC", "DETSECC", "AMPSECC", "PRESECD", "PRRSECD", "DATASECD",
            "TRIMSECD", "BIASSECD", "ORSECD", "CCDSECD", "DETSECD", "AMPSECD",
            "DAC0", "DAC1", "DAC2", "DAC3", "DAC4", "DAC5", "DAC6", "DAC7",
            "DAC8", "DAC9", "DAC10", "DAC11", "DAC12", "DAC13", "DAC14",
            "DAC15", "DAC16", "DAC17", "CLOCK0", "CLOCK1", "CLOCK2", "CLOCK3",
            "CLOCK4", "CLOCK5", "CLOCK6", "CLOCK7", "CLOCK8", "CLOCK9",
            "CLOCK10", "CLOCK11", "CLOCK12", "CLOCK13", "CLOCK14", "CLOCK15",
            "CLOCK16", "CLOCK17", "CLOCK18", "OFFSET0", "OFFSET1", "OFFSET2",
            "OFFSET3", "OFFSET4", "OFFSET5", "OFFSET6", "OFFSET7", "DELAYS",
            "CDSPARMS", "PGAGAIN", "OCSVER", "DOSVER", "CONSTVER"
    ]:
        if key in first_image_header:
            hdus[0].header[key] = (first_image_header[key],
                                   first_image_header.comments[key])

    hdus[0].header["BUNIT"] = "adu"
    hdus[0].header["EXTNAME"] = "BIAS"
    for filename in rawfiles:
        hdus[0].header["COMMENT"] = "Inc. {}".format(
            os.path.basename(filename))
    hdus.writeto(outfile, overwrite="True")

    log.info("done")
Пример #11
0
def preproc(rawimage, header, primary_header, bias=True, dark=True, pixflat=True, mask=True, bkgsub=False, nocosmic=False, cosmics_nsig=6, cosmics_cfudge=3., cosmics_c2fudge=0.5,ccd_calibration_filename=None, nocrosstalk=False, nogain=False):

    '''
    preprocess image using metadata in header

    image = ((rawimage-bias-overscan)*gain)/pixflat

    Args:
        rawimage : 2D numpy array directly from raw data file
        header : dict-like metadata, e.g. from FITS header, with keywords
            CAMERA, BIASSECx, DATASECx, CCDSECx
            where x = A, B, C, D for each of the 4 amplifiers
            (also supports old naming convention 1, 2, 3, 4).
        primary_header: dict-like metadata fit keywords EXPTIME, DOSVER
            DATE-OBS is also required if bias, pixflat, or mask=True

    Optional bias, pixflat, and mask can each be:
        False: don't apply that step
        True: use default calibration data for that night
        ndarray: use that array
        filename (str or unicode): read HDU 0 and use that

    Optional background subtraction with median filtering if bkgsub=True

    Optional disabling of cosmic ray rejection if nocosmic=True

    Optional tuning of cosmic ray rejection parameters:
        cosmics_nsig: number of sigma above background required
        cosmics_cfudge: number of sigma inconsistent with PSF required
        cosmics_c2fudge:  fudge factor applied to PSF

    Returns Image object with member variables:
        image : 2D preprocessed image in units of electrons per pixel
        ivar : 2D inverse variance of image
        mask : 2D mask of image (0=good)
        readnoise : 2D per-pixel readnoise of image
        meta : metadata dictionary
        TODO: define what keywords are included

    preprocessing includes the following steps:
        - bias image subtraction
        - overscan subtraction (from BIASSEC* keyword defined regions)
        - readnoise estimation (from BIASSEC* keyword defined regions)
        - gain correction (from GAIN* keywords)
        - pixel flat correction
        - cosmic ray masking
        - propagation of input known bad pixel mask
        - inverse variance estimation

    Notes:

    The bias image is subtracted before any other calculation to remove any
    non-uniformities in the overscan regions prior to calculating overscan
    levels and readnoise.

    The readnoise is an image not just one number per amp, because the pixflat
    image also affects the interpreted readnoise.

    The inverse variance is estimated from the readnoise and the image itself,
    and thus is biased.
    '''
    log=get_logger()

    header = header.copy()
    
    cfinder = None
    
    if ccd_calibration_filename is not False :
        cfinder = CalibFinder([header, primary_header], yaml_file=ccd_calibration_filename)
    
    #- TODO: Check for required keywords first

    #- Subtract bias image
    camera = header['CAMERA'].lower()

    #- convert rawimage to float64 : this is the output format of read_image
    rawimage = rawimage.astype(np.float64)

    bias = get_calibration_image(cfinder,"BIAS",bias)

    if bias is not False : #- it's an array
        if bias.shape == rawimage.shape  :
            log.info("subtracting bias")
            rawimage = rawimage - bias
        else:
            raise ValueError('shape mismatch bias {} != rawimage {}'.format(bias.shape, rawimage.shape))


    if cfinder and cfinder.haskey("AMPLIFIERS") :
        amp_ids=list(cfinder.value("AMPLIFIERS"))
    else :
        amp_ids=['A','B','C','D']

    #- check whether it's indeed CCDSECx with x in ['A','B','C','D']
    #  or older version with x in ['1','2','3','4']
    #  we can remove this piece of code at later times
    has_valid_keywords = True
    for amp in amp_ids :
        if not 'CCDSEC%s'%amp in header :
            log.warning("No CCDSEC%s keyword in header , will look for alternative naming CCDSEC{1,2,3,4} ..."%amp)
            has_valid_keywords = False
            break
    if not has_valid_keywords :
        amp_ids=['1','2','3','4']
        for amp in ['1','2','3','4'] :
            if not 'CCDSEC%s'%amp in header :
                log.error("No CCDSEC%s keyword, exit"%amp)
                raise KeyError("No CCDSEC%s keyword"%amp)

    #- Output arrays
    ny=0
    nx=0
    for amp in amp_ids :
        yy, xx = _parse_sec_keyword(header['CCDSEC%s'%amp])
        ny=max(ny,yy.stop)
        nx=max(nx,xx.stop)
    image = np.zeros( (ny,nx) )

    readnoise = np.zeros_like(image)



    #- Load mask
    mask = get_calibration_image(cfinder,"MASK",mask)

    if mask is False :
        mask = np.zeros(image.shape, dtype=np.int32)
    else :
        if mask.shape != image.shape :
            raise ValueError('shape mismatch mask {} != image {}'.format(mask.shape, image.shape))

    #- Load dark
    dark = get_calibration_image(cfinder,"DARK",dark)

    if dark is not False :
        if dark.shape != image.shape :
            log.error('shape mismatch dark {} != image {}'.format(dark.shape, image.shape))
            raise ValueError('shape mismatch dark {} != image {}'.format(dark.shape, image.shape))


        if cfinder and cfinder.haskey("EXPTIMEKEY") :
            exptime_key=cfinder.value("EXPTIMEKEY")
            log.info("Using exposure time keyword %s for dark normalization"%exptime_key)
        else :
            exptime_key="EXPTIME"
        exptime =  primary_header[exptime_key]

        log.info("Multiplying dark by exptime %f"%(exptime))
        dark *= exptime



    for amp in amp_ids :
        ii = _parse_sec_keyword(header['BIASSEC'+amp])

        if nogain :
            gain = 1.
        else :
            #- Initial teststand data may be missing GAIN* keywords; don't crash
            if 'GAIN'+amp in header:
                gain = header['GAIN'+amp]          #- gain = electrons / ADU
            else:
                if cfinder and cfinder.haskey('GAIN'+amp) :
                    gain = float(cfinder.value('GAIN'+amp))
                    log.info('Using GAIN{}={} from calibration data'.format(amp,gain))
                else :
                    gain = 1.0
                    log.warning('Missing keyword GAIN{} in header and nothing in calib data; using {}'.format(amp,gain))


        #- Add saturation level
        if 'SATURLEV'+amp in header:
            saturlev = header['SATURLEV'+amp]          # in electrons
        else:
            if cfinder and cfinder.haskey('SATURLEV'+amp) :
                saturlev = float(cfinder.value('SATURLEV'+amp))
                log.info('Using SATURLEV{}={} from calibration data'.format(amp,saturlev))
            else :
                saturlev = 200000
                log.warning('Missing keyword SATURLEV{} in header and nothing in calib data; using 200000'.format(amp,saturlev))

        overscan_image = rawimage[ii].copy()
        nrows=overscan_image.shape[0]
        log.info("nrows in overscan=%d"%nrows)
        overscan = np.zeros(nrows)
        rdnoise  = np.zeros(nrows)
        overscan_per_row = True
        if cfinder and cfinder.haskey('OVERSCAN'+amp) and cfinder.value("OVERSCAN"+amp).upper()=="PER_ROW" :
            log.info("Subtracting overscan per row for amplifier %s of camera %s"%(amp,camera))
            for j in range(nrows) :
                if np.isnan(np.sum(overscan_image[j])) :
                    log.warning("NaN values in row %d of overscan of amplifier %s of camera %s"%(j,amp,camera))
                    continue
                o,r =  _overscan(overscan_image[j])
                #log.info("%d %f %f"%(j,o,r))
                overscan[j]=o
                rdnoise[j]=r
        else :
            log.info("Subtracting average overscan for amplifier %s of camera %s"%(amp,camera))
            o,r =  _overscan(overscan_image)
            overscan += o
            rdnoise  += r

        rdnoise *= gain
        median_rdnoise  = np.median(rdnoise)
        median_overscan = np.median(overscan)
        log.info("Median rdnoise and overscan= %f %f"%(median_rdnoise,median_overscan))

        kk = _parse_sec_keyword(header['CCDSEC'+amp])
        for j in range(nrows) :
            readnoise[kk][j] = rdnoise[j]

        header['OVERSCN'+amp] = (median_overscan,'ADUs (gain not applied)')
        if gain != 1 :
            rdnoise_message = 'electrons (gain is applied)'
            gain_message    = 'e/ADU (gain applied to image)'
        else :
            rdnoise_message = 'ADUs (gain not applied)'
            gain_message    = 'gain not applied to image'
        header['OBSRDN'+amp] = (median_rdnoise,rdnoise_message)
        header['GAIN'+amp] = (gain,gain_message)
        
        #- Warn/error if measured readnoise is very different from expected if exists
        if 'RDNOISE'+amp in header:
            expected_readnoise = header['RDNOISE'+amp]
            if median_rdnoise < 0.5*expected_readnoise:
                log.error('Amp {} measured readnoise {:.2f} < 0.5 * expected readnoise {:.2f}'.format(
                    amp, median_rdnoise, expected_readnoise))
            elif median_rdnoise < 0.9*expected_readnoise:
                log.warning('Amp {} measured readnoise {:.2f} < 0.9 * expected readnoise {:.2f}'.format(
                    amp, median_rdnoise, expected_readnoise))
            elif median_rdnoise > 2.0*expected_readnoise:
                log.error('Amp {} measured readnoise {:.2f} > 2 * expected readnoise {:.2f}'.format(
                    amp, median_rdnoise, expected_readnoise))
            elif median_rdnoise > 1.2*expected_readnoise:
                log.warning('Amp {} measured readnoise {:.2f} > 1.2 * expected readnoise {:.2f}'.format(
                    amp, median_rdnoise, expected_readnoise))
        #else:
        #    log.warning('Expected readnoise keyword {} missing'.format('RDNOISE'+amp))

        log.info("Measured readnoise for AMP %s = %f"%(amp,median_rdnoise))

        #- subtract overscan from data region and apply gain
        jj = _parse_sec_keyword(header['DATASEC'+amp])

        data = rawimage[jj].copy()
        for k in range(nrows) :
            data[k] -= overscan[k]

        #- apply saturlev (defined in ADU), prior to multiplication by gain
        saturated = (rawimage[jj]>=saturlev)
        mask[kk][saturated] |= ccdmask.SATURATED

        #- subtract dark prior to multiplication by gain
        if dark is not False  :
            log.info("subtracting dark for amp %s"%amp)
            data -= dark[kk]

        image[kk] = data*gain


    if not nocrosstalk :
        #- apply cross-talk

        # the ccd looks like :
        # C D
        # A B
        # for cross talk, we need a symmetric 4x4 flip_matrix
        # of coordinates ABCD giving flip of both axis
        # when computing crosstalk of
        #    A   B   C   D
        #
        # A  AA  AB  AC  AD
        # B  BA  BB  BC  BD
        # C  CA  CB  CC  CD
        # D  DA  DB  DC  BB
        # orientation_matrix_defines change of orientation
        #
        fip_axis_0= np.array([[1,1,-1,-1],
                              [1,1,-1,-1],
                              [-1,-1,1,1],
                              [-1,-1,1,1]])
        fip_axis_1= np.array([[1,-1,1,-1],
                              [-1,1,-1,1],
                              [1,-1,1,-1],
                              [-1,1,-1,1]])

        for a1 in range(len(amp_ids)) :
            amp1=amp_ids[a1]
            ii1 = _parse_sec_keyword(header['CCDSEC'+amp1])
            a1flux=image[ii1]
            #a1mask=mask[ii1]

            for a2 in range(len(amp_ids)) :
                if a1==a2 :
                    continue
                amp2=amp_ids[a2]
                if cfinder is None : continue
                if not cfinder.haskey("CROSSTALK%s%s"%(amp1,amp2))  : continue
                crosstalk=cfinder.value("CROSSTALK%s%s"%(amp1,amp2))
                if crosstalk==0. : continue
                log.info("Correct for crosstalk=%f from AMP %s into %s"%(crosstalk,amp1,amp2))
                a12flux=crosstalk*a1flux.copy()
                #a12mask=a1mask.copy()
                if fip_axis_0[a1,a2]==-1 :
                    a12flux=a12flux[::-1]
                    #a12mask=a12mask[::-1]
                if fip_axis_1[a1,a2]==-1 :
                    a12flux=a12flux[:,::-1]
                    #a12mask=a12mask[:,::-1]
                ii2 = _parse_sec_keyword(header['CCDSEC'+amp2])
                image[ii2] -= a12flux
                # mask[ii2]  |= a12mask (not sure we really need to propagate the mask)

    #- Divide by pixflat image
    pixflat = get_calibration_image(cfinder,"PIXFLAT",pixflat)
    if pixflat is not False :
        if pixflat.shape != image.shape:
            raise ValueError('shape mismatch pixflat {} != image {}'.format(pixflat.shape, image.shape))

        if np.all(pixflat != 0.0):
            image /= pixflat
            readnoise /= pixflat
        else:
            good = (pixflat != 0.0)
            image[good] /= pixflat[good]
            readnoise[good] /= pixflat[good]
            mask[~good] |= ccdmask.PIXFLATZERO

        lowpixflat = (0 < pixflat) & (pixflat < 0.1)
        if np.any(lowpixflat):
            mask[lowpixflat] |= ccdmask.PIXFLATLOW

    #- Inverse variance, estimated directly from the data (BEWARE: biased!)
    var = image.clip(0) + readnoise**2
    ivar = np.zeros(var.shape)
    ivar[var>0] = 1.0 / var[var>0]

    if bkgsub :
        bkg = _background(image,header)
        image -= bkg


    img = Image(image, ivar=ivar, mask=mask, meta=header, readnoise=readnoise, camera=camera)

    #- update img.mask to mask cosmic rays

    if not nocosmic :
        cosmics.reject_cosmic_rays(img,nsig=cosmics_nsig,cfudge=cosmics_cfudge,c2fudge=cosmics_c2fudge)

    return img