Beispiel #1
0
 def covert_3dto2d(self, files):
     files = filenames2list(files, forcelist=True)
     for file in files:
         d, h = fits.getdata(file, header=True)
         d = d[0, :, :]
         h.remove('naxis3')
         fits.update(file, d, header=h)
Beispiel #2
0
def bias_file_by_file(fname, biasname, datahdus=0):
    hdulist = fits.open(fname)
    hdubias = fits.open(biasname)
    nhdus = len(hdulist)
    if nhdus > 1:
        istart = 1
    else:
        istart = 0
    hduindexes = list(range(nhdus))[istart:]
    if datahdus != 0:
        hduindexes = datahdus
    for i in hduindexes:
        data1 = ccdproc.CCDData(hdulist[i].data, unit="adu")
        data1.header = hdulist[i].header
        bias1 = ccdproc.CCDData(hdubias[i].data, unit="adu")
        bias1.header = hdubias[i].header
        commentstr = "Bias image is " + biasname
        # proc1 = ccdproc.subtract_bias(data1,bias1,add_keyword={'bias': True, 'calstat': 'OTZ', 'history':commentstr} )
        proc1 = ccdproc.subtract_bias(data1,
                                      bias1,
                                      add_keyword={
                                          'bias': True,
                                          'calstat': 'OTZ'
                                      })
        fits.update(fname, proc1.data, header=proc1.header, ext=i)
        # fits.update(fname, proc1.data, ext=i)
    hdulist.close()
    hdubias.close()
    mylog("Bias corrected {0} with {1}".format(fname, biasname))
    return
def fitsarith(input1, input2, output, operation, const1=None, const2=None):
    # read input files - must have same extension structure!
    hdu1 = fits.open(input1)
    hdu2 = fits.open(input2)
    # build output file by copying input1
    shutil.copyfile(input1,output)
    # loop through extensions, find image extensions only, and do arithmetic
    for i in range(len(hdu1)):
        exten1 = hdu1[i]
        exten2 = hdu2[i]     
        if not isinstance(exten1, (fits.hdu.image.ImageHDU, fits.hdu.compressed.CompImageHDU)):
            continue
        if const1 is None:
            data1 = exten1.data
        else:
            data1 = const1 * exten1.data
        if const2 is None:
            data2 = exten2.data
        else:
            data2 = const2* exten2.data
        if operation == "+":
            data_output = data1 + data2
        elif operation == "-":
            data_output = data1 - data2
        elif operation == "*":
            data_output = data1 * data2
        elif operation == "/":
            data_output = data1 / data2
        else:
            raise ValueError("Unknown operator %s" % operation)
        # update this extension in output fits file
        fits.update(output,data_output,i,header=exten1.header)
Beispiel #4
0
def cand_list_operation(filename, det_tab=None, mode='save'):
    """
    Creating CANDIDATES_LIST as an extension table of difference image FITS.

    Parameters:
    ----------
    filename: str
        filename of the difference image
    det_tab: pd.DataFrame / None
        1. DETECTION_TABLE or CANDIDATES_LIST DataFrame
        2. None if using mode='load' 
    mode: str
        ['save','load','update']
    """
    if mode == 'save':
        # create astropy table
        m = Table(det_tab.values, names=det_tab.columns)
        hdu = fits.table_to_hdu(m)
        # add extension table on top of the difference image
        with fits.open(filename, mode='update') as hdul0:
            hdul0.append(hdu)
            hdul0[-1].header['EXTNAME'] = 'CANDIDATES_LIST'
            hdul0.flush()

    elif mode == 'load':
        det_tab = getdata(filename, 'CANDIDATES_LIST')
        det_tab = pd.DataFrame(np.array(det_tab).byteswap().newbyteorder())
        return det_tab

    elif mode == 'update':
        m = Table(det_tab.values, names=det_tab.columns)
        hdr = getheader(filename, extname='CANDIDATES_LIST')
        update(filename, np.array(m), extname='CANDIDATES_LIST', header=hdr)
Beispiel #5
0
def run_sector_camera_chip(base_dir, output_dir, sector, camera, chip):
    pattern = os.path.join(base_dir, "tess", "ffi", "s{0:04d}".format(sector),
                           "*", "*", "{0:d}-{1:d}".format(camera,
                                                          chip), "*.fits")

    outdir = os.path.join(output_dir, "s{0:04d}".format(sector),
                          "{0:d}-{1:d}".format(camera, chip))
    os.makedirs(outdir, exist_ok=True)
    open(os.path.join(outdir, "index.auto"), "w").close()
    open(os.path.join(os.path.dirname(outdir), "index.auto"), "w").close()

    fns = list(sorted(glob.glob(pattern)))
    postcard_fns = make_postcards(fns, outdir)

    # Ensures no postcards have been repeated
    postcard_fns = np.unique(postcard_fns)

    # Writes in the background after making the postcards
    print("Computing backgrounds...")
    for fn in tqdm.tqdm(postcard_fns, total=len(postcard_fns)):
        with fits.open(fn) as hdu:
            bkg = calc_2dbkg(hdu[2].data, hdu[1].data['QUALITY'],
                             hdu[1].data['TSTART'])
            # Checks to make sure there isn't a background extension already
            if len(hdu) < 5:
                fits.append(fn, bkg)
            else:
                fits.update(fn, bkg, 4)
Beispiel #6
0
def fitsarith(inputlist, output, expression):

    if not inputlist:
        raise ValueError("Input is empty")

    hdus_in = [fits.open(fin) for fin in inputlist]
    use_expr = ""
    use_expr += expression
    n_in = len(hdus_in)

    # Do in reverse in case someone want 10 or more images
    for i in range(n_in - 1, -1, -1):
        from_string = "@%i" % i
        to_string = "h[%i]" % i
        use_expr = use_expr.replace(from_string, to_string)

    n_hdu = len(hdus_in[0])

    shutil.copyfile(inputlist[0], output)

    for j in range(n_hdu):

        if not isinstance(
                hdus_in[0][j],
            (fits.hdu.image.ImageHDU, fits.hdu.compressed.CompImageHDU)):
            continue
        h = [hdus[j].data for hdus in hdus_in]
        data_output = eval(use_expr)
        fits.update(output, data_output, j, header=hdus_in[0][j].header)
Beispiel #7
0
def flat_file_by_file(fname, flatname, datahdus=0):
    hdulist = fits.open(fname)
    hduflat = fits.open(flatname)
    nhdus = len(hdulist)
    if nhdus > 1:
        istart = 1
    else:
        istart = 0
    hduindexes = list(range(nhdus))[istart:]
    if datahdus != 0:
        hduindexes = datahdus
    for i in hduindexes:
        data1 = ccdproc.CCDData(hdulist[i].data, unit="adu")
        data1.header = hdulist[i].header
        flat1 = ccdproc.CCDData(hduflat[i].data, unit="adu")
        flat1.header = hduflat[i].header
        if i == 1:
            flatscale = np.mean(flat1)
        # flat1 = flat1/flatscale
        commentstr = "Flat image is " + flatname + " with scale" + str(
            flatscale)
        proc1 = ccdproc.flat_correct(data1,
                                     flat1,
                                     add_keyword={
                                         'flat': True,
                                         'calstat': 'OTZF',
                                         'history': commentstr
                                     })
        fits.update(fname, proc1.data, header=proc1.header, ext=i)
        # fits.update(fname, proc1.data, ext=i)
    hdulist.close()
    hduflat.close()
    mylog("Flat corrected {0} with {1}".format(fname, flatname))
    return
Beispiel #8
0
def create_data(imdir, rot, ov):
    """ imdir: directory for simulated fits image data
        rot: pupil rotation in degrees
        ov: oversample for simulation
        Writes sim data to fitsimdir
    """
    npix = 81
    wave = 4.3e-6  # SI 
    fnfmt = '/psf_nrm_{2:.1f}_{0}_{1}_rot{3:.3f}d.fits' # expects strings of  %(npix, holeshape, wave/um, rot_d)

    rot = utils.avoidhexsingularity(rot) # in utils
    affine_rot = utils.Affine2d(rotradccw=np.pi*rot/180.0, name="rot{0:.3f}d".format(rot)) # in utils

    jw = NRM_Model(mask='jwst', holeshape='hex', affine2d=affine_rot)
    jw.set_pixelscale(PIXELSCALE_r)
    jw.simulate(fov=81, bandpass=MONOF430M, over=ov)

    psffn = fnfmt.format(npix, 'hex', wave/um, rot)
    fits.writeto(imdir+psffn, jw.psf, overwrite=True)
    header = fits.getheader(imdir+psffn)
    header = utils.affinepars2header(header, affine_rot)
    fits.update(imdir+psffn, jw.psf, header=header)
    del jw

    return psffn  # filename only, not full path
def generate_sec_offsets_new(name):
    print name
    hdulist = pyfits.open('../AIS_GAL_SCAN/asprta/%s-asprta.fits'%name)
    initial = 0
    final = hdulist[1].data['T'].shape[0]-1

    centers = []
    center_time = []
    for i in range(initial, final+1):
        #center = np.load('../data/%s/cata/centroids_rot%d.npy'%(name, i))
        #if center.shape!=(2,3):
        #   print i, center.shape
        centers.append(np.load('../data/%s/cata/centroids_rot%d.npy'%(name, i)))
        center_time.append(np.load('../data/%s/cata/time_rot%d.npy'%(name, i)))
    centroids = np.concatenate(centers, axis=0)
    time =  np.concatenate(center_time, axis=0)

    print centroids.shape
    print time.shape
    output = '../plots/%s/cata/offsets_10_new_sec.pdf'%name
    dir = os.path.dirname(output)
    if not os.path.exists(dir):
      os.makedirs(dir)
    plt.plot(centroids[:,0], '.b')
    plt.savefig('../plots/%s/cata/offsets_10_new_sec.pdf'%name, dpi=190)
    plt.clf()

    np.save('../data/%s/cata/offsets%d_10_new_sec.npy'%(name, initial), centroids)
    np.save('../data/%s/cata/time%d_10_new_sec.npy'%(name, initial), time)
    
    co_data = hdulist[1].data
    T = co_data['T']
    ra = co_data['ra']
    dec = co_data['dec']
    roll = co_data['roll']
    ra_new = np.interp(time, T, ra) - centroids[:,0]
    dec_new = np.interp(time, T, dec) - centroids[:,1]
    roll_new = np.interp(time, T, roll) - centroids[:,2]

    other = np.zeros((time.shape[0], 8))
    array = np.concatenate([np.array([time, ra_new, dec_new, roll_new]).T, other], axis=1)
    data = np.core.records.fromarrays(array.transpose(), dtype=[('T', float), ('RA', float), ('DEC', float), ('ROLL', float),\
                 ('STATUS_FLAG', int), ('ROLL_RAD', float), ('X', float), ('Y', float), ('Z', float), ('XDOT', float), ('YDOT', float), ('ZDOT', float)])

    new_file = '../AIS_GAL_SCAN/asprta/%s-sec-asprta.fits'%(name)
    os.system('cp ../AIS_GAL_SCAN/asprta/%s-asprta.fits ../AIS_GAL_SCAN/asprta/%s-sec-asprta.fits'%(name, name))
    update(new_file, data, 1)
    hdu = pyfits.open(new_file)
    print hdu[1].data['RA'].shape
    print hdu[1].data['DEC'].shape
    hdu.close()

    tmp_files = glob.glob("../data/%s/cata/centroids_rot*"%name)
    for tmp_file in tmp_files:
        os.remove(tmp_file)

    tmp_files = glob.glob("../data/%s/cata/time_rot*"%name)
    for tmp_file in tmp_files:
        os.remove(tmp_file)
def generate_sec_offsets_new(name):
    print name
    hdulist = pyfits.open('../AIS_GAL_SCAN/asprta/%s-asprta.fits'%name)
    initial = 0
    final = hdulist[1].data['T'].shape[0]-1

    centers = []
    center_time = []
    for i in range(initial, final+1):
        #center = np.load('../data/%s/cata/centroids_rot%d.npy'%(name, i))
        #if center.shape!=(2,3):
        #   print i, center.shape
        centers.append(np.load('../data/%s/cata/centroids_rot%d.npy'%(name, i)))
        center_time.append(np.load('../data/%s/cata/time_rot%d.npy'%(name, i)))
    centroids = np.concatenate(centers, axis=0)
    time =  np.concatenate(center_time, axis=0)

    print centroids.shape
    print time.shape
    output = '../plots/%s/cata/offsets_10_new_sec.pdf'%name
    dir = os.path.dirname(output)
    if not os.path.exists(dir):
      os.makedirs(dir)
    plt.plot(centroids[:,0], '.b')
    plt.savefig('../plots/%s/cata/offsets_10_new_sec.pdf'%name, dpi=190)
    plt.clf()

    np.save('../data/%s/cata/offsets%d_10_new_sec.npy'%(name, initial), centroids)
    np.save('../data/%s/cata/time%d_10_new_sec.npy'%(name, initial), time)
    
    co_data = hdulist[1].data
    T = co_data['T']
    ra = co_data['ra']
    dec = co_data['dec']
    roll = co_data['roll']
    ra_new = np.interp(time, T, ra) - centroids[:,0]
    dec_new = np.interp(time, T, dec) - centroids[:,1]
    roll_new = np.interp(time, T, roll) - centroids[:,2]

    other = np.zeros((time.shape[0], 8))
    array = np.concatenate([np.array([time, ra_new, dec_new, roll_new]).T, other], axis=1)
    data = np.core.records.fromarrays(array.transpose(), dtype=[('T', float), ('RA', float), ('DEC', float), ('ROLL', float),\
                 ('STATUS_FLAG', int), ('ROLL_RAD', float), ('X', float), ('Y', float), ('Z', float), ('XDOT', float), ('YDOT', float), ('ZDOT', float)])

    new_file = '../AIS_GAL_SCAN/asprta/%s-sec-asprta.fits'%(name)
    os.system('cp ../AIS_GAL_SCAN/asprta/%s-asprta.fits ../AIS_GAL_SCAN/asprta/%s-sec-asprta.fits'%(name, name))
    update(new_file, data, 1)
    hdu = pyfits.open(new_file)
    print hdu[1].data['RA'].shape
    print hdu[1].data['DEC'].shape
    hdu.close()

    tmp_files = glob.glob("../data/%s/cata/centroids_rot*"%name)
    for tmp_file in tmp_files:
        os.remove(tmp_file)

    tmp_files = glob.glob("../data/%s/cata/time_rot*"%name)
    for tmp_file in tmp_files:
        os.remove(tmp_file)
Beispiel #11
0
 def update_fits(self, header_only=False):
     if header_only:
         hdu = fits.open(self.fitsfile, mode="update")
         hdu[0].header = self.header
         hdu.flush()
         hdu.close()
         return
     fits.update(self.fitsfile, self.data_array, self.header)
def find_rotation(
        imagedata,
        rotdegs,
        mx,
        my,
        sx,
        sy,
        xo,
        yo,  # for Affine2d
        pixel,
        npix,
        bandpass,
        over,
        holeshape,
        outdir=None):  # for nrm_model
    """ AS AZG 2018 08 Ann Arbor Develop the rotation loop first """

    vprint("Before Loop: ", rotdegs)

    #Extend this name to include multi-paramater searches?
    psffmt = 'psf_nrm_{0:d}_{1:s}_{2:.3f}um_r{3:.3f}deg.fits'
    # expect (npix, holeshape, bandpass/um, scl)

    if hasattr(rotdegs, '__iter__') is False:
        rotdegs = (rotdegs, )

    affine2d_list = create_afflist_rot(rotdegs, mx, my, sx, sy, xo, yo)
    crosscorr_rots = []

    for (rot, aff) in zip(rotdegs, affine2d_list):

        vprint(aff.name + "...")
        jw = NRM_Model(mask='jwst',
                       holeshape=holeshape,
                       over=over,
                       affine2d=aff)
        jw.set_pixelscale(pixel)
        jw.simulate(fov=npix, bandpass=bandpass, over=over)
        psffn = psffmt.format(npix, holeshape, bandpass / um, rot)
        if outdir:
            fits.PrimaryHDU(data=jw.psf).writeto(outdir + "/" + psffn,
                                                 overwrite=True)
            fits.writeto(psffn, jw.psf, overwrite=True)
            header = fits.getheader(psffn)
            utils.affinepars2header(header, aff)
            fits.update(psffn, jw.psf, header=header)

        crosscorr_rots.append(utils.rcrosscorrelate(imagedata, jw.psf).max())
        del jw

    vprint("Debug: ", crosscorr_rots, rotdegs)
    rot_measured_d, max_cor = utils.findpeak_1d(crosscorr_rots, rotdegs)
    vprint("Rotation measured: max correlation {1:.3e}", rot_measured_d,
           max_cor)

    # return convenient affine2d
    return utils.Affine2d(rotradccw=np.pi * rot_measured_d / 180.0,
                          name="{0:.4f}".format(rot_measured_d))
Beispiel #13
0
    def setUp(self):

        # directory containing the test data
        data_dir = os.path.join(os.path.dirname(__file__),
                                'test_data/affine2d_rot_psf')
        self.data_dir = data_dir
        print(data_dir)
        if not os.path.exists(data_dir):
            os.makedirs(data_dir)
        self.fnfmt = data_dir + '/psf_nrm_{2:.1f}_{0}_{1}_{3:.0f}.fits'  # expects strings of  %(imsize,hole)

        self.fmt = "    ({0:+.3f}, {1:+.3f}) -> ({2:+.3f}, {3:+.3f})"
        self.pixel = 0.0656 * u.arcsec.to(u.rad)
        self.npix = 87
        self.wave = 4.3e-6  # m
        self.over = 1

        mx, my = 1.0, 1.0
        sx, sy = 0.0, 0.0
        xo, yo = 0.0, 0.0
        affine_ideal = Affine2d(mx=mx,
                                my=my,
                                sx=sx,
                                sy=sy,
                                xo=xo,
                                yo=yo,
                                name="Ideal")

        for rot in (0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70,
                    75, 80, 85, 90, 95):
            """
            print("           rot degrees pre", rot, end='')
            diagnostic = rot/15.0 - int(rot/15.0)
            print("  diagnostic", diagnostic, end='')
            rot = avoidhexsingularity(rot) # in utils
            print("   rot degrees post", rot)
            """
            rot = avoidhexsingularity(rot)  # in utils
            affine_rot = Affine2d(rotradccw=np.pi * rot / 180.0,
                                  name="{0:.0f}".format(rot))  # in utils
            aff = affine_rot

            # hexonly g7s6 jwst
            self.jw = NRM_Model(mask='jwst', holeshape="hexonly", affine2d=aff)
            self.jw.set_pixelscale(self.pixel * arcsec2rad)
            self.jw.simulate(fov=self.npix, bandpass=self.wave, over=self.over)
            psffn = self.fnfmt.format(self.npix, 'hexonly', self.wave / um,
                                      rot)
            fits.writeto(psffn, self.jw.psf, overwrite=True)
            header = fits.getheader(psffn)
            header = affinepars2header(header, aff)
            fits.update(psffn, self.jw.psf, header=header)
            del self.jw

            del aff
            del affine_rot
Beispiel #14
0
def moving_wcs_fix(files, ref=None):
    """Correct IRS FITS WCS for the motion of the targeted moving object.

    Parameters
    ----------
    files : array of strings
      A list of files to update.  The files are updated in place.
    ref : tuple
      The "reference" RA and Dec of the target expressed as a tuple:
      `(ra_ref, dec_ref)`.  This is usually the position of the moving
      target at the start of the IRS observation.  The difference
      between ra_ref, dec_ref and the RA_REF, DEC_REF in the FITS
      headers is the motion of the target.  Set ref to `None` to use
      RA_REF and DEC_REF from the first file in the file list as the
      initial position.  [units: degrees]

    """

    from astropy.io import fits
    from ..util import spherical_coord_rotate
    
    assert np.iterable(files), "files must be an array of file names"

    ra_ref0, dec_ref0 = ref

    for f in files:
        im, h = fits.getdata(f, header=True)
        ra_ref1 = h["RA_REF"]
        dec_ref1 = h["DEC_REF"]

        # I found CRVALx missing in some LH files
        if h.get("CRVAL1") is not None:
            crval1, crval2 = spherical_coord_rotate(
                ra_ref1, dec_ref1, ra_ref0, dec_ref0,
                h["CRVAL1"], h["CRVAL2"])
        rarqst, decrqst = spherical_coord_rotate(
            ra_ref1, dec_ref1, ra_ref0, dec_ref0,
            h["RA_RQST"], h["DEC_RQST"])

        raslt, decslt = spherical_coord_rotate(
            ra_ref1, dec_ref1, ra_ref0, dec_ref0,
            h["RA_SLT"], h["DEC_SLT"])

        print("{} moved {:.3f} {:.3f}".format(f, (ra_ref1 - ra_ref0) * 3600.,
                                              (dec_ref1 - dec_ref0) * 3600.))

        if h.get("CRVAL1") is not None:
            h["CRVAL1"] = crval1
            h["CRVAL2"] = crval2
        h["RA_RQST"] = rarqst
        h["RA_SLT"] = raslt
        h["DEC_RQST"] = decrqst
        h["DEC_SLT"] = decslt
        h.add_history("WCS updated for moving target motion with mskpy.instrum3ents.spitzer.moving_wcs_fix")
        fits.update(f, im, h)
Beispiel #15
0
def generate_new_offsets_new(name, asprta, suffix, tmp_dir, num_p):
    print name

    try:
        centroids = np.load(tmp_dir + '/offsets_%s.npy' % (suffix))
        time = np.load(tmp_dir + '/time_%s.npy' % (suffix))
    except IOError:
        try:
            centroids = np.load(tmp_dir + '/offsets0_%s.npy' % (suffix))
            time = np.load(tmp_dir + '/time0_%s.npy' % (suffix))
        except IOError:
            print 'no file'
            return 0

    print centroids.shape
    print time.shape
    output = '../plots/1/%s-%s.pdf' % (name, suffix)
    dir = os.path.dirname(output)
    if not os.path.exists(dir):
        os.makedirs(dir)
    plt.plot(centroids[:, 0], '.k')
    plt.savefig('../plots/1/%s-%s_ra.pdf' % (name, suffix), dpi=190)
    plt.clf()

    plt.plot(centroids[:, 1], '.k')
    plt.savefig('../plots/1/%s-%s_dec.pdf' % (name, suffix), dpi=190)
    plt.clf()

    #np.save(tmp_dir+'/offsets_%s.npy'%(suffix), centroids)
    #np.save(tmp_dir+'/time_%s.npy'%(suffix), time)

    hdulist = pyfits.open(asprta)
    co_data = hdulist[1].data
    T = co_data['T']
    ra = co_data['ra']
    dec = co_data['dec']
    roll = co_data['roll']
    ra_new = np.interp(time, T, ra) - centroids[:, 0]
    dec_new = np.interp(time, T, dec) - centroids[:, 1]
    roll_new = np.interp(time, T, roll) - centroids[:, 2]

    other = np.zeros((time.shape[0], 8))
    array = np.concatenate(
        [np.array([time, ra_new, dec_new, roll_new]).T, other], axis=1)
    data = np.core.records.fromarrays(array.transpose(), dtype=[('T', float), ('RA', float), ('DEC', float), ('ROLL', float),\
                 ('STATUS_FLAG', int), ('ROLL_RAD', float), ('X', float), ('Y', float), ('Z', float), ('XDOT', float), ('YDOT', float), ('ZDOT', float)])

    new_file = re.split('-asprta.fits',
                        asprta)[0] + '-' + suffix + '-asprta.fits'
    os.system('cp {0} {1}'.format(asprta, new_file))
    update(new_file, data, 1)
    hdu = pyfits.open(new_file)
    print hdu[1].data['RA'].shape
    print hdu[1].data['DEC'].shape
    hdu.close()
Beispiel #16
0
    def predict_all(self,
                    filename,
                    extname={
                        'image': 'IMAGE',
                        'table': 'DETECTION_TABLE'
                    }):
        det_tab = pd.DataFrame(
            np.array(getdata(filename,
                             extname['table'])).byteswap().newbyteorder())

        # clean data with inf or NaN
        det_tab = det_tab.replace([np.inf, -np.inf], np.nan)
        det_tab.dropna(inplace=True)
        det_tab.reset_index(drop=True, inplace=True)

        # load image
        pix_val = getdata(filename, extname['image'])

        # create stamp DataFrame
        pixel_col = ['p' + str(i + 1) for i in np.arange(441)]
        stamps = [
            Cutout2D(pix_val, (det_tab.iloc[i]['X_IMAGE'] - 1,
                               det_tab.iloc[i]['Y_IMAGE'] - 1), (21, 21),
                     mode='partial').data.reshape(441)
            for i in np.arange(det_tab.shape[0])
        ]
        stamps = pd.DataFrame(stamps, columns=pixel_col)

        # scale detection stamps
        X = scaling(stamps)
        X.replace([np.inf, -np.inf], np.nan, inplace=True)
        X.dropna(inplace=True)
        det_tab = det_tab.loc[X.index.tolist()]

        if self.algorithm == 'Convolutional_Neural_Network':
            X = X.values.reshape(-1, 21, 21, 1)

        if self.algorithm == 'PCA_Random_Forest':
            X = self.pca.transform(X)
            X = self.kbest.transform(X)

        # make predictions
        if (self.algorithm == 'Random_Forest') or (self.algorithm
                                                   == 'PCA_Random_Forest'):
            pred = self.model.predict(X)
        elif (self.algorithm == 'Artificial_Neural_Network') or (
                self.algorithm == 'Convolutional_Neural_Network'):
            pred = self.model.predict(X)[:, 1]

        det_tab['real_bogus'] = pred

        # update the 'DETECTION_TABLE' including real-bogus score
        m = Table(det_tab.values, names=det_tab.columns)
        hdr = getheader(filename, extname='DETECTION_TABLE')
        update(filename, np.array(m), extname='DETECTION_TABLE', header=hdr)
def generate_new_offsets_new(name, asprta, suffix, tmp_dir, num_p):
    print name

    try:
        centroids = np.load(tmp_dir+'/offsets_%s.npy'%(suffix))
        time = np.load(tmp_dir+'/time_%s.npy'%(suffix))
    except IOError:
        try:
            centroids = np.load(tmp_dir+'/offsets0_%s.npy'%(suffix))
            time = np.load(tmp_dir+'/time0_%s.npy'%(suffix))
        except IOError:
            print 'no file'
            return 0

    print centroids.shape
    print time.shape
    output = '../plots/1/%s-%s.pdf'%(name, suffix)
    dir = os.path.dirname(output)
    if not os.path.exists(dir):
      os.makedirs(dir)
    plt.plot(centroids[:,0], '.k')
    plt.savefig('../plots/1/%s-%s_ra.pdf'%(name, suffix), dpi=190)
    plt.clf()

    plt.plot(centroids[:,1], '.k')
    plt.savefig('../plots/1/%s-%s_dec.pdf'%(name, suffix), dpi=190)
    plt.clf()

    #np.save(tmp_dir+'/offsets_%s.npy'%(suffix), centroids)
    #np.save(tmp_dir+'/time_%s.npy'%(suffix), time)
    
    hdulist = pyfits.open(asprta)
    co_data = hdulist[1].data
    T = co_data['T']
    ra = co_data['ra']
    dec = co_data['dec']
    roll = co_data['roll']
    ra_new = np.interp(time, T, ra) - centroids[:,0]
    dec_new = np.interp(time, T, dec) - centroids[:,1]
    roll_new = np.interp(time, T, roll) - centroids[:,2]

    other = np.zeros((time.shape[0], 8))
    array = np.concatenate([np.array([time, ra_new, dec_new, roll_new]).T, other], axis=1)
    data = np.core.records.fromarrays(array.transpose(), dtype=[('T', float), ('RA', float), ('DEC', float), ('ROLL', float),\
                 ('STATUS_FLAG', int), ('ROLL_RAD', float), ('X', float), ('Y', float), ('Z', float), ('XDOT', float), ('YDOT', float), ('ZDOT', float)])

    new_file = re.split('-asprta.fits', asprta)[0]+'-'+suffix+'-asprta.fits'
    os.system('cp {0} {1}'.format(asprta, new_file))
    update(new_file, data, 1)
    hdu = pyfits.open(new_file)
    print hdu[1].data['RA'].shape
    print hdu[1].data['DEC'].shape
    hdu.close()
Beispiel #18
0
def add_header(self,
               fromfile='telescope_obj1.fits',
               tofile='telescope_obj_p0.fits'):
    os.chdir('/home/dodkins/')
    hdu = pyfits.open(fromfile)
    hdr = hdu[0].header
    if fromfile == tofile: print(fromfile == tofile)
    hdu = pyfits.open(tofile, mode='update')
    scidata = hdu[0].data
    pyfits.update(tofile, scidata, hdr, 0)
    print(hdu[0].header)
    print('done')
Beispiel #19
0
def write_hduL(fitsfn, hduL):
    # If file doesn't exist, add the primary header
    hdu_primary = hduL[0]
    if os.path.exists(fitsfn) is False:
        fits.append(fitsfn, hdu_primary.data, header=hdu_primary.header)

    hduL = hduL[1:]
    for hdu in hduL:
        data = hdu.data
        extname = hdu.header['EXTNAME']
        header = hdu.header
        try:
            fits.update(fitsfn, data, extname, header=header)
        except KeyError:
            fits.append(fitsfn, data, header=header)
Beispiel #20
0
def write_hduL(fitsfn,hduL):
    # If file doesn't exist, add the primary header
    hdu_primary = hduL[0]
    if os.path.exists(fitsfn) is False:
        fits.append(fitsfn, hdu_primary.data, header=hdu_primary.header)

    hduL = hduL[1:]
    for hdu in hduL:
        data = hdu.data
        extname = hdu.header['EXTNAME']
        header = hdu.header
        try:
            fits.update(fitsfn, data, extname, header=header)
        except KeyError:
            fits.append(fitsfn, data, header=header)
Beispiel #21
0
def winbin_fits(image, win, bin, bintype, output=''):
    if output != '': output = filenames2list(output)
    else: output = filenames2list(image)
    for i, fn in enumerate(filenames2list(image, forcelist=True)):
        d, h = fits.getdata(fn, header=True)
        if win != 'full':
            d = d[win[2] - 1:win[3] + win[2] - 1,
                  win[0] - 1:win[0] + win[1] - 1]
        dc = bintype(d.reshape(d.shape[0], -1, bin[0]), 2)
        dr = bintype(dc.reshape(-1, bin[1], dc.shape[1]), 1)
        if (type(output) == str and output == image) or output[i] == fn:
            fits.update(fn, dr, header=h)
        elif type(output) == str:
            fits.writeto(output, dr, header=h)
        elif isinstance(output, (list, tuple)):
            fits.writeto(output[i], dr, header=h)
    return output
Beispiel #22
0
def oscan_trim_file(fname, datahdus=0):
    hdulist = fits.open(fname)
    nhdus = len(hdulist)
    if nhdus > 1:
        istart = 1
    else:
        istart = 0
    # loop from first-data to last HDU, unless datahdus is set
    hduindexes = list(range(nhdus))[istart:]
    if datahdus != 0:
        hduindexes = datahdus
    for i in hduindexes:
        hdulist = fits.open(fname)
        data1 = ccdproc.CCDData(hdulist[i].data, unit="adu")
        data1.header = hdulist[i].header
        # What happens if file is already overscan-subtracted?
        # We should probably default to using a model
        if modeling:
            oscan1 = ccdproc.subtract_overscan(
                data1,
                fits_section=data1.header['BIASSEC'],
                add_keyword={
                    'overscan': True,
                    'calstat': 'O'
                },
                model=models.Polynomial1D(1))
        else:
            oscan1 = ccdproc.subtract_overscan(
                data1,
                fits_section=data1.header['BIASSEC'],
                add_keyword={
                    'overscan': True,
                    'calstat': 'O'
                },
                model=None)

        trim1 = ccdproc.trim_image(oscan1,
                                   fits_section=oscan1.header['TRIMSEC'],
                                   add_keyword={
                                       'trimmed': True,
                                       'calstat': 'OT'
                                   })
        fits.update(fname, trim1.data, header=trim1.header, ext=i)
    hdulist.close()
    mylog("Overscan and trim {0}".format(fname))
    return
def store_frame_to_fits(fits_address, fits_hdu, ext_name):

    if fits_address.is_file():
        try:
            fits.update(fits_address,
                        data=fits_hdu.data,
                        header=fits_hdu.header,
                        extname=ext_name,
                        verify=True)
        except KeyError:
            fits.append(fits_address,
                        data=fits_hdu.data,
                        header=fits_hdu.header,
                        extname=ext_name)
    else:
        fits_hdu.writeto(fits_address, overwrite=True, output_verify='fix')

    return
Beispiel #24
0
    def test_image_extension_update_header(self):
        """
        Test that _makehdu correctly includes the header. For example in the
        fits.update convenience function.
        """
        filename = self.temp('twoextension.fits')

        hdus = [fits.PrimaryHDU(np.zeros((10, 10))),
                fits.ImageHDU(np.zeros((10, 10)))]

        fits.HDUList(hdus).writeto(filename)

        fits.update(filename,
                    np.zeros((10, 10)),
                    header=fits.Header([('WHAT', 100)]),
                    ext=1)
        h_out = fits.getheader(filename, ext=1)
        assert h_out['WHAT'] == 100
Beispiel #25
0
    def test_image_extension_update_header(self):
        """
        Test that _makehdu correctly includes the header. For example in the
        fits.update convenience function.
        """
        filename = self.temp('twoextension.fits')

        hdus = [fits.PrimaryHDU(np.zeros((10, 10))),
                fits.ImageHDU(np.zeros((10, 10)))]

        fits.HDUList(hdus).writeto(filename)

        fits.update(filename,
                    np.zeros((10, 10)),
                    header=fits.Header([('WHAT', 100)]),
                    ext=1)
        h_out = fits.getheader(filename, ext=1)
        assert h_out['WHAT'] == 100
Beispiel #26
0
def oscan_trim_file(fname):
    hdulist = fits.open(fname)
    nhdus = len(hdulist)
    if nhdus > 1:
        istart = 1
    else:
        istart = 0
    # loop from first-data to last HDU.
    for i in range(nhdus)[istart:]:
        hdulist = fits.open(fname)
        data1 = ccdproc.CCDData(hdulist[i].data, unit="adu")
        data1.header = hdulist[i].header
        # What happens if file is already overscan-subtracted?
        if modeling:
            oscan1 = ccdproc.subtract_overscan(
                data1,
                fits_section=data1.header['BIASSEC'],
                add_keyword={
                    'overscan': True,
                    'calstat': 'O'
                },
                model=models.Polynomial1D(1))
        else:
            oscan1 = ccdproc.subtract_overscan(
                data1,
                fits_section=data1.header['BIASSEC'],
                add_keyword={
                    'overscan': True,
                    'calstat': 'O'
                },
                model=None)

        trim1 = ccdproc.trim_image(oscan1,
                                   fits_section=oscan1.header['TRIMSEC'],
                                   add_keyword={
                                       'trimmed': True,
                                       'calstat': 'OT'
                                   })
        fits.update(fname, trim1.data, header=trim1.header, ext=i)
    hdulist.close()
    return
Beispiel #27
0
def run_sector_camera_chip(base_dir, sector, camera, chip):
    pattern = os.path.join(base_dir, "hlsp_eleanor_*.fits")
    outdir = base_dir
    open(os.path.join(outdir, "index.auto"), "w").close()
    open(os.path.join(os.path.dirname(outdir), "index.auto"), "w").close()

    fns = list(sorted(glob.glob(pattern)))

    # Ensures no postcards have been repeated
    fns = np.unique(fns)

    # Writes in the background after making the postcards
    print("Computing backgrounds...")
    for fn in tqdm.tqdm(fns, total=len(fns)):
        with fits.open(fn) as hdu:
            bkg = calc_2dbkg(hdu[2].data, hdu[1].data['QUALITY'],
                             hdu[1].data['TSTART'])
            # Checks to make sure there isn't a background extension already
            if len(hdu) < 5:
                fits.append(fn, bkg)
            else:
                fits.update(fn, bkg, 4)
Beispiel #28
0
def unbias(input_file,
           output_file,
           bias_method,
           bias_method_col=None,
           superbias_file=None):
    # build output file by copying input
    shutil.copyfile(input_file, output_file)

    ccd = get_ccd_from_id(None, input_file, [])
    hdulist = fits.open(input_file)

    if superbias_file is not None:
        superbias_ccd = get_ccd_from_id(None, superbias_file, [])
    else:
        superbias_ccd = None

    amps = get_amp_list(ccd)
    offset = get_amp_offset(ccd, superbias_ccd)

    for i, amp in enumerate(amps):
        regions = get_geom_regions(ccd, amp)
        serial_oscan = regions['serial_overscan']
        parallel_oscan = regions['parallel_overscan']
        img = get_raw_image(ccd, amp)
        if superbias_ccd is not None:
            superbias_im = get_raw_image(superbias_frame, amp + offset)
        else:
            superbias_im = None
        image = unbias_amp(img,
                           serial_oscan,
                           bias_type=bias_method,
                           superbias_im=superbias_im,
                           bias_type_col=bias_method_col,
                           parallel_oscan=parallel_oscan)
        fits.update(output_file,
                    image.image.array,
                    amp,
                    header=hdulist[amp].header)
Beispiel #29
0
def renormalize_by_flat(image, flat, read_from_file=False, datahdus=0):
    if read_from_file == True:
        hduimage = fits.open(image)
        hduflat = fits.open(flat)
    else:
        hduimage = image
        hduflat = flat
    nhdus = len(hdulist)
    # Do nothing if image only has 0-1 data extension
    if nhdus <= 1:
        mylog("Don't need to renormalize a 1-extension flat, returning")
        return
    secmeans = np.zeros(nhdus - 1)
    hduindexes = list(range(nhdus))[1:]
    if datahdus != 0:
        hduindexes = datahdus
    for i in hduindexes:
        secmeans[i - 1] = hduflat[i].data.mean()
    totmean = secmeans.mean()
    secmeans = secmeans / totmean
    mylog("Renormalized flat means by extension: {0} ".format(secmeans))
    for i in hduindexes:
        #hduimage[i].data = hduimage[i].data / secmeans[i-1]
        hduimage[i].divide(secmeans[i - 1] * hduimage[i].unit)
        if read_from_file == True:
            fits.update(image,
                        hduimage[i].data,
                        header=hduimage[i].header,
                        ext=i)
    if read_from_file == True:
        hduimage.close()
        hduflat.close()
    else:
        # This may be unnecessary if these are already pointing at identical structure
        image = hduimage
    # done?
    return
Beispiel #30
0
def update_fits_header(files, modify={}, remove=()):
    files = filenames2list(files, forcelist=True)
    for file in files:
        data, head = fits.getdata(file, header=True)
        for key in modify:
            if not isinstance(modify[key], dict):
                head[key] = modify[key]
            else:
                func = modify[key].get('func', None)
                args = modify[key].get('args', None)
                if func and args is None:
                    head[key] = func()
                if func is None and isinstance(args, str):
                    head[key] = head[args]
                if func is not None and args is not None:
                    if type(args) == str:
                        head[key] = func(head[args])
                    else:
                        head[key] = func(*[head[arg] for arg in args])
        if remove:
            for key in remove:
                if key in head:
                    head.remove(key)
        fits.update(file, data, header=head)
def generate_new_offsets_new(name, asprta, suffix, tmp_dir, num_p):
    print name
    #centers = []
    #center_time = []
    '''
    hdulist = pyfits.open('../AIS_GAL_SCAN/asprta/%s-asprta.fits'%name)
    initial = 0
    final = hdulist[1].data['T'].shape[0]-1
    hdulist.close()


    for i in range(initial, final+1):
        #center = np.load('../data/%s/cata/centroids_rot%d.npy'%(name, i))
        #if center.shape!=(2,3):
        #   print i, center.shape
        centers.append(np.load(tmp_dir+'/centroids_rot%d.npy'%(i)))
        center_time.append(np.load(tmp_dir+'/time_rot%d.npy'%(i)))
    '''
    '''
    for i in range(num_p):
        centers.append(np.load(tmp_dir+'/centroids_tmp%d.npy'%(i)))
        center_time.append(np.load(tmp_dir+'/time_tmp%d.npy'%(i)))
    centroids = np.concatenate(centers, axis=0)
    time =  np.concatenate(center_time, axis=0)
    '''
    try:
        centroids = np.load(tmp_dir+'/offsets_%s.npy'%(suffix))
        time = np.load(tmp_dir+'/time_%s.npy'%(suffix))
    except IOError:
        try:
            centroids = np.load(tmp_dir+'/offsets0_%s.npy'%(suffix))
            time = np.load(tmp_dir+'/time0_%s.npy'%(suffix))
        except IOError:
            print 'no file'
            return 0


    scst_file = pyfits.open('/scratch/dw1519/galex/AIS_GAL_SCAN/scst/%s-scst.fits'%name)
    scst_data = scst_file[1].data
    scst_time = scst_data['pktime'].copy()
    hv = scst_data['hvnom_nuv'].copy()
    scst_file.close()

    scst_ix = np.digitize(time, scst_time)-1
    ix_mask = scst_ix<scst_time.shape[0]
    #scst_ix = scst_ix[ix_mask]
    #time = time[ix_mask]
    #time2run = time_c[hv[scst_ix]>0]
    data_mask1 = hv[scst_ix]>0


    ct1 = np.trim_zeros(centroids[:,0],'f')
    lz = centroids[:,0].shape[0]-ct1.shape[0]
    tz = np.trim_zeros(centroids[:,0],'b').shape[0]
    data_mask2 = np.zeros(centroids.shape[0])
    data_mask2[lz:tz] = 1
    data_mask2 = data_mask2>0

    data_mask = data_mask1 & data_mask2
    '''
    rng = np.random.RandomState(42)
    clf = IsolationForest(max_samples=100, random_state=rng)
    X_train = np.column_stack([time[data_mask>0], centroids[data_mask>0,0:2]])
    clf.fit(X_train)
    y_pred_train = clf.predict(X_train)
    outlier = y_pred_train==-1
    '''
    outlier1 = (np.sum(centroids[data_mask],axis=1)==0)

    lim=10#6#6#5#3.5
    w = 50#40#40#30
    out_mask = np.zeros(np.sum(data_mask))
    z, mz = moving_stat(centroids[data_mask,0],out_mask,half_win=w)
    outlier = mz>lim

    z, mz = moving_stat(centroids[data_mask,1],out_mask,half_win=w)
    outlier = outlier | (mz>lim)

    a=np.zeros(np.sum(data_mask))
    a[0:100] = 1
    a = a.astype(bool)
    outlier = (outlier | outlier1) | a #| (centroids[data_mask,1]>.07) #| (((centroids[data_mask,0]>.03) | (centroids[data_mask,0]<0)) & a)
    c_new = np.zeros(centroids.shape)
    '''
    spl_ra = splrep(time[data_mask][~outlier], centroids[data_mask,0][~outlier])
    spl_dec = splrep(time[data_mask][~outlier], centroids[data_mask,1][~outlier])
    c_new[data_mask,0] = splev(time[data_mask], spl_ra)
    c_new[data_mask,1] = splev(time[data_mask], spl_dec)
    '''
    #fra = interpolate.interp1d(time[data_mask][~outlier], centroids[data_mask,0][~outlier])
    #fdec = interpolate.interp1d(time[data_mask][~outlier], centroids[data_mask,1][~outlier])

    if time[data_mask][-1]>time[data_mask][~outlier][-1]:
        ex_mask = time>time[data_mask][~outlier][-1]
        c_new[data_mask&ex_mask,0] = centroids[data_mask][~outlier][-1,0]
        c_new[data_mask&ex_mask,1] = centroids[data_mask][~outlier][-1,1]
        outlier = outlier[~ex_mask[data_mask]]
        data_mask = data_mask &(~ex_mask)
  
    if time[data_mask][0]<time[data_mask][~outlier][0]:
        ex_mask = time<time[data_mask][~outlier][0]
        c_new[data_mask&ex_mask,0] = centroids[data_mask][~outlier][0,0]
        c_new[data_mask&ex_mask,1] = centroids[data_mask][~outlier][0,1]
        outlier = outlier[~ex_mask[data_mask]]
        data_mask = data_mask &(~ex_mask)

    spl_ra = splrep(time[data_mask][~outlier], centroids[data_mask,0][~outlier])
    spl_dec = splrep(time[data_mask][~outlier], centroids[data_mask,1][~outlier])
    c_new[data_mask,0] = splev(time[data_mask], spl_ra)
    c_new[data_mask,1] = splev(time[data_mask], spl_dec) 
    '''
    else:
        #c_new[data_mask,0] = fra(time[data_mask])
        #c_new[data_mask,1] = fdec(time[data_mask])
        spl_ra = splrep(time[data_mask][~outlier], centroids[data_mask,0][~outlier])
        spl_dec = splrep(time[data_mask][~outlier], centroids[data_mask,1][~outlier])
        c_new[data_mask,0] = splev(time[data_mask], spl_ra)
        c_new[data_mask,1] = splev(time[data_mask], spl_dec)
    '''


    print centroids.shape
    print time.shape
    output = '../plots/0/%s-%s.pdf'%(name, suffix)
    dir = os.path.dirname(output)
    if not os.path.exists(dir):
      os.makedirs(dir)
    x = np.arange(centroids.shape[0])
    plt.plot(x[data_mask],centroids[data_mask,0], '.k', markersize=2)
    #plt.plot(x[data_mask][outlier], centroids[data_mask,0][outlier], '.r', markersize=2)
    plt.plot(x[data_mask],c_new[data_mask,0], '.r', markersize=2)
    #plt.plot(x[ex_mask], centroids[ex_mask,0], '.r', markersize=2)
    #plt.xlim(500,750)
    plt.tight_layout()
    plt.savefig('../plots/0/%s-%s_ra.pdf'%(name, suffix), dpi=190)
    plt.clf()

    plt.plot(x[data_mask], centroids[data_mask,1], '.k', markersize=2)
    #plt.plot(x[data_mask][outlier], centroids[data_mask,1][outlier], '.r', markersize=2)
    plt.plot(x[data_mask],c_new[data_mask,1], '.r', markersize=2)
    #plt.plot(x[ex_mask], centroids[ex_mask,1], '.r', markersize=2)
    #plt.xlim(500,750)
    plt.tight_layout()
    plt.savefig('../plots/0/%s-%s_dec.pdf'%(name, suffix), dpi=190)
    plt.clf()

    #np.save(tmp_dir+'/offsets_%s.npy'%(suffix), centroids)
    #np.save(tmp_dir+'/time_%s.npy'%(suffix), time)

    hdulist = pyfits.open(asprta)
    co_data = hdulist[1].data
    T = co_data['T']
    ra = co_data['ra']
    dec = co_data['dec']
    roll = co_data['roll']
    ra_new = np.interp(time, T, ra) - c_new[:,0]
    dec_new = np.interp(time, T, dec) - c_new[:,1]
    roll_new = np.interp(time, T, roll) - c_new[:,2]

    other = np.zeros((time.shape[0], 8))
    array = np.concatenate([np.array([time, ra_new, dec_new, roll_new]).T, other], axis=1)
    data = np.core.records.fromarrays(array.transpose(), dtype=[('T', float), ('RA', float), ('DEC', float), ('ROLL', float),\
                 ('STATUS_FLAG', int), ('ROLL_RAD', float), ('X', float), ('Y', float), ('Z', float), ('XDOT', float), ('YDOT', float), ('ZDOT', float)])

    new_file = re.split('-asprta.fits', asprta)[0]+'-'+suffix+'-asprta.fits'
    os.system('cp {0} {1}'.format(asprta, new_file))
    update(new_file, data, 1)
    hdu = pyfits.open(new_file)
    print hdu[1].data['RA'].shape
    print hdu[1].data['DEC'].shape
    hdu.close()
Beispiel #32
0
def find_scale(
        imagedata,
        affine_best,  # best current guess at data geometry cf analytical ideal
        scales,  # scales are near-unity
        pixel,
        npix,
        bandpass,
        over,
        holeshape,
        outdir=None):  # for nrm_model
    """  Preserve incoming "pixel" value, put the scale correction into the Affine2d object
         Is that kosher???  Should we change the pixel scale and leave affine2d the same?
         Affine2d can also incorporate unequal x and y scales, shears...
         For now place scale corrections into the Affine2d object     
         
         Note - placing isotropic scale change into Affine2d is equivalent to changing
         the effective image distance in the optical train while insisting that the
         mask physical geometry does not change, and the wavelength is perfectly knowm

         AS 2018 10  """

    affine_best.show("\tfind_scale")
    vprint("\tBefore Loop: ", scales)

    #Extend this name to include multi-paramater searches?
    psffmt = 'psf_nrm_{0:d}_{1:s}_{2:.3f}um_scl{3:.3f}.fits'
    # expect (npix, holeshape, bandpass/um, scl)

    if hasattr(scales, '__iter__') is False:
        scales = (scales, )

    affine2d_list = create_afflist_scales(scales, affine_best.mx,
                                          affine_best.my, affine_best.sx,
                                          affine_best.sy, affine_best.xo,
                                          affine_best.yo)
    crosscorrs = []

    for (scl, aff) in zip(scales, affine2d_list):

        vprint(aff.name + "...")
        jw = NRM_Model(mask='jwst',
                       holeshape=holeshape,
                       over=over,
                       affine2d=aff)
        jw.set_pixelscale(pixel)
        jw.simulate(fov=npix, bandpass=bandpass, over=over)
        psffn = psffmt.format(npix, holeshape, bandpass[:, 1][0] / um, scl)
        if outdir:
            fits.PrimaryHDU(data=jw.psf).writeto(outdir + "/" + psffn,
                                                 overwrite=True)
            fits.writeto(psffn, jw.psf, overwrite=True)
            header = fits.getheader(psffn)
            utils.affinepars2header(header, aff)
            fits.update(psffn, jw.psf, header=header)

        crosscorrs.append(utils.rcrosscorrelate(imagedata, jw.psf).max())
        del jw

    vprint("\tfind_affine2d_parameters: crosscorrelations", crosscorrs)
    vprint("\tfind_affine2d_parameters:            scales", scales)
    scl_measured, max_cor = utils.findpeak_1d(crosscorrs, scales)
    vprint(
        "\tfind_affine2d_parameters factor measured {0:.5f}  Max correlation {1:.3e}"
        .format(scl_measured, max_cor))
    vprint("\tfind_affine2d_parameters pitch from header  {0:.3f} mas".format(
        pixel * rad2mas))
    vprint(
        "\tfind_affine2d_parameters pitch  {0:.3f} mas (implemented using affine2d)"
        .format(scl_measured * pixel * rad2mas))

    # return convenient affine2d
    return utils.Affine2d(affine_best.mx * scl_measured,
                          affine_best.my * scl_measured,
                          affine_best.sx * scl_measured,
                          affine_best.sy * scl_measured,
                          affine_best.xo * scl_measured,
                          affine_best.yo * scl_measured,
                          name="scale_{0:.4f}".format(scl))
Beispiel #33
0
 def update_fits(self):
     data = np.vstack((self.wav,self.spec)) if self.wav else self.spec
     fits.update(self.file, data, self.header)    
Beispiel #34
0
    def setUp(self):

        allholes = ('b4', 'c2', 'b5', 'b2', 'c1', 'b6', 'c6')
        b4, c2, b5, b2, c1, b6, c6 = allholes
        self.hc = (b2, b6, b5)  # holechoices
        self.hstr = holes2string(self.hc)
        self.holeshape = "hex"

        # directory containing the test data
        data_dir = os.path.join(os.path.dirname(__file__),
                                'test_data/affine2d_makemodel_rot')
        self.data_dir = data_dir
        print(data_dir)
        if not os.path.exists(data_dir):
            os.makedirs(data_dir)

    # expects strings of  % (self.npix, self.holeshape, self.wave/um, rot, self.hstr)
        self.fnfmt = data_dir + '/psf_nrm_{2:.1f}_{0}_{1}_{3:.0f}_{4:s}.fits'

        self.fmt = "    ({0:+.3f}, {1:+.3f}) -> ({2:+.3f}, {3:+.3f})"
        self.pixel = 0.0656
        self.npix = 87
        self.wave = 4.3e-6  # m
        self.over = 11

        mx, my = 1.0, 1.0
        sx, sy = 0.0, 0.0
        xo, yo = 0.0, 0.0
        affine_ideal = Affine2d(mx=mx,
                                my=my,
                                sx=sx,
                                sy=sy,
                                xo=xo,
                                yo=yo,
                                name="Ideal")

        rots = (0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75,
                80, 85, 90, 95)
        rots = (
            0.000,
            10.000,
        )
        for rot in rots:

            rot = avoidhexsingularity(rot)  # in utils
            affine_rot = Affine2d(rotradccw=np.pi * rot / 180.0,
                                  name="{0:.0f}".format(rot))  # in utils
            aff = affine_rot

            # holeshape is hex or circ g7s6 jwst
            # because model_array requires
            # a primary beam for one slice of the model
            self.jw = NRM_Model(
                mask='jwst',
                holeshape=self.holeshape,
                #chooseholes=self.hc,
                affine2d=aff)
            self.jw.set_pixelscale(self.pixel * arcsec2rad)
            self.jw.simulate(fov=self.npix, bandpass=self.wave, over=self.over)
            # write psf
            psffn = self.fnfmt.format(self.npix, self.holeshape,
                                      self.wave / um, rot, self.hstr)
            fits.writeto(psffn, self.jw.psf, overwrite=True)
            header = fits.getheader(psffn)
            header = affinepars2header(header, aff)
            fits.update(psffn, self.jw.psf, header=header)
            print("test:  psf shape", self.jw.psf.shape)

            modelslices = self.jw.make_model(fov=self.npix,
                                             bandpass=self.wave,
                                             over=self.over)
            print("test:  modelslices type", type(modelslices))
            print("test:  modelslices shape", modelslices.shape)
            modelfn = psffn.replace("psf_nrm", "model")

            # write model
            model_for_fitsfile = np.zeros(
                (modelslices.shape[2], modelslices.shape[0],
                 modelslices.shape[1]))
            for sl in range(modelslices.shape[2]):
                model_for_fitsfile[sl, :, :] = modelslices[:, :, sl]
            print("test:  model_for_fitsfile type", type(model_for_fitsfile))
            print("test:  model_for_fitsfile shape", model_for_fitsfile.shape)
            fits.writeto(modelfn, model_for_fitsfile[6, :, :], overwrite=True)
            header = fits.getheader(modelfn)
            header = affinepars2header(header, aff)
            fits.update(modelfn, model_for_fitsfile[6, :, :], header=header)

            del self.jw

            del aff
            del affine_rot
Beispiel #35
0
logf.write(str(datetime.now() - startTime)+" - Done writing ASCII\n")

# Write FITS files with simulated data (in units of electrons)

hdu0=fits.PrimaryHDU()
hdu1=fits.ImageHDU()
hdu2=fits.ImageHDU()
hdu3=fits.ImageHDU()
hdu4=fits.ImageHDU()
hdul=fits.HDUList([hdu0, hdu1, hdu2, hdu3, hdu4])


hdul.writeto(tmpdir+'/lcoetc_out.fits', clobber=1)

fits.update(tmpdir+'/lcoetc_out.fits', np.transpose(neobj2d+nesky2d+mkrandnoise(nenoise2d)), 1)
fits.update(tmpdir+'/lcoetc_out.fits', np.transpose(neobj2d), 2)
fits.update(tmpdir+'/lcoetc_out.fits', np.transpose(nesky2d), 3)
fits.update(tmpdir+'/lcoetc_out.fits', np.transpose(nenoise2d), 4)

hdul=fits.open(tmpdir+'/lcoetc_out.fits')

for i in np.arange(4):
    hdul[i+1].header.set('DISPAXIS', '1')
    hdul[i+1].header.set('CTYPE1', 'LINEAR')
    hdul[i+1].header.set('CUNIT1', 'Angstroms')
    hdul[i+1].header.set('CRPIX1', '1')
    hdul[i+1].header.set('CRVAL1', str(lam[0]))
    hdul[i+1].header.set('CD1_1', str(ddisp))

def generate_first_offsets(name):
    print name
    hdulist = pyfits.open('../AIS_GAL_SCAN/asprta/%s-asprta.fits'%name)
    initial = 1
    final = hdulist[1].data['T'].shape[0]-1

    centers = []
    center_time = []
    for i in range(initial, final+1):
        c = np.load('../data/%s/cata/centroids_rot%d.npy'%(name, i))
        #if c.shape == (1,3):
        #   c = c[:,:2]
        centers.append(c)
        center_time.append(np.load('../data/%s/cata/time_rot%d.npy'%(name, i)))
        print c.shape
    centroids = np.concatenate(centers, axis=0)
    time =  np.concatenate(center_time, axis=0)

    print centroids.shape

    out_mask = np.zeros(centroids.shape[0])

    z, mz = moving_stat(centroids[:,0], out_mask, half_win=100)
    outliers = np.zeros(centroids.shape[0])
    outliers[mz>3.5] = 1
    outliers[out_mask>0] = 1
    outliers = outliers>0
    index = np.arange(centroids.shape[0])
    centroids[outliers, 0] = np.interp(index[outliers], index[~outliers], centroids[~outliers,0])

    z, mz = moving_stat(centroids[:,1], out_mask, half_win=100)
    outliers = np.zeros(centroids.shape[0])
    outliers[mz>3.5] = 1
    outliers[out_mask>0] = 1
    outliers = outliers>0
    index = np.arange(centroids.shape[0])
    centroids[outliers, 1] = np.interp(index[outliers], index[~outliers], centroids[~outliers,1])


    output = "../plots/%s/cata/output.csv"%(name)
    dir = os.path.dirname(output)
    if not os.path.exists(dir):
        os.makedirs(dir)

    plt.plot(centroids[:,0], '.b')
    plt.savefig('../plots/%s/cata/offsets_10_new_half.pdf'%name, dpi=190)
    plt.clf()

    np.save('../data/%s/cata/time%d_10_new_half.npy'%(name, initial), time)
    np.save('../data/%s/cata/offsets%d_10_new_half.npy'%(name, initial), centroids)

    co_data = hdulist[1].data
    T = co_data['T']
    ra = co_data['ra']
    dec = co_data['dec']
    roll = co_data['roll']
    ra_new = np.interp(time, T, ra) - centroids[:,0]
    dec_new = np.interp(time, T, dec) - centroids[:,1]
    roll_new = np.interp(time, T, roll) - centroids[:,2]

    other = np.zeros((time.shape[0], 8))
    array = np.concatenate([np.array([time, ra_new, dec_new, roll_new]).T, other], axis=1)
    data = np.core.records.fromarrays(array.transpose(), dtype=[('T', float), ('RA', float), ('DEC', float), ('ROLL', float),\
                 ('STATUS_FLAG', int), ('ROLL_RAD', float), ('X', float), ('Y', float), ('Z', float), ('XDOT', float), ('YDOT', float), ('ZDOT', float)])

    new_file = '../AIS_GAL_SCAN/asprta/%s-cal-asprta.fits'%(name)
    os.system('cp ../AIS_GAL_SCAN/asprta/%s-asprta.fits ../AIS_GAL_SCAN/asprta/%s-cal-asprta.fits'%(name, name))
    update(new_file, data, 1)
    hdu = pyfits.open(new_file)
    print hdu[1].data['RA'].shape
    print hdu[1].data['DEC'].shape
    hdu.close()

    #asp_cal.interpolate_offsets(name, 1., centroids)
    tmp_files = glob.glob("../data/%s/cata/centroids_rot*"%name)
    for tmp_file in tmp_files:
        os.remove(tmp_file)

    tmp_files = glob.glob("../data/%s/cata/time_rot*"%name)
    for tmp_file in tmp_files:
        os.remove(tmp_file)

    '''
Beispiel #37
0
def main(folder, quiet=0):
    """analysis.main(folder, quiet=0)

    This script gives the number of "observed" stars from the sampled datafiles in "folder"/ 
    according to the selection criteria from Yusef-Zadeh et al

    Parameters
    ----------
    folder  String:
        Specifiecs the folder where the files are
    quiet   boolean:
        =1 suppresses all standard output

    Returns
    -------
    returns a file named __expected_number in which contains a numpy array of the simulation parameters
    number, A_v, Aperature_size, Age and the expected 'detected' number
    The first line of the file is an ','-seperated head of the contained informations
    """

    if quiet:
        output_stream = StringIO()
    else:
        output_stream = sys.stdout



    color1 = "I4"             #filter system for first color of CMD
    color2 = "M1"             #filter system for second color of CMD
    zeromagc1 = zero.zero_mag[color1]
    zeromagc2 = zero.zero_mag[color2]
    min_mag = 8.              #minimal observation limit
    max_mag = 0.              #maximal observation limit

#getting file list
    files = sorted(os.listdir('%s/%s' % (os.getcwdu(), folder))) 
    out = []

    for fil in files:
#only using files created by the automated simulation
        if fil.startswith('sim_') and not 'settings' in fil.encode("ascii"):
            print("%s/%s" % (folder,fil.encode("ascii")), file=output_stream)
      

      # Read in
            hdulist = fits.open('%s/%s' %(folder,fil))
            data = hdulist[1].data

      #calculating magnitudes from fluxes and converting to CMD-data
            x = -2.5*(np.log10(data['c%s' % color1]/zeromagc1) - np.log10(data['c%s' % color2]/zeromagc2))
            y = -2.5*(np.log10(data['c%s' % color2]/zeromagc2))

            
            sel = np.logical_and( (y > -10./3. * (x-1.) + 10.), np.logical_and(max_mag < y, y < min_mag))
            sel = np.logical_and(sel, y < -x + 12.)
            n = sum(sel)
            t = Table(hdulist[1].data)
            if 'sel' in t.columns:
                t.remove_column('sel')
            t.add_column(Column(name='sel', data=sel.astype('int')))
            
            hdulist[1].data = np.array(t)
            tmp, av, apera, age = fil.split('_')
            fits.update('%s/%s' %(folder,fil), np.array(t), ext = 1, clobber=True)
            out.append([av, apera, age, n])

    #writing obtained data to "folder/__expected_number"
    head = ['#', 'AV', 'Aperature_size', 'Age', 'Expected_number']
    f = open('%s/__expected_number' % folder, 'w')
    f.write(','.join(head)+'\n' )
    np.savetxt(f, np.asarray(out).astype(int))
    f.close()
   
    print ("Analysed %s files and saved output to %s" % (len(out),'%s/__expected_number' % folder), file=output_stream)
Beispiel #38
0
def main(folder, quiet=0):
    """analysis.main(folder, quiet=0)

    This script gives the number of "observed" stars from the sampled datafiles in "folder"/ 
    according to the selection criteria from Yusef-Zadeh et al

    Parameters
    ----------
    folder  String:
        Specifiecs the folder where the files are
    quiet   boolean:
        =1 suppresses all standard output

    Returns
    -------
    returns a file named __expected_number in which contains a numpy array of the simulation parameters
    number, A_v, Aperature_size, Age and the expected 'detected' number
    The first line of the file is an ','-seperated head of the contained informations
    """

    if quiet:
        output_stream = StringIO()
    else:
        output_stream = sys.stdout

    color1 = "I4"  #filter system for first color of CMD
    color2 = "M1"  #filter system for second color of CMD
    zeromagc1 = zero.zero_mag[color1]
    zeromagc2 = zero.zero_mag[color2]
    min_mag = 8.  #minimal observation limit
    max_mag = 0.  #maximal observation limit

    #getting file list
    files = sorted(os.listdir('%s/%s' % (os.getcwdu(), folder)))
    out = []

    for fil in files:
        #only using files created by the automated simulation
        if fil.startswith('sim_') and not 'settings' in fil.encode("ascii"):
            print("%s/%s" % (folder, fil.encode("ascii")), file=output_stream)

            # Read in
            hdulist = fits.open('%s/%s' % (folder, fil))
            data = hdulist[1].data

            #calculating magnitudes from fluxes and converting to CMD-data
            x = -2.5 * (np.log10(data['c%s' % color1] / zeromagc1) -
                        np.log10(data['c%s' % color2] / zeromagc2))
            y = -2.5 * (np.log10(data['c%s' % color2] / zeromagc2))

            sel = np.logical_and((y > -10. / 3. * (x - 1.) + 10.),
                                 np.logical_and(max_mag < y, y < min_mag))
            sel = np.logical_and(sel, y < -x + 12.)
            n = sum(sel)
            t = Table(hdulist[1].data)
            if 'sel' in t.columns:
                t.remove_column('sel')
            t.add_column(Column(name='sel', data=sel.astype('int')))

            hdulist[1].data = np.array(t)
            tmp, av, apera, age = fil.split('_')
            fits.update('%s/%s' % (folder, fil),
                        np.array(t),
                        ext=1,
                        clobber=True)
            out.append([av, apera, age, n])

    #writing obtained data to "folder/__expected_number"
    head = ['#', 'AV', 'Aperature_size', 'Age', 'Expected_number']
    f = open('%s/__expected_number' % folder, 'w')
    f.write(','.join(head) + '\n')
    np.savetxt(f, np.asarray(out).astype(int))
    f.close()

    print("Analysed %s files and saved output to %s" %
          (len(out), '%s/__expected_number' % folder),
          file=output_stream)
def generate_new_offsets_new(name, asprta, suffix, tmp_dir, num_p):
    print name
    #centers = []
    #center_time = []
    '''
    hdulist = pyfits.open('../AIS_GAL_SCAN/asprta/%s-asprta.fits'%name)
    initial = 0
    final = hdulist[1].data['T'].shape[0]-1
    hdulist.close()


    for i in range(initial, final+1):
        #center = np.load('../data/%s/cata/centroids_rot%d.npy'%(name, i))
        #if center.shape!=(2,3):
        #   print i, center.shape
        centers.append(np.load(tmp_dir+'/centroids_rot%d.npy'%(i)))
        center_time.append(np.load(tmp_dir+'/time_rot%d.npy'%(i)))
    '''
    '''
    for i in range(num_p):
        centers.append(np.load(tmp_dir+'/centroids_tmp%d.npy'%(i)))
        center_time.append(np.load(tmp_dir+'/time_tmp%d.npy'%(i)))
    centroids = np.concatenate(centers, axis=0)
    time =  np.concatenate(center_time, axis=0)
    '''
    try:
        centroids = np.load(tmp_dir+'/offsets_%s.npy'%(suffix))
        time = np.load(tmp_dir+'/time_%s.npy'%(suffix))
    except IOError:
        try:
            centroids = np.load(tmp_dir+'/offsets0_%s.npy'%(suffix))
            time = np.load(tmp_dir+'/time0_%s.npy'%(suffix))
        except IOError:
            print 'no file'
            return 0

    ct1 = np.trim_zeros(centroids[:,0],'f')
    lz = centroids[:,0].shape[0]-ct1.shape[0]
    tz = np.trim_zeros(centroids[:,0],'b').shape[0]
    data_mask = np.zeros(centroids.shape[0])
    data_mask[lz:tz] = 1

    lim=10
    out_mask = np.zeros(tz-lz)
    z, mz = moving_stat(centroids[data_mask>0,0],out_mask,half_win=100)
    outlier = mz>lim

    z, mz = moving_stat(centroids[data_mask>0,1],out_mask,half_win=100)
    outlier = outlier | (mz>lim)

    c_new = np.zeros(centroids.shape)
    spl_ra = splrep(time[data_mask>0][~outlier], centroids[data_mask>0,0][~outlier])
    spl_dec = splrep(time[data_mask>0][~outlier], centroids[data_mask>0,1][~outlier])
    c_new[data_mask>0,0] = splev(time[data_mask>0], spl_ra)
    c_new[data_mask>0,1] = splev(time[data_mask>0], spl_dec)

    print centroids.shape
    print time.shape
    output = '../plots/0/%s-%s.pdf'%(name, suffix)
    dir = os.path.dirname(output)
    if not os.path.exists(dir):
      os.makedirs(dir)
    plt.plot(centroids[:,0], '.k')
    plt.plot(c_new[:,0], '.r')
    plt.savefig('../plots/0/%s-%s_ra.pdf'%(name, suffix), dpi=190)
    plt.clf()

    plt.plot(centroids[:,1], '.k')
    plt.plot(c_new[:,1], '.r')
    plt.savefig('../plots/0/%s-%s_dec.pdf'%(name, suffix), dpi=190)
    plt.clf()

    #np.save(tmp_dir+'/offsets_%s.npy'%(suffix), centroids)
    #np.save(tmp_dir+'/time_%s.npy'%(suffix), time)
    
    hdulist = pyfits.open(asprta)
    co_data = hdulist[1].data
    T = co_data['T']
    ra = co_data['ra']
    dec = co_data['dec']
    roll = co_data['roll']
    ra_new = np.interp(time, T, ra) - c_new[:,0]
    dec_new = np.interp(time, T, dec) - c_new[:,1]
    roll_new = np.interp(time, T, roll) - c_new[:,2]

    other = np.zeros((time.shape[0], 8))
    array = np.concatenate([np.array([time, ra_new, dec_new, roll_new]).T, other], axis=1)
    data = np.core.records.fromarrays(array.transpose(), dtype=[('T', float), ('RA', float), ('DEC', float), ('ROLL', float),\
                 ('STATUS_FLAG', int), ('ROLL_RAD', float), ('X', float), ('Y', float), ('Z', float), ('XDOT', float), ('YDOT', float), ('ZDOT', float)])

    new_file = re.split('-asprta.fits', asprta)[0]+'-'+suffix+'-asprta.fits'
    os.system('cp {0} {1}'.format(asprta, new_file))
    update(new_file, data, 1)
    hdu = pyfits.open(new_file)
    print hdu[1].data['RA'].shape
    print hdu[1].data['DEC'].shape
    hdu.close()
Beispiel #40
0
extname = f'{151}-{153}_linelog'
cols = fits.ColDefs(list_columns)
hdu = fits.BinTableHDU.from_columns(cols, name=extname)

# Safe extra data in header
for param, value in fit_results['Fitting_results'].items():
    hdu.header[f'hierarch {param}'] = value[0]
    hdu.header[f'hierarch {param}_err'] = value[1]

for i, label in enumerate(inputLabels):
    hdu.header[f'hierarch flux_{label}'] = inputFlux[i]
    hdu.header[f'hierarch err_{label}'] = inputErr[i]

fits.update(db_fits,
            data=hdu.data,
            header=hdu.header,
            extname=extname,
            verify=True)

if db_fits.is_file():
    try:
        print('Updating')
        fits.update(db_fits,
                    data=hdu.data,
                    header=hdu.header,
                    extname=extname,
                    verify=True)
    except:
        print('Appending')
        fits.append(db_fits, data=hdu.data, header=hdu.header, extname=extname)
else:
Beispiel #41
0
def clean (usr_imgfile, usr_outfile):
    #numerixenv.check() #Temporary NUMERIX environment check
    print("puftcorr version %s" %__version__)
    print("Input file:  %s" %usr_imgfile)
    print("Output file: %s" %usr_outfile)

    imgfile = osfn(usr_imgfile)
    outfile = osfn(usr_outfile)

    # check for existence of output file
    if os.access(outfile,os.F_OK):
       s = "\nERROR: Output file %s already exists\n" %(outfile)
       sys.stdout.write(s)
       return

    # create the output file as a copy of the input raw file
    shutil.copyfile (imgfile, outfile)

    # retrieve the input file
    img = InputFile(imgfile)

    # set correction parameter values
    pars = params(img.camera)

    # loop over readouts in the input file
    for i in range(img.nsamp):
        imset = i+1

        if imset == 1:
           s = " processing imset 1"
        elif imset == img.nsamp:
           s = " %d\n" % imset
        else:
           s = " %d" % imset
        sys.stdout.write(s)
        sys.stdout.flush()

        # get individual readout
        im = Readout(img,imset)

        # rotate, if necessary
        if img.camera == 1:
            im.data = ndimage.rotate(im.data,-90,mode='nearest')
        elif img.camera == 3:
            im.data = ndimage.rotate(im.data,180,mode='nearest')

        # get correction image
        im = get_corr(im, pars)

        # rotate back, if necessary
        if img.camera == 1:
            im.data = ndimage.rotate(im.data,+90,mode='nearest')
        elif img.camera == 3:
            im.data = ndimage.rotate(im.data,180,mode='nearest')

        # subtract correction image from original raw image
        im.data = img.f['sci',imset].data - im.data

        # make sure corrected pixel values don't go off the
        # ends of the Int16 data range before writing to output
        im.data = np.clip(im.data,-32768.0,32767.0)

        # write corrected image to output file
        pyfits.update(outfile,im.data.astype(np.dtype('int16')),
                      'sci',imset,
                      header=im.header)


    # close the input files
    img.f.close()
    img.dark.close()

    return
Beispiel #42
0
def generate_first_offsets(name):
    print name
    hdulist = pyfits.open('../AIS_GAL_SCAN/asprta/%s-asprta.fits' % name)
    initial = 1
    final = hdulist[1].data['T'].shape[0] - 1

    centers = []
    center_time = []
    for i in range(initial, final + 1):
        c = np.load('../data/%s/cata/centroids_rot%d.npy' % (name, i))
        #if c.shape == (1,3):
        #   c = c[:,:2]
        centers.append(c)
        center_time.append(
            np.load('../data/%s/cata/time_rot%d.npy' % (name, i)))
        print c.shape
    centroids = np.concatenate(centers, axis=0)
    time = np.concatenate(center_time, axis=0)

    print centroids.shape

    out_mask = np.zeros(centroids.shape[0])

    z, mz = moving_stat(centroids[:, 0], out_mask, half_win=100)
    outliers = np.zeros(centroids.shape[0])
    outliers[mz > 3.5] = 1
    outliers[out_mask > 0] = 1
    outliers = outliers > 0
    index = np.arange(centroids.shape[0])
    centroids[outliers, 0] = np.interp(index[outliers], index[~outliers],
                                       centroids[~outliers, 0])

    z, mz = moving_stat(centroids[:, 1], out_mask, half_win=100)
    outliers = np.zeros(centroids.shape[0])
    outliers[mz > 3.5] = 1
    outliers[out_mask > 0] = 1
    outliers = outliers > 0
    index = np.arange(centroids.shape[0])
    centroids[outliers, 1] = np.interp(index[outliers], index[~outliers],
                                       centroids[~outliers, 1])

    output = "../plots/%s/cata/output.csv" % (name)
    dir = os.path.dirname(output)
    if not os.path.exists(dir):
        os.makedirs(dir)

    plt.plot(centroids[:, 0], '.b')
    plt.savefig('../plots/%s/cata/offsets_10_new_half.pdf' % name, dpi=190)
    plt.clf()

    np.save('../data/%s/cata/time%d_10_new_half.npy' % (name, initial), time)
    np.save('../data/%s/cata/offsets%d_10_new_half.npy' % (name, initial),
            centroids)

    co_data = hdulist[1].data
    T = co_data['T']
    ra = co_data['ra']
    dec = co_data['dec']
    roll = co_data['roll']
    ra_new = np.interp(time, T, ra) - centroids[:, 0]
    dec_new = np.interp(time, T, dec) - centroids[:, 1]
    roll_new = np.interp(time, T, roll) - centroids[:, 2]

    other = np.zeros((time.shape[0], 8))
    array = np.concatenate(
        [np.array([time, ra_new, dec_new, roll_new]).T, other], axis=1)
    data = np.core.records.fromarrays(array.transpose(), dtype=[('T', float), ('RA', float), ('DEC', float), ('ROLL', float),\
                 ('STATUS_FLAG', int), ('ROLL_RAD', float), ('X', float), ('Y', float), ('Z', float), ('XDOT', float), ('YDOT', float), ('ZDOT', float)])

    new_file = '../AIS_GAL_SCAN/asprta/%s-cal-asprta.fits' % (name)
    os.system(
        'cp ../AIS_GAL_SCAN/asprta/%s-asprta.fits ../AIS_GAL_SCAN/asprta/%s-cal-asprta.fits'
        % (name, name))
    update(new_file, data, 1)
    hdu = pyfits.open(new_file)
    print hdu[1].data['RA'].shape
    print hdu[1].data['DEC'].shape
    hdu.close()

    #asp_cal.interpolate_offsets(name, 1., centroids)
    tmp_files = glob.glob("../data/%s/cata/centroids_rot*" % name)
    for tmp_file in tmp_files:
        os.remove(tmp_file)

    tmp_files = glob.glob("../data/%s/cata/time_rot*" % name)
    for tmp_file in tmp_files:
        os.remove(tmp_file)
    '''