Example #1
0
def normalize_quartzes(quartz_list):
    '''Normalizes the files in quartz_list using noao>twodspec>longslit>response'''
    #This needs more intelligent Failure Handling; current version is alpha.
    irf_prm.set_response(iraf.response)
    for quartz in quartz_list:
        iraf.response(calibrat=quartz,normaliz=quartz,response='n'+quartz)
    return
Example #2
0
def normalise_flats(flatdir):

    """
    Normalise flats
    """

    print 'In directory ' + flatdir
    print 'Normalising combinined flats...'

    if os.path.exists( os.path.join(flatdir,'nFlat.fits') ):
        os.remove(os.path.join(flatdir,'nFlat.fits') )
        print 'Removing file ' + os.path.join(flatdir,'nFlat.fits')

    iraf.twodspec(_doprint=0)
    iraf.longslit(_doprint=0)

    iraf.response.setParam('calibration', os.path.join(flatdir,'Flat.fits'))
    iraf.response.setParam('normalization', os.path.join( flatdir, 'Flat.fits' ))
    iraf.response.setParam('response', os.path.join( flatdir, 'nFlat') )
    iraf.response.setParam('low_reject', 3.)
    iraf.response.setParam('high_reject', 3.)
    iraf.response.setParam('order',40)

    iraf.response()

    return None
Example #3
0
def combine_flat(lstfile):
    if os.path.isfile('Halogen.fits'):
        print 'remove Halogen.fits'
        os.remove('Halogen.fits')
    if os.path.isfile('Resp.fits'):
        print 'remove Resp.fits'
        os.remove('Resp.fits')
    iraf.noao()
    iraf.imred()
    iraf.ccdred()
    iraf.flatcombine(input='tbo//@' + lstfile,
                     output='Halogen',
                     combine='average',
                     reject='crreject',
                     ccdtype='',
                     process=False,
                     subsets=False,
                     delete=False,
                     clobber=False,
                     scale='mode',
                     statsec='',
                     nlow=1,
                     nhigh=1,
                     nkeep=1,
                     mclip=True,
                     lsigma=3.0,
                     hsigma=3.0,
                     rdnoise='rdnoise',
                     gain='gain',
                     snoise=0.0,
                     pclip=-0.5,
                     blank=1.0)
    iraf.twodspec()
    iraf.longslit(dispaxis=2,
                  nsum=1,
                  observatory='Lijiang',
                  extinction=func.config_path + os.sep + 'LJextinct.dat',
                  caldir=func.std_path + os.sep,
                  interp='poly5')
    iraf.response(calibration='Halogen',
                  normalization='Halogen',
                  response='Resp',
                  interactive=True,
                  threshold='INDEF',
                  sample='*',
                  naverage=1,
                  function='spline3',
                  order=25,
                  low_reject=10.0,
                  high_reject=10.0,
                  niterate=1,
                  grow=0.0,
                  graphics='stdgraph',
                  cursor='')
Example #4
0
def combine_flat(filename):
	outname = filename.replace('.lst','.fits')
	print 'run function flatcombine...'
	print 'make file', outname
	iraf.flatcombine(input = 'tbo//@' + filename
		, output = outname, combine = 'average', reject = 'avsigclip'
		, ccdtype = '', process = False, subsets = True
		, delete = False, clobber = False, scale = 'mode'
		, statsec = '', nlow = 1, nhigh = 1, nkeep = 1
		, mclip = True, lsigma = 3.0, hsigma = 3.0
		, rdnoise = 9.4, gain = 0.35, snoise = 0.0
		, pclip = -0.5, blank = 1.0)
	iraf.noao()
	iraf.twodspec()
	iraf.longslit()
	print 'run function response...'
	print 'make file', 're' + outname
	iraf.response(calibration = outname
		, normalization = outname, response = 're' + outname
		, interactive = True, threshold = 'INDEF', sample = '*'
		, naverage = 1, function = 'spline3', order = 7
		, low_reject = 0.0, high_reject = 0.0, niterate = 1
		, grow = 0.0, graphics = 'stdgraph', cursor = '')
	print 'run function illumination...'
	print 'make file', 'il' + outname
	iraf.illumination(images = 're' + outname
		, illuminations = 'il' + outname, interactive = False
		, bins = '', nbins = 5, sample = '*', naverage = 1
		, function = 'spline3', order = 1, low_reject = 0.0
		, high_reject = 0.0, niterate = 1, grow = 0.0
		, interpolator = 'poly3', graphics = 'stdgraph', cursor = '')
	print 'run function imarith...'
	print 'make file', 'per' + outname
	iraf.imarith(operand1 = 're' + outname
		, op = '/', operand2 = 'il' + outname, result = 'per' + outname
		, title = '', divzero = 0.0, hparams = '', pixtype = ''
		, calctype = '', verbose = True, noact = False)
        return outname, 're' + outname, 'il' + outname, 'per' + outname
Example #5
0
def gen_Resp_2016():
    iraf.twodspec()
    iraf.longslit(dispaxis=2,
                  nsum=1,
                  observatory='observatory',
                  extinction=func.extinction_file,
                  caldir=func.std_path + os.sep,
                  interp='poly5')
    iraf.response(calibration='Halogen',
                  normalization='Halogen',
                  response='Resp',
                  interactive=True,
                  threshold='INDEF',
                  sample='*',
                  naverage=1,
                  function='spline3',
                  order=45,
                  low_reject=10.0,
                  high_reject=10.0,
                  niterate=1,
                  grow=0.0,
                  graphics='stdgraph',
                  cursor='')
Example #6
0
def combine_flat(lstfile):
    iraf.noao()
    iraf.imred()
    iraf.ccdred()
    iraf.flatcombine(input = 'tbo//@' + lstfile
    	, output = 'Halogen', combine = 'average', reject = 'crreject'
    	, ccdtype = '', process = False, subsets = False
    	, delete = False, clobber = False, scale = 'mode'
    	, statsec = '', nlow = 1, nhigh = 1, nkeep = 1
    	, mclip = True, lsigma = 3.0, hsigma = 3.0
    	, rdnoise = 'rdnoise', gain = 'gain', snoise = 0.0
    	, pclip = -0.5, blank = 1.0)
    script_path = os.path.split(os.path.realpath(__file__))[0]
    iraf.twodspec()
    iraf.longslit(dispaxis = 2, nsum = 1, observatory = 'observatory'
        , extinction = script_path + os.sep + 'LJextinct.dat'
        , caldir = script_path + os.sep + 'standarddir' + os.sep, interp = 'poly5')
    iraf.response(calibration = 'Halogen'
    	, normalization = 'Halogen', response = 'Resp'
    	, interactive = True, threshold = 'INDEF', sample = '*'
    	, naverage = 1, function = 'spline3', order = 25
    	, low_reject = 10.0, high_reject = 10.0, niterate = 1
    	, grow = 0.0, graphics = 'stdgraph', cursor = '')
Example #7
0
    flat1d = np.median(flat1dtop / flat1dbot, axis=0)
    # close the illumination file because we don't need it anymore
    illumhdu.close()

    # File stage m1d for median 1-D
    flat1dfname = 'flats/flt%05.2fm1d.fits' % (ga)
    tofits(flat1dfname, flat1d, hdr=combinehdu[0].header.copy())

    # run response
    # r1d = response1d
    resp1dfname = 'flats/flt%05.2fr1d.fits' % (ga)
    iraf.response(flat1dfname,
                  flat1dfname,
                  resp1dfname,
                  order=31,
                  interactive=False,
                  naverage=-5,
                  low_reject=3.0,
                  high_reject=3.0,
                  niterate=5,
                  mode='hl')

    resp1dhdu = pyfits.open(resp1dfname)
    resp1d = resp1dhdu[0].data.copy()
    resp1dhdu.close()

    # After response divide out the response function
    # normalize the 1d resp to its median
    resp1d /= np.median(resp1d)

    # Chuck any outliers
    flatsig = np.std(resp1d - 1.0)
Example #8
0
def process():
    ovrsc = '[1:4,*]'
    trim = '[200:2046,50:200]'
    imglist = glob.glob('FORS2*.fits')
#    objdict = {}
#    objdict.clear()
    for img in imglist:
        hdu = pyfits.getheader(img)
        if 'OBJECT' in hdu:
            obj = hdu['OBJECT']
            if obj not in objdict:
                objdict[obj] = [img]
            else:
                objdict[obj].append(img)

    if 'STD' not in objdict:
        os.system('cp /dark/jsamuel/agn/standards/FORS2.2016-05-12T09:15:14.678.fits ./')
        objdict['STD'] = ['FORS2.2016-05-12T09:15:14.678.fits']
        os.system('cp /dark/jsamuel/agn/standards/FORS2.2016-05-12T04:24:38.547.fits ./')
        objdict['STD'].append('FORS2.2016-05-12T04:24:38.547.fits')

    stars = {'specphot-LTT7379':'l7379','specphot-LDS749B':'lds749b','specphot-EG274':'eg274','specphot-G138-31':'g13831','specphot-C-32d9927':'cd32','specphot-LTT9491':'l9491','specphot-LTT7987':'l7987'}

    i = 0
    for key in objdict.keys():
        if 'STD' in key:
            for img in objdict[key]:
                i = i + 1
                numstars = len(objdict['STD'])
                hds = pyfits.getheader(img)
                _starname = stars[hds['HIERARCH ESO OBS NAME']]
                if _starname in ['lds749b','g13831']:
                    print 'Bad standard, copying from 2016-05-12'
                    if not os.path.isdir('badstd'):
                        os.mkdir('badstd')
                    os.system('mv '+img+' ./badstd')
                    if i >= numstars:
                        objdict.pop('STD')
                        os.system('cp /dark/jsamuel/agn/standards/FORS2.2016-05-12T09:15:14.678.fits ./')
                        objdict['STD'] = ['FORS2.2016-05-12T09:15:14.678.fits']
                        os.system('cp /dark/jsamuel/agn/standards/FORS2.2016-05-12T04:24:38.547.fits ./')
                        objdict['STD'].append('FORS2.2016-05-12T04:24:38.547.fits')



    

    if os.path.isfile('biaslist'):
        os.remove('biaslist')

    if os.path.isfile('masterbias.fits'):
        os.remove('masterbias.fits')

    f = open('biaslist','w')
    for img in objdict['BIAS']:
        f.write(img+'\n')
    f.close()

    imglist = '@biaslist'
    name = 'masterbias.fits'
    hdb = pyfits.getheader(objdict['BIAS'][0])
    _gain = hdb['HIERARCH ESO DET OUT1 GAIN']
    _ron = hdb['HIERARCH ESO DET OUT1 RON']
    iraf.zerocombine(imglist,output=name,combine='average',reject='minmax',ccdtype='none',process='no',gain=_gain,rdnoise=_ron,Stdout=1)



    if os.path.isfile('flatlist'):
        os.remove('flatlist')

    if os.path.isfile('sciflatlist'):
        os.remove('sciflatlist')

    if os.path.isfile('stdflatlist'):
        os.remove('stdflatlist')

    if os.path.isfile('masterflat.fits'):
        os.remove('masterflat.fits')

    if os.path.isfile('scimasterflat.fits'):
        os.remove('scimasterflat.fits')

    if os.path.isfile('stdmasterflat.fits'):
        os.remove('stdmasterflat.fits')



        
    f = open('sciflatlist','w')
    for img in objdict['FLAT,LAMP']:
        hdu = pyfits.getheader(img)
        if hdu['HIERARCH ESO DPR TECH'] == 'SPECTRUM':
            f.write(img+'\n')
    f.close()

    j = 0
    f = open('stdflatlist','w')
    for img in objdict['FLAT,LAMP']:
        hdu = pyfits.getheader(img)
        if hdu['HIERARCH ESO DPR TECH'] == 'MOS':
            f.write(img+'\n')
            j = j + 1
    f.close()





    imglist = '@sciflatlist'
    name = 'scimasterflat.fits'
    hdf = pyfits.getheader(objdict['FLAT,LAMP'][0])
    _gain = hdf['HIERARCH ESO DET OUT1 GAIN']
    _ron = hdf['HIERARCH ESO DET OUT1 RON']
    iraf.flatcombine(imglist,output=name,combine='average',reject='avsigclip',ccdtype='none',process='no',subsets='yes',delete='no',clobber='no',gain=_gain,rdnoise=_ron,Stdout=1)


    if j == 0:
        imglist = '@sciflatlist'
    elif j >= 1:
        imglist = '@stdflatlist'
    name = 'stdmasterflat.fits'
    hdf = pyfits.getheader(objdict['FLAT,LAMP'][0])
    _gain = hdf['HIERARCH ESO DET OUT1 GAIN']
    _ron = hdf['HIERARCH ESO DET OUT1 RON']
    iraf.flatcombine(imglist,output=name,combine='average',reject='avsigclip',ccdtype='none',process='no',subsets='yes',delete='no',clobber='no',gain=_gain,rdnoise=_ron,Stdout=1)





    if os.path.isfile('tmasterbias.fits'):
        os.remove('tmasterbias.fits')

    fits = 'masterbias.fits'
    name = 'tmasterbias.fits'
    zcor = 'no'
    fcor = 'no'
    _zero = ''
    _flat = ''
    iraf.ccdproc(fits,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=_flat,Stdout=1)


    if os.path.isfile('tmasterflat.fits'):
        os.remove('tmasterflat.fits')

    if os.path.isfile('tscimasterflat.fits'):
        os.remove('tscimasterflat.fits')

    if os.path.isfile('tstdmasterflat.fits'):
        os.remove('tstdmasterflat.fits')



    fits = 'scimasterflat.fits'
    name = 'tscimasterflat.fits'
    zcor = 'yes'
    fcor = 'no'
    _zero = 'tmasterbias.fits'
    _flat = ''
    iraf.ccdproc(fits,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=_flat,Stdout=1)

    fits = 'stdmasterflat.fits'
    name = 'tstdmasterflat.fits'
    zcor = 'yes'
    fcor = 'no'
    _zero = 'tmasterbias.fits'
    _flat = ''
    iraf.ccdproc(fits,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=_flat,Stdout=1)
                




    if os.path.isfile('ntmasterflat.fits'):
        os.remove('ntmasterflat.fits')

    if os.path.isfile('ntscimasterflat.fits'):
        os.remove('ntscimasterflat.fits')

    if os.path.isfile('ntstdmasterflat.fits'):
        os.remove('ntstdmasterflat.fits')

    cal = 'tscimasterflat.fits'
    resp = 'ntscimasterflat.fits'
    _order = 100
    iraf.response(calibration=cal,normalization=cal,response=resp,interactive='no',function='legendre',order=_order,graphics='stdgraph')

    cal = 'tstdmasterflat.fits'
    resp = 'ntstdmasterflat.fits'
    _order = 100
    iraf.response(calibration=cal,normalization=cal,response=resp,interactive='no',function='legendre',order=_order,graphics='stdgraph')




##########

    ovrsc = '[1:4,*]'
    trim = '[200:2046,50:200]'
    
    fimg = 'ntscimasterflat'
    _area = '[1:150,*]'
    _value = 1.0
    iraf.imreplace(images=fimg+_area,value=_value)

    fimg = 'ntstdmasterflat'
    _area = '[1:150,*]'
    _value = 1.0
    iraf.imreplace(images=fimg+_area,value=_value)
    

    alist = glob.glob('*AGN*.fits')
    for f in alist:
        os.remove(f)
    slist = glob.glob('STD*.fits')
    for f in slist:
        os.remove(f)
    llist = glob.glob('LAMP*.fits')
    for f in llist:
        os.remove(f)


    zcor = 'yes'
    fcor = 'yes'
    _zero = 'tmasterbias.fits'
    sciflat = 'ntscimasterflat.fits'
    stdflat = 'ntstdmasterflat.fits'

    for key in objdict.keys():
        if 'AGN' in key:
            for img in objdict[key]:
                hds = pyfits.getheader(img)
                if 'EXPTIME' in hds:
                    if hds['EXPTIME'] >= 50.0:
                        num = img.rsplit('.',2)[1]
                        name = key+'_'+num
                        '''
                        iraf.ccdproc(img,output='trim_'+name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor='no',darkcor='no',flatcor='no',illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero='',flat='',Stdout=1)

                        _gain = hds['HIERARCH ESO DET OUT1 GAIN']
                        _ron = hds['HIERARCH ESO DET OUT1 RON']
                        cosmics.lacos('trim_'+name+'.fits', output='c_'+name+'.fits', gain=_gain, readn=_ron, xorder=9, yorder=9, sigclip=4.5, sigfrac=0.5, objlim=1, verbose=True, interactive=False)

                        iraf.ccdproc('c_'+name,output=name,ccdtype='',noproc='no',fixpix='no',overscan='no',trim='no',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec='',trimsec='',zero=_zero,flat=sciflat,Stdout=1)
                        '''

                        iraf.ccdproc(img,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=sciflat,Stdout=1)

                    else:
                        pass


        elif 'STD' in key:
            for img in objdict[key]:
                num = img.rsplit('.',2)[1]
                name = key+'_'+num
                iraf.ccdproc(img,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=stdflat,Stdout=1)

        elif 'WAVE' in key:
            for img in objdict[key]:
                num = img.rsplit('.',2)[1]
                name = 'LAMP_'+num
                iraf.ccdproc(img,output=name,ccdtype='',noproc='no',fixpix='no',overscan='yes',trim='yes',zerocor=zcor,darkcor='no',flatcor=fcor,illumcor='no',fringecor='no',readcor='no',scancor='no',biassec=ovrsc,trimsec=trim,zero=_zero,flat=stdflat,Stdout=1)
Example #9
0
os.system("ls halogen*b.fits > flat.in")
iraf.imcombine('@flat.in',
               'mflat.fits',
               combine="median",
               rdnoise=4.8,
               gain=1.22)

#making master flat.
iraf.response("mflat",
              "mflat",
              "nmflat",
              interactive="yes",
              threshold="INDEF",
              sample="*",
              naverage=1,
              function="spline3",
              order=20,
              low_reject=0.,
              high_reject=0.,
              niterate=1,
              grow=0.,
              graphics="stdgraph",
              cursor="")

iraf.imstat('nmflat.fits')
os.system("ls J105829*b.fits GD248*b.fits > flatf.in")
os.system("sed s/b.fits/bf.fits/g flatf.in > flatf.out")
iraf.imarith('@flatf.in', '/', 'nmflat.fits', '@flatf.out')
print("***********************************************************")
print("Flat correction done!")
Example #10
0
def makeflats(fs=None):
    # Note the list of files need to not include any paths relative to
    # the work directory.
    # Maybe not the greatest convention, but we can update this later
    iraf.cd('work')
    if fs is None:
        fs = glob('pysalt/bxgp*.fits')
    if len(fs) == 0:
        print "WARNING: No flat-fields to combine and normalize."
        # Fail gracefully by going up a directory
        iraf.cd('..')
        return
    # make a flats directory
    if not os.path.exists('flats'):
        os.mkdir('flats')

    # Figure out which images are flats and which grating angles were used
    allflats, grangles = get_ims(fs, 'flat')

    # For each grating angle
    for ga in np.unique(grangles):
        # grab the flats for this gr angle
        flats = allflats[grangles == ga]

        # For each chip
        for c in range(1, 7):
            # run imcombine with average and sigclip, weighted by exposure time
            flatlist = ''
            for f in flats:
                flatlist += '%s[%i],' % (f, c)
                # Add the exptime keyword to each extension
                pyfits.setval(f, 'EXPTIME', ext=c,
                              value=pyfits.getval(f, 'EXPTIME'))

            # set the output combined file name
            combineoutname = 'flats/flt%05.2fcomc%i.fits' % (ga, c)
            if os.path.exists(combineoutname):
                os.remove(combineoutname)
            # initialize the iraf command
            iraf.unlearn(iraf.imcombine)
            print(flatlist)
            # don't forget to remove the last comma in the filelist
            iraf.imcombine(input=flatlist[:-1], output=combineoutname,
                           combine='average', reject='sigclip', lsigma=3.0,
                           hsigma=3.0, weight='exposure', expname='EXPTIME')

            pyfits.setval(combineoutname, 'DISPAXIS', value=1)
            # We want to make an illumination correction file
            # before running response:
            illumoutname = 'flats/flt%05.2fillc%i.fits' % (ga, c)
            iraf.unlearn(iraf.illumination)
            iraf.illumination(images=combineoutname,
                              illuminations=illumoutname, interactive=False,
                              naverage=-40, order=11, low_reject=3.0,
                              high_reject=3.0, niterate=5, mode='hl')

            # Flag any pixels in the illumination correction< 0.1
            illumhdu = pyfits.open(illumoutname, mode='update')
            illumhdu[0].data[illumhdu[0].data <= 0.1] = 0.0
            illumhdu.flush()

            # Get 40 pixels out of the middle of the image and
            # median them to run response
            combinehdu = pyfits.open(combineoutname)
            ny = combinehdu[0].data.shape[0]
            # divide out the illumination correction before running response
            flat1d = np.median(combinehdu[0].data[ny / 2 - 21: ny / 2 + 20, :]
                               / illumhdu[0].data[ny / 2 - 21: ny / 2 + 20, :],
                               axis=0)
            # close the illumination file because we don't need it anymore
            illumhdu.close()

            # File stage m1d for median 1-D
            flat1dfname = 'flats/flt%05.2fm1dc%i.fits' % (ga, c)
            tofits(flat1dfname, flat1d, hdr=combinehdu[0].header.copy())

            # run response
            # r1d = response1d
            resp1dfname = 'flats/flt%05.2fr1dc%i.fits' % (ga, c)
            iraf.response(flat1dfname, flat1dfname, resp1dfname, order=31,
                          interactive=False, naverage=-5, low_reject=3.0,
                          high_reject=3.0, niterate=5, mode='hl')

            resp1dhdu = pyfits.open(resp1dfname)
            resp1d = resp1dhdu[0].data.copy()
            resp1dhdu.close()

            # After response divide out the response function
            # normalize the 1d resp to its median
            resp1d /= np.median(resp1d)

            # Chuck any outliers
            flatsig = np.std(resp1d - 1.0)
            resp1d[abs(resp1d - 1.0) > 5.0 * flatsig] = 1.0
            resp = flat1d / resp1d

            resp2dfname = 'flats/flt%05.2fresc%i.fits' % (ga, c)
            resp2d = combinehdu[0].data.copy() / resp
            tofits(resp2dfname, resp2d, hdr=combinehdu[0].header.copy())
            combinehdu.close()

            # close the combined flat because we don't need it anymore
            combinehdu.close()

            pyfits.setval(resp2dfname, 'DISPAXIS', value=1)

            # Reset any pixels in the flat field correction< 0.1
            # We could flag bad pixels here if we want, but not right now
            flathdu = pyfits.open(resp2dfname, mode='update')
            flathdu[0].data[flathdu[0].data <= 0.1] = 0.0
            flathdu.flush()
            flathdu.close()
    # Step back up to the top directory
    iraf.cd('..')
Example #11
0
def makeflats(fs=None):
    # Note the list of files need to not include any paths relative to
    # the work directory.
    # Maybe not the greatest convention, but we can update this later
    iraf.cd('work')
    if fs is None:
        fs = glob('pysalt/bxgp*.fits')
    if len(fs) == 0:
        print "WARNING: No flat-fields to combine and normalize."
        # Fail gracefully by going up a directory
        iraf.cd('..')
        return
    # make a flats directory
    if not os.path.exists('flats'):
        os.mkdir('flats')

    # Figure out which images are flats and which grating angles were used
    allflats, grangles = get_ims(fs, 'flat')

    # For each grating angle
    for ga in np.unique(grangles):
        # grab the flats for this gr angle
        flats = allflats[grangles == ga]

        # For each chip
        for c in range(1, 7):
            # run imcombine with average and sigclip, weighted by exposure time
            flatlist = ''
            for f in flats:
                flatlist += '%s[%i],' % (f, c)
                # Add the exptime keyword to each extension
                pyfits.setval(f, 'EXPTIME', ext=c,
                              value=pyfits.getval(f, 'EXPTIME'))

            # set the output combined file name
            combineoutname = 'flats/flt%05.2fcomc%i.fits' % (ga, c)
            if os.path.exists(combineoutname):
                os.remove(combineoutname)
            # initialize the iraf command
            iraf.unlearn(iraf.imcombine)
            print(flatlist)
            # don't forget to remove the last comma in the filelist
            iraf.imcombine(input=flatlist[:-1], output=combineoutname,
                           combine='average', reject='sigclip', lsigma=3.0,
                           hsigma=3.0, weight='exposure', expname='EXPTIME')

            pyfits.setval(combineoutname, 'DISPAXIS', value=1)
            # We want to make an illumination correction file
            # before running response:
            illumoutname = 'flats/flt%05.2fillc%i.fits' % (ga, c)
            iraf.unlearn(iraf.illumination)
            iraf.illumination(images=combineoutname,
                              illuminations=illumoutname, interactive=False,
                              naverage=-40, order=11, low_reject=3.0,
                              high_reject=3.0, niterate=5, mode='hl')

            # Flag any pixels in the illumination correction< 0.1
            illumhdu = pyfits.open(illumoutname, mode='update')
            illumhdu[0].data[illumhdu[0].data <= 0.1] = 0.0
            illumhdu.flush()

            # Get 40 pixels out of the middle of the image and
            # median them to run response
            combinehdu = pyfits.open(combineoutname)
            ny = combinehdu[0].data.shape[0]
            # divide out the illumination correction before running response
            flat1d = np.median(combinehdu[0].data[ny / 2 - 21: ny / 2 + 20, :]
                               / illumhdu[0].data[ny / 2 - 21: ny / 2 + 20, :],
                               axis=0)
            # close the illumination file because we don't need it anymore
            illumhdu.close()

            # File stage m1d for median 1-D
            flat1dfname = 'flats/flt%05.2fm1dc%i.fits' % (ga, c)
            tofits(flat1dfname, flat1d, hdr=combinehdu[0].header.copy())

            # run response
            # r1d = response1d
            resp1dfname = 'flats/flt%05.2fr1dc%i.fits' % (ga, c)
            iraf.response(flat1dfname, flat1dfname, resp1dfname, order=31,
                          interactive=False, naverage=-5, low_reject=3.0,
                          high_reject=3.0, niterate=5, mode='hl')

            resp1dhdu = pyfits.open(resp1dfname)
            resp1d = resp1dhdu[0].data.copy()
            resp1dhdu.close()

            # After response divide out the response function
            # normalize the 1d resp to its median
            resp1d /= np.median(resp1d)

            # Chuck any outliers
            flatsig = np.std(resp1d - 1.0)
            resp1d[abs(resp1d - 1.0) > 5.0 * flatsig] = 1.0
            resp = flat1d / resp1d

            resp2dfname = 'flats/flt%05.2fresc%i.fits' % (ga, c)
            resp2d = combinehdu[0].data.copy() / resp
            tofits(resp2dfname, resp2d, hdr=combinehdu[0].header.copy())
            combinehdu.close()

            # close the combined flat because we don't need it anymore
            combinehdu.close()

            pyfits.setval(resp2dfname, 'DISPAXIS', value=1)

            # Reset any pixels in the flat field correction< 0.1
            # We could flag bad pixels here if we want, but not right now
            flathdu = pyfits.open(resp2dfname, mode='update')
            flathdu[0].data[flathdu[0].data <= 0.1] = 0.0
            flathdu.flush()
            flathdu.close()
    # Step back up to the top directory
    iraf.cd('..')
Example #12
0
def test(instr=None,rawdata_dir='UT160327',reduction_dir=None,dograting=None,dofilter=None,silent=False):
    print('='*70)
    print("---------- Drew's APO 3.5m reduction script ----------")
    print('='*70)

    print('loading some basic iraf packages...')
    iraf.noao(); iraf.imred(); iraf.ccdred()
    print('='*70)

##########################################################################################################
    print('1. INSTRUMENT ESTABLISHMENT\n')
    if instr is None:
        print('Which APO 3.5m instrument did you use (default is ARCTIC)? Choices:')
        print(' - enter 1 for ARCTIC (default)')
        print(' - enter 2 for DIS')
        print(' - enter 3 for something else')
        a=raw_input('')
        if a=='':  a='1'
        if a[:1]=='1': instr='arctic'
        if a[:1]=='2': instr='dis'
        if (a[:1]!='1') & (a[:1]!='2'): sys.exit("You're out of luck for now. Sorry!")
        print('\nOk. '+instr.upper()+' data will be reduced')
        print('='*70)

##########################################################################################################
    print('2. RAW DATA DIRECTORY ESTABLISHMENT\n')
    if rawdata_dir is None:
        a=raw_input('Enter the name of the directory where your images are stored: ')
        if a=='': sys.exit('You must specify a raw data directory. Exiting the APO 3.5m reduction script.')
        if a[-1]!='/': a=a+'/'
        rawdata_dir=a
        check=glob.glob(rawdata_dir)
        if len(check)==0: sys.exit('The directory "'+rawdata_dir+'" does not exist. Exiting the APO 3.5m reduction script.')
    else:
        if rawdata_dir[-1:]!='/': rawdata_dir=rawdata_dir+'/'
        check=glob.glob(rawdata_dir)
        if len(check)==0: sys.exit('The directory "'+rawdata_dir+'" does not exist. Exiting the APO 3.5m reduction script.')

    print('Ok. The raw data directory will be "'+rawdata_dir+'"')
    print('='*70)

##########################################################################################################
    print('3. RAW DATA CHECK\n')
    rawfiles = np.array(glob.glob(rawdata_dir+'*fits')); nraw=len(rawfiles)
    if nraw==0: sys.exit('No FITS files found in the raw data directory. Exiting the APO 3.5m reduction script.')
    instrlist=[]; imtype=[]
    for rawfile in rawfiles:
        head=fits.getheader(rawfile)
        instrlist.append(head['instrume'].lower())
    instrlist=np.array(instrlist)
    gd=np.where(instrlist==instr); ngd=len(gd[0])
    if ngd==0: 
        sys.exit('No FITS files from '+instr.upper()+' were found in the raw data directory. Exiting the APO 3.5m reduction script.')
    else:
        rawfiles=rawfiles[gd]
        print('Ok. Found '+str(ngd)+' FITS images with instrument='+instr.upper()+' in the raw data directory.')
        print('='*70)

##########################################################################################################
    print('4. REDUCTION DIRECTORY ESTABLISHMENT\n')
    if reduction_dir is not None:
        check=glob.glob(reduction_dir)
        if len(check)==0: 
            a=raw_input('Specified reduction directory does not exist. Create it (Y/N)? ')
            if (a=='') | (a[:1].lower()=='y'): 
                cmnd='mkdir '+reduction_dir
                os.system(cmnd)
                print('"'+reduction_dir+'" has been created.')
    else:
        reduction_dir=rawdata_dir[:-1]+'_reduction/'
        print('The reduction directory will be '+reduction_dir)
        print(' - hit ENTER to accept')
        print(' - otherwise enter a different reduction directory name: ')
        a=raw_input('')
        if a=='':
            check=glob.glob(reduction_dir)
            if len(check)==0:
                cmnd='mkdir '+reduction_dir
                os.system(cmnd)
                print('\nOk. "'+reduction_dir+'" has been created.')
        else:
            reduction_dir=a
            check=glob.glob(reduction_dir)
            if len(check)==0:
                cmnd='mkdir '+reduction_dir 
                os.system(cmnd)
                print('\nOk. "'+reduction_dir+'" has been created.')

    print('\nOk. Reduction products will be stored in '+reduction_dir+'.')
    print('='*70)

##########################################################################################################
    if instr=='dis':
        print('loading some more IRAF packages for DIS reduction...')
        iraf.twod(); iraf.apex(); iraf.longs(); iraf.oned()
        print('='*70)
        print('5. DIS PARAMETER ESTABLISHMENT\n')
        grating=[]; imagetypes=[]
        for rawfile in rawfiles:
            head=fits.getheader(rawfile)
            grating.append(head['grating'])
            imagetypes.append(head['imagetyp'].lower())
        imagetypes=np.array(imagetypes)
        grating=np.array(grating)
        uniquegrating=np.unique(grating)

        if dograting is None:
            print("Which DIS grating do you want to reduce? Here's what you have:")
            for i in range(len(uniquegrating)): 
                if i==0: 
                    print(' - enter '+str(i+1)+' for '+uniquegrating[i]+' (default)')
                else:
                    print(' - enter '+str(i+1)+' for '+uniquegrating[i])
            a=raw_input('')
            if a=='': a='1'
            dograting=uniquegrating[int(a[:1])-1]
            gd=np.where(grating==dograting); ngd=len(gd[0])
            rawfiles=rawfiles[gd]; imagetypes=imagetypes[gd]
            print('\nOk. '+dograting+' data will be reduced: '+str(ngd)+' total images')
        else:
            gd=np.where(grating==dograting); ngd=len(gd[0])
            if ngd==0:
                sys.exit('No '+dograting+' images found. Exiting the APO 3.5m reduction script.')
            else:
                rawfiles=rawfiles[gd]; imagetypes=imagetypes[gd]
                print('\nOk. '+dograting+' images will be reduced: '+str(ngd)+' total images')

        bias=np.where((imagetypes=='bias') | (imagetypes=='zero')); nbias=len(bias[0])
        zerocor='yes'
        if nbias==0:
            print('\nNo bias images were found. Proceed without bias correction? (y/n)')
            a=raw_input('')
            if (a[:1]=='y') | (a==''): 
                zerocor='no'; biasfiles=''; nbias=0
            else:
                sys.exit('Exiting the APO 3.5m reduction script.')
        else:
            biasfiles=rawfiles[bias]; nbias=len(bias[0])

        # get values depending on R vs B detector
        detector=dograting[:1].lower()
        if detector=='b':
            gain=1.68
            rdnoise=4.9
            badpixfile=''
            fixpix='no'
        if detector=='r':
            gain=1.88
            rdnoise=4.6
            badpixfile='badpix_disR.txt'
            fixpix='yes'

        head=fits.getheader(rawfiles[0])
        biassec=head['biassec']
        datasec=head['datasec']

        genericlabel=dograting

        print('\nOk. DIS parameters established.')
        print('='*70)

##########################################################################################################
    if instr=='arctic':
        print('5. ARCTIC PARAMETER ESTABLISHMENT\n')
        gain=2.0
        rdnoise=3.7
        badpixfile=''
        fixpix='no'

        filters=[]; imagetypes=[]
        for rawfile in rawfiles:
            head=fits.getheader(rawfile)
            filters.append(head['filter'])
            imagetypes.append(head['imagetyp'].lower())
        imagetypes=np.array(imagetypes)
        filters=np.array(filters)
        uniquefilters=np.unique(filters)

        bias=np.where(imagetypes=='bias'); nbias=len(bias[0])
        zcor='yes'
        if nbias==0:
            print('\nNo bias images were found. Proceed without bias correction? (y/n)')
            a=raw_input('')
            if (a[:1]=='y') | (a==''): 
                zcor='no'; biasfiles=''
            else:
                sys.exit('Exiting the APO 3.5m reduction script.')
        else:
            biasfiles=rawfiles[bias]; nbias=len(bias[0])

        # establish ARCTIC filter
        if dofilter is None:
            print('Which ARCTIC filter do you want to reduce? ')
            for i in range(len(uniquefilters)): 
                if i==0: 
                    print(' - enter '+str(i+1)+' for '+uniquefilters[i]+' (default)')
                else:
                    print(' - enter '+str(i+1)+' for '+uniquefilters[i])
            a=raw_input('')
            if a=='': a='1'
            dofilter=uniquefilters[int(a[:1])-1]
            gd=np.where(filters==dofilter); ngd=len(gd[0])
            rawfiles=rawfiles[gd]
            print('\nOk. '+dofilter+' filter images will be reduced: '+str(ngd)+' total images')
        else:
            gd=np.where(filters==dofilter); ngd=len(gd[0])
            if ngd==0:
                sys.exit('No '+dofilter+' images found. Exiting the APO 3.5m reduction script.')
            else:
                rawfiles=rawfiles[gd]
                print('\nOk. '+dofilter+' filter images will be reduced: '+str(ngd)+' total images')


        head=fits.getheader(rawfiles[0])
        biassec=head['bsec11']
        datasec=head['dsec11']

        genericlabel=dofilter.replace(' ','_')
        genericlabel=genericlabel.replace('/','_')

        print('\nOk. ARCTIC parameters established.')
        print('='*70)

##########################################################################################################
    print('6. GET IMAGE TYPES\n')
    nrawfiles = len(rawfiles)

    # gather some more info from headers
    imtype = []; exptime = []; ra = []; dec = []
    for rawfile in rawfiles:
        header = fits.getheader(rawfile)
        imtype.append(header['IMAGETYP'].lower())
        exptime.append(header['EXPTIME'])
        ra.append(header['RA'])
        dec.append(header['DEC'])
    imtype=np.array(imtype)
    exptime=np.array(exptime); ra=np.array(ra)
    dec=np.array(dec)

    # separate the files into biases, flat, objs, and comps
    flat=np.where(imtype=='flat');  nflat=len(flat[0]); flatfiles=rawfiles[flat]
    obj=np.where(imtype=='object'); nobj=len(obj[0]);   objfiles=rawfiles[obj]
    if instr=='dis':
        comp=np.where(imtype=='comp'); ncomp=len(comp[0]); compfiles=rawfiles[comp]
        print('Ok. You have '+str(nbias)+' biases, '+str(nflat)+' flats, '+str(ncomp)+' comps, and '+str(nobj)+' objects.')
    if instr=='arctic':
        print('Ok. You have '+str(nbias)+' biases, '+str(nflat)+' flats, and '+str(nobj)+' objects.')
    print('='*70)


##########################################################################################################
    print('7. MAKE MASTER BIAS\n')
    if zcor=='yes':
        masterzero=reduction_dir+'Zero_'+genericlabel+'.fits'
        a=raw_input('Average combine biases into master bias (y/n)? ')
        if (a[:1]=='y') | (a==''):
            iraf.imcombine(','.join(biasfiles),output=masterzero,combine='average',reject='avsigclip',lsigma='3',
                           hsigma='3',rdnoise=rdnoise,gain=gain)
            print('\nOk. A master bias has been made: '+masterzero)
        print('='*70)
    else:
        masterzero=''
        print('\nOk, you have no bias files. Proceeding')


##########################################################################################################
    print('8. BIAS AND OVERSCAN CORRECTION\n')
    a=raw_input('Overscan and bias-subtraction of the flats, objs, and if DIS, comps. Do it (y/n)? ')
    if (a[:1]=='y') | (a==''):
        # set up a list of the input files (flats, comps, objs)
        if instr=='dis': infiles=np.concatenate((flatfiles,compfiles,objfiles),axis=0)
        if instr=='arctic': infiles=np.concatenate((flatfiles,objfiles),axis=0)

        infilesfile=reduction_dir+'inlist_'+genericlabel+'_ccdproc1'
        np.savetxt(infilesfile,infiles,fmt='%s')
        
        # We're making changes to the images now, so need to set up a list of output files
        outfiles=[]
        for infile in infiles:
            tmp=infile.split('/')
            tmp1=tmp[len(tmp)-1].split('.fits')
            outfiles.append(reduction_dir+tmp1[0]+'_cproc1.fits')
        outfiles=np.array(outfiles)
        outfilesfile=reduction_dir+'outlist_'+genericlabel+'_ccdproc1'
        np.savetxt(outfilesfile,outfiles,fmt='%s')

        iraf.ccdproc('@'+infilesfile,output='@'+outfilesfile,ccdtype='',fixpix=fixpix,oversca='yes',trim='yes',
                     zerocor=zcor,darkcor='no',flatcor='no',fixfile=badpixfile,biassec=biassec,trimsec=datasec,
                     zero=masterzero,interac='no',low_rej='3',high_re='3')
        print('\nOk. Overscan and bias subtracting done.')
    print('='*70)

##########################################################################################################
    print('9. MAKE MASTER FLAT\n')
    masterflat=reduction_dir+'Flat_'+genericlabel+'.fits'
    a=raw_input('Median combine the flats (y/n)? ')
    if (a[:1]=='y') | (a==''):
        procfiles=np.array(glob.glob(reduction_dir+'*_cproc1.fits'))
        imtyp=[]
        for i in range(len(procfiles)):
            head=fits.getheader(procfiles[i])
            imtyp.append(head['imagetyp'].lower())
        imtyp=np.array(imtyp)
        gd=np.where(imtyp=='flat')
        flats=procfiles[gd]

        iraf.imcombine(','.join(flats),output=masterflat,combine='median',reject='avsigclip',lsigma='2',
                       hsigma='2',rdnoise=rdnoise,gain=gain)
        print('\nOk. A master flat has been made: '+masterflat)
    print('='*70)

##########################################################################################################
    print('10. MAKE NORMALIZED MASTER FLAT\n')
    masternormflat=reduction_dir+'NormFlat_'+genericlabel+'.fits'
    a=raw_input('Normalize the response of the master flat (y/n)? ')
    if (a[:1]=='y') | (a==''):
        if instr=='arctic':
            im=fits.getdata(masterflat)
            imdim=len(im)
            meanval=np.mean(im[imdim-100:imdim+100,imdim-100:imdim+100])
            iraf.imarith(operand1=masterflat,op='/',operand2=meanval,result=masternormflat)
            print('\nOk. A normalized master flat has been created: '+masternormflat)
        if instr=='dis':
            iraf.response(calibrat=masterflat,normaliz=masterflat+'[*,400:500]',response=masternormflat,
                          interac='no',functio='spline3',order='5',low_rej='3',high_rej='3')
            print('\nOk. A normalized master flat has been created: '+masternormflat)
    print('='*70)

##########################################################################################################
    print('11. FLAT FIELD THE IMAGES\n')

    a=raw_input('Flat field the objs & comps by the normalized master flat (y/n)? ')
    if (a[:1]=='y') | (a==''):
        tmpfiles=np.array(glob.glob(reduction_dir+'*_cproc1.fits'))
        imtyp=[]
        for i in range(len(tmpfiles)):
            head=fits.getheader(tmpfiles[i])
            imtyp.append(head['imagetyp'].lower())
        imtyp=np.array(imtyp)
        gd1=np.where(imtyp=='object')
        if instr=='dis':
            gd2=np.where(imtyp=='comp')
            infiles=np.concatenate((tmpfiles[gd1],tmpfiles[gd2]),axis=0)
        if instr=='arctic':
            infiles=tmpfiles[gd1]
        infilesfile=reduction_dir+'inlist_'+genericlabel+'_ccdproc2'
        np.savetxt(infilesfile,infiles,fmt='%s')

        outfiles=[]
        for infile in infiles: 
            outfiles.append(infile.replace('cproc1','cproc2'))
        outfiles=np.array(outfiles)
        outfilesfile=reduction_dir+'outlist_'+genericlabel+'_ccdproc2'
        np.savetxt(outfilesfile,outfiles,fmt='%s')

        iraf.ccdproc('@'+infilesfile,output='@'+outfilesfile,ccdtype='',fixpix='no',oversca='no',trim='no',
                     zerocor='no',darkcor='no',flatcor='yes',fixfile='',biassec=biassec,trimsec=datasec,
                     flat=masternormflat,interac='no',low_rej='3',high_re='3')
        print('\nOk. The objs and comps have been flat fielded.')
    print('='*70)


##########################################################################################################
    if instr=='dis':
        print('Next steps (done manually for now):')
        print('='*70)
        print('12. Extract apertures via APALL.')
        print('13. Run IDENTIFY on one or more comps.')
        print('14. (optional) Run REIDENTIFY to transfer solution among comps.')
        print('15. Assign solution(s) to objs using REFSPECTRA.')
        print('16. Run DISPCOR on the objs to apply the wavelength solution.')
        print('='*70)
        print('Other things that should be done:')
        print('='*70)
        print('step ?1. flux calibration.')
        print('step ?2. sky/background subtraction.')
        print('step ?3. heliocentric velocity correction.')


    return imtyp
Example #13
0
                 rdnoise='rdnoise',
                 gain='gain')

iraf.imstat(flatcup)
iraf.imstat('Flat')

iraf.imstat(flati)
iraf.imstat('iFlat')

iraf.imexamine('Flat')
iraf.imexamine('iFlat')

iraf.response.unlearn()
iraf.response(calibration='Flat',
              normalization='Flat',
              response='nFlat',
              function='legendre',
              order='1')

iraf.response.unlearn()
iraf.response(calibration='iFlat',
              normalization='iFlat',
              response='niFlat',
              function='legendre',
              order='1')

iraf.imstat('nFlat')
iraf.imexamine('nFlat')

iraf.imstat('niFlat')
iraf.imexamine('niFlat')