コード例 #1
0
def subzero(imagesre, zero='Zero'):
    '''Run ccdproc remove Zero level noise'''

    imageslist = glob.glob(imagesre)
    imagesin = ', '.join(imageslist)

    # Load packages
    iraf.imred()
    iraf.ccdred()
    # Unlearn previouse settings
    iraf.ccdred.ccdproc.unlearn()
    iraf.ccdred.combine.unlearn()
    # setup and run task
    iraf.ccdred.ccdproc.ccdtype = ''
    iraf.ccdred.ccdproc.noproc = False
    iraf.ccdred.ccdproc.fixpix = False
    iraf.ccdred.ccdproc.overscan = False
    iraf.ccdred.ccdproc.darkcor = False
    iraf.ccdred.ccdproc.illumcor = False
    iraf.ccdred.ccdproc.fringecor = False
    iraf.ccdred.ccdproc.readcor = False
    iraf.ccdred.ccdproc.scancor = False
    iraf.ccdred.ccdproc.trim = False
    iraf.ccdred.ccdproc.trimsec = ''
    iraf.ccdred.ccdproc.readaxis = 'line'
    iraf.ccdred.ccdproc.zerocor = True
    iraf.ccdred.ccdproc.zero = zero
    iraf.ccdred.ccdproc.flatcor = False
    iraf.ccdred.ccdproc.flat = ''
    iraf.ccdred.ccdproc(images=imagesin)
コード例 #2
0
def divflat(imagesre, flat='Flat'):
    '''Run ccdproc task to images'''

    imageslist = glob.glob(imagesre)
    imagesin = ', '.join(imageslist)

    # Load packages
    iraf.imred()
    iraf.ccdred()
    # Unlearn settings
    iraf.ccdred.ccdproc.unlearn()
    iraf.ccdred.combine.unlearn()
    # Setup and run task
    iraf.ccdred.ccdproc.ccdtype = ''
    iraf.ccdred.ccdproc.noproc = False
    iraf.ccdred.ccdproc.fixpix = False
    iraf.ccdred.ccdproc.overscan = False
    iraf.ccdred.ccdproc.darkcor = False
    iraf.ccdred.ccdproc.illumcor = False
    iraf.ccdred.ccdproc.fringecor = False
    iraf.ccdred.ccdproc.readcor = False
    iraf.ccdred.ccdproc.scancor = False
    iraf.ccdred.ccdproc.trim = False
    iraf.ccdred.ccdproc.trimsec = ''
    iraf.ccdred.ccdproc.readaxis = 'line'
    iraf.ccdred.ccdproc.zerocor = False
    iraf.ccdred.ccdproc.zero = ''
    iraf.ccdred.ccdproc.flatcor = True
    iraf.ccdred.ccdproc.flat = flat
    iraf.ccdred.ccdproc(images=imagesin)
コード例 #3
0
def masterbias(biasre, output='Zero', combine='median', reject='minmax',
        ccdtype='', rdnoise='rdnoise', gain='gain'):
    '''run the task ccdred.zerocombine with chosen parameters

    Input:
    -------
     str biasre: regular expression to identify zero level images

    Output:
    -------
     file Zero.fits: combined zerolevel images
    '''

    biaslist = glob.glob(biasre)
    biasstring = ', '.join(biaslist)

    # load packages
    iraf.imred()
    iraf.ccdred()
    # unlearn settings
    iraf.imred.unlearn()
    iraf.ccdred.unlearn()
    iraf.ccdred.ccdproc.unlearn()
    iraf.ccdred.combine.unlearn()
    iraf.ccdred.zerocombine.unlearn()
    iraf.ccdred.setinstrument.unlearn()
    # setup task
    iraf.ccdred.zerocombine.output = output
    iraf.ccdred.zerocombine.combine = combine
    iraf.ccdred.zerocombine.reject = reject
    iraf.ccdred.zerocombine.ccdtype = ccdtype
    iraf.ccdred.zerocombine.rdnoise = rdnoise
    iraf.ccdred.zerocombine.gain = gain
    # run task
    iraf.ccdred.zerocombine(input=biasstring)
コード例 #4
0
def correctimages(imagesre, zero='Zero', flat='nFlat'):
    '''Run ccdproc task to correct images'''

    imageslist = glob.glob(imagesre)
    imagesin = ', '.join(imageslist)

    trimsection = str(raw_input('Enter trim section (or Hit <Enter>): '))
    trimquery = True
    if trimsection == '':
        trimquery = False

    # Load Packages
    iraf.imred()
    iraf.ccdred()
    # Unlearn Settings
    iraf.ccdred.ccdproc.unlearn()
    iraf.ccdred.combine.unlearn()
    # Setup and run task
    iraf.ccdred.ccdproc.ccdtype = ''
    iraf.ccdred.ccdproc.noproc = False
    iraf.ccdred.ccdproc.fixpix = False
    iraf.ccdred.ccdproc.overscan = False
    iraf.ccdred.ccdproc.darkcor = False
    iraf.ccdred.ccdproc.illumcor = False
    iraf.ccdred.ccdproc.fringecor = False
    iraf.ccdred.ccdproc.readcor = False
    iraf.ccdred.ccdproc.scancor = False
    iraf.ccdred.ccdproc.trim = trimquery
    iraf.ccdred.ccdproc.trimsec = trimsection
    iraf.ccdred.ccdproc.readaxis = 'line'
    iraf.ccdred.ccdproc.zerocor = True
    iraf.ccdred.ccdproc.zero = zero
    iraf.ccdred.ccdproc.flatcor = True
    iraf.ccdred.ccdproc.flat = flat
    iraf.ccdred.ccdproc(images=imagesin)
コード例 #5
0
ファイル: cor_ftbo.py プロジェクト: zzxihep/spec2015_1
def coroverbiastrim(lstfile):
    iraf.noao()
    iraf.imred()
    iraf.ccdred()
    x1,x2,y1,y2 = get_trim_sec()
    iraf.ccdproc(images = '@' + lstfile + '//[1]'
        , output = '%bo%bo%@' + lstfile
        , ccdtype = '', max_cache = 0, noproc = False
        , fixpix = False, overscan = True, trim = False
        , zerocor = True, darkcor = False, flatcor = False
        , illumcor = False, fringecor = False, readcor = False
        , scancor = False, readaxis = 'line', fixfile = ''
        , biassec = '[5:45,%s:%s]'%(y1,y2), trimsec = '[%s:%s,%s:%s]'%(x1,x2,y1,y2)
        , zero = 'Zero', dark = '', flat = '', illum = '', fringe = ''
        , minreplace = 1.0, scantype = 'shortscan', nscan = 1
        , interactive = False, function = 'chebyshev', order = 1
        , sample = '*', naverage = 1, niterate = 1
        , low_reject = 3.0, high_reject = 3.0, grow = 1.0)
    iraf.ccdproc(images = '%bo%bo%@' + lstfile
        , output = '%tbo%tbo%@' + lstfile
        , ccdtype = '', max_cache = 0, noproc = False
        , fixpix = False, overscan = False, trim = True
        , zerocor = False, darkcor = False, flatcor = False
        , illumcor = False, fringecor = False, readcor = False
        , scancor = False, readaxis = 'line', fixfile = ''
        , biassec = '[5:45,%s:%s]'%(y1,y2), trimsec = '[%s:%s,%s:%s]'%(x1,x2,y1,y2)
        , zero = 'Zero', dark = '', flat = '', illum = '', fringe = ''
        , minreplace = 1.0, scantype = 'shortscan', nscan = 1
        , interactive = False, function = 'chebyshev', order = 1
        , sample = '*', naverage = 1, niterate = 1
        , low_reject = 3.0, high_reject = 3.0, grow = 1.0)
    iraf.flpr()
コード例 #6
0
ファイル: gen_zero.py プロジェクト: zzxihep/spec2015_1
def combinebias(filename):
    iraf.noao()
    iraf.imred()
    iraf.ccdred()
    iraf.zerocombine(input = 'o//@' + filename
	, output = 'Zero', combine = 'average', reject = 'minmax'
	, ccdtype = '', process = False, delete = False
	, clobber = False, scale = 'none', statsec = ''
	, nlow = 0, nhigh = 1, nkeep = 1, mclip = True
	, lsigma = 3.0, hsigma = 3.0, rdnoise = 'rdnoise'
	, gain = 'gain', snoise = 0.0, pclip = -0.5, blank = 0.0)
コード例 #7
0
ファイル: correct_fits_ftbo.py プロジェクト: zzxihep/spec2015
def main():
	iraf.noao()
	iraf.imred()
	iraf.ccdred()
        print '=' * 20, 'Overscan', '=' * 20
	correct_overscan('spec.lst')
	print '=' * 20, 'combine bias', '=' * 20
	combine_bias('bias.lst')
	print '=' * 20, 'correct bias', '=' * 20
	correct_bias('spec_no_bias.lst', 'bias_spec.fits')
        name = os.popen('ls object*.lst').readlines()
        name = [i.split()[0] for i in name]
        for i in name:
            ntrim_flat(i)
コード例 #8
0
def flatresponse(input='Flat', output='nFlat'):
    ''' normalize Flat to correct illumination patterns'''

    iraf.imred()
    iraf.ccdred()
    iraf.specred()

    iraf.ccdred.combine.unlearn()
    iraf.ccdred.ccdproc.unlearn()
    iraf.specred.response.unlearn()
    iraf.specred.response.interactive = True
    iraf.specred.response.function = 'chebyshev'
    iraf.specred.response.order = 1
    iraf.specred.response(calibration=input, normalization=input,
            response=output)
コード例 #9
0
ファイル: FXCOR.py プロジェクト: jmccormac01/Spectroscopy
def ImportPackages():
	iraf.noao(_doprint=0)
	iraf.rv(_doprint=0)
	iraf.imred(_doprint=0)
	iraf.kpnoslit(_doprint=0)	
	iraf.ccdred(_doprint=0)
	iraf.astutil(_doprint=0)
	
	iraf.keywpars.setParam('ra','CAT-RA') 
	iraf.keywpars.setParam('dec','CAT-DEC')
	iraf.keywpars.setParam('ut','UT')
	iraf.keywpars.setParam('utmiddl','UT-M_E')
	iraf.keywpars.setParam('exptime','EXPTIME')
	iraf.keywpars.setParam('epoch','CAT-EPOC')
	iraf.keywpars.setParam('date_ob','DATE-OBS')
	iraf.keywpars.setParam('hjd','HJD')
	iraf.keywpars.setParam('mjd_obs','MJD-OBS')
	iraf.keywpars.setParam('vobs','VOBS')
	iraf.keywpars.setParam('vrel','VREL')
	iraf.keywpars.setParam('vhelio','VHELIO')
	iraf.keywpars.setParam('vlsr','VLSR')
	iraf.keywpars.setParam('vsun','VSUN')
	iraf.keywpars.setParam('mode','ql')
	iraf.fxcor.setParam('continu','both')
	iraf.fxcor.setParam('filter','none')
	iraf.fxcor.setParam('rebin','smallest')
	iraf.fxcor.setParam('pixcorr','no')
	iraf.fxcor.setParam('apodize','0.2')
	iraf.fxcor.setParam('function','gaussian')
	iraf.fxcor.setParam('width','INDEF')
	iraf.fxcor.setParam('height','0.')
	iraf.fxcor.setParam('peak','no')
	iraf.fxcor.setParam('minwidt','3.')
	iraf.fxcor.setParam('maxwidt','21.')
	iraf.fxcor.setParam('weights','1.')
	iraf.fxcor.setParam('backgro','0.')
	iraf.fxcor.setParam('window','INDEF')
	iraf.fxcor.setParam('wincent','INDEF')
	iraf.fxcor.setParam('verbose','long')
	iraf.fxcor.setParam('imupdat','no')
	iraf.fxcor.setParam('graphic','stdgraph')
	iraf.fxcor.setParam('interac','yes')
	iraf.fxcor.setParam('autowri','yes')
	iraf.fxcor.setParam('autodra','yes')
	iraf.fxcor.setParam('ccftype','image')
	iraf.fxcor.setParam('observa','lapalma')
	iraf.fxcor.setParam('mode','ql')
	return 0
コード例 #10
0
ファイル: cor_ftbo.py プロジェクト: zzxihep/spec2015_1
def corhalogen(lstfile):
    iraf.noao()
    iraf.imred()
    iraf.ccdred()
    iraf.ccdproc(images = '[email protected]' + lstfile
        , output = '%ftbo%ftbo%@' + lstfile
        , ccdtype = '', max_cache = 0, noproc = False
        , fixpix = False, overscan = False, trim = False
        , zerocor = False, darkcor = False, flatcor = True
        , illumcor = False, fringecor = False, readcor = False
        , scancor = False, readaxis = 'line', fixfile = ''
        , biassec = '', trimsec = ''
        , zero = 'Zero', dark = '', flat = 'Resp', illum = '', fringe = ''
        , minreplace = 1.0, scantype = 'shortscan', nscan = 1
        , interactive = False, function = 'chebyshev', order = 1
        , sample = '*', naverage = 1, niterate = 1
        , low_reject = 3.0, high_reject = 3.0, grow = 1.0)
    iraf.flpr()
コード例 #11
0
ファイル: gen_zero.py プロジェクト: zzxihep/spec2015_1
def coroverscan(filename):
    iraf.noao()
    iraf.imred()
    iraf.ccdred()
    iraf.ccdproc(images = '@' + filename + '//[1]'
    	, output = '%o%o%@' + filename
	, ccdtype = '', max_cache = 0, noproc = False
	, fixpix = False, overscan = True, trim = False
	, zerocor = False, darkcor = False, flatcor = False
	, illumcor = False, fringecor = False, readcor = False
	, scancor = False, readaxis = 'line', fixfile = ''
	, biassec = '[5:45,1:4612]', trimsec = '', zero = ''
	, dark = '', flat = '', illum = '', fringe = ''
	, minreplace = 1.0, scantype = 'shortscan', nscan = 1
	, interactive = False, function = 'chebyshev', order = 1
	, sample = '*', naverage = 1, niterate = 1
	, low_reject = 3.0, high_reject = 3.0, grow = 1.0)
    iraf.flpr()
コード例 #12
0
def runapall(imagesre, gain='gain', rdnoise='rdnoise'):
    '''Extract aperture spectra for science images ...'''

    imageslist = glob.glob(imagesre)
    imagesin = ', '.join(imageslist)

    # load packages
    iraf.imred()
    iraf.ccdred()
    iraf.specred()
    # unlearn previous settings
    iraf.ccdred.combine.unlearn()
    iraf.ccdred.ccdproc.unlearn()
    iraf.specred.apall.unlearn()
    # setup and run task
    iraf.specred.apall.format = 'onedspec'
    iraf.specred.apall.readnoise = rdnoise
    iraf.specred.apall.gain = gain
    iraf.specred.apall(input=imagesin)
コード例 #13
0
ファイル: cor_ftbo.py プロジェクト: zzxihep/spec2015_1
def combine_flat(lstfile):
    iraf.noao()
    iraf.imred()
    iraf.ccdred()
    iraf.flatcombine(input = 'tbo//@' + lstfile
    	, output = 'Halogen', combine = 'average', reject = 'crreject'
    	, ccdtype = '', process = False, subsets = False
    	, delete = False, clobber = False, scale = 'mode'
    	, statsec = '', nlow = 1, nhigh = 1, nkeep = 1
    	, mclip = True, lsigma = 3.0, hsigma = 3.0
    	, rdnoise = 'rdnoise', gain = 'gain', snoise = 0.0
    	, pclip = -0.5, blank = 1.0)
    script_path = os.path.split(os.path.realpath(__file__))[0]
    iraf.twodspec()
    iraf.longslit(dispaxis = 2, nsum = 1, observatory = 'observatory'
        , extinction = script_path + os.sep + 'LJextinct.dat'
        , caldir = script_path + os.sep + 'standarddir' + os.sep, interp = 'poly5')
    iraf.response(calibration = 'Halogen'
    	, normalization = 'Halogen', response = 'Resp'
    	, interactive = True, threshold = 'INDEF', sample = '*'
    	, naverage = 1, function = 'spline3', order = 25
    	, low_reject = 10.0, high_reject = 10.0, niterate = 1
    	, grow = 0.0, graphics = 'stdgraph', cursor = '')
コード例 #14
0
def masterflat(flatre, output='Flat', combine='median', reject='sigclip',
               scale='mode', rdnoise='rdnoise', gain='gain'):
    '''combine flat images with the task ccdred.flatcombine

    Input:
    -------
     str: flatre - regular expression to bias files in the current directory

    Output:
    -------
     file: Flat.fits - combined flat field images
    '''

    flatlist = glob.glob(flatre)
    flatstring = ', '.join(flatlist)

    # load packages
    iraf.imred()
    iraf.ccdred()
    # unlearn settings
    iraf.imred.unlearn()
    iraf.ccdred.unlearn()
    iraf.ccdred.ccdproc.unlearn()
    iraf.ccdred.combine.unlearn()
    iraf.ccdred.flatcombine.unlearn()
    iraf.ccdred.setinstrument.unlearn()
    # setup task
    iraf.ccdred.flatcombine.output = output
    iraf.ccdred.flatcombine.combine = combine
    iraf.ccdred.flatcombine.reject = reject
    iraf.ccdred.flatcombine.ccdtype = ''
    iraf.ccdred.flatcombine.process = 'no'
    iraf.ccdred.flatcombine.subsets = 'yes'
    iraf.ccdred.flatcombine.scale = scale
    iraf.ccdred.flatcombine.rdnoise = rdnoise
    iraf.ccdred.flatcombine.gain = gain
    iraf.ccdred.flatcombine(input=flatstring)
コード例 #15
0
ファイル: random_tests.py プロジェクト: CarlosBacigalupo/ipn
cur.execute("SELECT spec_path,name from fields where ymd=140825 and ccd='ccd_1' and obstype='BIAS'")

# <codecell>

objs=cur.fetchall()

# <codecell>

from pyraf import iraf

# <codecell>

iraf.noao(_doprint=0,Stdout="/dev/null")
iraf.imred(_doprint=0,Stdout="/dev/null")
iraf.ccdred(_doprint=0,Stdout="/dev/null")

# <codecell>

iraf.ccdproc(images='tmp/flats/25aug10034.fits', ccdtype='', fixpix='no', oversca='no', trim='no', zerocor='yes', darkcor='no', flatcor='no', zero='tmp/masterbias',Stdout="/dev/null")

# <codecell>

pwd

# <codecell>

cd ~/Documents/workspace/GAP/IrafReduction/140825/ccd11/

# <codecell>
コード例 #16
0
ファイル: rcred.py プロジェクト: scizen9/kpy
def reduce_image(image, flatdir=None, biasdir=None, cosmic=False, astrometry=True, channel='rc', target_dir='reduced', overwrite=False):
    '''
    Applies Flat field and bias calibrations to the image.
    
    Steps:
    
    1. - Solve astrometry on the entire image.
    2. - Computes cosmic ray rejectionon the entire image.
    3. - Compute master bias (if it does not exist) and de-bias the image.
    4. - Separate the image into 4 filters.
    5. - Compute flat field for each filter (if it does not exist) and apply flat fielding on the image.
    6. - Compute the image zeropoint.

    '''
    
    logger.info("Reducing image %s"% image)    

    print "Reducing image ", image    

    
    image = os.path.abspath(image)
    imname = os.path.basename(image).replace(".fits", "")
    try:
        objectname = fitsutils.get_par(image, "NAME").replace(" ","")+"_"+fitsutils.get_par(image, "FILTER")
    except:
        logger.error( "ERROR, image "+ image + " does not have a NAME or a FILTER!!!")
        return

    print "For object", objectname
    logger.info( "For object %s"% objectname)


    
    
    #Change to image directory
    mydir = os.path.dirname(image)
    if mydir=="": mydir = "."
    mydir = os.path.abspath(mydir)
    os.chdir(mydir)
    #Create destination directory
    if (not os.path.isdir(target_dir)):
        os.makedirs(target_dir)

    #If we don't want to overwrite the already extracted images, we check wether they exist.
    if (not overwrite):
        existing = True
        for band in ['u', 'g', 'r', 'i']:
            destfile = os.path.join(target_dir, imname + "_f_b_a_%s_%s_0.fits"%(objectname, band))
            logger.info( "Looking if file %s exists: %s"%( destfile, \
                (os.path.isfile(destfile) ) ) )
            existing = existing and (os.path.isfile( destfile ) )
        if existing:
            return []


        
    #Initialize the basic parameters.
    init_header_reduced(image)

    astro = ""
    if (astrometry):
        logger.info( "Solving astometry for the whole image...")
        img = solve_astrometry(image)
        if (os.path.isfile(img)):
            astro="a_"
            fitsutils.update_par(img, "IQWCS", 1)
        else:
            logger.error( "ASTROMETRY DID NOT SOLVE ON IMAGE %s"% image)
            img = image

    #Update noise parameters needed for cosmic reection
    if (fitsutils.get_par(img, "ADCSPEED")==2):
        fitsutils.update_par(img, "RDNOISE", 20.)
    else:
        fitsutils.update_par(img, "RDNOISE", 4.)

    if (cosmic):
        logger.info( "Correcting for cosmic rays...")
        # Correct for cosmics each filter
        cleanimg = clean_cosmic(os.path.join(os.path.abspath(mydir), img))
        img = cleanimg
    
    
    
    #Get basic statistics for the image
    nsrc, fwhm, ellip, bkg = sextractor.get_image_pars(img)
    
    logger.info( "Sextractor statistics: nscr %d, fwhm (pixel) %.2f, ellipticity %.2f"% (nsrc, fwhm, ellip))
    print "Sextractor statistics: nscr %d, fwhm (pixel) %.2f, ellipticity %.2f"% (nsrc, fwhm, ellip)

    
    dic = {"SEEPIX": fwhm/0.394, "NSRC":nsrc, "ELLIP":ellip}
    #Update the seeing information from sextractor
    fitsutils.update_pars(img, dic)
    
    
    #Compute BIAS
    if (biasdir is None or biasdir==""): biasdir = "."
    create_masterbias(biasdir)
    
    bias_slow = os.path.join(biasdir, "Bias_%s_%s.fits"%(channel, 'slow'))
    bias_fast = os.path.join(biasdir, "Bias_%s_%s.fits"%(channel, 'fast'))
    
    # Running IRAF to DE-BIAS
    iraf.noao(_doprint=0)
    iraf.imred(_doprint=0)
    iraf.ccdred(_doprint=0)
    
    #Compute flat field
    if (flatdir is None or flatdir==""): flatdir = "."
    create_masterflat(flatdir, biasdir)
    
    #New names for the object.
    debiased = os.path.join(os.path.dirname(img), "b_" + os.path.basename(img))
    logger.info( "Creating debiased file, %s"%debiased)
    
    if ( (fitsutils.get_par(img, "ADCSPEED")==0.1 and not os.path.isfile(bias_slow)) \
        or (fitsutils.get_par(img, "ADCSPEED")==2 and not os.path.isfile(bias_fast)) ):
        logger.warn( "Master bias not found! Tryting to copy from reference folder...")
        copy_ref_calib(mydir, "Bias")
        if ( (fitsutils.get_par(img, "ADCSPEED")==0.1 and not os.path.isfile(bias_slow)) \
        or (fitsutils.get_par(img, "ADCSPEED")==2 and not os.path.isfile(bias_fast)) ):
            logger.error( "Bias not found in reference folder")
            return


    #Clean first
    if (os.path.isfile(debiased)):
        os.remove(debiased)
        
    #Debias
    if (fitsutils.get_par(img, "ADCSPEED")==2):
        iraf.imarith(img, "-", bias_fast, debiased)
        fitsutils.update_par(debiased, "BIASFILE", bias_fast)
        fitsutils.update_par(debiased, "RDNOISE", 20.)

    else:
        iraf.imarith(img, "-", bias_slow, debiased)
        fitsutils.update_par(debiased, "BIASFILE", bias_slow)
        fitsutils.update_par(debiased, "RDNOISE", 4.)

    #Set negative counts to zero
    hdu = fits.open(debiased)
    header = hdu[0].header
    hdu[0].data[hdu[0].data<0] = 0
    hdu.writeto(debiased, clobber=True)

    #Slicing the image for flats  
    slice_names = slice_rc(debiased)
    print "Creating sliced files, ", slice_names

    
    #Remove un-sliced image
    os.remove(debiased)

    # DE-flat each filter and store under object name
    for i, debiased_f in enumerate(slice_names):
        b = fitsutils.get_par(debiased_f, 'filter')
        
        deflatted = os.path.join(os.path.dirname(image), target_dir, imname + "_f_b_" + astro + objectname + "_%s.fits"%b)

        #Flat to be used for that filter
        flat = os.path.join(flatdir, "Flat_%s_%s_norm.fits"%(channel, b))

        if (not os.path.isfile(flat)):
            logger.warn( "Master flat not found in %s"% flat)
            copy_ref_calib(mydir, "Flat_%s_%s_norm"%(channel, b))
            continue
        else:
            logger.info( "Using flat %s"%flat)
            
        #Cleans the deflatted file if exists
        if (os.path.isfile(deflatted)):
            os.remove(deflatted)

        if (os.path.isfile(debiased_f) and os.path.isfile(flat)):
            logger.info( "Storing de-flatted %s as %s"%(debiased_f, deflatted))
            time.sleep(1)
            iraf.imarith(debiased_f, "/", flat, deflatted)
        else:
            logger.error( "SOMETHING IS WRONG. Error when dividing %s by the flat field %s!"%(debiased_f, flat))
        
        #Removes the de-biased file
        os.remove(debiased_f)
        
        logger.info( "Updating header with original filename and flat field used.")
        fitsutils.update_par(deflatted, "ORIGFILE", os.path.basename(image))
        fitsutils.update_par(deflatted, "FLATFILE", flat)

        slice_names[i] = deflatted
            
                
    #Moving files to the target directory
    for image in slice_names:
        bkg = get_median_bkg(image)
        fitsutils.update_par(image, "SKYBKG", bkg)
        #shutil.move(name, newname)

        
    #Compute the zeropoints
    for image in slice_names:
        zeropoint.calibrate_zeropoint(image)
        
    return slice_names
コード例 #17
0
ファイル: rcred.py プロジェクト: scizen9/kpy
def create_masterbias(biasdir=None, channel='rc'):
    '''
    Combines slow and fast readout mode biases for the specified channel.
    '''
    
    iraf.noao(_doprint=0)
    iraf.imred(_doprint=0)
    iraf.ccdred(_doprint=0)
    
    if (biasdir is None) or biasdir=="": biasdir = "."
        
    outs = "Bias_%s_slow.fits"%channel
    outf = "Bias_%s_fast.fits"%channel

    doslow = True
    dofast = True
    if (os.path.isfile(os.path.join(biasdir,outs))): 
        logger.warn( "%s master Bias exists!"%outs)
        doslow = False
    if ( os.path.isfile(os.path.join(biasdir,outf))):
        logger.warn("%s master Bias exists!"%outs)
        dofast = False
    if(doslow or dofast):
        logger.info("Starting the Master Bias creation!")
    else:
        return

    os.chdir(biasdir)        
        
    lfastbias = []
    lslowbias = []
    
    #Select all filts that are Bias with same instrument
    for f in glob.glob("rc*fits"):
        try:
            if ( "BIAS" in str.upper(fitsutils.get_par(f, "IMGTYPE")) ):
                if (fitsutils.get_par(f, "ADCSPEED")==2):
                    lfastbias.append(f)
                else:
                    lslowbias.append(f)
        except:
            pass
                
    logger.info("Files for bias SLOW mode: %s"% lslowbias)
    logger.info( "Files for bias FAST mode: %s"% lfastbias)
    
    if len(lfastbias) > 0 and dofast:
        bfile_fast ="lbias_fast_"+channel
        np.savetxt(bfile_fast, np.array(lfastbias), fmt="%s")
        if (os.path.isfile("Bias_stats_fast")): os.remove("Bias_stats_fast")
        iraf.imstat("@"+bfile_fast, Stdout="Bias_stats_fast")
        
        st = np.genfromtxt("Bias_stats_fast", names=True, dtype=None)
        logger.info("%s"%st)
        
        iraf.imcombine(input = "@"+bfile_fast, \
                    output = outf, \
                    combine = "median",\
                    scale = "mode")
        os.remove(bfile_fast)
        
        #copy into the reference folder with current date
        newdir = os.path.join("../../refphot/", os.path.basename(os.path.abspath(biasdir)))
        if (not os.path.isdir(newdir)):
            os.makedirs(newdir)
        shutil.copy(outf, os.path.join(newdir, os.path.basename(outf)) )
    else:
        copy_ref_calib(biasdir, outf)


    if len(lslowbias) > 0 and doslow:

        bfile_slow ="lbias_slow_"+channel
        np.savetxt(bfile_slow, np.array(lslowbias), fmt="%s")
        if (os.path.isfile("Bias_stats_slow")): os.remove("Bias_stats_slow")
        iraf.imstat("@"+bfile_slow, Stdout="Bias_stats_slow")
        
        st = np.genfromtxt("Bias_stats_slow", names=True, dtype=None)
        logger.info("%s"%st)
        
        iraf.imcombine(input = "@"+bfile_slow, \
                    output = outs, \
                    combine = "median",\
                    scale = "mode")
        os.remove(bfile_slow)
        
        #copy into the reference folder with current date
        newdir = os.path.join("../../refphot/", os.path.basename(os.path.abspath(biasdir)))
        if (not os.path.isdir(newdir)):
            os.makedirs(newdir)
        shutil.copy(outs, os.path.join(newdir, os.path.basename(outs)) )  
    else:
        copy_ref_calib(biasdir, outs)
コード例 #18
0
ファイル: rcred.py プロジェクト: scizen9/kpy
def create_superflat(imdir, filters=["u", "g", "r", "i"]):
    #Locate images for each filter
    imlist = glob.glob("rc*fits")
    
  
    #Run sextractor to locate bright sources
  
    sexfiles = sextractor.run_sex(imlist, overwrite=False)
    maskfiles = []
    
    for i, im in enumerate(imlist): 
        #Create a mask and store it int he mask directory
        maskfile = mask_stars(im, sexfiles[i])        
        maskfiles.append(maskfile)
        fitsutils.update_par(im, "BPM", os.path.relpath(maskfile))
        
    
        
    for filt in filters:
        fimlist = [im for im in imlist if fitsutils.get_par(im, "FILTER") == filt]
        fmasklist = [im for im in maskfiles if fitsutils.get_par(im, "FILTER") == filt]
        
        if len(fimlist) == 0:
            continue
        
        fsfile ="lflat_%s"%filt
        msfile = "lmask_%s"%filt
        np.savetxt(fsfile, np.array(fimlist), fmt="%s")
        np.savetxt(msfile, np.array(fmasklist), fmt="%s")
        
        
        '''masklist = []
        
        for m in fmasklist:
            hdulist = fits.open(m)
            data = hdulist[0].data
            masklist.append(data)
            
            
        masklist = np.array(masklist)
        
        hdu = fits.PrimaryHDU(masklist)
        hdulist = fits.HDUList([hdu])
        hdulist.writeto("mastermask_%s.fits"%filt)'''           
                
        # Running IRAF
        iraf.noao(_doprint=0)
        iraf.imred(_doprint=0)
        iraf.ccdred(_doprint=0)
        
        iraf.imarith("@"+fsfile, "*", "@"+msfile, "[email protected]"+fsfile)


        #Combine flats
        iraf.imcombine(input = "[email protected]"+fsfile, \
                        output = "superflat_%s.fits"%filt, \
                        combine = "median",\
                        scale = "mode", \
                        masktype="badvalue",\
                        maskvalue = 0)
                        
        iraf.imstat("superflat_%s.fits"%filt, fields="image,npix,mean,stddev,min,max,mode", Stdout="Flat_stats")
        time.sleep(0.1)
        st = np.genfromtxt("Flat_stats", names=True, dtype=None)
        #Normalize flats
        iraf.imarith("superflat_%s.fits"%filt, "/", st["MODE"], "superflat_%s_norm.fits"%filt)
コード例 #19
0
ファイル: rcred.py プロジェクト: scizen9/kpy
def create_masterflat(flatdir=None, biasdir=None, channel='rc'):
    '''
    Creates a masterflat from both dome flats and sky flats if the number of counts in the given filter
    is not saturated and not too low (between 3000 and 40000). 
    '''
    
    
    if (flatdir == None or flatdir==""): flatdir = "."
        
    if (biasdir == None or biasdir==""): biasdir = "."
        
    os.chdir(flatdir)
    
    if (len(glob.glob("Flat_%s*norm.fits"%channel)) == 4):
        logger.info( "Master Flat exists!")
        return 
    if (len(glob.glob("Flat_%s*norm.fits"%channel)) > 0):
        logger.info( "Some Master Flat exist!")
        return 
    else:
        logger.info( "Starting the Master Flat creation!")

    bias_slow = "Bias_%s_slow.fits"%channel
    bias_fast = "Bias_%s_fast.fits"%channel
    
    if (not os.path.isfile(bias_slow) and not os.path.isfile(bias_fast) ):
        create_masterbias(biasdir)
     
    lsflat = []
    lfflat = []
    
    obj = ""
    imtype = ""
    
    #Select all filts that are Flats with same instrument
    for f in glob.glob(channel+"*fits"):
        try:
            if fitsutils.has_par(f, "OBJECT"):
                obj = str.upper(fitsutils.get_par(f, "OBJECT"))
            else:
                continue

            if fitsutils.has_par(f, "IMGTYPE"):
                imtype = str.upper(fitsutils.get_par(f, "IMGTYPE"))
            else:
                continue
        
            #if ("RAINBOW CAM" in str.upper(fitsutils.get_par(f, "CAM_NAME")) and  ("DOME" in  obj or "FLAT" in obj or "Twilight" in obj or "TWILIGHT" in imtype or "DOME" in imtype)):
            if ( "TWILIGHT" in imtype):

                if (fitsutils.get_par(f, "ADCSPEED")==2):
                    lfflat.append(f)
                else:
                    lsflat.append(f)
        except:
            logger.error( "Error with retrieving parameters for file %s"% f)
            pass
                
    logger.info( "Files for slow flat %s"% lsflat)
    logger.info( "Files for fast flat %s"% lfflat)
    
    fsfile ="lflat_slow_"+channel
    np.savetxt(fsfile, np.array(lsflat), fmt="%s")
    fffile ="lflat_fast_"+channel
    np.savetxt(fffile, np.array(lfflat), fmt="%s")


    # Running IRAF
    iraf.noao(_doprint=0)
    iraf.imred(_doprint=0)
    iraf.ccdred(_doprint=0)
    
    #Remove bias from the flat
    if len(lsflat) >0:
        iraf.imarith("@"+fsfile, "-", bias_slow, "[email protected]"+fsfile)
    
    if len(lfflat) >0:
        iraf.imarith("@"+fffile, "-", bias_fast, "[email protected]"+fffile)    
    
    #Remove the list files
    os.remove(fsfile)
    os.remove(fffile)
    
    #Slices the flats.
    debiased_flats = glob.glob("b_*.fits")
    for f in debiased_flats:
        logger.info( "Slicing file %s"% f)
        try:
            slice_rc(f)
        except:
            logger.error("Error when slicing file... deleting the unsliced one...")
        #Remove the un-sliced file
        os.remove(f)
        
    #Selects the ones that are suitable given the number of counts and combines them.
    bands = ['u', 'g', 'r', 'i']
    for b in bands:
        out = "Flat_%s_%s.fits"%(channel, b)
        out_norm = out.replace(".fits","_norm.fits")

        if (os.path.isfile(out_norm)):
            logger.error( "Master Flat for filter %s exists. Skipping..."%b)
            continue
        
        lfiles = []
        for f in glob.glob('b_*_%s.fits'%b):
            d = fits.open(f)[0].data
            if np.percentile(d, 90)>4000 and np.percentile(d, 90)<40000:
                lfiles.append(f)

        if len(lfiles) == 0:
            logger.error( "WARNING!!! Could not find suitable flats for band %s"%b)
            continue
        if len(lfiles) < 3:
            logger.error( "WARNING!!! Could find less than 3 flats for band %s. Skipping, as it is not reliable..."%b)
            continue
        ffile ="lflat_"+b
        np.savetxt(ffile, np.array(lfiles), fmt="%s")
    
        
        #Cleaning of old files
        if(os.path.isfile(out)): os.remove(out)
        if(os.path.isfile(out_norm)): os.remove(out_norm)
        if(os.path.isfile("Flat_stats")): os.remove("Flat_stats")
        
        
        #Combine flats
        iraf.imcombine(input = "@"+ffile, \
                        output = out, \
                        combine = "median",\
                        scale = "mode",
                        weight = "exposure")
        iraf.imstat(out, fields="image,npix,mean,stddev,min,max,mode", Stdout="Flat_stats")
        st = np.genfromtxt("Flat_stats", names=True, dtype=None)
        #Normalize flats
        iraf.imarith(out, "/", st["MODE"], out_norm)
        
        #Do some cleaning
        logger.info( 'Removing from lfiles')
        for f in glob.glob('b_*_%s.fits'%b):
            os.remove(f)

        os.remove(ffile)
        
        
        if os.path.isfile(fsfile):
            os.remove(fsfile)
        if os.path.isfile(fffile):
            os.remove(fffile)

        #copy into the reference folder with current date
        newdir = os.path.join("../../refphot/", os.path.basename(os.path.abspath(flatdir)))
        if (not os.path.isdir(newdir)):
            os.makedirs(newdir)
        shutil.copy(out_norm, os.path.join(newdir, os.path.basename(out_norm)) )   
コード例 #20
0
def floydsautoredu(files,_interactive,_dobias,_doflat,_listflat,_listbias,_listarc,_cosmic,_ext_trace,_dispersionline,liststandard,listatmo,_automaticex,_classify=False,_verbose=False,smooth=1,fringing=1):
    import floyds
    import string,re,os,glob,sys,pickle
    from numpy import array, arange, mean,pi,arccos,sin,cos,argmin
    from astropy.io import fits
    from pyraf import iraf
    import datetime
    os.environ["PYRAF_BETA_STATUS"] = "1"
    iraf.set(direc=floyds.__path__[0]+'/')
    _extinctdir='direc$standard/extinction/'
    _tel=floyds.util.readkey3(floyds.util.readhdr(re.sub('\n','',files[0])),'TELID')
    if _tel=='fts':
        _extinction='ssoextinct.dat'
        _observatory='sso'
    elif _tel=='ftn':
        _extinction='maua.dat' 
        _observatory='cfht'   
    else: sys.exit('ERROR: observatory not recognised')
    dv=floyds.util.dvex()
    scal=pi/180.
    iraf.noao(_doprint=0)
    iraf.imred(_doprint=0)
    iraf.ccdred(_doprint=0)
    iraf.twodspec(_doprint=0)
    iraf.longslit(_doprint=0)
    iraf.specred(_doprint=0)
    toforget = ['ccdred.flatcombine','ccdred.zerocombine','ccdproc','specred.apall','longslit.identify','longslit.reidentify',\
                    'specred.standard','longslit.fitcoords','specred.transform','specred.response']
    for t in toforget: iraf.unlearn(t)
    iraf.longslit.dispaxi=2
    iraf.longslit.mode='h'
    iraf.identify.fwidth=7 
    iraf.identify.order=2 
    iraf.specred.dispaxi=2
    iraf.specred.mode='h'
    iraf.ccdproc.darkcor='no'
    iraf.ccdproc.fixpix='no'
    iraf.ccdproc.trim='no'
    iraf.ccdproc.flatcor='no'
    iraf.ccdproc.overscan='no'
    iraf.ccdproc.zerocor='no'
    iraf.ccdproc.biassec=''
    iraf.ccdproc.ccdtype=''
    iraf.ccdred.instrument = "/dev/null"
    if _verbose: 
        iraf.ccdred.verbose='yes'
        iraf.specred.verbose='yes'
    else: 
        iraf.specred.verbose='no'
        iraf.ccdred.verbose='no'
    now=datetime.datetime.now()
    datenow=now.strftime('20%y%m%d%H%M')
    MJDtoday=55928+(datetime.date.today()-datetime.date(2012, 01, 01)).days
    outputlist=[]
    hdra=floyds.util.readhdr(re.sub('\n','',files[0]))
    _gain=floyds.util.readkey3(hdra,'gain')
    _rdnoise=floyds.util.readkey3(hdra,'ron')
    std,rastd,decstd,magstd=floyds.util.readstandard('standard_floyds_mab.txt')
    _naxis2=hdra.get('NAXIS2')
    _naxis1=hdra.get('NAXIS1')
    if not _naxis1: _naxis1=2079
    if not _naxis2: 
        if not hdr0.get('HDRVER'):   _naxis1=511
        else:                        _naxis1=512
    _overscan='[2049:'+str(_naxis1)+',1:'+str(_naxis2)+']'
    _biassecblu='[380:2048,325:'+str(_naxis2)+']'    
    _biassecred='[1:1800,1:350]'    
    lista={}
    objectlist={}
    biaslist={}
    flatlist={}
    flatlistd={}
    arclist={}
    max_length=14
    for img in files:
        hdr0=floyds.util.readhdr(img)
        if  floyds.util.readkey3(hdr0,'naxis2')>=500:
            if 'blu' not in lista: lista['blu']=[]
            if 'red' not in lista: lista['red']=[]
            _object0=floyds.util.readkey3(hdr0,'object')
            _object0 = re.sub(':', '', _object0) # colon
            _object0 = re.sub('/', '', _object0) # slash
            _object0 = re.sub('\s', '', _object0) # any whitespace
            _object0 = re.sub('\(', '', _object0) # open parenthesis
            _object0 = re.sub('\[', '', _object0) # open square bracket
            _object0 = re.sub('\)', '', _object0) # close parenthesis
            _object0 = re.sub('\]', '', _object0) # close square bracket
            _object0 = _object0.replace(r'\t', '') # Any tab characters
            _object0 = _object0.replace('*', '') # Any asterisks

            if len(_object0) > max_length:
                _object0 = _object0[:max_length]
            _date0=floyds.util.readkey3(hdr0,'date-night')
            _tel=floyds.util.readkey3(hdr0,'TELID')
            _type=floyds.util.readkey3(hdr0,'OBSTYPE')
            if not _type:    _type=floyds.util.readkey3(hdr0,'imagetyp')
            _slit=floyds.util.readkey3(hdr0,'slit')
            if _type:
                _type = _type.lower()
                if _type in ['sky','spectrum','expose']:
                    nameoutb=str(_object0)+'_'+_tel+'_'+str(_date0)+'_blue_'+str(_slit)+'_'+str(MJDtoday)
                    nameoutr=str(_object0)+'_'+_tel+'_'+str(_date0)+'_red_'+str(_slit)+'_'+str(MJDtoday)
                elif _type in ['lamp','arc','l']:
                    nameoutb='arc_'+str(_object0)+'_'+_tel+'_'+str(_date0)+'_blue_'+str(_slit)+'_'+str(MJDtoday)
                    nameoutr='arc_'+str(_object0)+'_'+_tel+'_'+str(_date0)+'_red_'+str(_slit)+'_'+str(MJDtoday)
                elif _type in ['flat','f','lampflat','lamp-flat']:
                    nameoutb='flat_'+str(_object0)+'_'+_tel+'_'+str(_date0)+'_blue_'+str(_slit)+'_'+str(MJDtoday)
                    nameoutr='flat_'+str(_object0)+'_'+_tel+'_'+str(_date0)+'_red_'+str(_slit)+'_'+str(MJDtoday)
                else:
                    nameoutb=str(_type.lower())+'_'+str(_object0)+'_'+_tel+'_'+str(_date0)+'_blue_'+str(_slit)+'_'+str(MJDtoday)
                    nameoutr=str(_type.lower())+'_'+str(_object0)+'_'+_tel+'_'+str(_date0)+'_red_'+str(_slit)+'_'+str(MJDtoday)

                bimg=floyds.util.name_duplicate(img,nameoutb,'')
                rimg=floyds.util.name_duplicate(img,nameoutr,'')
####
                floyds.util.delete(bimg)
                floyds.util.delete(rimg)
                iraf.imcopy(img,bimg,verbose='no')
                iraf.imcopy(img,rimg,verbose='no')

                aaa=iraf.hedit(bimg,'CCDSEC',delete='yes',update='yes',verify='no',Stdout=1)
                aaa=iraf.hedit(bimg,'TRIMSEC',delete='yes',update='yes',verify='no',Stdout=1)
                aaa=iraf.hedit(rimg,'CCDSEC',delete='yes',update='yes',verify='no',Stdout=1)
                aaa=iraf.hedit(rimg,'TRIMSEC',delete='yes',update='yes',verify='no',Stdout=1)

                iraf.ccdproc(bimg,output='', overscan="yes", trim="yes", zerocor='no', flatcor='no', zero='', ccdtype='',\
                                 fixpix='no', trimsec=_biassecblu, biassec=_overscan, readaxi='line', Stdout=1)
                iraf.ccdproc(rimg,output='', overscan="yes", trim="yes", zerocor='no', flatcor='no', zero='', ccdtype='',\
                                 fixpix='no', trimsec=_biassecred, biassec=_overscan, readaxi='line', Stdout=1)
                floyds.util.updateheader(bimg,0,{'GRISM':['blu',' blue order']})
                floyds.util.updateheader(rimg,0,{'GRISM':['red',' blue order']})
                floyds.util.updateheader(bimg,0,{'arcfile':[img,'file name in the archive']})
                floyds.util.updateheader(rimg,0,{'arcfile':[img,'file name in the archive']})
                lista['blu'].append(bimg)
                lista['red'].append(rimg)
            else: 
                print 'warning type not defined'
    for arm in lista.keys():
        for img in lista[arm]:
            print img
            hdr=floyds.util.readhdr(img)
            _type=floyds.util.readkey3(hdr,'OBSTYPE')
            if _type=='EXPOSE':  
                      _type=floyds.util.readkey3(hdr,'imagetyp')
                      if not _type: _type='EXPOSE'

            if _type=='EXPOSE':  
                print 'warning obstype still EXPOSE, are this old data ?  run manually floydsfixheader'

            _slit=floyds.util.readkey3(hdr,'slit')
            _grpid=floyds.util.readkey3(hdr,'grpid')
            if _type.lower() in ['flat','f','lamp-flat','lampflat'] :
                if (arm,_slit) not in flatlist:  flatlist[arm,_slit]={}
                if _grpid not in flatlist[arm,_slit]: flatlist[arm,_slit][_grpid]=[img]
                else: flatlist[arm,_slit][_grpid].append(img)
            elif _type.lower() in ['lamp','l','arc']:
                if (arm,_slit) not in arclist:  arclist[arm,_slit]={}
                if _grpid not in arclist[arm,_slit]: arclist[arm,_slit][_grpid]=[img]
                else: arclist[arm,_slit][_grpid].append(img)
            elif _type in ['bias','b']:
                if arm not in biaslist: biaslist[arm]=[]
                biaslist[arm].append(img)
            elif _type.lower() in ['sky','s','spectrum']:
                try:
                    _ra=float(floyds.util.readkey3(hdr,'RA'))
                    _dec=float(floyds.util.readkey3(hdr,'DEC'))
                except:
                    ra00=string.split(floyds.util.readkey3(hdr,'RA'),':')
                    ra0,ra1,ra2=float(ra00[0]),float(ra00[1]),float(ra00[2])
                    _ra=((ra2/60.+ra1)/60.+ra0)*15.
                    dec00=string.split(floyds.util.readkey3(hdr,'DEC'),':')
                    dec0,dec1,dec2=float(dec00[0]),float(dec00[1]),float(dec00[2])
                    if '-' in str(dec0):       _dec=(-1)*((dec2/60.+dec1)/60.+((-1)*dec0))
                    else:                      _dec=(dec2/60.+dec1)/60.+dec0
                dd=arccos(sin(_dec*scal)*sin(decstd*scal)+cos(_dec*scal)*cos(decstd*scal)*cos((_ra-rastd)*scal))*((180/pi)*3600)
                if _verbose:
                    print _ra,_dec
                    print std[argmin(dd)],min(dd)
                if min(dd)<5200: _typeobj='std'
                else: _typeobj='obj'
                if min(dd)<5200:
                    floyds.util.updateheader(img,0,{'stdname':[std[argmin(dd)],'']})
                    floyds.util.updateheader(img,0,{'magstd':[float(magstd[argmin(dd)]),'']})
                if _typeobj not in objectlist:      objectlist[_typeobj]={}

                if (arm,_slit) not in objectlist[_typeobj]:     objectlist[_typeobj][arm,_slit]=[img]
                else: objectlist[_typeobj][arm,_slit].append(img)
    if _verbose:
        print 'object'
        print objectlist
        print 'flat'
        print flatlist
        print 'bias'
        print biaslist
        print 'arc'
        print arclist

    if liststandard and 'std' in objectlist.keys():  
        print 'external standard, raw standard not used'
        del objectlist['std']

    sens={}
    outputfile={}
    atmo={}
    for tpe in objectlist:
      if tpe not in outputfile:  outputfile[tpe]={}
      for setup in objectlist[tpe]:
        if setup not in sens:   sens[setup]=[]
        print '\n### setup= ',setup,'\n### objects= ',objectlist[tpe][setup],'\n'
        for img in objectlist[tpe][setup]:
              print '\n\n### next object= ',img,' ',floyds.util.readkey3(floyds.util.readhdr(img),'object'),'\n'
              hdr=floyds.util.readhdr(img)
              archfile=floyds.util.readkey3(hdr,'arcfile')
              _gain=floyds.util.readkey3(hdr,'gain')
              _rdnoise=floyds.util.readkey3(hdr,'ron')
              _grism=floyds.util.readkey3(hdr,'grism')
              _grpid=floyds.util.readkey3(hdr,'grpid')
              if archfile not in outputfile[tpe]: outputfile[tpe][archfile]=[]
#####################      flat   ###############
              if _listflat:   flatgood=_listflat    # flat list from reducer
              elif setup in flatlist:  
                  if _grpid in flatlist[setup]:
                      print '\n###FLAT WITH SAME GRPID'
                      flatgood= flatlist[setup][_grpid]     # flat in the  raw data
                  else:  
                      flatgood=[]
                      for _grpid0 in flatlist[setup].keys():
                          for ii in flatlist[setup][_grpid0]:
                              flatgood.append(ii)
              else: flatgood=[]
              if len(flatgood)!=0:
                  if len(flatgood)>1:
                      f=open('_oflatlist','w')
                      for fimg in flatgood:
                          print fimg
                          f.write(fimg+'\n')
                      f.close()
                      floyds.util.delete('flat'+img)
                      iraf.ccdred.flatcombine('"@_oflatlist"',output='flat'+img,combine='average',reject='none',ccdtype=' ',rdnoise=_rdnoise,gain=_gain, process='no', Stdout=1)
                      floyds.util.delete('_oflatlist')
                      flatfile='flat'+img
                  elif len(flatgood)==1:
                      os.system('cp '+flatgood[0]+' flat'+img)
                      flatfile='flat'+img
              else: flatfile=''
##########################   find arcfile            #######################
              arcfile=''
              if _listarc:       arcfile= [floyds.util.searcharc(img,_listarc)[0]][0]   # take arc from list 
              if not arcfile and setup in arclist.keys():
                    if _grpid in arclist[setup]:  
                        print '\n###ARC WITH SAME GRPID'
                        arcfile= arclist[setup][_grpid]     # flat in the  raw data
                    else:  
                        arcfile=[]
                        for _grpid0 in arclist[setup].keys():
                            for ii in arclist[setup][_grpid0]:
                                arcfile.append(ii)                   
              if arcfile:
                  if len(arcfile)>1:                           # more than one arc available
                      print arcfile
#                     _arcclose=floyds.util.searcharc(imgex,arcfile)[0]   # take the closest in time 
                      _arcclose=floyds.sortbyJD(arcfile)[-1]               #  take the last arc of the sequence
                      if _interactive.upper() in ['YES','Y']:
                              for ii in floyds.floydsspecdef.sortbyJD(arcfile):
                                  print '\n### ',ii 
                              arcfile=raw_input('\n### more than one arcfile available, which one to use ['+str(_arcclose)+'] ? ')
                              if not arcfile: arcfile=_arcclose
                      else: arcfile=_arcclose
                  else: arcfile=arcfile[0]
              else:   print '\n### Warning: no arc found'

###################################################################   rectify 
              if setup[0]=='red':
                  fcfile=floyds.__path__[0]+'/standard/ident/fcrectify_'+_tel+'_red'
                  fcfile1=floyds.__path__[0]+'/standard/ident/fcrectify1_'+_tel+'_red'
                  print fcfile
              else:
                  fcfile=floyds.__path__[0]+'/standard/ident/fcrectify_'+_tel+'_blue'
                  fcfile1=floyds.__path__[0]+'/standard/ident/fcrectify1_'+_tel+'_blue'
                  print fcfile
              print img,arcfile,flatfile
              img0=img
              if img      and img not in outputfile[tpe][archfile]: outputfile[tpe][archfile].append(img)
              if arcfile  and arcfile not in outputfile[tpe][archfile]: outputfile[tpe][archfile].append(arcfile)
              if flatfile and flatfile not in outputfile[tpe][archfile]: outputfile[tpe][archfile].append(flatfile)

              img,arcfile,flatfile=floyds.floydsspecdef.rectifyspectrum(img,arcfile,flatfile,fcfile,fcfile1,'no',_cosmic)
              if img      and img not in outputfile[tpe][archfile]: outputfile[tpe][archfile].append(img)
              if arcfile  and arcfile not in outputfile[tpe][archfile]: outputfile[tpe][archfile].append(arcfile)
              if flatfile and flatfile not in outputfile[tpe][archfile]: outputfile[tpe][archfile].append(flatfile)
###################################################################         check wavecalib  
              if tpe=='std' or floyds.util.readkey3(floyds.util.readhdr(img),'exptime') < 300:
                  if setup[0]=='red':
                      print '\n### check standard wave calib'
                      data, hdr = fits.getdata(img, 0, header=True) 
                      y=data.mean(1)
                      import numpy as np
                      if np.argmax(y) < 80 and np.argmax(y) > 15:                      
                          y2=data[np.argmax(y)-3:np.argmax(y)+3].mean(0)
                          yy2=data[np.argmax(y)-9:np.argmax(y)-3].mean(0)
                          floyds.util.delete('_std.fits')
                          fits.writeto('_std.fits', np.float32(y2-yy2), hdr)
                          shift=floyds.floydsspecdef.checkwavestd('_std.fits',_interactive,2)
                          zro=hdr['CRVAL1']
                          floyds.util.updateheader(img,0,{'CRVAL1':[zro+int(shift),'']})
                          floyds.util.updateheader(img,0,{'shift':[float(shift),'']})
                          floyds.util.delete('_std.fits')
                      else:
                          print 'object not found'
                  else: 
                      print '\n### warning check in wavelength not possible for short exposure in the blu range '
              else:
                    print '\n### check object wave calib'
                    _skyfile=floyds.__path__[0]+'/standard/ident/sky_'+setup[0]+'.fits'
                    data, hdr = fits.getdata(img, 0, header=True) 
                    y=data.mean(1)
                    import numpy as np
                    if np.argmax(y) < 80 and np.argmax(y) > 15:
                        yy1=data[10:np.argmax(y)-9].mean(0)
                        yy2=data[np.argmax(y)+9:-10].mean(0)
                        floyds.util.delete('_sky.fits')
                        fits.writeto('_sky.fits', np.float32(yy1+yy2), hdr)
                        shift=floyds.floydsspecdef.checkwavelength_obj('_sky.fits',_skyfile,_interactive,usethirdlayer=False)
                        floyds.util.delete('_sky.fits')
                        zro=hdr['CRVAL1']
                        floyds.util.updateheader(img,0,{'CRVAL1':[zro+int(shift),'']})
                        floyds.util.updateheader(img,0,{'shift':[float(shift),'']})
                    else:  print 'object not found'
####################################################     flat field
              if img and flatfile and setup[0]=='red':
                      imgn='n'+img
                      hdr1 = floyds.readhdr(img)
                      hdr2 = floyds.readhdr(flatfile)
                      _grpid1=floyds.util.readkey3(hdr1,'grpid')
                      _grpid2=floyds.util.readkey3(hdr2,'grpid')
                      if _grpid1==_grpid2:
                          print flatfile,img,setup[0]
                          imgn=floyds.fringing_classicmethod2(flatfile,img,'no','*',15,setup[0])
                      else:
                          print 'Warning flat not the same OB'
                          imgex=floyds.floydsspecdef.extractspectrum(img,dv,_ext_trace,_dispersionline,_interactive,tpe,automaticex=_automaticex)
                          floyds.delete('flat'+imgex)
                          iraf.specred.apsum(flatfile,output='flat'+imgex,referen=img,interac='no',find='no',recente='no',resize='no',\
                                             edit='no',trace='no',fittrac='no',extract='yes',extras='no',review='no',backgro='none')
                          fringingmask=floyds.normflat('flat'+imgex)
                          print '\n### fringing correction'
                          print imgex,fringingmask
                          imgex,scale,shift=floyds.correctfringing_auto(imgex,fringingmask)  #  automatic correction
                          shift=int(.5+float(shift)/3.5)        # shift from correctfringing_auto in Angstrom
                          print '\n##### flat scaling: ',str(scale),str(shift)
########################################################
                          datax, hdrx = fits.getdata(flatfile, 0, header=True)
                          xdim=hdrx['NAXIS1']
                          ydim=hdrx['NAXIS2']
                          iraf.specred.apedit.nsum=15 
                          iraf.specred.apedit.width=100.  
                          iraf.specred.apedit.line=1024 
                          iraf.specred.apfind.minsep=20.  
                          iraf.specred.apfind.maxsep=1000.  
                          iraf.specred.apresize.bkg='no' 
                          iraf.specred.apresize.ylevel=0.5 
                          iraf.specred.aptrace.nsum=10
                          iraf.specred.aptrace.step=10
                          iraf.specred.aptrace.nlost=10
                          floyds.util.delete('n'+flatfile)
                          floyds.util.delete('norm.fits')
                          floyds.util.delete('n'+img)
                          floyds.util.delete(re.sub('.fits','c.fits',flatfile))
                          iraf.imcopy(flatfile+'[500:'+str(xdim)+',*]',re.sub('.fits','c.fits',flatfile),verbose='no')
                          iraf.imarith(flatfile,'/',flatfile,'norm.fits',verbose='no')
                          flatfile=re.sub('.fits','c.fits',flatfile)
                          floyds.util.delete('n'+flatfile)
                          iraf.unlearn(iraf.specred.apflatten)
                          floyds.floydsspecdef.aperture(flatfile)
                          iraf.specred.apflatten(flatfile,output='n'+flatfile,interac=_interactive,find='no',recenter='no', resize='no',edit='no',trace='no',\
                                                 fittrac='no',fitspec='no', flatten='yes', aperture='',\
                                                 pfit='fit2d',clean='no',function='legendre',order=15,sample = '*', mode='ql')
                          iraf.imcopy('n'+flatfile,'norm.fits[500:'+str(xdim)+',*]',verbose='no')
                          floyds.util.delete('n'+flatfile)
                          floyds.util.delete('n'+img)
                          iraf.imrename('norm.fits','n'+flatfile,verbose='no')
                          imgn=floyds.floydsspecdef.applyflat(img,'n'+flatfile,'n'+img,scale,shift)
              else:                  imgn=''

              if imgn and imgn not in outputfile[tpe][archfile]: outputfile[tpe][archfile].append(imgn)

###################################################      2D flux calib
              hdr=floyds.util.readhdr(img)
              _sens=''
              if liststandard:  _sens=floyds.util.searchsens(img,liststandard)[0]   # search in the list from reducer
              if not _sens:
                  try:      _sens=floyds.util.searchsens(img,sens[setup])[0]        # search in the reduced data
                  except:   _sens=floyds.util.searchsens(img,'')[0]              # search in tha archive
              if _sens:
                  if _sens[0]=='/': 
                      os.system('cp '+_sens+' .')
                      _sens=string.split(_sens,'/')[-1]
                  imgd=fluxcalib2d(img,_sens)
                  if imgn:     imgdn=fluxcalib2d(imgn,_sens)
                  else: imgdn=''
                  if _sens not in outputfile[tpe][archfile]: outputfile[tpe][archfile].append(_sens)
                  else:        imgdn='' 
                  print '\n### do 2D calibration'
              else:
                  imgd=''
                  imgdn=''
################    extraction         ####################################
              if imgdn:
                  try:
                      imgdnex=floyds.floydsspecdef.extractspectrum(imgdn,dv,_ext_trace,_dispersionline,_interactive,tpe,automaticex=_automaticex)
                  except Exception as e:
                      print 'failed to extract', imgdn
                      print e
                      imgdnex=''
              else:       
                  imgdnex=''
              if imgd:
                  try:
                      imgdex=floyds.floydsspecdef.extractspectrum(imgd,dv,_ext_trace,_dispersionline,_interactive,tpe,automaticex=_automaticex)  
                  except Exception as e:
                      print 'failed to extract', imgd
                      print e
                      imgdex=''
              else:
                  imgdex=''
              if imgd    and imgd    not in outputfile[tpe][archfile]: outputfile[tpe][archfile].append(imgd)
              if imgdn   and imgdn   not in outputfile[tpe][archfile]: outputfile[tpe][archfile].append(imgdn)
              if imgdnex and imgdnex not in outputfile[tpe][archfile]: outputfile[tpe][archfile].append(imgdnex)
              if imgdex  and imgdex  not in outputfile[tpe][archfile]: outputfile[tpe][archfile].append(imgdex)
              if tpe=='std':
                  if imgn:
                      try:
                          imgnex=floyds.floydsspecdef.extractspectrum(imgn,dv,_ext_trace,_dispersionline,_interactive,tpe,automaticex=_automaticex)  
                      except Exception as e:
                          print 'failed to extract', imgn
                          print e
                          imgnex=''
                  elif img:
                      try:
                          imgnex=floyds.floydsspecdef.extractspectrum(img,dv,_ext_trace,_dispersionline,_interactive,tpe,automaticex=_automaticex)  
                      except Exception as e:
                          print 'failed to extract', img
                          print e
                          imgnex=''
                  if imgnex:
                    hdrs=floyds.util.readhdr(imgnex)
                    _tel=floyds.util.readkey3(hdrs,'TELID')
                    try:
                      _outputsens2='sens_'+_tel+'_'+str(floyds.util.readkey3(hdrs,'date-night'))+'_'+str(floyds.util.readkey3(hdrs,'grism'))+\
                          '_'+re.sub('.dat','',floyds.util.readkey3(hdrs,'stdname'))+'_'+str(MJDtoday)
                    except:  sys.exit('Error: missing header -stdname- in standard '+str(standardfile)+'  ')                          
                    print '\n### compute sensitivity function and atmofile'
                    if setup[0]=='red':
                          atmofile=floyds.floydsspecdef.telluric_atmo(imgnex)
                          if atmofile and atmofile not in outputfile[tpe][archfile]:    outputfile[tpe][archfile].append(atmofile)
                          stdusedclean=re.sub('_ex.fits','_clean.fits',imgnex)
                          floyds.util.delete(stdusedclean)
                          _function='spline3'
                          iraf.specred.sarith(input1=imgnex,op='/',input2=atmofile,output=stdusedclean, format='multispec')
                          try:
                              _outputsens2=floyds.floydsspecdef.sensfunction(stdusedclean,_outputsens2,_function,8,_interactive)
                          except:
                              print 'Warning: problem computing sensitivity function'
                              _outputsens2=''
                          if setup not in atmo: atmo[setup]=[atmofile]
                          else: atmo[setup].append(atmofile)
                    else:
                          _function='spline3'
                          try:
                              _outputsens2=floyds.floydsspecdef.sensfunction(imgnex,_outputsens2,_function,12,_interactive,'3400:4700')#,3600:4300')
                          except:
                              print 'Warning: problem computing sensitivity function'
                              _outputsens2=''
                    if _outputsens2  and _outputsens2 not in outputfile[tpe][archfile]:    outputfile[tpe][archfile].append(_outputsens2)
    ###################################################
    if 'obj' in outputfile:
      for imm in outputfile['obj']:
        lista = []
        tt_red = ''
        ntt_red = ''
        tt_blue = ''
        for f in outputfile['obj'][imm]:
            if '_ex.fits' in f and '_blue_' in f:
                tt_blue = f
            elif '_ex.fits' in f and f[:3] == 'ntt':
                ntt_red = f
            elif '_ex.fits' in f and f[:2] == 'tt':
                tt_red = f
            else:
                lista.append(f)
        merged = ntt_red.replace('_red_', '_merge_')
        if tt_blue and ntt_red:
            floyds.floydsspecdef.combspec2(tt_blue, ntt_red, merged, scale=True, num=None)
        if os.path.isfile(merged):
            lista.append(merged)
            floyds.util.delete(tt_blue)
            floyds.util.delete(tt_red)
            floyds.util.delete(ntt_red)
        else:
            if tt_blue: lista.append(tt_blue)
            if tt_red:  lista.append(tt_red)
            if ntt_red: lista.append(ntt_red)
        outputfile['obj'][imm] = lista
    readme=floyds.floydsspecauto.writereadme()
    return outputfile,readme
コード例 #21
0
ファイル: sofispec2Ddef.py プロジェクト: svalenti/pessto
def sofispecreduction(files, _interactive, _doflat, listflat, _docross, _verbose=False):
    # print "LOGX:: Entering `sofispecreduction` method/function in
    # %(__file__)s" % globals()
    import ntt
    from ntt.util import delete, readhdr, readkey3, correctcard, rangedata
    import string, re, sys, os, glob

    try:        
        from astropy.io import fits as pyfits
    except:     
        import pyfits

    from pyraf import iraf
    from numpy import argmin, array, min, isnan, arange, mean, sum
    from numpy import sqrt, pi

    iraf.noao(_doprint=0)
    iraf.imred(_doprint=0)
    iraf.ccdred(_doprint=0)
    iraf.twodspec(_doprint=0)
    iraf.longslit(_doprint=0)
    iraf.specred(_doprint=0)
    toforget = ['ccdred.flatcombine', 'ccdproc', 'specred.apall', 'longslit.identify', 'longslit.reidentify',
                'longslit.fitcoords', 'specred.transform', 'specred.response', 'imutil.hedit']
    for t in toforget:
        iraf.unlearn(t)
    iraf.longslit.dispaxi = 2
    iraf.longslit.mode = 'h'
    iraf.specred.dispaxi = 2
    iraf.specred.mode = 'h'
    iraf.ccdproc.darkcor = 'no'
    iraf.ccdproc.fixpix = 'no'
    iraf.ccdproc.flatcor = 'no'
    iraf.ccdproc.zerocor = 'no'
    iraf.ccdproc.overscan = 'no'
    iraf.ccdproc.ccdtype = ''
    iraf.ccdred.instrument = "/dev/null"

    iraf.set(direc=ntt.__path__[0] + '/')

    if _interactive:
        _interact = 'yes'
    else:
        _interact = 'no'
    if _verbose:
        iraf.ccdred.verbose = 'yes'
        iraf.specred.verbose = 'yes'
    else:
        iraf.specred.verbose = 'no'
        iraf.ccdred.verbose = 'no'
    import datetime
    import time

    now = datetime.datetime.now()
    datenow = now.strftime('20%y%m%d%H%M')
    MJDtoday = 55927 + (datetime.date.today() - datetime.date(2012, 01, 01)).days
    # if they are not sorted the fieldlist dict could crash
    files = ntt.sofiphotredudef.sortbyJD(files)
    outputlist = []
    setup = []
    fieldlist = {}
    OBID = {}
    RA = {}
    DEC = {}
    objects = {}
    flats = {}
    lamps1 = {}
    _rdnoise = readkey3(readhdr(re.sub('\n', '', files[0])), 'ron')
    _gain = readkey3(readhdr(re.sub('\n', '', files[0])), 'gain')
    for img in files:
        img = re.sub('\n', '', img)
        hdr = readhdr(img)
        _object = readkey3(hdr, 'object')
        _filter = readkey3(hdr, 'filter')
        _date = readkey3(hdr, 'date-night')
        _exptime = readkey3(hdr, 'exptime')
        _grism = readkey3(hdr, 'grism')
        _obsmode = readkey3(hdr, 'obsmode')
        _type = ''
        if _grism.lower() not in ['gr', 'gb']:
            _type = 'image'
        if not _type:
            if _object.lower() == 'flat':
                _type = 'flat'
                if _date not in flats:
                    flats[_date] = {}
                if _grism not in flats[_date]:
                    flats[_date][_grism] = [img]
                else:
                    flats[_date][_grism].append(img)
            elif _object.lower() == 'lamp':
                _lampid = (readkey3(hdr, 'esoid'), readkey3(hdr, 'grism'))
                if _lampid not in lamps1:
                    lamps1[_lampid] = [None, None]
                if readkey3(hdr, 'lamp1') == 'Xenon':
                    lamps1[_lampid][0] = img
                else:
                    lamps1[_lampid][1] = img
                _type = 'lamp'
                # if readkey3(hdr,'lamp1')=='Xenon':
            #                     _type='lamp'
            #                     if _grism not in lamps:
            #                         lamps[_grism]=[img]
            #                     else:
            #                         lamps[_grism].append(img)
            #                 else:
            #                     _type='notgood'
        if not _type:
            _ra = readkey3(hdr, 'RA')
            _dec = readkey3(hdr, 'DEC')
            _object_name = readkey3(hdr, 'object')
            _OBID = (readkey3(hdr, 'esoid'), _grism)
            if string.count(_object_name, '/') or string.count(_object_name, '.') or string.count(_object_name, ' '):
                nameobj = string.split(_object_name, '/')[0]
                nameobj = string.split(nameobj, ' ')[0]
                nameobj = string.split(nameobj, '.')[0]
            else:
                nameobj = _object_name
            if _grism not in fieldlist:
                fieldlist[_grism] = {}
            if _OBID not in OBID:
                count = 1
                nameobj0 = nameobj + '_' + str(count)
                answ = 'yes'
                while answ == 'yes':
                    if nameobj0 in fieldlist[_grism]:
                        count = count + 1
                        nameobj0 = nameobj + '_' + str(count)
                    else:
                        answ = 'no'
                fieldlist[_grism][nameobj0] = []
                OBID[readkey3(hdr, 'esoid'), _grism] = nameobj0
            fieldlist[_grism][nameobj0].append(img)

        if _verbose:
            print img
            print _type, _object, _filter
            print 'lamps', lamps1

    lamps = {}
    for _lampid in lamps1:
        lamp = ''
        output = 'arc_' + str(_lampid[0]) + '_' + str(_lampid[1]) + '.fits'
        if lamps1[_lampid][0] and lamps1[_lampid][1]:
            print lamps1[_lampid][0], lamps1[_lampid][1]
            # try:
            ntt.util.delete(output)
            iraf.imarith(lamps1[_lampid][0], '-', lamps1[_lampid]
                         [1], result=output, verbose='yes')
            #            except:
            #                print 'warning, lamp file not ON/OFF'
            #                os.system('cp '+lamps1[_lampid][0]+' '+output)

            lamp = output
        elif lamps1[_lampid][0] and not lamps1[_lampid][1]:
            os.system('cp ' + lamps1[_lampid][0] + ' ' + output)
            lamp = output
        if lamp:
            if _lampid[1] not in lamps:
                lamps[_lampid[1]] = [lamp]
            else:
                lamps[_lampid[1]].append(lamp)

    if _verbose:
        print '\n### FIELDS\n', fieldlist
        print '\n### OBID\n', OBID
        print '\n### FLATS\n', flats
        print '\n### LAMPS\n', lamps

#    if not flats:
#        sys.exit('\n### error: spectroscopic flat not available, add flats in the directory and try again')
#    if not lamps:
# sys.exit('\n### error: spectroscopic lamp not available, add lamps in
# the directory and try again')

    if not listflat:
        print '\n### list of available spectroscopic flats (ON,OFF):'
        for _date in flats:
            for _grism in flats[_date]:
                for img in flats[_date][_grism]:
                    if pyfits.open(img)[0].data.mean() >= 2000:
                        print img, _grism, _date, 'ON ? '
                    else:
                        print img, _grism, _date, 'OFF ? '
        for _date in flats:
            for _grism in flats[_date]:
                flat = {'ON': [], 'OFF': []}
                for img in flats[_date][_grism]:
                    _type = ''
                    if readkey3(hdr, 'lamp3'):
                        print '\n### header lamp3 found: flat ON ', str(img)
                        _type = 'ON'
                    else:
                        if pyfits.open(img)[0].data.mean() >= 2000:
                            _type = 'ON'
                        else:
                            _type = 'OFF'
                    aa, bb, cc = ntt.util.display_image(img, 1, '', '', False)
                    print '\n### number of flat already selected (ON,OFF): \n ### please select same number ' \
                          'of ON and OFF flats \n' + \
                        str(len(flat['ON'])) + '  ' + str(len(flat['OFF']))
                    print '\n### image ' + str(img)
                    answ = raw_input(
                        'ON/OFF/REJECT/STOP [' + str(_type) + ']  ok (ON[n]/OFF[f]/r/s) [' + _type + '] ? ')
                    if not answ:
                        answ = _type
                    if answ in ['ON', 'on', 'n']:
                        _type = 'ON'
                    if answ in ['OFF', 'off', 'f']:
                        _type = 'OFF'
                    if answ in ['s', 'S', 'STOP', 'stop', 'Stop']:
                        _type = 'stop'
                    if answ in ['r', 'R', 'reject']:
                        _type = 'r'
                    if _type in ['ON', 'OFF']:
                        flat[_type].append(img)
                    elif _type == 'stop':
                        if len(flat['ON']) == len(flat['OFF']) and len(flat['OFF']) >= 2:
                            break
                        elif len(flat['ON']) == len(flat['OFF']) and len(flat['OFF']) == 0:
                            break
                        else:
                            print '\n### Warning: you can stop only if the numbers of ON and OFF are the same'
                print len(flat['ON']), len(flat['OFF'])
                if len(flat['ON']) == len(flat['OFF']) and len(flat['OFF']) >= 2:
                    ff = open('_flatlist', 'w')
                    for ii in range(0, len(flat['OFF'])):
                        delete('flat_' + str(_date) + '_' + str(_grism) +
                               '_' + str(MJDtoday) + '_' + str(ii) + '.fits')
                        iraf.imarith(flat['ON'][ii], '-', flat['OFF'][ii],
                                     result='flat_' + str(_date) + '_' + str(_grism) + '_' + str(MJDtoday) + '_' + str(
                                         ii) + '.fits', verbose='no')
                        ff.write(
                            'flat_' + str(_date) + '_' + str(_grism) + '_' + str(MJDtoday) + '_' + str(ii) + '.fits\n')
                    ff.close()
                    masterflat = 'flat_' + \
                        str(_date) + '_' + str(_grism) + \
                        '_' + str(MJDtoday) + '.fits'
                    delete(masterflat)
                    _order = '80'
                    iraf.ccdred.flatcombine(input='@_flatlist', output=masterflat, combine='median', rdnoise=_rdnoise,
                                            gain=_gain, ccdtype='')
                    hdr = readhdr(masterflat)
                    matching = [s for s in hdr.keys() if "IMCMB" in s]
                    for imcmb in matching:
                        aaa = iraf.hedit(masterflat, imcmb, delete='yes', update='yes',
                                         verify='no', Stdout=1)
                    delete('_flatlist')
                    print masterflat
                    correctcard(masterflat)
                    if masterflat not in outputlist:
                        outputlist.append(masterflat)
                    ntt.util.updateheader(masterflat, 0, {'FILETYPE': [41102, 'flat field'],
                                                          'SINGLEXP': [False, 'TRUE if resulting from single exposure'],
                                                          'M_EPOCH': [False, 'TRUE if resulting from multiple epochs']})

                    print '\n###  master flat ........... done '
                    delete('n' + masterflat)
                    iraf.specred.response(masterflat, normaliz=masterflat + '[100:900,*]',
                                          response='n' + masterflat, interac=_interact, thresho='INDEF', sample='*',
                                          naverage=2,
                                          function='spline3', low_rej=3, high_rej=3, order=_order, niterat=20, grow=0,
                                          graphic='stdgraph', mode='q')
                    listflat.append('n' + masterflat)
                    if 'n' + masterflat not in outputlist:
                        outputlist.append('n' + masterflat)
                    ntt.util.updateheader('n' + masterflat, 0, {'FILETYPE': [41203, 'normalized flat field'],
                                                                'TRACE1': [masterflat, 'Originating file']})
                    # ntt.util.updateheader('n'+masterflat,0,{'TRACE1':[masterflat,'']})

                    flattot = flat['ON'] + flat['OFF']
                    num = 0
                    for img in flattot:
                        num = num + 1
                        ntt.util.updateheader(masterflat, 0, {
                            'PROV' + str(num): [readkey3(readhdr(img), 'ARCFILE'), 'Originating file'],
                            'TRACE' + str(num): [readkey3(readhdr(img), 'ARCFILE'), 'Originating file']})
                        ntt.util.updateheader('n' + masterflat, 0, {
                            'PROV' + str(num): [readkey3(readhdr(img), 'ARCFILE'), 'Originating file']})

                    if listflat:
                        print '\n### flat available:\n### ' + str(listflat), '\n'
                elif len(flat['ON']) == len(flat['OFF']) and len(flat['OFF']) == 0:
                    print '\n### no good flats in this set ......'
                else:
                    sys.exit('\n### Error: number of ON and OFF not the same')

    for _grism in fieldlist:
        obj0 = fieldlist[_grism][fieldlist[_grism].keys()[0]][0]
        # #############              arc              #########################
        if _grism not in lamps:
            print '\n### take arc from archive '
            arcfile = ntt.util.searcharc(obj0, '')[0]
            if arcfile[0] == '/':
                os.system('cp ' + arcfile + ' ' +
                          string.split(arcfile, '/')[-1])
                arcfile = string.split(arcfile, '/')[-1]
            lamps[_grism] = [arcfile]

        if _grism in lamps:
            arclist = lamps[_grism]
            if arclist:
                arcfile = ntt.util.searcharc(obj0, arclist)[0]
            else:
                arcfile = ntt.util.searcharc(obj0, '')[0]

            print arcfile
            if arcfile:
                print arcfile
                datea = readkey3(readhdr(arcfile), 'date-night')
                if arcfile[0] == '/':
                    os.system('cp ' + arcfile + ' ' +
                              string.split(arcfile, '/')[-1])
                    arcfile = string.split(arcfile, '/')[-1]

                if _doflat:
                    if listflat:
                        flat0 = ntt.util.searchflat(arcfile, listflat)[0]
                    else:
                        flat0 = ''
                else:
                    flat0 = ''

                if flat0:
                    _flatcor = 'yes'
                else:
                    _flatcor = 'no'
                    _doflat = False

                ntt.util.delete('arc_' + datea + '_' + _grism +
                                '_' + str(MJDtoday) + '.fits')

                print arcfile, flat0, _flatcor, _doflat

                if _doflat:
                    iraf.noao.imred.ccdred.ccdproc(arcfile,
                                                   output='arc_' + datea + '_' + _grism +
                                                   '_' +
                                                   str(MJDtoday) + '.fits',
                                                   overscan='no', trim='no', zerocor='no', flatcor=_flatcor, flat=flat0)
                else:
                    os.system('cp ' + arcfile + ' ' + 'arc_' + datea +
                              '_' + _grism + '_' + str(MJDtoday) + '.fits')

                iraf.noao.imred.ccdred.ccdproc('arc_' + datea + '_' + _grism + '_' + str(MJDtoday) + '.fits', output='',
                                               overscan='no', trim='yes', zerocor='no', flatcor='no', flat='',
                                               trimsec='[30:1000,1:1024]')

                arcfile = 'arc_' + datea + '_' + \
                    _grism + '_' + str(MJDtoday) + '.fits'

                ntt.util.correctcard(arcfile)
                print arcfile

                if arcfile not in outputlist:
                    outputlist.append(arcfile)

                ntt.util.updateheader(arcfile, 0, {'FILETYPE': [41104, 'pre-reduced 2D arc'],
                                                   'SINGLEXP': [True, 'TRUE if resulting from single exposure'],
                                                   'M_EPOCH': [False, 'TRUE if resulting from multiple epochs'],
                                                   'PROV1': [readkey3(readhdr(arcfile), 'ARCFILE'), 'Originating file'],
                                                   'TRACE1': [readkey3(readhdr(arcfile), 'ARCFILE'),
                                                              'Originating file']})

                arcref = ntt.util.searcharc(obj0, '')[0]
                if not arcref:
                    identific = iraf.longslit.identify(images=arcfile, section='column 10',
                                                       coordli='direc$standard/ident/Lines_XeAr_SOFI.dat', nsum=10,
                                                       fwidth=7, order=3, mode='h', Stdout=1, verbose='yes')
                else:
                    print arcref
                    os.system('cp ' + arcref + ' .')
                    arcref = string.split(arcref, '/')[-1]
                    if not os.path.isdir('database/'):
                        os.mkdir('database/')
                    if os.path.isfile(ntt.util.searcharc(obj0, '')[1] + '/database/id' + re.sub('.fits', '', arcref)):
                        os.system('cp ' + ntt.util.searcharc(obj0, '')[1] + '/database/id' + re.sub('.fits', '',
                                                                                                    arcref) + ' database/')

                    print arcref, arcfile
                    #                        time.sleep(5)
                    #                        os.system('rm -rf database/idarc_20130417_GR_56975')
                    #                        raw_input('ddd')
                    identific = iraf.longslit.reidentify(referenc=arcref, images=arcfile, interac='NO',  # _interact,
                                                         section='column 10', shift=0.0,
                                                         coordli='direc$standard/ident/Lines_XeAr_SOFI.dat',
                                                         overrid='yes', step=0, newaps='no', nsum=5, nlost=2,
                                                         mode='h', verbose='yes', Stdout=1)
                    #                        print identific
                    #                        raw_input('ddd')
                    identific = iraf.longslit.reidentify(referenc=arcref, images=arcfile, interac=_interact,
                                                         section='column 10', shift=1.0,
                                                         coordli='direc$standard/ident/Lines_XeAr_SOFI.dat',
                                                         overrid='yes', step=0, newaps='no', nsum=5, nlost=2,
                                                         mode='h', verbose='yes', Stdout=1)
                    #                        fitsfile = ntt.efoscspec2Ddef.continumsub('new3.fits', 6, 1)
                    # I need to run twice I don't know why
                    #                        print identific
                    #                        raw_input('ddd')
                    if _interactive:
                        answ = raw_input(
                            '\n### do you like the identification [[y]/n]')
                        if not answ:
                            answ = 'y'
                    else:
                        answ = 'y'
                    if answ in ['n', 'N', 'no', 'NO', 'No']:
                        yy1 = pyfits.open(arcref)[0].data[:, 10:20].mean(1)
                        xx1 = arange(len(yy1))
                        yy2 = pyfits.open(arcfile)[0].data[:, 10:20].mean(1)
                        xx2 = arange(len(yy2))

                        ntt.util.delete('_new3.fits')
                        hdu = pyfits.PrimaryHDU(yy1)
                        hdulist = pyfits.HDUList([hdu])
                        hdulist.writeto('_new3.fits')

                        fitsfile = ntt.efoscspec2Ddef.continumsub('_new3.fits', 4, 1)
                        yy1 = pyfits.open(fitsfile)[0].data

                        ntt.util.delete('_new3.fits')
                        hdu = pyfits.PrimaryHDU(yy2)
                        hdulist = pyfits.HDUList([hdu])
                        hdulist.writeto('_new3.fits')

                        fitsfile = ntt.efoscspec2Ddef.continumsub('_new3.fits', 4, 1)
                        yy2 = pyfits.open(fitsfile)[0].data

                        _shift = ntt.efoscspec2Ddef.checkwavelength_arc(
                            xx1, yy1, xx2, yy2, '', '') * (-1)

                        print arcref, arcfile, _shift
                        identific = iraf.longslit.reidentify(referenc=arcref, images=arcfile, interac='YES',
                                                             section='column 10', shift=_shift,
                                                             coordli='direc$standard/ident/Lines_XeAr_SOFI.dat',
                                                             overrid='yes', step=0, newaps='no', nsum=5, nlost=2,
                                                             mode='h', verbose='yes', Stdout=1)

                        answ = raw_input('\n### is it ok now ? [[y]/n] ')
                        if not answ:
                            answ = 'y'
                        if answ in ['n', 'N', 'no', 'NO', 'No']:
                            sys.exit(
                                '\n### Warning: line identification with some problems')
                iraf.longslit.reidentify(referenc=arcfile, images=arcfile, interac='NO', section='column 10',
                                         coordli='direc$standard/ident/Lines_XeAr_SOFI.dat', overrid='yes', step=10,
                                         newaps='yes', nsum=5, nlost=2, mode='h', verbose='no')
                iraf.longslit.fitcoords(images=re.sub('.fits', '', arcfile), fitname=re.sub('.fits', '', arcfile),
                                        interac='no', combine='yes', databas='database',
                                        function='legendre', yorder=4, logfile='', plotfil='', mode='h')
                if identific:
                    _rms = float(identific[-1].split()[-1])
                    _num = float(identific[-1].split()[2].split('/')[0])
                    hdr = ntt.util.readhdr(arcfile)
                    hedvec = {'LAMRMS': [_rms * .1, 'residual RMS [nm]'],
                              'LAMNLIN': [_num, 'Nb of arc lines used in the fit of the wavel. solution'],
                              'SPEC_ERR': [(_rms * .1) / sqrt(float(_num)), 'statistical uncertainty'],
                              'SPEC_SYE': [0.1, 'systematic error']}
                    ntt.util.updateheader(arcfile, 0, hedvec)
            else:
                sys.exit('Warning: arcfile not found')
        else:
            print 'here'
        # ########################################################################################################
        for field in fieldlist[_grism]:
            listaobj = fieldlist[_grism][field]
            listaobj = ntt.sofiphotredudef.sortbyJD(listaobj)
            listatemp = listaobj[:]
            # ##############             flat            ######################
            if listflat and _doflat:
                flat0 = ntt.util.searchflat(listaobj[0], listflat)[0]
            else:
                flat0 = ''
            if flat0:
                _flatcor = 'yes'
            else:
                _flatcor = 'no'

            ##########   crosstalk        ###########################

            listatemp2 = []
            _date = readkey3(readhdr(listatemp[0]), 'date-night')
            for img in listatemp:
                #                    num2=listatemp.index(listasub[j])
                imgout = field + '_' + str(_date) + '_' + str(_grism) + '_' + str(MJDtoday) + '_' + str(
                    listatemp.index(img)) + '.fits'
                print '\n### input image: ' + str(img)
                delete(imgout)
                listatemp2.append(imgout)
                if _docross:
                    print '### correct for cross talk   .....   done'
                    ntt.sofiphotredudef.crosstalk(img, imgout)
                    correctcard(imgout)
                    ntt.util.updateheader(
                        imgout, 0, {'CROSSTAL': ['True', '']})
                else:
                    os.system('cp ' + img + ' ' + imgout)
                    correctcard(imgout)
                if _flatcor == 'yes':
                    print '### correct for flat field   .....   done'
                    try:
                        iraf.noao.imred.ccdred.ccdproc(imgout, output='', overscan='no', trim='no', zerocor='no',
                                                       flatcor=_flatcor, flat=flat0)
                    except:
                        iraf.imutil.imreplace(
                            images=flat0, value=0.01, lower='INDEF', upper=0.01, radius=0)
                        iraf.noao.imred.ccdred.ccdproc(imgout, output='', overscan='no', trim='no', zerocor='no',
                                                       flatcor=_flatcor, flat=flat0)
                iraf.noao.imred.ccdred.ccdproc(imgout, output='', overscan='no', trim='yes', zerocor='no',
                                               flatcor='no', flat='', trimsec='[30:1000,1:1024]')
                ntt.util.updateheader(
                    imgout, 0, {'FLATCOR': [flat0, 'flat correction']})

                if imgout not in outputlist:
                    outputlist.append(imgout)
                ntt.util.updateheader(imgout, 0, {'FILETYPE': [42104, 'pre-reduced frame'],
                                                  'SINGLEXP': [True, 'TRUE if resulting from single exposure'],
                                                  'M_EPOCH': [False, 'TRUE if resulting from multiple epochs'],
                                                  'PROV1': [readkey3(readhdr(imgout), 'ARCFILE'), 'Originating file'],
                                                  'TRACE1': [readkey3(readhdr(imgout), 'ARCFILE'), 'Originating file']})
                print '### output image: ' + str(imgout)

            listatemp = listatemp2[:]
            #########    differences object images  #####################
            listasub = ntt.sofispec2Ddef.findsubimage(listatemp)
            reduced = []
            print '\n### Select Frames to be subtracted (eg A-B, B-A, C-D, D-C, ....) '
            print '###    frame1 \t  frame2  \t   offset1  \t   offset2  \t  JD1  \t    JD2\n'
            if len(listatemp) >= 2 and len(listasub) >= 2:
                for j in range(0, len(listatemp)):
                    print '### ', listatemp[j], listasub[j], str(readkey3(readhdr(listatemp[j]), 'xcum')), str(
                        readkey3(readhdr(listasub[j]), 'xcum')), \
                        str(readkey3(readhdr(listatemp[j]), 'JD')), str(
                            readkey3(readhdr(listatemp[j]), 'JD'))
                    if _interactive:
                        answ = raw_input('\n### ok [[y]/n] ? ')
                        if not answ:
                            answ = 'y'
                    else:
                        answ = 'y'
                    num1 = j
                    image1 = listatemp[j]
                    _date = readkey3(readhdr(image1), 'date-night')
                    if answ == 'y':
                        num2 = listatemp.index(listasub[j])
                        image2 = listasub[j]
                    else:
                        image2 = raw_input(
                            'which image do you want to subtract')
                        num2 = listatemp.index(image2)
                    imgoutsub = field + '_' + str(_date) + '_' + str(_grism) + '_' + str(MJDtoday) + '_' + str(
                        num1) + '_' + str(num2) + '.fits'
                    delete(imgoutsub)
                    iraf.images.imutil.imarith(
                        operand1=image1, op='-', operand2=image2, result=imgoutsub, verbose='no')
                    ntt.util.updateheader(imgoutsub, 0, {'skysub': [image2, 'sky image subtracted'],
                                                         'FILETYPE': [42115, 'pre-reduced frame sky subtracted'],
                                                         'TRACE1': [image1, 'Originating file'],
                                                         'PROV2': [readkey3(readhdr(image2), 'ARCFILE'),
                                                                   'Originating file'],
                                                         'TRACE2': [image2, 'Originating file']})

                    reduced.append(imgoutsub)
                    if imgoutsub not in outputlist:
                        outputlist.append(imgoutsub)
            ########################     2D wavelengh calibration      ########
            for img in reduced:
                if arcfile:
                    hdra = ntt.util.readhdr(arcfile)
                    delete('t' + img)
                    iraf.specred.transform(input=img, output='t' + img, minput='',
                                           fitnames=re.sub('.fits', '', arcfile), databas='database',
                                           x1='INDEF', x2='INDEF', y1='INDEF', y2='INDEF', flux='yes', mode='h',
                                           logfile='logfile')
                    ntt.util.updateheader('t' + img, 0,
                                          {'ARC': [arcfile, ''], 'FILETYPE': [42106, 'wavelength calibrate 2D frames'],
                                           'TRACE1': [img, 'Originating file']})
                    ntt.util.updateheader(
                        't' + img, 0, {'TRACE1': [img, 'Originating file']})
                    ntt.util.updateheader('t' + img, 0,
                                          {'LAMRMS': [ntt.util.readkey3(hdra, 'LAMRMS'), 'residual RMS [nm]'],
                                           'LAMNLIN': [ntt.util.readkey3(hdra, 'LAMNLIN'), 'number of arc lines'],
                                           'SPEC_ERR': [ntt.util.readkey3(hdra, 'SPEC_ERR'), 'statistical uncertainty'],
                                           'SPEC_SYE': [ntt.util.readkey3(hdra, 'SPEC_SYE'), 'systematic error']})
                    ###########################
                    delete('t' + arcfile)
                    iraf.specred.transform(input=arcfile, output='t' + arcfile, minput='',
                                           fitnames=re.sub('.fits', '', arcfile), databas='database',
                                           x1='INDEF', x2='INDEF', y1='INDEF', y2='INDEF', flux='yes', mode='h',
                                           logfile='logfile')
                    specred = ntt.util.spectraresolution2(arcfile, 50)
                    if specred:
                        ntt.util.updateheader(
                            't' + img, 0, {'SPEC_RES': [specred, 'Spectral resolving power']})
                    delete('t' + arcfile)
                    ###########################
                    iraf.hedit('t' + img, 'TRACE2', delete='yes',
                               update='yes', verify='no', Stdout=1)

                    if 't' + img not in outputlist:
                        outputlist.append('t' + img)
                    print '\n### 2D frame t' + str(img) + ' wavelengh calibrated  ............ done'

                    _skyfile = ntt.__path__[
                        0] + '/standard/ident/sky_' + _grism + '.fits'  # check in wavelengh   #########
                    hdr = ntt.util.readhdr(img)
                    if glob.glob(_skyfile) and readkey3(hdr, 'exptime') > 20.:
                        _original = readkey3(hdr, 'ORIGFILE')
                        _archive = readkey3(hdr, 'ARCFILE')
                        if os.path.isfile(_archive):
                            imgstart = _archive
                        elif os.path.isfile(_original):
                            imgstart = _original
                        else:
                            imgstart = ''
                        if imgstart:
                            delete('_tmp.fits')
                            print imgstart, arcfile
                            iraf.specred.transform(input=imgstart, output='_tmp.fits', minput='',
                                                   fitnames=re.sub('.fits', '', arcfile), databas='database',
                                                   x1='INDEF', x2='INDEF', y1='INDEF', y2='INDEF', flux='yes', mode='h',
                                                   logfile='logfile')

                            shift = ntt.sofispec2Ddef.skysofifrom2d('_tmp.fits', _skyfile)
                            zro = pyfits.open('_tmp.fits')[0].header.get('CRVAL2')

                            delete('_tmp.fits')
                            if _interactive:
                                answ = raw_input(
                                    'do you want to correct the wavelengh calibration with this shift: ' + str(
                                        shift) + ' [[y]/n] ? ')
                                if not answ:
                                    answ = 'y'
                            else:
                                answ = 'y'
                            if answ.lower() in ['y', 'yes']:
                                ntt.util.updateheader('t' + img, 0,
                                                      {'CRVAL2': [zro + int(shift), ''], 'shift': [float(shift), '']})
                            #                                    ntt.util.updateheader('t'+img,0,{'shift':[float(shift),'']})
                            print '\n### check wavelengh calibration with sky lines ..... done'
                    try:
                        hdrt = ntt.util.readhdr('t' + img)
                        wavelmin = float(readkey3(hdrt, 'CRVAL2')) + (0.5 - float(readkey3(hdrt, 'CRPIX2'))) * float(
                            readkey3(hdrt, 'CDELT2'))
                        wavelmax = float(readkey3(hdrt, 'CRVAL2')) + (
                            (float(readkey3(hdrt, 'NAXIS2')) + 0.5 - float(readkey3(hdrt, 'CRPIX2'))) * float(
                                readkey3(hdrt, 'CDELT2')))
                        hedvec = {}
                        hedvec['WAVELMIN'] = [
                            wavelmin * .1, '[nm] minimum wavelength']
                        hedvec['WAVELMAX'] = [
                            wavelmax * .1, ' [nm] maximum wavelength']
                        hedvec['XMIN'] = [wavelmin, '[A] minimum wavelength']
                        hedvec['XMAX'] = [wavelmax, '[A]  maximum wavelength']
                        hedvec['SPEC_BW'] = [
                            (wavelmax * .1) - (wavelmin * .1), '[nm] Bandpass Width Wmax - Wmin']
                        hedvec['SPEC_VAL'] = [
                            ((wavelmax * .1) + (wavelmin * .1)) / 2., '[nm] Mean Wavelength']
                        hedvec['SPEC_BIN'] = [
                            ((wavelmax * .1) - (wavelmin * .1)) /
                            (float(readkey3(hdr, 'NAXIS2')) - 1),
                            'Wavelength bin size [nm/pix]']
                        hedvec['VOCLASS'] = ['SPECTRUM V1.0', 'VO Data Model']
                        hedvec['VOPUB'] = ['ESO/SAF',
                                           'VO Publishing Authority']
                        #                            hedvec['APERTURE']=[float(re.sub('slit','',readkey3(hdrt,'slit'))),'aperture width']
                        ntt.util.updateheader('t' + img, 0, hedvec)
                    except:
                        pass
                else:
                    print '\n### Warning: arc not found for the image ' + str(img) + ' with setup ' + str(_grism)

    reduceddata = rangedata(outputlist)
    print '\n### adding keywords for phase 3 ....... '
    f = open('logfile_spec2d_' + str(reduceddata) +
             '_' + str(datenow) + '.raw.list', 'w')
    for img in outputlist:
        if img[-4:] == 'fits':
            hdr = readhdr(img)
            # ###############################################
            # cancel pc matrix
            if 'PC1_1' in hdr.keys():
                aaa = iraf.hedit(img, 'PC1_1', delete='yes',
                                 update='yes', verify='no', Stdout=1)
            if 'PC2_2' in hdr.keys():
                aaa = iraf.hedit(img, 'PC2_2', delete='yes',
                                 update='yes', verify='no', Stdout=1)
            if 'PC1_2' in hdr.keys():
                aaa = iraf.hedit(img, 'PC1_2', delete='yes',
                                 update='yes', verify='no', Stdout=1)
            if 'PC2_1' in hdr.keys():
                aaa = iraf.hedit(img, 'PC2_1', delete='yes',
                                 update='yes', verify='no', Stdout=1)
            #################
            # added for DR2
            print img

            if 'NCOMBINE' in hdr:
                _ncomb = readkey3(hdr, 'NCOMBINE')
            else:
                _ncomb = 1.0

            ntt.util.updateheader(
                img, 0, {'DETRON ': [12, 'Readout noise per output (e-)']})
            ntt.util.updateheader(img, 0, {'EFFRON': [12. * (1 / sqrt(readkey3(hdr, 'ndit') * _ncomb)) * sqrt(pi / 2),
                                                      'Effective readout noise per output (e-)']})
            ntt.util.phase3header(img)  # phase 3 definitions
            ############################
            #  change for DR2
            ############################
            texp = float(readkey3(hdr, 'dit')) * float(readkey3(hdr, 'ndit'))
            mjdend = float(readkey3(hdr, 'MJD-OBS')) + (float(readkey3(hdr, 'ndit')) * (
                float(readkey3(hdr, 'dit')) + 1.8)) / (60. * 60. * 24.)
            strtexp = time.strftime('%H:%M:%S', time.gmtime(texp))
            _telapse = (mjdend - float(readkey3(hdr, 'MJD-OBS'))) * \
                60. * 60 * 24.
            # tmid=_telapse/2.
            tmid = (mjdend + float(readkey3(hdr, 'MJD-OBS'))) / 2
            ntt.util.updateheader(img, 0, {'quality': ['Final', 'fast or rapid reduction'],
                                           'BUNIT': ['ADU', 'Physical unit of array values'],
                                           'DIT': [readkey3(hdr, 'dit'), 'Detector Integration Time'],
                                           'NDIT': [readkey3(hdr, 'ndit'), 'Number of sub-integrations'],
                                           'TEXPTIME': [texp, 'Total integration time of all exposures (s)'],
                                           'EXPTIME': [texp, 'Total integration time. ' + strtexp],
                                           'MJD-END': [mjdend, 'End of observations (days)'],
                                           'TELAPSE': [_telapse, 'Total elapsed time [days]'],
                                           'TMID': [tmid, '[d] MJD mid exposure'],
                                           'TITLE': [readkey3(hdr, 'object'), 'Dataset title'],
                                           #'TITLE':[str(tmid)[0:9]+' '+str(readkey3(hdr,'object'))+' '+str(readkey3(hdr,'grism'))+' '+\
                                           # str(readkey3(hdr,'filter'))+'
                                           # '+str(readkey3(hdr,'slit')),'Dataset
                                           # title'],\
                                           'EXT_OBJ': [False, 'TRUE if extended'],
                                           'CONTNORM': [False, 'spectrum normalized to the continuum'],
                                           'TOT_FLUX': [False, 'TRUE if phot cond and all src flux is captured'],
                                           'SPECSYS': ['TOPOCENT', 'Reference frame for spectral coordinate'],
                                           'FLUXCAL': ['ABSOLUTE', 'type of flux calibration'],
                                           'FLUXERR': [34.7, 'Fractional uncertainty of the flux [%]'],
                                           'DISPELEM': ['Gr#' + re.sub('Gr', '', readkey3(hdr, 'grism')),
                                                        'Dispersive element name']})
            if readkey3(hdr, 'tech'):
                ntt.util.updateheader(
                    img, 0, {'PRODCATG': ['SCIENCE.IMAGE', 'Data product category']})
            aaa = str(readkey3(hdr, 'arcfiles')) + '\n'
            f.write(aaa)
            try:
                ntt.util.airmass(img)  # phase 3 definitions
            except:
                print '\n### airmass not computed for image: ', img
        else:
            print img + ' is not a fits image'
    f.close()
    return outputlist, 'logfile_spec2d_' + str(reduceddata) + '_' + str(datenow) + '.raw.list'
コード例 #22
0
ファイル: rcred.py プロジェクト: nblago/kpy
def create_masterflat(flatdir=None, biasdir=None, channel='rc'):
    '''
    Creates a masterflat from both dome flats and sky flats if the number of counts in the given filter
    is not saturated and not too low (between 1500 and 40000). 
    '''
    
    
    if (flatdir == None or flatdir==""): flatdir = "."
        
    if (biasdir == None or biasdir==""): biasdir = "."
        
    os.chdir(flatdir)
    
    if (len(glob.glob("Flat_%s*norm.fits"%channel)) == 4):
        print "Master Flat exists!"
        return 
    else:
        print "Starting the Master Flat creation!"

    bias_slow = "Bias_%s_fast.fits"%channel
    bias_fast = "Bias_%s_fast.fits"%channel
    
    if (not os.path.isfile(bias_slow) and not os.path.isfile(bias_fast) ):
        create_masterbias(biasdir)
     
    lsflat = []
    lfflat = []
    
    #Select all filts that are Flats with same instrument
    for f in glob.glob("*fits"):
        #try:
        if fitsutils.has_par(f, "OBJECT"):
            obj = str.upper(fitsutils.get_par(f, "OBJECT"))
        else:
            continue
        
        if ( ("DOME" in  obj or "FLAT" in obj) and (channel == fitsutils.get_par(f, "CHANNEL"))):
            if (fitsutils.get_par(f, "ADCSPEED")==2):
                lfflat.append(f)
            else:
                lsflat.append(f)
        #except:
        #    print "Error with retrieving parameters for file", f
        #    pass
                
    print "Files for slow flat", lsflat
    print "Files for fast flat", lfflat
    
    fsfile ="lflat_slow_"+channel
    np.savetxt(fsfile, np.array(lsflat), fmt="%s")
    fffile ="lflat_fast_"+channel
    np.savetxt(fffile, np.array(lfflat), fmt="%s")



    # Running IRAF
    iraf.noao(_doprint=0)
    iraf.imred(_doprint=0)
    iraf.ccdred(_doprint=0)
    
    #Remove bias from the flat
    if len(lsflat) >0:
        iraf.imarith("@"+fsfile, "-", bias_slow, "[email protected]"+fsfile)
    
    if len(lfflat) >0:
        iraf.imarith("@"+fffile, "-", bias_fast, "[email protected]"+fffile)    
    
    #Slices the flats.
    debiased_flats = glob.glob("b_*.fits")
    for f in debiased_flats:
        print "Slicing file", f
        slice_rc(f)
        #Remove the un-sliced file
        os.remove(f)
        
    #Selects the ones that are suitable given the number of counts and combines them.
    bands = ['u', 'g', 'r', 'i']
    for b in bands:
        out = "Flat_%s_%s.fits"%(channel, b)
        out_norm = out.replace(".fits","_norm.fits")

        if (os.path.isfile(out_norm)):
            print "Master Flat for filter %s exists. Skipping..."%b
            continue
        
        lfiles = []
        for f in glob.glob('b_*_%s.fits'%b):
            d = pf.open(f)[0].data
            if np.percentile(d, 90)>1500 and np.percentile(d, 90)<40000:
                lfiles.append(f)

        if len(lfiles) == 0:
            print "WARNING!!! Could not find suitable flats for band %s"%b
            continue
        ffile ="lflat_"+b
        np.savetxt(ffile, np.array(lfiles), fmt="%s")
    
        
        #Cleaning of old files
        if(os.path.isfile(out)): os.remove(out)
        if(os.path.isfile(out_norm)): os.remove(out_norm)
        if(os.path.isfile("Flat_stats")): os.remove("Flat_stats")
        
        
        #Combine flats
        iraf.imcombine(input = "@"+ffile, \
                        output = out, \
                        combine = "median",\
                        scale = "mode",
                        weight = "exposure")
        iraf.imstat(out, fields="image,npix,mean,stddev,min,max,mode", Stdout="Flat_stats")
        st = np.genfromtxt("Flat_stats", names=True, dtype=None)
        #Normalize flats
        iraf.imarith(out, "/", st["MODE"], out_norm)
        
        #Do some cleaning
        print 'Removing from lfiles'
        for f in glob.glob('b_*_%s.fits'%b):
            os.remove(f)

        os.remove(ffile)
        
        
        if os.path.isfile(fsfile):
            os.remove(fsfile)
        if os.path.isfile(fffile):
            os.remove(fffile)
コード例 #23
0
ファイル: kaitutils.py プロジェクト: cenko/python
from pyraf import iraf
import copy, os, shutil, glob, sys, string, re, math, operator, time
import pyfits
from types import *
from mx.DateTime import *

from iqpkg import *
import ephem

# Necessary packages
iraf.images()
iraf.immatch()
iraf.imfilter()
iraf.noao()
iraf.imred()
iraf.ccdred()
iraf.digiphot()
iraf.apphot()

yes=iraf.yes
no=iraf.no
INDEF=iraf.INDEF
hedit=iraf.hedit
imgets=iraf.imgets
imcombine=iraf.imcombine

pyrafdir="python/pyraf/"
pyrafdir_key='PYRAFPARS'

if os.environ.has_key(pyrafdir_key):
    pardir=os.environ[pyrafdir_key]
コード例 #24
0
ファイル: apored.py プロジェクト: drewchojnowski/ASTR535
def test(instr=None,rawdata_dir='UT160327',reduction_dir=None,dograting=None,dofilter=None,silent=False):
    print('='*70)
    print("---------- Drew's APO 3.5m reduction script ----------")
    print('='*70)

    print('loading some basic iraf packages...')
    iraf.noao(); iraf.imred(); iraf.ccdred()
    print('='*70)

##########################################################################################################
    print('1. INSTRUMENT ESTABLISHMENT\n')
    if instr is None:
        print('Which APO 3.5m instrument did you use (default is ARCTIC)? Choices:')
        print(' - enter 1 for ARCTIC (default)')
        print(' - enter 2 for DIS')
        print(' - enter 3 for something else')
        a=raw_input('')
        if a=='':  a='1'
        if a[:1]=='1': instr='arctic'
        if a[:1]=='2': instr='dis'
        if (a[:1]!='1') & (a[:1]!='2'): sys.exit("You're out of luck for now. Sorry!")
        print('\nOk. '+instr.upper()+' data will be reduced')
        print('='*70)

##########################################################################################################
    print('2. RAW DATA DIRECTORY ESTABLISHMENT\n')
    if rawdata_dir is None:
        a=raw_input('Enter the name of the directory where your images are stored: ')
        if a=='': sys.exit('You must specify a raw data directory. Exiting the APO 3.5m reduction script.')
        if a[-1]!='/': a=a+'/'
        rawdata_dir=a
        check=glob.glob(rawdata_dir)
        if len(check)==0: sys.exit('The directory "'+rawdata_dir+'" does not exist. Exiting the APO 3.5m reduction script.')
    else:
        if rawdata_dir[-1:]!='/': rawdata_dir=rawdata_dir+'/'
        check=glob.glob(rawdata_dir)
        if len(check)==0: sys.exit('The directory "'+rawdata_dir+'" does not exist. Exiting the APO 3.5m reduction script.')

    print('Ok. The raw data directory will be "'+rawdata_dir+'"')
    print('='*70)

##########################################################################################################
    print('3. RAW DATA CHECK\n')
    rawfiles = np.array(glob.glob(rawdata_dir+'*fits')); nraw=len(rawfiles)
    if nraw==0: sys.exit('No FITS files found in the raw data directory. Exiting the APO 3.5m reduction script.')
    instrlist=[]; imtype=[]
    for rawfile in rawfiles:
        head=fits.getheader(rawfile)
        instrlist.append(head['instrume'].lower())
    instrlist=np.array(instrlist)
    gd=np.where(instrlist==instr); ngd=len(gd[0])
    if ngd==0: 
        sys.exit('No FITS files from '+instr.upper()+' were found in the raw data directory. Exiting the APO 3.5m reduction script.')
    else:
        rawfiles=rawfiles[gd]
        print('Ok. Found '+str(ngd)+' FITS images with instrument='+instr.upper()+' in the raw data directory.')
        print('='*70)

##########################################################################################################
    print('4. REDUCTION DIRECTORY ESTABLISHMENT\n')
    if reduction_dir is not None:
        check=glob.glob(reduction_dir)
        if len(check)==0: 
            a=raw_input('Specified reduction directory does not exist. Create it (Y/N)? ')
            if (a=='') | (a[:1].lower()=='y'): 
                cmnd='mkdir '+reduction_dir
                os.system(cmnd)
                print('"'+reduction_dir+'" has been created.')
    else:
        reduction_dir=rawdata_dir[:-1]+'_reduction/'
        print('The reduction directory will be '+reduction_dir)
        print(' - hit ENTER to accept')
        print(' - otherwise enter a different reduction directory name: ')
        a=raw_input('')
        if a=='':
            check=glob.glob(reduction_dir)
            if len(check)==0:
                cmnd='mkdir '+reduction_dir
                os.system(cmnd)
                print('\nOk. "'+reduction_dir+'" has been created.')
        else:
            reduction_dir=a
            check=glob.glob(reduction_dir)
            if len(check)==0:
                cmnd='mkdir '+reduction_dir 
                os.system(cmnd)
                print('\nOk. "'+reduction_dir+'" has been created.')

    print('\nOk. Reduction products will be stored in '+reduction_dir+'.')
    print('='*70)

##########################################################################################################
    if instr=='dis':
        print('loading some more IRAF packages for DIS reduction...')
        iraf.twod(); iraf.apex(); iraf.longs(); iraf.oned()
        print('='*70)
        print('5. DIS PARAMETER ESTABLISHMENT\n')
        grating=[]; imagetypes=[]
        for rawfile in rawfiles:
            head=fits.getheader(rawfile)
            grating.append(head['grating'])
            imagetypes.append(head['imagetyp'].lower())
        imagetypes=np.array(imagetypes)
        grating=np.array(grating)
        uniquegrating=np.unique(grating)

        if dograting is None:
            print("Which DIS grating do you want to reduce? Here's what you have:")
            for i in range(len(uniquegrating)): 
                if i==0: 
                    print(' - enter '+str(i+1)+' for '+uniquegrating[i]+' (default)')
                else:
                    print(' - enter '+str(i+1)+' for '+uniquegrating[i])
            a=raw_input('')
            if a=='': a='1'
            dograting=uniquegrating[int(a[:1])-1]
            gd=np.where(grating==dograting); ngd=len(gd[0])
            rawfiles=rawfiles[gd]; imagetypes=imagetypes[gd]
            print('\nOk. '+dograting+' data will be reduced: '+str(ngd)+' total images')
        else:
            gd=np.where(grating==dograting); ngd=len(gd[0])
            if ngd==0:
                sys.exit('No '+dograting+' images found. Exiting the APO 3.5m reduction script.')
            else:
                rawfiles=rawfiles[gd]; imagetypes=imagetypes[gd]
                print('\nOk. '+dograting+' images will be reduced: '+str(ngd)+' total images')

        bias=np.where((imagetypes=='bias') | (imagetypes=='zero')); nbias=len(bias[0])
        zerocor='yes'
        if nbias==0:
            print('\nNo bias images were found. Proceed without bias correction? (y/n)')
            a=raw_input('')
            if (a[:1]=='y') | (a==''): 
                zerocor='no'; biasfiles=''; nbias=0
            else:
                sys.exit('Exiting the APO 3.5m reduction script.')
        else:
            biasfiles=rawfiles[bias]; nbias=len(bias[0])

        # get values depending on R vs B detector
        detector=dograting[:1].lower()
        if detector=='b':
            gain=1.68
            rdnoise=4.9
            badpixfile=''
            fixpix='no'
        if detector=='r':
            gain=1.88
            rdnoise=4.6
            badpixfile='badpix_disR.txt'
            fixpix='yes'

        head=fits.getheader(rawfiles[0])
        biassec=head['biassec']
        datasec=head['datasec']

        genericlabel=dograting

        print('\nOk. DIS parameters established.')
        print('='*70)

##########################################################################################################
    if instr=='arctic':
        print('5. ARCTIC PARAMETER ESTABLISHMENT\n')
        gain=2.0
        rdnoise=3.7
        badpixfile=''
        fixpix='no'

        filters=[]; imagetypes=[]
        for rawfile in rawfiles:
            head=fits.getheader(rawfile)
            filters.append(head['filter'])
            imagetypes.append(head['imagetyp'].lower())
        imagetypes=np.array(imagetypes)
        filters=np.array(filters)
        uniquefilters=np.unique(filters)

        bias=np.where(imagetypes=='bias'); nbias=len(bias[0])
        zcor='yes'
        if nbias==0:
            print('\nNo bias images were found. Proceed without bias correction? (y/n)')
            a=raw_input('')
            if (a[:1]=='y') | (a==''): 
                zcor='no'; biasfiles=''
            else:
                sys.exit('Exiting the APO 3.5m reduction script.')
        else:
            biasfiles=rawfiles[bias]; nbias=len(bias[0])

        # establish ARCTIC filter
        if dofilter is None:
            print('Which ARCTIC filter do you want to reduce? ')
            for i in range(len(uniquefilters)): 
                if i==0: 
                    print(' - enter '+str(i+1)+' for '+uniquefilters[i]+' (default)')
                else:
                    print(' - enter '+str(i+1)+' for '+uniquefilters[i])
            a=raw_input('')
            if a=='': a='1'
            dofilter=uniquefilters[int(a[:1])-1]
            gd=np.where(filters==dofilter); ngd=len(gd[0])
            rawfiles=rawfiles[gd]
            print('\nOk. '+dofilter+' filter images will be reduced: '+str(ngd)+' total images')
        else:
            gd=np.where(filters==dofilter); ngd=len(gd[0])
            if ngd==0:
                sys.exit('No '+dofilter+' images found. Exiting the APO 3.5m reduction script.')
            else:
                rawfiles=rawfiles[gd]
                print('\nOk. '+dofilter+' filter images will be reduced: '+str(ngd)+' total images')


        head=fits.getheader(rawfiles[0])
        biassec=head['bsec11']
        datasec=head['dsec11']

        genericlabel=dofilter.replace(' ','_')
        genericlabel=genericlabel.replace('/','_')

        print('\nOk. ARCTIC parameters established.')
        print('='*70)

##########################################################################################################
    print('6. GET IMAGE TYPES\n')
    nrawfiles = len(rawfiles)

    # gather some more info from headers
    imtype = []; exptime = []; ra = []; dec = []
    for rawfile in rawfiles:
        header = fits.getheader(rawfile)
        imtype.append(header['IMAGETYP'].lower())
        exptime.append(header['EXPTIME'])
        ra.append(header['RA'])
        dec.append(header['DEC'])
    imtype=np.array(imtype)
    exptime=np.array(exptime); ra=np.array(ra)
    dec=np.array(dec)

    # separate the files into biases, flat, objs, and comps
    flat=np.where(imtype=='flat');  nflat=len(flat[0]); flatfiles=rawfiles[flat]
    obj=np.where(imtype=='object'); nobj=len(obj[0]);   objfiles=rawfiles[obj]
    if instr=='dis':
        comp=np.where(imtype=='comp'); ncomp=len(comp[0]); compfiles=rawfiles[comp]
        print('Ok. You have '+str(nbias)+' biases, '+str(nflat)+' flats, '+str(ncomp)+' comps, and '+str(nobj)+' objects.')
    if instr=='arctic':
        print('Ok. You have '+str(nbias)+' biases, '+str(nflat)+' flats, and '+str(nobj)+' objects.')
    print('='*70)


##########################################################################################################
    print('7. MAKE MASTER BIAS\n')
    if zcor=='yes':
        masterzero=reduction_dir+'Zero_'+genericlabel+'.fits'
        a=raw_input('Average combine biases into master bias (y/n)? ')
        if (a[:1]=='y') | (a==''):
            iraf.imcombine(','.join(biasfiles),output=masterzero,combine='average',reject='avsigclip',lsigma='3',
                           hsigma='3',rdnoise=rdnoise,gain=gain)
            print('\nOk. A master bias has been made: '+masterzero)
        print('='*70)
    else:
        masterzero=''
        print('\nOk, you have no bias files. Proceeding')


##########################################################################################################
    print('8. BIAS AND OVERSCAN CORRECTION\n')
    a=raw_input('Overscan and bias-subtraction of the flats, objs, and if DIS, comps. Do it (y/n)? ')
    if (a[:1]=='y') | (a==''):
        # set up a list of the input files (flats, comps, objs)
        if instr=='dis': infiles=np.concatenate((flatfiles,compfiles,objfiles),axis=0)
        if instr=='arctic': infiles=np.concatenate((flatfiles,objfiles),axis=0)

        infilesfile=reduction_dir+'inlist_'+genericlabel+'_ccdproc1'
        np.savetxt(infilesfile,infiles,fmt='%s')
        
        # We're making changes to the images now, so need to set up a list of output files
        outfiles=[]
        for infile in infiles:
            tmp=infile.split('/')
            tmp1=tmp[len(tmp)-1].split('.fits')
            outfiles.append(reduction_dir+tmp1[0]+'_cproc1.fits')
        outfiles=np.array(outfiles)
        outfilesfile=reduction_dir+'outlist_'+genericlabel+'_ccdproc1'
        np.savetxt(outfilesfile,outfiles,fmt='%s')

        iraf.ccdproc('@'+infilesfile,output='@'+outfilesfile,ccdtype='',fixpix=fixpix,oversca='yes',trim='yes',
                     zerocor=zcor,darkcor='no',flatcor='no',fixfile=badpixfile,biassec=biassec,trimsec=datasec,
                     zero=masterzero,interac='no',low_rej='3',high_re='3')
        print('\nOk. Overscan and bias subtracting done.')
    print('='*70)

##########################################################################################################
    print('9. MAKE MASTER FLAT\n')
    masterflat=reduction_dir+'Flat_'+genericlabel+'.fits'
    a=raw_input('Median combine the flats (y/n)? ')
    if (a[:1]=='y') | (a==''):
        procfiles=np.array(glob.glob(reduction_dir+'*_cproc1.fits'))
        imtyp=[]
        for i in range(len(procfiles)):
            head=fits.getheader(procfiles[i])
            imtyp.append(head['imagetyp'].lower())
        imtyp=np.array(imtyp)
        gd=np.where(imtyp=='flat')
        flats=procfiles[gd]

        iraf.imcombine(','.join(flats),output=masterflat,combine='median',reject='avsigclip',lsigma='2',
                       hsigma='2',rdnoise=rdnoise,gain=gain)
        print('\nOk. A master flat has been made: '+masterflat)
    print('='*70)

##########################################################################################################
    print('10. MAKE NORMALIZED MASTER FLAT\n')
    masternormflat=reduction_dir+'NormFlat_'+genericlabel+'.fits'
    a=raw_input('Normalize the response of the master flat (y/n)? ')
    if (a[:1]=='y') | (a==''):
        if instr=='arctic':
            im=fits.getdata(masterflat)
            imdim=len(im)
            meanval=np.mean(im[imdim-100:imdim+100,imdim-100:imdim+100])
            iraf.imarith(operand1=masterflat,op='/',operand2=meanval,result=masternormflat)
            print('\nOk. A normalized master flat has been created: '+masternormflat)
        if instr=='dis':
            iraf.response(calibrat=masterflat,normaliz=masterflat+'[*,400:500]',response=masternormflat,
                          interac='no',functio='spline3',order='5',low_rej='3',high_rej='3')
            print('\nOk. A normalized master flat has been created: '+masternormflat)
    print('='*70)

##########################################################################################################
    print('11. FLAT FIELD THE IMAGES\n')

    a=raw_input('Flat field the objs & comps by the normalized master flat (y/n)? ')
    if (a[:1]=='y') | (a==''):
        tmpfiles=np.array(glob.glob(reduction_dir+'*_cproc1.fits'))
        imtyp=[]
        for i in range(len(tmpfiles)):
            head=fits.getheader(tmpfiles[i])
            imtyp.append(head['imagetyp'].lower())
        imtyp=np.array(imtyp)
        gd1=np.where(imtyp=='object')
        if instr=='dis':
            gd2=np.where(imtyp=='comp')
            infiles=np.concatenate((tmpfiles[gd1],tmpfiles[gd2]),axis=0)
        if instr=='arctic':
            infiles=tmpfiles[gd1]
        infilesfile=reduction_dir+'inlist_'+genericlabel+'_ccdproc2'
        np.savetxt(infilesfile,infiles,fmt='%s')

        outfiles=[]
        for infile in infiles: 
            outfiles.append(infile.replace('cproc1','cproc2'))
        outfiles=np.array(outfiles)
        outfilesfile=reduction_dir+'outlist_'+genericlabel+'_ccdproc2'
        np.savetxt(outfilesfile,outfiles,fmt='%s')

        iraf.ccdproc('@'+infilesfile,output='@'+outfilesfile,ccdtype='',fixpix='no',oversca='no',trim='no',
                     zerocor='no',darkcor='no',flatcor='yes',fixfile='',biassec=biassec,trimsec=datasec,
                     flat=masternormflat,interac='no',low_rej='3',high_re='3')
        print('\nOk. The objs and comps have been flat fielded.')
    print('='*70)


##########################################################################################################
    if instr=='dis':
        print('Next steps (done manually for now):')
        print('='*70)
        print('12. Extract apertures via APALL.')
        print('13. Run IDENTIFY on one or more comps.')
        print('14. (optional) Run REIDENTIFY to transfer solution among comps.')
        print('15. Assign solution(s) to objs using REFSPECTRA.')
        print('16. Run DISPCOR on the objs to apply the wavelength solution.')
        print('='*70)
        print('Other things that should be done:')
        print('='*70)
        print('step ?1. flux calibration.')
        print('step ?2. sky/background subtraction.')
        print('step ?3. heliocentric velocity correction.')


    return imtyp
コード例 #25
0
ファイル: soficalibdef.py プロジェクト: svalenti/pessto
def makeillumination(lista,flatfield):#,outputfile,illum_frame):
    import os,glob,string,re
    from astropy.io import fits as pyfits
    import ntt
    from ntt.util import readhdr, readkey3, delete, display_image, defsex,  name_duplicate, correctcard
    from numpy import compress, array, argmax, argmin, min, argsort, float32
    import datetime
    MJDtoday = 55927 + (datetime.date.today() - datetime.date(2012, 01, 01)).days
    _date = readkey3(readhdr(lista[0]), 'date-night')
    _filter = readkey3(readhdr(lista[0]), 'filter')
    illum_frame = name_duplicate(
        lista[0], 'illum_' + _date + '_' + _filter + '_' + str(MJDtoday), '')
    from pyraf import iraf
    iraf.images(_doprint=0)
    iraf.imutil(_doprint=0)
    iraf.utilities(_doprint=0)
    iraf.noao(_doprint=0)
    iraf.imred(_doprint=0)
    iraf.ccdred(_doprint=0)
    iraf.digiphot(_doprint=0)
    iraf.daophot(_doprint=0)
    iraf.generic(_doprint=0)
    toforget = ['digiphot.daophot', 'imutil.imarith',
                'image', 'utilities.surfit']
    for t in toforget:
        iraf.unlearn(t)
    n = len(lista)
#   start loop to read image names from the input file
    lista1 = []
    iraf.ccdred.verbose = 'no'
    ff = open('templist.lst', 'w')
    for i in range(0, len(lista)):
        ff.write('C' + lista[i] + '\n')
        delete('C' + lista[i])
        delete('C' + re.sub('.fits', '_sub.fits', lista[i]))
        ntt.sofiphotredudef.crosstalk(lista[i], 'C' + lista[i])
        iraf.noao.imred.ccdred.ccdproc('C' + lista[i], output='', overscan="no", trim="yes", ccdtype='', darkcor='no', fixpix='no', zerocor="no", flatcor='yes',
                                       illumco='no', trimsec='[1:1024,1:1007]', biassec='', flat=flatfield, illum='')
        correctcard('C' + lista[i])
        lista1.append('C' + lista[i])
    ff.close()
    print '\n### prereducing STD frames to compute illumination correction ........'
    lista2, skyfile = ntt.sofiphotredudef.skysub(lista1, readkey3(
        readhdr(lista1[0]), 'ron'), readkey3(readhdr(lista1[0]), 'gain'), True)
    lista2 = ntt.sofiphotredudef.sortbyJD(lista2)
    print '\n### use x on the star and q  to continue....'
    display_image(lista2[0], 2, '', '', False)
    delete('tmpone.coo')
    iraf.image.tv.imexamine(lista2[0], 2, logfile='tmpone.coo',
                            keeplog='yes', xformat='', yformat='', wcs='logical')
    iraf.tvmark(2, 'tmpone.coo', mark="circle", number='yes',
                label='no', radii=8, nxoffse=5, nyoffse=5, color=204, txsize=2)
    xycoo = iraf.proto.fields('tmpone.coo', '1,2', Stdout=1)
    x0, y0 = string.split(xycoo[0])
    x0 = float(x0)
    y0 = float(y0)
    xcum0 = readkey3(readhdr(lista2[0]), 'xcum')
    ycum0 = readkey3(readhdr(lista2[0]), 'ycum')
    iraf.digiphot(_doprint=0)
    iraf.daophot(_doprint=0)
    iraf.noao.digiphot.daophot.datapars.datamin = -1000
    iraf.noao.digiphot.daophot.datapars.datamax = 60000
    iraf.noao.digiphot.daophot.daopars.function = 'gauss'
    iraf.noao.digiphot.daophot.photpars.zmag = 0
    namesex = defsex('default.sex')
    for i in range(0, len(lista2)):
        j = i + 1
        xcum = readkey3(readhdr(lista2[i]), 'xcum')
        ycum = readkey3(readhdr(lista2[i]), 'ycum')
        xx = x0 - xcum0 + xcum
        yy = y0 - ycum0 + ycum
        # sex objects
        os.system('sex ' + lista2[i] + ' -c ' + namesex + '>  _logsex')
        delete('_logsex')
        xpix = iraf.proto.fields('detections.cat', fields='2', Stdout=1)
        ypix = iraf.proto.fields('detections.cat', fields='3', Stdout=1)
        cm = iraf.proto.fields('detections.cat', fields='4', Stdout=1)
        cm = compress((array(xpix) != ''), array(cm, float))
        ypix = compress((array(xpix) != ''), array(ypix, float))
        xpix = compress((array(xpix) != ''), array(xpix, float))
        if len(xpix) > 300:
            num = 300
        else:
            num = len(xpix) - 1
        xpix = xpix[argsort(cm)][0:num]
        ypix = ypix[argsort(cm)][0:num]
        distance = (ypix - yy)**2 + (xpix - xx)**2
        xx1, yy1 = xpix[argmin(distance)], ypix[argmin(distance)]
        f = open('tmpone.coo', 'w')
        f.write(str(xx1) + ' ' + str(yy1) + '\n')
        f.close()
        display_image(lista2[i], 1, '', '', False)
        iraf.tvmark(1, 'tmpone.coo', mark="circle", number='yes',
                    label='no', radii=8, nxoffse=5, nyoffse=5, color=204, txsize=2)
        answ = 'n'
        while answ != 'y':
            answ = raw_input('selected the right one [[y]/n] ?')
            if not answ:
                answ = 'y'
            if answ in ['y', 'YES', 'yes', 'Y']:
                print lista2[i]
                delete('pippo.' + str(j) + '.mag')
                gggg = iraf.digiphot.daophot.phot(
                    lista2[i], "tmpone.coo", output="pippo." + str(j) + ".mag", verify='no', interac='no', Stdout=1)
                try:
                    float(string.split(gggg[0])[3])
                    answ = 'y'
                except:
                    print '\n### warning'
                    answ = 'n'
            else:
                print '\n### select the std star'
                display_image(lista2[i], 1, '', '', False)
                iraf.image.tv.imexamine(lista2[
                                        i], 1, logfile='tmpone.coo', keeplog='yes', xformat='', yformat='', wcs='logical')
                xycoo = iraf.proto.fields('tmpone.coo', '1,2', Stdout=1)
                x2, y2 = string.split(xycoo[0])
                f = open('tmpone.coo', 'w')
                f.write(str(x2) + ' ' + str(y2) + '\n')
                f.close()
                delete('pippo.' + str(j) + '.mag')
                print '###### new selection ' + str(x2), str(y2)
                gggg = iraf.digiphot.daophot.phot(
                    lista2[i], "tmpone.coo", output='pippo.' + str(j) + '.mag', verify='no', interac='no', Stdout=1)
                try:
                    float(string.split(gggg[0])[3])
                    answ = 'y'
                except:
                    print '\n### warning'
                    answ = 'n'

    os.system('ls pippo.*.mag > tempmag.lst')
    tmptbl0 = iraf.txdump(textfile="@tempmag.lst",
                          fields="XCENTER,YCENTER,FLUX", expr='yes', Stdout=1)
    ff = open('magnitudini', 'w')
    for i in tmptbl0:
        ff.write(i + '\n')
    ff.close()
#   delete the temporary images and files
    delete("temp*.fits")
    delete('temp*.lst')
    delete(illum_frame)
    print '\n### fitting the illumination surface...'
    aaa = iraf.utilities.surfit('magnitudini', image=illum_frame, function="polynomial",
                                xorder=2, yorder=2, xterms="full", ncols=1024, nlines=1024, Stdout=1)
    iraf.noao.imred.generic.normalize(illum_frame)
    correctcard(lista[0])
    data, hdr = pyfits.getdata(illum_frame, 0, header=True)
    data0, hdr0 = pyfits.getdata(lista[0], 0, header=True)
    delete(illum_frame)
    pyfits.writeto(illum_frame, float32(data), hdr0)
    flatfield0 = string.split(flatfield, '/')[-1]
    ntt.util.updateheader(
        illum_frame, 0, {'MKILLUM': [flatfield0, 'flat field']})
    display_image(illum_frame, 1, '', '', False)
    for i in range(0, len(lista)):  # in lista:
        img = lista[i]
        delete('pippo.' + str(i) + '.mag')
        delete('C' + img)
        delete('C' + re.sub('.fits', '_sky.fits', img))
#    delete('C*.fits.mag.1')
#    iraf.hedit(illum_frame,'MKILLUM','Illum. corr. created '+flatfield,add='yes',update='yes',verify='no')
    return illum_frame
コード例 #26
0
ファイル: bench_reduce.py プロジェクト: eigenbrot/snakes
import sys
import os
import glob

glob = glob.glob

# Load the IRAF packages we'll need
try:
    current_dir = os.getcwd()
    if os.getlogin() == "Arthur":
        os.chdir("/Users/Arthur/Ureka/iraf/local")
    from pyraf import iraf

    os.chdir(current_dir)
    iraf.imred(_doprint=0)
    iraf.ccdred(_doprint=0)
except Exception as e:
    print "Failure: could not find pyraf/iraf"
    sys.exit(1)


def make_lists(raw_dir):

    all_raw = glob("{}/*.fits".format(raw_dir))
    all_ot = ["{}_ot.fits".format(os.path.basename(i).split(".fits")[0]) for i in all_raw]
    comps = [i for i in all_ot if "comp" in i]
    darks = [i for i in all_ot if "dark" in i]
    flats = [i for i in all_ot if "dflat" in i]
    zero = [i for i in all_ot if "zero" in i]
    all_otz = [i for i in all_ot if "zero" not in i]
コード例 #27
0
ファイル: rcred.py プロジェクト: nblago/kpy
def create_masterbias(biasdir=None, channel='rc'):
    '''
    Combines slow and fast readout mode biases for the specified channel.
    '''
    
    iraf.noao(_doprint=0)
    iraf.imred(_doprint=0)
    iraf.ccdred(_doprint=0)
    
    if (biasdir == None) or biasdir=="": biasdir = "."
        
    outs = "Bias_%s_slow.fits"%channel
    outf = "Bias_%s_fast.fits"%channel

    if (os.path.isfile(os.path.join(biasdir,outs)) and os.path.isfile(os.path.join(biasdir,outf))):
        print "Master Bias exists!"
        return
    else:
        print "Starting the Master Bias creation!"

    os.chdir(biasdir)        
        
    lfastbias = []
    lslowbias = []
    
    #Select all filts that are Bias with same instrument
    for f in glob.glob("*fits"):
        try:
            if (channel == fitsutils.get_par(f, "CHANNEL") and "BIAS" in str.upper(fitsutils.get_par(f, "OBJECT")) ):
                if (fitsutils.get_par(f, "ADCSPEED")==2):
                    lfastbias.append(f)
                else:
                    lslowbias.append(f)
        except:
            pass
                
    print "Files for bias SLOW mode: ", lslowbias
    print "Files for bias FAST mode: ", lfastbias
    
    if len(lfastbias) > 0:
        bfile_fast ="lbias_fast_"+channel
        np.savetxt(bfile_fast, np.array(lfastbias), fmt="%s")
        if (os.path.isfile("Bias_stats_fast")): os.remove("Bias_stats_fast")
        iraf.imstat("@"+bfile_fast, Stdout="Bias_stats_fast")
        
        st = np.genfromtxt("Bias_stats_fast", names=True, dtype=None)
        print st
        
        iraf.imcombine(input = "@"+bfile_fast, \
                    output = outf, \
                    combine = "median",\
                    scale = "mode")
        os.remove(bfile_fast)

    if len(lslowbias) > 0:

        bfile_slow ="lbias_slow_"+channel
        np.savetxt(bfile_slow, np.array(lslowbias), fmt="%s")
        if (os.path.isfile("Bias_stats_slow")): os.remove("Bias_stats_slow")
        iraf.imstat("@"+bfile_slow, Stdout="Bias_stats_slow")
        
        st = np.genfromtxt("Bias_stats_slow", names=True, dtype=None)
        print st
        
        iraf.imcombine(input = "@"+bfile_slow, \
                    output = outs, \
                    combine = "median",\
                    scale = "mode")
        os.remove(bfile_slow)
コード例 #28
0
ファイル: ext_spec_line.py プロジェクト: zzxihep/spec2015
def main():
    iraf.noao()
    iraf.imred()
    iraf.ccdred()
    iraf.crutil()
    iraf.twodspec()
    iraf.apextract()
    name = os.popen('ls object*.lst').readlines()
    name = [i.split('\n')[0] for i in name]
    if len(name) == 0:
        print 'no object, the script end'
    for i in name:
        print 'open file ' + i + '...'
        f = open(i)
        l = f.readlines()
        f.close()
        l = [i2.split('\n')[0] for i2 in l]
        for i2 in l:
            print i2
        objs = []
        calibs = []
        standards = []
        fno = l.index('object') + 1
        while fno < len(l):
            if '.fits' not in l[fno]:
                break
            objs.append('ftbo' + l[fno])
            fno += 1
        fno = l.index('calibration') + 1
        while fno < len(l):
            if '.fits' not in l[fno]:
                break
            calibs.append('ftbo' + l[fno])
            fno += 1
        fno = l.index('standard') + 1
        while fno < len(l):
            if '.fits' not in l[fno]:
                break
            standards.append('ftbo' + l[fno])
            fno += 1
        print '=' * 20, 'reject cosmic ray', '=' * 20
        #cobj = shift_align(objs)
        #cstandard = shift_align(standards)
        cobjs = []
        cstandards = []
        for eobj in objs:
            cobjs.append(cos_clear([eobj]))
        for eobj in standards:
            cstandards.append(cos_clear([eobj]))
        print 'The fits having rejected cosmic ray is: '
        print cobjs
        print cstandards
        print '=' * 20, 'get spectrum line', '=' * 20
        acobjs = []
        acstandards = []
        for cobj in cobjs:
            acobjs.append(get_spec_line(cobj))
        for cobj in cstandards:
            acstandards.append(get_spec_line(cobj))
        acalibs = []
        for i3 in calibs:
            acalibs.append(get_cal_spec_line(i3, cobj))
        print '=' * 20, 'wavelength identify', '=' * 20
        iacalibs = wl_identy(acalibs)
        print '=' * 20, 'wavelength calibrate', '=' * 20
        inamepart = i.replace('.','_').split('_')
        objnames = []
        stdobjnames = []
        for numb in xrange(len(acobjs)):
            objnames.append( 'iac' + inamepart[1] + '_' + inamepart[2] + '_' + inamepart[3] +
                    '_' + str(numb) + '.fits')
        for numb in xrange(len(acstandards)):
            stdobjnames.append('iacstd' + inamepart[1] + '_' + inamepart[2] + '_' + inamepart[3] +
                    '_' + str(numb) + '.fits')
        for numb in xrange(len(objnames)):
            wl_cal(acobjs[numb], objnames[numb], iacalibs)
        for numb in xrange(len(stdobjnames)):
            wl_cal(acstandards[numb], stdobjnames[numb], iacalibs)
        print '=' * 20, 'flux calibrate', '=' * 20
        for objname in objnames:
            final_objname = flux_cal_new(objname, stdobjnames)
        print 'the final object name is', final_objname
コード例 #29
0
ファイル: rcred.py プロジェクト: nblago/kpy
def reduce_image(img, flatdir=None, biasdir=None, cosmic=True, astrometry=True, channel='rc', target_dir='reduced'):
    '''
    Applies Flat field and bias calibrations to the image.
    
    Steps:
    
    1. - Solve astrometry on the entire image.
    2. - Compute master bias and de-bias the image.
    3. - Separate the image into 4 filters.
    4. - Compute flat field for each filter and apply flat fielding on the image.
    5. - Computes cosmic ray rejectionon the entire image.
    6. - Compute zeropoint for each image and store in a log file.
    7. - Plot zeropoint for the night.
    '''
    
    print "Reducing image ", img    

    objectname = fitsutils.get_par(img, "OBJECT").replace(" ", "").replace("]","").replace("[", "")

    print "For object", objectname
    
    #Change to image directory
    mydir = os.path.dirname(img)
    if mydir=="": mydir = "."
    mydir = os.path.abspath(mydir)
    os.chdir(mydir)
    #Create destination directory
    if (not os.path.isdir(target_dir)):
        os.makedirs(target_dir)

    #Rename to the image name only
    img = os.path.basename(img)


    if (astrometry):
        print "Solving astometry for the whole image..."
        img = solve_astrometry(img)
        astro = "a_"
    else:
        astro = ""
        
    
    #Compute BIAS
    if (biasdir == None or biasdir==""): biasdir = "."
    create_masterbias(biasdir)
    
    bias_slow = os.path.join(biasdir, "Bias_%s_%s.fits"%(channel, 'slow'))
    bias_fast = os.path.join(biasdir, "Bias_%s_%s.fits"%(channel, 'fast'))
    
    # Running IRAF to DE-BIAS
    iraf.noao(_doprint=0)
    iraf.imred(_doprint=0)
    iraf.ccdred(_doprint=0)
    
    #Compute flat field
    if (flatdir == None or flatdir==""): flatdir = "."
    create_masterflat(flatdir, biasdir)
    
    #New names for the object.
    debiased = "b_" + astro + img
    print "Creating debiased file, ",debiased
    
    if (not os.path.isfile(bias_slow) or not os.path.isfile(bias_fast)):
        print "Master bias not found!"
        return

    #Debias
    if (fitsutils.get_par(img, "ADCSPEED")==2):
        iraf.imarith(img, "-", bias_fast, debiased)
        fitsutils.update_par(debiased, "BIASFILE", bias_fast)
        fitsutils.update_par(debiased, "RDNOISE", 20.)

    else:
        iraf.imarith(img, "-", bias_slow, debiased)
        fitsutils.update_par(debiased, "BIASFILE", bias_slow)
        fitsutils.update_par(debiased, "RDNOISE", 4.)

    #Set negative counts to zero
    hdu = pf.open(debiased)
    header = hdu[0].header
    hdu[0].data[hdu[0].data<0] = 0
    hdu.writeto(debiased, clobber=True)

    #Slicing the image for flats  
    slice_names = slice_rc(debiased)

    
    #Remove un-sliced image
    os.remove(debiased)

    # DE-flat each filter and store under object name
    for i, debiased_f in enumerate(slice_names):
        b = fitsutils.get_par(debiased_f, 'filter')
        
        deflatted = "f_b_" + astro + objectname + "_%s.fits"%b

        #Flat to be used for that filter
        flat = os.path.join(flatdir, "Flat_%s_%s_norm.fits"%(channel, b))

        if (not os.path.isfile(flat)):
            print "Master flat not found in", flat
            return
        #Cleans the deflatted file if exists
        if (os.path.isfile(deflatted)):
            os.remove(deflatted)
            
        iraf.imarith(debiased_f, "/", flat, deflatted)
        
        #Removes the de-biased file
        os.remove(debiased_f)
        
        print "Updating header with original filename and flat field used."
        fitsutils.update_par(deflatted, "ORIGFILE", img)
        fitsutils.update_par(deflatted, "FLATFILE", flat)

        slice_names[i] = deflatted
            
            
    if (cosmic):
        print "Correcting for cosmic rays..."
        # Correct for cosmics each filter
        for i, deflatted in enumerate(slice_names):
            cclean = "c_" +name
            clean_cosmic(os.path.join(os.path.abspath(mydir), deflatted), cclean)
            slice_names[i] = cclean
           
    #Moving files to the target directory
    for name in slice_names:
        if (os.path.isfile(name)):
            shutil.move(name, os.path.join(target_dir, name))
コード例 #30
0
ファイル: wcal2d.py プロジェクト: zzxihep/spec2015_1
def combine_lamp(lstfile):
    iraf.noao()
    iraf.imred()
    iraf.ccdred()
    iraf.imcombine(input = '%ftbo%ftbo%@'+lstfile
        , output = 'Lamp', combine = 'sum', reject = 'none')