示例#1
0
def combine_flats(refresh='2', method='2'):
    if method == '1':
        meta = 'med'
        source = 'Trimmed_Flat/subflatsmed'
        dest = 'Master_Files/mflat_median.fits'
    elif method == '2':
        meta = 'sig'
        source = 'Trimmed_Flat/subflatssig'
        dest = 'Master_Files/mflat.fits'
    subflatcollection = ImageFileCollection(source)
    combtime = 0
    if refresh == '1':
        print('found', len(subflatcollection.values('file')), 'subflats')
        start = time.time()
        if method == '1':
            mflat = ccdp.combine(subflatcollection.files_filtered(
                imtype='subflat', include_path=True),
                                 method='median')
            mflat.meta['flatcom'] = 'median'
            combtime = time.time() - start
            print('combination took', combtime, 'seconds')
        elif method == '2':
            mflat = ccdp.combine(subflatcollection.files_filtered(
                imtype='subflat', include_path=True),
                                 sigma_clip=True,
                                 sigma_clip_low_thresh=5,
                                 sigma_clip_high_thresh=5,
                                 sigma_clip_func=np.nanmedian,
                                 sigma_clip_dev_func=mad_std)
            mflat.meta['flatcom'] = 'sigma'
            combtime = time.time() - start
            print('combination took', combtime, 'seconds')
        mflat.meta['normmed'] = (np.nanmedian(mflat),
                                 'nanmedian of the master flat')
        mflat.meta['subflats'] = meta
        mflat.write(dest[0:-5] + '_' + meta + '.fits', overwrite=True)
    else:
        try:
            if method == '1':
                mflat = CCDData.read('Master_Files/mflat_median_med.fits',
                                     unit='adu')

            elif method == '2':
                mflat = CCDData.read('Master_Files/mflat_sig.fits', unit='adu')
        except:
            print('can\'t locate master flat, create or check directory')
            sys.exit()
    return subflatcollection, mflat, dest, combtime
示例#2
0
def get_files(filenames,
              location='.',
              imagetyp=None,
              filter=None,
              fexptime='*'):
    '''Gather files from input directories, choosing by image type filter
    if specified'''

    # for summary display only
    keywords = ('object', 'date-obs', 'IMAGETYP', 'FILTER', 'EXPTIME')

    collection = ImageFileCollection(filenames=filenames,
                                     location=location,
                                     keywords=keywords)

    if imagetyp in ['light', 'bias', 'dark']:
        imagetyp = '%s frame' % imagetyp
    else:
        imagetyp = '*'

    if fexptime is None:
        fexptime = '*'

    if imagetyp == 'bias frame':
        fexptime = 0.

    # filter and reread
    if filter is None:
        filelist = collection.files_filtered(IMAGETYP=imagetyp,
                                             EXPTIME=fexptime,
                                             include_path=True)
    else:
        filelist = collection.files_filtered(IMAGETYP=imagetyp,
                                             EXPTIME=fexptime,
                                             FILTER=filter,
                                             include_path=True)

    if not filelist:
        raise RuntimeError('No matching files found.')

    location = os.path.join(collection.location, os.path.dirname(filenames[0]))

    collection = ImageFileCollection(filenames=filelist,
                                     location=location,
                                     keywords=keywords)

    collection.summary.pprint()
    return collection
示例#3
0
def sub_bias(refresh='2', bias='2'):
    tflatcollection = ImageFileCollection('Trimmed_Flat')
    if bias == '1':
        biaspath = 'Master_Files/mbias_median.fits'
        dest = 'Trimmed_Flat/subflatsmed/'
    elif bias == '2':
        biaspath = 'Master_Files/mbias.fits'
        dest = 'Trimmed_Flat/subflatssig/'
    if refresh == '1':
        subflatpathlist = []
        mbias = CCDData.read(biaspath, unit='adu')
        for ccdf, flatn in tflatcollection.ccds(imtype='trimmed flat',
                                                return_fname=True):
            subflat = ccdp.subtract_bias(ccdf, mbias, add_keyword='subbias')
            subflat.meta['imtype'] = ('subflat', 'bias subtracted flat')
            subflat.write(dest + flatn[0:8] + '_subbias.fits', overwrite=True)
            subflatpathlist.append(dest + flatn[0:8] + '_subbias.fits')
    else:
        try:
            subflatcollection = ImageFileCollection(dest)
            subflatpathlist = subflatcollection.files_filtered(
                imtype='subflat', include_path=True)
            print('found', len(subflatpathlist), 'subflats')
        except:
            print('can\'t locate subflats, create or check directory')
            sys.exit()
    return tflatcollection, subflatpathlist
示例#4
0
def trim_bias(refresh='2'):
    biascollection = ImageFileCollection('HD115709/bias', ext=4)
    flag = 0
    if refresh == '1':
        tbiaspathlist = []
        for ccdb, biasn in biascollection.ccds(return_fname=True,
                                               ccd_kwargs={'unit': 'adu'}):
            if flag == 0:
                print('all biases will be trimmed to :', ccdb.meta['trimsec'])
                flag = 1

            print('trimming', biasn)

            tbias = ccdp.trim_image(ccdb,
                                    fits_section=str(ccdb.meta['trimsec']))
            tbias.meta['imtype'] = ('trimmed bias', 'type of image')
            tbias.meta['taxis1'] = (2048, 'dimension1')
            tbias.meta['taxis2'] = (4096, 'dimension2')
            tbias.write('Trimmed_Bias/' + biasn[0:8] + '_trim.fits',
                        overwrite=True)
            tbiaspathlist.append('Trimmed_Bias/' + biasn[0:8] + '_trim.fits')
        print('created', len(tbiaspathlist), 'trimmed biases')
    else:
        try:
            tbiascollection = ImageFileCollection('Trimmed_Bias')
            tbiaspathlist = tbiascollection.files_filtered(
                imtype='trimmed bias', include_path=True)
            print('found', len(tbiaspathlist), 'trimmed bias')
        except:
            print('can\'t locate trimmed biases, create or check directory')
            sys.exit()
    return biascollection, tbiaspathlist
示例#5
0
def trim_flat(refresh='2'):
    flatcollection = ImageFileCollection('HD115709/flat_SII', ext=4)
    flag = 0
    tflatpathlist = []
    if refresh == '1':
        for ccdf, flatn in flatcollection.ccds(return_fname=True,
                                               ccd_kwargs={'unit': 'adu'}):
            if flag == 0:
                print('all flats will be trimmed to :', ccdf.meta['trimsec'])
                flag = 1

            print('trimming', flatn)

            tflat = ccdp.trim_image(ccdf,
                                    fits_section=str(ccdf.meta['trimsec']))
            tflat.meta['imtype'] = ('trimmed flat', 'type of image')
            tflat.meta['taxis1'] = (2048, 'dimension1')
            tflat.meta['taxis2'] = (4096, 'dimension2')
            tflat.write('Trimmed_Flat/' + flatn[0:8] + '_trim.fits',
                        overwrite=True)
            tflatpathlist.append('Trimmed_Flat/' + flatn[0:8] + '_trim.fits')
        print('created', len(tflatpathlist), 'trimmed flats')
    elif refresh == '2':
        try:
            tflatcollection = ImageFileCollection('Trimmed_Flat')
            tflatpathlist = tflatcollection.files_filtered(
                imtype='trimmed flat', include_path=True)
            print('found', len(tflatpathlist), 'trimmed flats')
        except:
            print('can\'t locate trimmed flats, create or check directory')
            sys.exit(0)
    return flatcollection, tflatpathlist
示例#6
0
def fix_headers(folder):
    """
    Find FITS files in folder that have WCS CTYPE ``RA--TAN`` and ``DEC--TAN``
    and add ``-SIP`` to the end. Necessary because astropy handling of SIP
    distortion keywords changed in v1.2.

    Parameters
    ----------

    folder : str
        Path to folder with the images to fix.
    """
    ic = ImageFileCollection(folder)
    ic.summary['ctype1']

    files = ic.files_filtered(ctype1='RA---TAN')

    if not len(files):
        print('No files to fix in {}'.format(folder))

    for file in files:
        fname = os.path.join(folder, file)
        print('Fixing file {}'.format(file))
        with fits.open(fname) as f:
            f[0].header['ctype1'] = f[0].header['ctype1'] + '-SIP'
            f[0].header['ctype2'] = f[0].header['ctype2'] + '-SIP'
            f.writeto(fname, clobber=True)
示例#7
0
def bias_combine(refresh='2', method='2'):
    tbiascollection = ImageFileCollection('Trimmed_Bias')
    combtime = 0
    if refresh == '1':
        print('found', len(tbiascollection.values('file')), 'trimmed biases')
        start = time.time()
        if method == '1':
            combined_bias = ccdp.combine(tbiascollection.files_filtered(
                imtype='trimmed bias', include_path=True),
                                         method='median')
            combbiaspath = 'Master_Files/mbias_median.fits'
            combined_bias.meta['combined'] = 'median'
            combtime = time.time() - start
            print('combination took', combtime, 'seconds')
            combined_bias.write(combbiaspath, overwrite=True)
        elif method == '2':
            combined_bias = ccdp.combine(tbiascollection.files_filtered(
                imtype='trimmed bias', include_path=True),
                                         sigma_clip=True,
                                         sigma_clip_low_thresh=5,
                                         sigma_clip_high_thresh=5,
                                         sigma_clip_func=np.nanmedian,
                                         sigma_clip_dev_func=mad_std)
            combbiaspath = 'Master_Files/mbias.fits'
            combined_bias.meta['combined'] = 'sigma_clip average'
            combtime = time.time() - start
            print('combination took', combtime, 'seconds')
            combined_bias.write(combbiaspath, overwrite=True)
    else:
        try:
            if method == '1':
                combined_bias = CCDData.read('Master_Files/mbias_median.fits',
                                             unit='adu')
                combbiaspath = 'Master_Files/mbias_median.fits'
            elif method == '2':
                combined_bias = CCDData.read('Master_Files/mbias.fits',
                                             unit='adu')
                combbiaspath = 'Master_Files/mbias.fits'
        except:
            print('can\'t locate master bias, create or check directory')
            sys.exit()

    return tbiascollection, combined_bias, combbiaspath, combtime
示例#8
0
def bias_combine(refresh=0):
    tbiascollection = ImageFileCollection('Trimmed_Bias')
    print('found', len(tbiascollection.values('file')), 'trimmed biases')
    combined_bias = ccdp.combine(tbiascollection.files_filtered(
        imtype='trimmed bias', include_path=True),
                                 sigma_clip=True,
                                 sigma_clip_low_thresh=5,
                                 sigma_clip_high_thresh=5,
                                 sigma_clip_func=np.nanmedian,
                                 sigma_clip_dev_func=mad_std)
    combined_bias.meta['combined'] = True
    combined_bias.write('Master_Files/mbias.fits', overwrite=True)
    return tbiascollection
示例#9
0
def bias_combine(refresh=False, method=1):
    if refresh == 0:
        tbiascollection = ImageFileCollection('Trimmed_Bias')
        print('found', len(tbiascollection.values('file')), 'trimmed biases')
        start=time.time()
        if method == 1:
            combined_bias = ccdp.combine(tbiascollection.files_filtered(
            imtype='trimmed bias', include_path=True),
            method='median')
            combbiaspath = 'Master_Files/mbias_median.fits'
            combined_bias.meta['combined'] = 'median'
        elif method == 2:
            combined_bias = ccdp.combine(tbiascollection.files_filtered(
            imtype='trimmed bias', include_path=True),
            sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5,
            sigma_clip_func=np.nanmedian, sigma_clip_dev_func=mad_std)
            combbiaspath = 'Master_Files/mbias.fits'
            combined_bias.meta['combined'] = 'sigma_clip average'
        combtime=time.time()-start
        print('combination took',combtime,'seconds')
        combined_bias.write('Master_Files/mbias_median.fits',
                            overwrite=True)
    return tbiascollection,combined_bias,combbiaspath,combtime
示例#10
0
def get_files(filenames,location='.',imagetyp=None,filter=None,fexptime='*'):
    '''Gather files from input directories, choosing by image type filter
    if specified'''

    # for summary display only
    keywords = ('object','date-obs','IMAGETYP','FILTER','EXPTIME')
    
    collection = ImageFileCollection(filenames=filenames,location=location,
                                     keywords=keywords)
    
    if imagetyp in ['light','bias','dark']:
        imagetyp = '%s frame' % imagetyp
    else:
        imagetyp = '*'

    if fexptime is None:
        fexptime = '*'

    if imagetyp == 'bias frame':
        fexptime = 0.

    # filter and reread
    if filter is None:
        filelist = collection.files_filtered(IMAGETYP=imagetyp,EXPTIME=fexptime,include_path=True)
    else:
        filelist = collection.files_filtered(IMAGETYP=imagetyp,EXPTIME=fexptime,FILTER=filter,include_path=True)

    if not filelist:
        raise RuntimeError('No matching files found.')

    location = os.path.join(collection.location,os.path.dirname(filenames[0]))

    collection = ImageFileCollection(filenames=filelist,location=location,
                                     keywords=keywords)

    collection.summary.pprint()
    return collection
示例#11
0
def load_files(target_dir, config, files, logger):
    from ccdproc import ImageFileCollection

    keywords = [
        'object', 'imagetyp', 'date-obs', 'telra', 'teldec', 'airmass',
        'filter', 'oairtemp', 'relhum', 'subbias', 'flatcor'
    ]
    ic = ImageFileCollection(location=target_dir,
                             filenames=files,
                             keywords=keywords)
    nbias = len(ic.files_filtered(imagetyp='bias'))
    nflat = len(ic.files_filtered(imagetyp='flat'))
    ndata = len(ic.files_filtered(imagetyp='object'))

    filters = []
    for h in ic.headers():
        if h['IMAGETYP'] in ['object', 'flat']:
            if h['FILTER'] not in filters:
                filters.append(h['FILTER'])

    filters.sort()
    #    flat_breakdown = []
    #    data_breakdown = []
    #    for filt in filters:
    #        for k in config.flats.values():
    #            flat_breakdown.append('{} {} {}'.format(
    #                len(ic.files_filtered(
    #                    imagetyp='flat', object=k, filter=filt)),
    #                filt, k))
    #        data_breakdown.append('{} {}'.format(
    #            len(ic.files_filtered(imagetyp='OBJECT', filter=filt)), filt))

    logger.info('{} files: {} bias, {} flats, {} object ({} filters)'.format(
        len(ic.files), nbias, nflat, ndata, len(filters)))

    return ic
示例#12
0
    def test_run_astrometry_with_dest_does_not_modify_source(self):

        destination = self.test_dir.make_numbered_dir()
        list_before = self.test_dir.listdir(sort=True)
        arglist = [
            '--destination-dir', destination.strpath, self.test_dir.strpath
        ]
        run_astrometry.main(arglist)
        list_after = self.test_dir.listdir(sort=True)
        # nothing should change in the source directory
        assert (list_before == list_after)
        # for each light file in the destination directory we should have a
        # file with the same basename but an extension of blind
        ic = ImageFileCollection(destination.strpath, keywords=['IMAGETYP'])
        for image in ic.files_filtered(imagetyp='LIGHT'):
            image_path = destination.join(image)
            print(image_path.purebasename)
            blind_path = destination.join(image_path.purebasename + '.blind')
            print(blind_path.strpath)
            assert (blind_path.check())
示例#13
0
ic1 = ImageFileCollection(indir.replace('sci2', '20160115'))

#create an array of dates
arm = 'Red arm'
file_list = []
date_list=[]
for hdu, fname in ic1.hdus(obstype='Arc', isiarm=arm, return_fname=True):
    if os.path.isfile('w_arc_'+os.path.basename(fname)):
        d = hdu.header['DATE-OBS'] + ' ' + hdu.header['UT']
        d = datetime.strptime(d, '%Y-%m-%d %H:%M:%S.%f')
        file_list.append(fname)
        date_list.append(d)
date_arr = np.array(date_list)

#reduce the object frames
for filename in ic1.files_filtered(obstype='TARGET', isiarm=arm):
    hdu = fits.open('obj_' + filename)

    #find the closest two arcs in time
    #open them, based on the time difference 
    #average their frames, and then use that as the 
    #wave map for the object file

    date = hdu[0].header['DATE-OBS'] + ' ' + hdu[0].header['UT']
    date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S.%f')
    d = abs(date_arr - date)
    i = d.argmin()
    if date_arr[i] < date:
       j = i+1
    else:
       j = i-1
示例#14
0
import glob
from astropy import units as u
from astropy.io import fits
from ccdproc import CCDData, Combiner
import numpy as np
from ccdproc import Combiner, ImageFileCollection, fits_ccddata_writer
from t120_init import t120

listimg = ImageFileCollection(
    t120.t120_flat_dir)  #,glob_include='*.fit',glob_exclude='*.fits')

list_filters = listimg.values('filter', unique=True)
#for filter_name in listimg.values('filter',unique=True):
for filter_name in list_filters:
    t120.log.info('*** filter: ' + filter_name)
    my_files = listimg.files_filtered(filter=filter_name)
    t120.log.info('my_files=' + my_files)

pouet

for filter_name in listimg.values('filter', unique=True):
    t120.log.info('*** filter: ' + filter_name)
    listccd = []
    for ccd, file_name in listimg.ccds(ccd_kwargs={'unit': 'adu'},
                                       filter=filter_name,
                                       return_fname=True):
        t120.log.info('now considering file ' + file_name)
        listccd.append(ccd)
    """
    t120.log.info('now making the Combiner object')
    combiner = Combiner(listccd)
示例#15
0
            header['HISTORY'] = '= Cosmic rays rejected using ccdproc.cosmicray_lacosmic '
            fits.writeto('z'+f,crimage,header)
            hdu1.close()
        i += 1
        print ('\n')

# make an image collection of all the files in the data directory
# z is prefix for galaxies that already have cosmic ray rejection
ic = ImageFileCollection(os.getcwd(),keywords='*',glob_include='*.fit*')


# combine bias frames

if args.zerocombine:
    # select all files with imagetyp=='bias'
    bias_files = ic.files_filtered(imagetyp = ccdkeyword['bias'])
    # feed list into ccdproc.combine, output bias

    master_bias = ccdproc.combine(bias_files,method='average',sigma_clip=True,unit=u.adu)
    gaincorrected_master_bias = ccdproc.gain_correct(master_bias,float(gain))
    print('writing fits file for master bias')
    gaincorrected_master_bias.write('bias-combined.fits',overwrite=True)
else:
    if args.bias:
        print('not combining zeros')
        print('\t reading in bias-combined.fits instead')
        hdu1 = fits.open('bias-combined.fits')
        header = hdu1[0].header
        gaincorrected_master_bias = CCDData(hdu1[0].data, unit=u.electron, meta=header)
        hdu1.close()
    else:
if len(sys.argv)!=3: 
   print('Usage:\npython wht_basic_rection.py [full_path_to_raw_data] [full_path_to_reduced_data]\n')
   exit()

indir = sys.argv[1]
outdir = sys.argv[2]

if not os.path.isdir(outdir): os.mkdir(outdir)
os.chdir(outdir)

#change this to point to your raw data directory
ic1 = ImageFileCollection(indir)

#create the bias frames
blue_bias_list = []
for filename in ic1.files_filtered(obstype='Bias', isiarm='Blue arm'):
    print ic1.location + filename
    ccd = CCDData.read(ic1.location + filename, unit = u.adu)
    #this has to be fixed as the bias section does not include the whole section that will be trimmed
    ccd = ccdproc.subtract_overscan(ccd, median=True,  overscan_axis=0, fits_section='[1:966,4105:4190]')
    ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] )
    blue_bias_list.append(ccd)
master_bias_blue = ccdproc.combine(blue_bias_list, method='median')
master_bias_blue.write('master_bias_blue.fits', clobber=True)

red_bias_list = []
for filename in ic1.files_filtered(obstype='Bias', isiarm='Red arm'):
    print ic1.location + filename
    ccd = CCDData.read(ic1.location + filename, unit = u.adu)
    #this has to be fixed as the bias section does not include the whole section that will be trimmed
    ccd = ccdproc.subtract_overscan(ccd, median=True,  overscan_axis=0, fits_section='[1:966,4105:4190]')
示例#17
0
        'Usage:\npython wht_basic_rection.py [full_path_to_raw_data] [full_path_to_reduced_data]\n'
    )
    exit()

indir = sys.argv[1]
outdir = sys.argv[2]

if not os.path.isdir(outdir): os.mkdir(outdir)
os.chdir(outdir)

#change this to point to your raw data directory
ic1 = ImageFileCollection(indir)

#create the bias frames
blue_bias_list = []
for filename in ic1.files_filtered(obstype='Bias', isiarm='Blue arm'):
    print ic1.location + filename
    ccd = CCDData.read(ic1.location + filename, unit=u.adu)
    #this has to be fixed as the bias section does not include the whole section that will be trimmed
    ccd = ccdproc.subtract_overscan(ccd,
                                    median=True,
                                    overscan_axis=0,
                                    fits_section='[1:966,4105:4190]')
    ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'])
    blue_bias_list.append(ccd)
master_bias_blue = ccdproc.combine(blue_bias_list, method='median')
master_bias_blue.write('master_bias_blue.fits', clobber=True)

red_bias_list = []
for filename in ic1.files_filtered(obstype='Bias', isiarm='Red arm'):
    print ic1.location + filename
示例#18
0
文件: core.py 项目: mkelley/robotools
def find_files(path, db, config, logger):
    files = glob(os.sep.join((path, config.file_template)))
    files = sorted([os.path.basename(f) for f in files])

    keywords = [
        'object', 'imagetyp', 'date-obs', 'ut', 'telra', 'teldec', 'airmass',
        'filter', 'exptime', 'oairtemp', 'relhum', 'seeing', 'subbias',
        'flatcor'
    ]
    ic = ImageFileCollection(location=path, filenames=files, keywords=keywords)

    files = sorted(list(ic.files_filtered(imagetyp='object')))
    logger.debug('Found {} object files in {}.'.format(len(files), path))
    if len(files) == 0:
        raise ObjectFilesError('No object files found in {}.'.format(path))

    ic = ImageFileCollection(location=path, filenames=files, keywords=keywords)

    rows = []
    for row in ic.summary:
        fn = row['file'].replace('.fits', '')
        c = db.execute('SELECT * FROM obs WHERE filename=?', [fn])
        if c.fetchone() is None or config.update_obs:
            with fits.open(os.sep.join((path, row['file']))) as hdu:
                wcs = WCS(hdu[0])
                shape = hdu[0].data.shape

            if ((wcs.wcs.crpix[0] == 0) or (wcs.wcs.crpix[0] == 1)
                    or ('PIXEL' in wcs.wcs.ctype)):
                # missing WCS solution
                ra, dec = '0', '0'
            else:
                ra, dec = wcs.all_pix2world(shape[1] / 2, shape[0] / 2, 0)
                c = SkyCoord(ra, dec, unit='deg')
                ra, dec = c.to_string('hmsdms', sep=':').split()

            rows.append(
                (fn, row['object'], row['date-obs'] + ' ' + row['ut'],
                 row['airmass'], row['filter'], row['exptime'], ra, dec,
                 row['oairtemp'], row['relhum'], row['seeing'] / 2))

    if len(rows) > 0:
        if config.update_obs:
            cols = list(zip(*rows))
            update = zip(*(cols + [cols[0]]))
            logger.info('Updating with FITS header meta data.')
            db.executemany(
                '''
            UPDATE OR IGNORE obs SET
              filename=?,object=?,obsdate=?,airmass=?,filter=?,exptime=?,
              ra=?,dec=?,oairtemp=?,relhum=?,seeing=?
            WHERE filename=?
            ''', update)

        # insert whatever wasn't updated
        db.executemany(
            '''
        INSERT OR IGNORE INTO obs
          (filename,object,obsdate,airmass,filter,exptime,ra,dec,
          oairtemp,relhum,seeing)
        VALUES
          (?,?,?,?,?,?,?,?,?,?,?)''', rows)

    return ic
示例#19
0
def triage_fits_files(dir=None, file_info_to_keep=None):
    """
    Check FITS files in a directory for deficient headers

    `dir` is the name of the directory to search for files.

    `file_info_to_keep` is a list of the FITS keywords to get values
    for for each FITS file in `dir`.
    """
    dir = dir or '.'
    all_file_info = file_info_to_keep or [
        'imagetyp', 'object', 'filter', 'wcsaxes'
    ]
    feder = Feder()
    RA = feder.RA
    if ((not (set(RA.names) <= set(all_file_info)))
            and (all_file_info != '*')):
        all_file_info.extend(RA.names)

    images = ImageFileCollection(dir, keywords=all_file_info)
    file_info = images.summary

    # check for bad image type and halt until that is fixed.
    if contains_maximdl_imagetype(images):
        raise ValueError(
            'Correct MaxImDL-style image types before proceeding.')

    file_needs_filter = \
        list(images.files_filtered(imagetyp='light',
                                   filter=None))
    file_needs_filter += \
        list(images.files_filtered(imagetyp='flat',
                                   filter=None))

    file_needs_object_name = \
        list(images.files_filtered(imagetyp='light',
                                   object=None))

    lights = file_info[file_info['imagetyp'] == 'LIGHT']
    file_needs_pointing = []
    file_needs_astrometry = []
    if lights:
        has_no_ra = np.array([True] * len(lights))
        has_no_ha = np.array([True] * len(lights))
        for ra_name in RA.names:
            col_name = \
                get_column_name_case_insensitive(ra_name, lights.colnames)
            try:
                has_no_ra &= (lights[col_name].mask)
            except KeyError:
                pass
        for ha_name in feder.HA.names:
            col_name = \
                get_column_name_case_insensitive(ha_name, lights.colnames)
            try:
                has_no_ha &= lights[col_name].mask
            except KeyError:
                pass

        file_needs_astrometry = list(lights['file'][lights['wcsaxes'].mask])
        needs_minimal_pointing = has_no_ha | has_no_ra
        file_needs_pointing = list(lights['file'][needs_minimal_pointing])

    full_path = os.path.abspath(dir)
    path_column = Column(data=[full_path] * len(file_info), name='Source path')
    containing_dir = os.path.basename(full_path)
    containing_dir_col = Column(data=[containing_dir] * len(file_info),
                                name='Source directory')
    file_info.add_columns([path_column, containing_dir_col])

    dir_info = {
        'files': file_info,
        'needs_filter': file_needs_filter,
        'needs_pointing': file_needs_pointing,
        'needs_object_name': file_needs_object_name,
        'needs_astrometry': file_needs_astrometry
    }

    return dir_info
示例#20
0
文件: lbcproc.py 项目: jchowk/LBCgo
def lbcgo(raw_directory='./raw/',
          image_directory='./',
          lbc_chips=True,
          lbcr=True,
          lbcb=True,
          filter_names=None,
          bias_proc=False,
          do_astrometry=True,
          scamp_iterations=3,
          verbose=True,
          clean=True):
    """Process a directory of LBC data.

    By default, all raw data are to be in the ./raw/ directory before processing (raw_directory='./raw/').
    By default, all new images are written in the CWD (image_directory='./').

    """

    # It will go something like this...

    # Make sure the input directories have trailing slashes:
    if raw_directory[-1] != '/':
        raw_directory += '/'
    if image_directory[-1] != '/':
        image_directory += '/'

    ########## Collect basic image information
    # Find the raw files we'll use
    if lbcb & lbcr:
        lbc_file_base = 'lbc?.*.*.fits*'
    elif lbcb:
        lbc_file_base = 'lbcb.*.*.fits*'
    elif lbcr:
        lbc_file_base = 'lbcr.*.*.fits*'

    # What information do we want from the headers?
    keywds = [
        'object', 'filter', 'exptime', 'imagetyp', 'propid', 'lbcobnam',
        'airmass', 'HA', 'objra', 'objdec'
    ]

    ##### Create an ImageFileCollection object to hold the raw data list.
    if np.int(ccdproc.__version__[0]) == 2:
        ic0 = ImageFileCollection(raw_directory,
                                  keywords=keywds,
                                  glob_include=lbc_file_base)

        num_images = np.size(ic0.summary['object'])
        # Exit if (for some reason) no images are found in the raw_directory.
        if num_images == 0:
            print('WARNING: No images found.')
            return None
    else:
        raw_lbc_files = glob(raw_directory + lbc_file_base)
        ic0 = ImageFileCollection(raw_directory,
                                  keywords=keywds,
                                  filenames=raw_lbc_files)
        num_images = np.size(ic0.summary['object'])

        # Exit if (for some reason) no images are found in the raw_directory.
        if num_images == 0:
            print('WARNING: No images found.')
            return None

    ######### Create the master bias frame (if requested)
    if bias_proc == True:
        make_bias(ic0,
                  image_directory=image_directory,
                  raw_directory=raw_directory)

    ###### Define which chips to extract if default is chosen:
    if lbc_chips == True:
        lbc_chips = [1, 2, 3, 4]

    ###### Per filter:
    #
    # Step through the filters in the ImageCollection unless user gives filter
    # list.
    #
    # For now nights with V-band filters in both LBCB and LBCR require running
    # the code with 'lbcr=False' then 'lbcb=False' to avoid coadding the two
    # images.

    # TODO: Loop B/R to avoid issues w/V-band from LBCB/LBCR?
    # TODO: Check for co-pointing files
    if filter_names == None:
        icX = ImageFileCollection(
            raw_directory,
            keywords=keywds,
            filenames=(ic0.files_filtered(imagetyp='object')).tolist())
        filter_names = icX.values('filter', unique=True)

    # Loop through each of the filters
    for filter in filter_names:
        # Find the images with the filter of interest.
        if verbose == True:
            print('Processing {0} files.'.format(filter))

        # List of images in the current filter
        ic1 = ImageFileCollection(
            raw_directory,
            keywords=keywds,
            filenames=(ic0.files_filtered(filter=filter)).tolist())

        # Make master flat fields.
        # Check to see if flat exists; if so, skip making the flat (time
        # consuming)
        flatname = 'flat.' + filter + '.fits'
        if not os.path.lexists(flatname):
            make_flatfield(ic1,
                           verbose=verbose,
                           raw_directory=raw_directory,
                           image_directory=image_directory)
        else:
            print('Using existing {0}'.format(flatname))

        # Remove overscan, trim object files.
        overfiles = go_overscan(ic1,
                                lbc_chips=lbc_chips,
                                verbose=verbose,
                                raw_directory=raw_directory,
                                image_directory=image_directory)

        # Apply bias frames
        if bias_proc == True:
            # zero_files = go_bias(verbose=verbose)
            print('')

        # Image collection of object frames overscanned & bias subtracted
        ic2 = ImageFileCollection(image_directory,
                                  keywords=keywds,
                                  filenames=overfiles)

        # Apply the flatfields
        flatfiles = go_flatfield(ic2,
                                 return_files=True,
                                 image_directory=image_directory,
                                 verbose=verbose,
                                 cosmiccorrect=False)

        # Image collection of object frames overscanned & bias subtracted + flattened
        ic3 = ImageFileCollection(image_directory,
                                  keywords=keywds,
                                  filenames=flatfiles)

        # from IPython import embed ; embed()
        # Create directories for extracting individual chips.
        tgt_dirs, fltr_dirs = make_targetdirectories(
            ic3, image_directory=image_directory, verbose=verbose)

        # Extract individual chips
        go_extractchips(fltr_dirs, lbc_chips, verbose=verbose)

        # Register and coadd the images
        if do_astrometry:
            go_register(fltr_dirs,
                        lbc_chips=lbc_chips,
                        scamp_iterations=scamp_iterations)

    # Let's do some clean-up.
    if clean:
        # We will eventually ...
        #remove _over, _zero, _flat files.
        print('')
    else:
        # We will eventually ...
        # Move _over, _zero, _flat files.
        print('')
示例#21
0
def run_memory_profile(n_files, sampling_interval, size=None, sigma_clip=False,
                       combine_method=None, memory_limit=None):
    """
    Try opening a bunch of files with a relatively low limit on the number
    of open files.

    Parameters
    ----------

    n_files : int
        Number of files to combine.

    sampling_interval : float
        Time, in seconds, between memory samples.

    size : int, optional
        Size of one side of the image (the image is always square).

    sigma_clip : bool, optional
        If true, sigma clip the data before combining.

    combine_method : str, optional
        Should be one of the combine methods accepted by
        ccdproc.combine

    memory_limit : int, optional
        Cap on memory use during image combination.
    """
    # Do a little input validation
    if n_files <= 0:
        raise ValueError("Argument 'n' must be a positive integer")

    proc = psutil.Process()

    print('Process ID is: ', proc.pid, flush=True)
    ic = ImageFileCollection(str(TMPPATH))
    files = ic.files_filtered(for_prof='yes', include_path=True)

    kwargs = {'method': combine_method}

    if sigma_clip:
        kwargs.update(
            {'sigma_clip': True,
             'sigma_clip_low_thresh': 5,
             'sigma_clip_high_thresh': 5,
             'sigma_clip_func': np.ma.median,
             'sigma_clip_dev_func': median_absolute_deviation}
        )

    ccd = CCDData.read(files[0])
    expected_img_size = _calculate_size_of_image(ccd, None)

    if memory_limit:
        kwargs['mem_limit'] = memory_limit

    pre_mem_use = memory_usage(-1, interval=sampling_interval, timeout=1)
    baseline = np.mean(pre_mem_use)
    print('Subtracting baseline memory before profile: {}'.format(baseline))
    mem_use = memory_usage((combine, (files,), kwargs),
                           interval=sampling_interval, timeout=None)
    mem_use = [m - baseline for m in mem_use]
    return mem_use, expected_img_size