Exemple #1
0
def bias_combine(refresh=0):
    tbiascollection = ImageFileCollection('Trimmed_Bias')
    print('found', len(tbiascollection.values('file')), 'trimmed biases')
    combined_bias = ccdp.combine(tbiascollection.files_filtered(
        imtype='trimmed bias', include_path=True),
                                 sigma_clip=True,
                                 sigma_clip_low_thresh=5,
                                 sigma_clip_high_thresh=5,
                                 sigma_clip_func=np.nanmedian,
                                 sigma_clip_dev_func=mad_std)
    combined_bias.meta['combined'] = True
    combined_bias.write('Master_Files/mbias.fits', overwrite=True)
    return tbiascollection
def combine_flats(refresh='2', method='2'):
    if method == '1':
        meta = 'med'
        source = 'Trimmed_Flat/subflatsmed'
        dest = 'Master_Files/mflat_median.fits'
    elif method == '2':
        meta = 'sig'
        source = 'Trimmed_Flat/subflatssig'
        dest = 'Master_Files/mflat.fits'
    subflatcollection = ImageFileCollection(source)
    combtime = 0
    if refresh == '1':
        print('found', len(subflatcollection.values('file')), 'subflats')
        start = time.time()
        if method == '1':
            mflat = ccdp.combine(subflatcollection.files_filtered(
                imtype='subflat', include_path=True),
                                 method='median')
            mflat.meta['flatcom'] = 'median'
            combtime = time.time() - start
            print('combination took', combtime, 'seconds')
        elif method == '2':
            mflat = ccdp.combine(subflatcollection.files_filtered(
                imtype='subflat', include_path=True),
                                 sigma_clip=True,
                                 sigma_clip_low_thresh=5,
                                 sigma_clip_high_thresh=5,
                                 sigma_clip_func=np.nanmedian,
                                 sigma_clip_dev_func=mad_std)
            mflat.meta['flatcom'] = 'sigma'
            combtime = time.time() - start
            print('combination took', combtime, 'seconds')
        mflat.meta['normmed'] = (np.nanmedian(mflat),
                                 'nanmedian of the master flat')
        mflat.meta['subflats'] = meta
        mflat.write(dest[0:-5] + '_' + meta + '.fits', overwrite=True)
    else:
        try:
            if method == '1':
                mflat = CCDData.read('Master_Files/mflat_median_med.fits',
                                     unit='adu')

            elif method == '2':
                mflat = CCDData.read('Master_Files/mflat_sig.fits', unit='adu')
        except:
            print('can\'t locate master flat, create or check directory')
            sys.exit()
    return subflatcollection, mflat, dest, combtime
def bias_combine(refresh='2', method='2'):
    tbiascollection = ImageFileCollection('Trimmed_Bias')
    combtime = 0
    if refresh == '1':
        print('found', len(tbiascollection.values('file')), 'trimmed biases')
        start = time.time()
        if method == '1':
            combined_bias = ccdp.combine(tbiascollection.files_filtered(
                imtype='trimmed bias', include_path=True),
                                         method='median')
            combbiaspath = 'Master_Files/mbias_median.fits'
            combined_bias.meta['combined'] = 'median'
            combtime = time.time() - start
            print('combination took', combtime, 'seconds')
            combined_bias.write(combbiaspath, overwrite=True)
        elif method == '2':
            combined_bias = ccdp.combine(tbiascollection.files_filtered(
                imtype='trimmed bias', include_path=True),
                                         sigma_clip=True,
                                         sigma_clip_low_thresh=5,
                                         sigma_clip_high_thresh=5,
                                         sigma_clip_func=np.nanmedian,
                                         sigma_clip_dev_func=mad_std)
            combbiaspath = 'Master_Files/mbias.fits'
            combined_bias.meta['combined'] = 'sigma_clip average'
            combtime = time.time() - start
            print('combination took', combtime, 'seconds')
            combined_bias.write(combbiaspath, overwrite=True)
    else:
        try:
            if method == '1':
                combined_bias = CCDData.read('Master_Files/mbias_median.fits',
                                             unit='adu')
                combbiaspath = 'Master_Files/mbias_median.fits'
            elif method == '2':
                combined_bias = CCDData.read('Master_Files/mbias.fits',
                                             unit='adu')
                combbiaspath = 'Master_Files/mbias.fits'
        except:
            print('can\'t locate master bias, create or check directory')
            sys.exit()

    return tbiascollection, combined_bias, combbiaspath, combtime
def bias_combine(refresh=False, method=1):
    if refresh == 0:
        tbiascollection = ImageFileCollection('Trimmed_Bias')
        print('found', len(tbiascollection.values('file')), 'trimmed biases')
        start=time.time()
        if method == 1:
            combined_bias = ccdp.combine(tbiascollection.files_filtered(
            imtype='trimmed bias', include_path=True),
            method='median')
            combbiaspath = 'Master_Files/mbias_median.fits'
            combined_bias.meta['combined'] = 'median'
        elif method == 2:
            combined_bias = ccdp.combine(tbiascollection.files_filtered(
            imtype='trimmed bias', include_path=True),
            sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5,
            sigma_clip_func=np.nanmedian, sigma_clip_dev_func=mad_std)
            combbiaspath = 'Master_Files/mbias.fits'
            combined_bias.meta['combined'] = 'sigma_clip average'
        combtime=time.time()-start
        print('combination took',combtime,'seconds')
        combined_bias.write('Master_Files/mbias_median.fits',
                            overwrite=True)
    return tbiascollection,combined_bias,combbiaspath,combtime
import glob
from astropy import units as u
from astropy.io import fits
from ccdproc import CCDData, Combiner
import numpy as np
from ccdproc import Combiner, ImageFileCollection, fits_ccddata_writer
from t120_init import t120

listimg = ImageFileCollection(
    t120.t120_flat_dir)  #,glob_include='*.fit',glob_exclude='*.fits')

list_filters = listimg.values('filter', unique=True)
#for filter_name in listimg.values('filter',unique=True):
for filter_name in list_filters:
    t120.log.info('*** filter: ' + filter_name)
    my_files = listimg.files_filtered(filter=filter_name)
    t120.log.info('my_files=' + my_files)

pouet

for filter_name in listimg.values('filter', unique=True):
    t120.log.info('*** filter: ' + filter_name)
    listccd = []
    for ccd, file_name in listimg.ccds(ccd_kwargs={'unit': 'adu'},
                                       filter=filter_name,
                                       return_fname=True):
        t120.log.info('now considering file ' + file_name)
        listccd.append(ccd)
    """
    t120.log.info('now making the Combiner object')
    combiner = Combiner(listccd)
Exemple #6
0
        print('not combining zeros')
        print('\t reading in bias-combined.fits instead')
        hdu1 = fits.open('bias-combined.fits')
        header = hdu1[0].header
        gaincorrected_master_bias = CCDData(hdu1[0].data, unit=u.electron, meta=header)
        hdu1.close()
    else:
        print('not using a bias frame')
###################################################
# COMBINE DARKS
###################################################
# identify darks and combine darks with longest exposure time

# change to use the dark with the closest exposure time

exptimes = np.array(ic.values('exptime'))

image_types = np.array(ic.values('imagetyp'))

if args.darkcombine:

    # select all files with imagetyp=='dark'
    # want to read in one set of long exposure dark frames, like 120 s
    # observers should take a set of darks that correspond to longest exposure time
    # e.g. 120s

    set_exptime=set(exptimes[image_types == ccdkeyword['dark']])
    dark_exptimes_set = np.array(list(set_exptime),'f')
    max_exposure = max(exptimes[image_types == ccdkeyword['dark']])
    '''
    * subtract bias from combined dark frames
Exemple #7
0
def na_back_pipeline(directory=None,
                     glob_include='Jupiter*',
                     calibration=None,
                     photometry=None,
                     n_back_boxes=N_BACK_BOXES,
                     num_processes=None,
                     outdir=None,
                     create_outdir=True,
                     **kwargs):

    outdir = outdir or reduced_dir(directory, create=False)
    collection = ImageFileCollection(directory, glob_include=glob_include)
    if collection is None:
        return []
    summary_table = collection.summary
    #print(summary_table['raoff'])
    try:
        raoffs = collection.values('raoff', unique=True)
        decoffs = collection.values('decoff', unique=True)
    except Exception as e:
        log.error(f'Problem with RAOFF/DECOFF in {directory}: {e}')
        return []
    f_pairs = []
    for raoff in raoffs:
        for decoff in decoffs:
            try:
                subc = collection.filter(raoff=raoff, decoff=decoff)
            except:
                log.debug(f'No match for RAOFF = {raoff} DECOFF = {decoff}')
                continue
            fp = closest_in_time(subc, ('Na_on', 'Na_off'),
                                 valid_long_exposure,
                                 directory=directory)
            f_pairs.extend(fp)
    if len(f_pairs) == 0:
        log.warning(f'No matching set of Na background files found '
                    f'in {directory}')
        return []

    if calibration is None:
        calibration = Calibration(reduce=True)
    if photometry is None:
        photometry = Photometry(precalc=True,
                                n_back_boxes=n_back_boxes,
                                **kwargs)

    # Eventually put  back in
    cmp = CorMultiPipe(auto=True,
                       calibration=calibration,
                       photometry=photometry,
                       create_outdir=create_outdir,
                       post_process_list=[
                           multi_filter_proc, combine_masks, na_back_process,
                           no_outfile
                       ],
                       num_processes=num_processes,
                       process_expand_factor=15,
                       **kwargs)

    # but get ready to write to reduced directory if necessary
    #pout = cmp.pipeline([f_pairs[0]], outdir=outdir, overwrite=True)
    pout = cmp.pipeline(f_pairs, outdir=outdir, overwrite=True)
    pout, _ = prune_pout(pout, f_pairs)
    return pout
Exemple #8
0
def go_swarp(inputfiles,
             output_filename = None,
             configfile = None,
             verbose = True):
    """Do SWARP"""

    # Make sure we have a configuration file:
    if configfile == None:
        # TODO: replace this with default config file in LBCgo directories.
        configfile = 'swarp.lbc.conf'

    # Gather some information about the input files
    keywds = ['object', 'filter', 'exptime', 'imagetyp', 'propid', 'lbcobnam',
                  'airmass', 'HA', 'objra', 'objdec']
    ic_swarp = ImageFileCollection('./', keywords=keywds,
                                    filenames = inputfiles)

    # Check all are the same filter:
    # TODO: Set exception if not all are same filter rather than just bail.
    fltrs = ic_swarp.values('filter',unique=True)
    if np.size(fltrs) != 1:
        print('Warning: not all files in SWARP call use the same filter.')
        return None

    # Calculate mean airmass (weighted by exposure time)
    exp_airmass = np.array(ic_swarp.values('airmass'))
    exp_time = np.array(ic_swarp.values('exptime'))
    airmass = np.average(exp_airmass,weights=exp_time)

    # from IPython import embed ; embed()

    # For now grab the information from the first header:
    if output_filename == None:
        imhead = fits.getheader(inputfiles[0])
        # Shorten the filter names used:
        filter_text = imhead['FILTER']
        filter_text = filter_text.replace('-SLOAN','').replace('-BESSEL','').replace('SDT_Uspec','Uspec')
        # Create final output filename
        # TODO: Check if file exists.
        output_filename = (imhead['object']).\
            replace(' ','')+'.'+filter_text+'.mos.fits'

    # Rename the weight image
    weight_filename = output_filename.replace('.mos.fits','.mos.weight.fits')

    # Create the list of input files:
    inputfile_text = ''
    for fl in inputfiles: inputfile_text = inputfile_text+' '+fl

    cmd_flags = ' -c '+configfile+ \
        ' -IMAGEOUT_NAME '+ output_filename + \
        ' -WEIGHTOUT_NAME '+ weight_filename + \
        ' -WEIGHT_TYPE NONE '+ \
        ' -HEADER_SUFFIX ".head"'+ \
        ' -FSCALE_KEYWORD NONE -FSCALE_DEFAULT 1.0 '+\
        ' -CELESTIAL_TYPE EQUATORIAL -CENTER_TYPE ALL '+\
        ' -COMBINE_BUFSIZE 4096 ' +\
        ' -COPY_KEYWORDS '+\
        ' OBJECT,OBJRA,OBJDEC,OBJEPOCH,PROPID,PI_NAME,'+\
        'FILTER,SATURATE,RDNOISE,GAIN,EXPTIME,AIRMASS,TIME-OBS'

    # Create the final command:
    cmd = 'swarp ' + inputfile_text + cmd_flags

    try:
        if verbose:
            print('########### SWARP image combination '
                  '########### \n')
            print(cmd)
            swarp = Popen(shlex.split(cmd),
                          close_fds=True)
        else:
            swarp = Popen(shlex.split(cmd),
                          stdout=DEVNULL,
                          stderr=DEVNULL,
                          close_fds=True)
    except Exception as e:
        print('Oops: SWARP call:', (e))
        return None

    swarp.wait()

    # Add airmass to header:
    fits.setval(output_filename,'AIRMASS',value=airmass)
Exemple #9
0
def lbcgo(raw_directory='./raw/',
          image_directory='./',
          lbc_chips=True,
          lbcr=True,
          lbcb=True,
          filter_names=None,
          bias_proc=False,
          do_astrometry=True,
          scamp_iterations=3,
          verbose=True,
          clean=True):
    """Process a directory of LBC data.

    By default, all raw data are to be in the ./raw/ directory before processing (raw_directory='./raw/').
    By default, all new images are written in the CWD (image_directory='./').

    """

    # It will go something like this...

    # Make sure the input directories have trailing slashes:
    if raw_directory[-1] != '/':
        raw_directory += '/'
    if image_directory[-1] != '/':
        image_directory += '/'

    ########## Collect basic image information
    # Find the raw files we'll use
    if lbcb & lbcr:
        lbc_file_base = 'lbc?.*.*.fits*'
    elif lbcb:
        lbc_file_base = 'lbcb.*.*.fits*'
    elif lbcr:
        lbc_file_base = 'lbcr.*.*.fits*'

    # What information do we want from the headers?
    keywds = [
        'object', 'filter', 'exptime', 'imagetyp', 'propid', 'lbcobnam',
        'airmass', 'HA', 'objra', 'objdec'
    ]

    ##### Create an ImageFileCollection object to hold the raw data list.
    if np.int(ccdproc.__version__[0]) == 2:
        ic0 = ImageFileCollection(raw_directory,
                                  keywords=keywds,
                                  glob_include=lbc_file_base)

        num_images = np.size(ic0.summary['object'])
        # Exit if (for some reason) no images are found in the raw_directory.
        if num_images == 0:
            print('WARNING: No images found.')
            return None
    else:
        raw_lbc_files = glob(raw_directory + lbc_file_base)
        ic0 = ImageFileCollection(raw_directory,
                                  keywords=keywds,
                                  filenames=raw_lbc_files)
        num_images = np.size(ic0.summary['object'])

        # Exit if (for some reason) no images are found in the raw_directory.
        if num_images == 0:
            print('WARNING: No images found.')
            return None

    ######### Create the master bias frame (if requested)
    if bias_proc == True:
        make_bias(ic0,
                  image_directory=image_directory,
                  raw_directory=raw_directory)

    ###### Define which chips to extract if default is chosen:
    if lbc_chips == True:
        lbc_chips = [1, 2, 3, 4]

    ###### Per filter:
    #
    # Step through the filters in the ImageCollection unless user gives filter
    # list.
    #
    # For now nights with V-band filters in both LBCB and LBCR require running
    # the code with 'lbcr=False' then 'lbcb=False' to avoid coadding the two
    # images.

    # TODO: Loop B/R to avoid issues w/V-band from LBCB/LBCR?
    # TODO: Check for co-pointing files
    if filter_names == None:
        icX = ImageFileCollection(
            raw_directory,
            keywords=keywds,
            filenames=(ic0.files_filtered(imagetyp='object')).tolist())
        filter_names = icX.values('filter', unique=True)

    # Loop through each of the filters
    for filter in filter_names:
        # Find the images with the filter of interest.
        if verbose == True:
            print('Processing {0} files.'.format(filter))

        # List of images in the current filter
        ic1 = ImageFileCollection(
            raw_directory,
            keywords=keywds,
            filenames=(ic0.files_filtered(filter=filter)).tolist())

        # Make master flat fields.
        # Check to see if flat exists; if so, skip making the flat (time
        # consuming)
        flatname = 'flat.' + filter + '.fits'
        if not os.path.lexists(flatname):
            make_flatfield(ic1,
                           verbose=verbose,
                           raw_directory=raw_directory,
                           image_directory=image_directory)
        else:
            print('Using existing {0}'.format(flatname))

        # Remove overscan, trim object files.
        overfiles = go_overscan(ic1,
                                lbc_chips=lbc_chips,
                                verbose=verbose,
                                raw_directory=raw_directory,
                                image_directory=image_directory)

        # Apply bias frames
        if bias_proc == True:
            # zero_files = go_bias(verbose=verbose)
            print('')

        # Image collection of object frames overscanned & bias subtracted
        ic2 = ImageFileCollection(image_directory,
                                  keywords=keywds,
                                  filenames=overfiles)

        # Apply the flatfields
        flatfiles = go_flatfield(ic2,
                                 return_files=True,
                                 image_directory=image_directory,
                                 verbose=verbose,
                                 cosmiccorrect=False)

        # Image collection of object frames overscanned & bias subtracted + flattened
        ic3 = ImageFileCollection(image_directory,
                                  keywords=keywds,
                                  filenames=flatfiles)

        # from IPython import embed ; embed()
        # Create directories for extracting individual chips.
        tgt_dirs, fltr_dirs = make_targetdirectories(
            ic3, image_directory=image_directory, verbose=verbose)

        # Extract individual chips
        go_extractchips(fltr_dirs, lbc_chips, verbose=verbose)

        # Register and coadd the images
        if do_astrometry:
            go_register(fltr_dirs,
                        lbc_chips=lbc_chips,
                        scamp_iterations=scamp_iterations)

    # Let's do some clean-up.
    if clean:
        # We will eventually ...
        #remove _over, _zero, _flat files.
        print('')
    else:
        # We will eventually ...
        # Move _over, _zero, _flat files.
        print('')
Exemple #10
0
def make_targetdirectories(image_collection,
                           image_directory='./',
                           object_names=None,
                           verbose=True):
    """ Make directories to store data from individual targets and move the data for those targets into the directories.

    :param image_collection:
    :param image_directory:
    :param object_names:
    :param verbose:
    :return: object_directories, filter_directories
    """

    # If we specify objects, use only those objects
    if object_names == None:
        object_names = np.unique(image_collection.summary['object'])

    # Return a list of the directories
    object_directories = []
    filter_directories = []

    # Loop through the objects
    for obj in object_names:
        # Name of the target directory to be created
        dirname = image_directory + obj.replace(' ', '') + '/'
        object_directories.append(dirname)

        # Test that the directory doesn't already exist
        if not os.path.lexists(dirname):
            os.makedirs(dirname)

            if verbose == True:
                print('Creating directory for object {0}.'.format(obj))
        else:
            if verbose == True:
                print('Directory exists for object {0}.'.format(obj))

        # Move the data for each object into its directory.
        # object_files = image_collection.files_filtered(object=obj)
        object_files = \
          image_collection.summary['file'][(image_collection.summary['object'] == obj)]
        for fl in object_files:
            cmd = 'mv {0} {1}'.format(fl, dirname)
            crap = call(cmd, shell=True)

        # Create filter-specific directories and fill them. For now this just
        # assumes that every object has the same filter list. This is fine,
        # since we're normally going through this step on a filter-by-filter
        # basis anyway (see proc.py).

        # The following should select only the filters appropriate for this object:
        keywds = ['object', 'filter']
        image_collectionObj = ImageFileCollection(dirname, keywords=keywds,
                            filenames = \
                            (image_collection.files_filtered(object=obj.replace('+','\+').replace('-','\-'))).tolist())

        # debug
        # from IPython import embed ; embed()

        # Now select the unique filters for this objects
        filters = image_collectionObj.values('filter', unique=True)

        # Step through each filter, creating directories and moving files.
        for filter in filters:
            filter_dirname = dirname + filter + '/'
            filter_directories.append(filter_dirname)

            # Test that the directory doesn't already exist
            if not os.path.lexists(filter_dirname):
                os.makedirs(filter_dirname)

                if verbose == True:
                    print('Creating {1} directory for object {0}.'.format(
                        obj, filter))
            else:
                if verbose == True:
                    print('Directory exists for {1} for object {0}.'.format(
                        obj, filter))

            filter_files = image_collection.files_filtered(object=obj.replace(
                '+', '\+').replace('-', '\-'),
                                                           filter=filter)
            for fltfl in filter_files:
                cmd = 'mv {0} {1}'.format(dirname + fltfl, filter_dirname)
                print(cmd)
                crap = call(cmd, shell=True)

    return object_directories, filter_directories