コード例 #1
0
def build_master_images_table_main(png_file_list, reproc, reproc_sets):
    '''
    The main controller.    
    '''
    logging.info('Beginning script')
    logging.info('-filelist setting is {}'.format(png_file_list))
    logging.info('-reproc setting is {}'.format(reproc))
    logging.info('-reproc_sets setting is {}'.format(reproc_sets))
    png_file_list = glob.glob(png_file_list)
    logging.info('filelist returned {} files'.format(len(png_file_list)))

    # Trim the input_png_list according to the reproc settings
    if reproc:
        png_file_list = png_file_list
    elif not reproc:
        logging.info('Removing existing records from filelist')
        existing_png_files = session.query(distinct(MasterImages.name)).all()
        existing_png_files = set([item[0] for item in existing_png_files])
        png_file_list = [item for item in png_file_list
                         if os.path.basename(item) not in existing_png_files]
    logging.info('Processing {} files'.format(len(png_file_list)))

    # Get Existing set information from the DB
    set_info_query = session.query(MasterImages.set_id, 
                                   MasterImages.set_index, 
                                   MasterImages.project_id,
                                   MasterImages.visit,
                                   MasterImages.orbit,
                                   MasterImages.drz_mode,
                                   MasterImages.cr_mode).\
                             filter(MasterImages.visit != None).\
                             filter(MasterImages.orbit != None).\
                             all()
    existing_set_dict = {(record.project_id, 
                          record.visit, 
                          record.orbit,
                          MasterImages.drz_mode,
                          MasterImages.cr_mode): {'set_id':record.set_id, 
                                                  'set_index':record.set_index} 
                         for record in set_info_query}
    
    # Get the max set_id value
    max_set_id = session.query(func.max(MasterImages.set_id)).one()[0]
    if max_set_id == None:
        max_set_id = 0
            
    # Build the new records     
    for png_file in png_file_list:
        logging.info('Processing {}'.format(png_file))
        fits_file = png_file.replace('png/','').replace('_linear.png','.fits')
        with fits.open(fits_file) as hdulist:
            header = hdulist[0].header
        master_images = MasterImages(header, fits_file, png_file)
        existing_set_dict, max_set_id = master_images.set_set_values(existing_set_dict, max_set_id)
        session.add(master_images)
    logging.info('Committing records to database')
    session.commit()
    logging.info('Committing records to database complete')
    logging.info('Script completed')
def test_build_master_images_table():
    """Nosetests testing function.

    This function iterates over the test_dict_list list created at the 
    module level of of this module to generate the values for testing.

    This pattern is described in the `nosetests` docs: 
    https://nose.readthedocs.org/en/latest/writing_tests.html#test-generators

    Parameters: 
        nothing

    Returns: 
        nothing

    Yields:
        check_value : function
            Yields instances of the check_value function so that each 
            assert will count as a seperate test. 
    """
    for test_dict in test_dict_list:

        # Set the non-file specific variables for this set of tests.
        max_set_id = 0
        existing_set_dict = {}
        path = '/astro/mtpipeline/mtpipeline_outputs/wfpc2/06741_mars/'
        fits_file_list = test_dict['fits_file_list']

        # Build the MasterImages instance for each test iteration. 
        for fits_file in fits_file_list:
            fits_file = os.path.join(path, fits_file)
            png_file = fits_file.replace('.fits','_linear.png')
            png_file = os.path.join(path, 'png', os.path.basename(png_file))
            mi = MasterImages(header, fits_file, png_file)
            existing_set_dict, max_set_id = mi.set_set_values(existing_set_dict, max_set_id)

            # Run the tests. 
            true_dict = test_dict[os.path.basename(fits_file)]
            for key in true_dict:
                yield (check_value, getattr(mi, key), true_dict[key], key, 
                       os.path.basename(fits_file))