def build_master_images_table_main(png_file_list, reproc, reproc_sets):
'''
The main controller.
'''
logging.info('Beginning script')
logging.info('-filelist setting is {}'.format(png_file_list))
logging.info('-reproc setting is {}'.format(reproc))
logging.info('-reproc_sets setting is {}'.format(reproc_sets))
png_file_list = glob.glob(png_file_list)
logging.info('filelist returned {} files'.format(len(png_file_list)))
# Trim the input_png_list according to the reproc settings
if reproc:
png_file_list = png_file_list
elif not reproc:
logging.info('Removing existing records from filelist')
existing_png_files = session.query(distinct(MasterImages.name)).all()
existing_png_files = set([item[0] for item in existing_png_files])
png_file_list = [item for item in png_file_list
if os.path.basename(item) not in existing_png_files]
logging.info('Processing {} files'.format(len(png_file_list)))
# Get Existing set information from the DB
set_info_query = session.query(MasterImages.set_id,
MasterImages.set_index,
MasterImages.project_id,
MasterImages.visit,
MasterImages.orbit,
MasterImages.drz_mode,
MasterImages.cr_mode).\
filter(MasterImages.visit != None).\
filter(MasterImages.orbit != None).\
all()
existing_set_dict = {(record.project_id,
record.visit,
record.orbit,
MasterImages.drz_mode,
MasterImages.cr_mode): {'set_id':record.set_id,
'set_index':record.set_index}
for record in set_info_query}
# Get the max set_id value
max_set_id = session.query(func.max(MasterImages.set_id)).one()[0]
if max_set_id == None:
max_set_id = 0
# Build the new records
for png_file in png_file_list:
logging.info('Processing {}'.format(png_file))
fits_file = png_file.replace('png/','').replace('_linear.png','.fits')
with fits.open(fits_file) as hdulist:
header = hdulist[0].header
master_images = MasterImages(header, fits_file, png_file)
existing_set_dict, max_set_id = master_images.set_set_values(existing_set_dict, max_set_id)
session.add(master_images)
logging.info('Committing records to database')
session.commit()
logging.info('Committing records to database complete')
logging.info('Script completed')
def test_build_master_images_table():
"""Nosetests testing function.
This function iterates over the test_dict_list list created at the
module level of of this module to generate the values for testing.
This pattern is described in the `nosetests` docs:
https://nose.readthedocs.org/en/latest/writing_tests.html#test-generators
Parameters:
nothing
Returns:
nothing
Yields:
check_value : function
Yields instances of the check_value function so that each
assert will count as a seperate test.
"""
for test_dict in test_dict_list:
# Set the non-file specific variables for this set of tests.
max_set_id = 0
existing_set_dict = {}
path = '/astro/mtpipeline/mtpipeline_outputs/wfpc2/06741_mars/'
fits_file_list = test_dict['fits_file_list']
# Build the MasterImages instance for each test iteration.
for fits_file in fits_file_list:
fits_file = os.path.join(path, fits_file)
png_file = fits_file.replace('.fits','_linear.png')
png_file = os.path.join(path, 'png', os.path.basename(png_file))
mi = MasterImages(header, fits_file, png_file)
existing_set_dict, max_set_id = mi.set_set_values(existing_set_dict, max_set_id)
# Run the tests.
true_dict = test_dict[os.path.basename(fits_file)]
for key in true_dict:
yield (check_value, getattr(mi, key), true_dict[key], key,
os.path.basename(fits_file))