def build_master_images_table_main(png_file_list, reproc, reproc_sets):
'''
The main controller.
'''
logging.info('Beginning script')
logging.info('-filelist setting is {}'.format(png_file_list))
logging.info('-reproc setting is {}'.format(reproc))
logging.info('-reproc_sets setting is {}'.format(reproc_sets))
png_file_list = glob.glob(png_file_list)
logging.info('filelist returned {} files'.format(len(png_file_list)))
# Trim the input_png_list according to the reproc settings
if reproc:
png_file_list = png_file_list
elif not reproc:
logging.info('Removing existing records from filelist')
existing_png_files = session.query(distinct(MasterImages.name)).all()
existing_png_files = set([item[0] for item in existing_png_files])
png_file_list = [item for item in png_file_list
if os.path.basename(item) not in existing_png_files]
logging.info('Processing {} files'.format(len(png_file_list)))
# Get Existing set information from the DB
set_info_query = session.query(MasterImages.set_id,
MasterImages.set_index,
MasterImages.project_id,
MasterImages.visit,
MasterImages.orbit,
MasterImages.drz_mode,
MasterImages.cr_mode).\
filter(MasterImages.visit != None).\
filter(MasterImages.orbit != None).\
all()
existing_set_dict = {(record.project_id,
record.visit,
record.orbit,
MasterImages.drz_mode,
MasterImages.cr_mode): {'set_id':record.set_id,
'set_index':record.set_index}
for record in set_info_query}
# Get the max set_id value
max_set_id = session.query(func.max(MasterImages.set_id)).one()[0]
if max_set_id == None:
max_set_id = 0
# Build the new records
for png_file in png_file_list:
logging.info('Processing {}'.format(png_file))
fits_file = png_file.replace('png/','').replace('_linear.png','.fits')
with fits.open(fits_file) as hdulist:
header = hdulist[0].header
master_images = MasterImages(header, fits_file, png_file)
existing_set_dict, max_set_id = master_images.set_set_values(existing_set_dict, max_set_id)
session.add(master_images)
logging.info('Committing records to database')
session.commit()
logging.info('Committing records to database complete')
logging.info('Script completed')
def insert_record(record_dict, tableclass_instance):
'''
Insert the value into the database using SQLAlchemy.
This is being phased out in favor of a proper __init__ method for
the mapped classes.
'''
record = tableclass_instance
check_type(record_dict, dict)
for key in record_dict.keys():
setattr(record, key, record_dict[key])
session.add(record)
session.commit()
def add_new_record(record, region):
'''
Make a dictionary of all the data for a record.
'''
finders = Finders()
finders.sub_images_id = session.query(SubImages.id)\
.filter(SubImages.master_images_id == record.MasterFinders.master_images_id)\
.filter(SubImages.region == region)\
.one().id
finders.master_finders_id = record.MasterFinders.id
finders.object_name = record.MasterFinders.object_name
finders.x = record.MasterFinders.ephem_x - ((record.MasterFinders.ephem_x // 425) * 425)
finders.y = record.MasterFinders.ephem_y - ((record.MasterFinders.ephem_y // 425) * 425)
session.add(finders)
