def build_master_images_table_main(png_file_list, reproc, reproc_sets): ''' The main controller. ''' logging.info('Beginning script') logging.info('-filelist setting is {}'.format(png_file_list)) logging.info('-reproc setting is {}'.format(reproc)) logging.info('-reproc_sets setting is {}'.format(reproc_sets)) png_file_list = glob.glob(png_file_list) logging.info('filelist returned {} files'.format(len(png_file_list))) # Trim the input_png_list according to the reproc settings if reproc: png_file_list = png_file_list elif not reproc: logging.info('Removing existing records from filelist') existing_png_files = session.query(distinct(MasterImages.name)).all() existing_png_files = set([item[0] for item in existing_png_files]) png_file_list = [item for item in png_file_list if os.path.basename(item) not in existing_png_files] logging.info('Processing {} files'.format(len(png_file_list))) # Get Existing set information from the DB set_info_query = session.query(MasterImages.set_id, MasterImages.set_index, MasterImages.project_id, MasterImages.visit, MasterImages.orbit, MasterImages.drz_mode, MasterImages.cr_mode).\ filter(MasterImages.visit != None).\ filter(MasterImages.orbit != None).\ all() existing_set_dict = {(record.project_id, record.visit, record.orbit, MasterImages.drz_mode, MasterImages.cr_mode): {'set_id':record.set_id, 'set_index':record.set_index} for record in set_info_query} # Get the max set_id value max_set_id = session.query(func.max(MasterImages.set_id)).one()[0] if max_set_id == None: max_set_id = 0 # Build the new records for png_file in png_file_list: logging.info('Processing {}'.format(png_file)) fits_file = png_file.replace('png/','').replace('_linear.png','.fits') with fits.open(fits_file) as hdulist: header = hdulist[0].header master_images = MasterImages(header, fits_file, png_file) existing_set_dict, max_set_id = master_images.set_set_values(existing_set_dict, max_set_id) session.add(master_images) logging.info('Committing records to database') session.commit() logging.info('Committing records to database complete') logging.info('Script completed')
def insert_record(record_dict, tableclass_instance): ''' Insert the value into the database using SQLAlchemy. This is being phased out in favor of a proper __init__ method for the mapped classes. ''' record = tableclass_instance check_type(record_dict, dict) for key in record_dict.keys(): setattr(record, key, record_dict[key]) session.add(record) session.commit()
def add_new_record(record, region): ''' Make a dictionary of all the data for a record. ''' finders = Finders() finders.sub_images_id = session.query(SubImages.id)\ .filter(SubImages.master_images_id == record.MasterFinders.master_images_id)\ .filter(SubImages.region == region)\ .one().id finders.master_finders_id = record.MasterFinders.id finders.object_name = record.MasterFinders.object_name finders.x = record.MasterFinders.ephem_x - ((record.MasterFinders.ephem_x // 425) * 425) finders.y = record.MasterFinders.ephem_y - ((record.MasterFinders.ephem_y // 425) * 425) session.add(finders)