def build_master_images_table_main(png_file_list, reproc, reproc_sets): ''' The main controller. ''' logging.info('Beginning script') logging.info('-filelist setting is {}'.format(png_file_list)) logging.info('-reproc setting is {}'.format(reproc)) logging.info('-reproc_sets setting is {}'.format(reproc_sets)) png_file_list = glob.glob(png_file_list) logging.info('filelist returned {} files'.format(len(png_file_list))) # Trim the input_png_list according to the reproc settings if reproc: png_file_list = png_file_list elif not reproc: logging.info('Removing existing records from filelist') existing_png_files = session.query(distinct(MasterImages.name)).all() existing_png_files = set([item[0] for item in existing_png_files]) png_file_list = [item for item in png_file_list if os.path.basename(item) not in existing_png_files] logging.info('Processing {} files'.format(len(png_file_list))) # Get Existing set information from the DB set_info_query = session.query(MasterImages.set_id, MasterImages.set_index, MasterImages.project_id, MasterImages.visit, MasterImages.orbit, MasterImages.drz_mode, MasterImages.cr_mode).\ filter(MasterImages.visit != None).\ filter(MasterImages.orbit != None).\ all() existing_set_dict = {(record.project_id, record.visit, record.orbit, MasterImages.drz_mode, MasterImages.cr_mode): {'set_id':record.set_id, 'set_index':record.set_index} for record in set_info_query} # Get the max set_id value max_set_id = session.query(func.max(MasterImages.set_id)).one()[0] if max_set_id == None: max_set_id = 0 # Build the new records for png_file in png_file_list: logging.info('Processing {}'.format(png_file)) fits_file = png_file.replace('png/','').replace('_linear.png','.fits') with fits.open(fits_file) as hdulist: header = hdulist[0].header master_images = MasterImages(header, fits_file, png_file) existing_set_dict, max_set_id = master_images.set_set_values(existing_set_dict, max_set_id) session.add(master_images) logging.info('Committing records to database') session.commit() logging.info('Committing records to database complete') logging.info('Script completed')
def update_record(record_dict, query): ''' Update a record in the database using SQLAlchemy. See insert_record for details. ''' check_type(record_dict, dict) count = query.update(record_dict) session.commit()
def insert_record(record_dict, tableclass_instance): ''' Insert the value into the database using SQLAlchemy. This is being phased out in favor of a proper __init__ method for the mapped classes. ''' record = tableclass_instance check_type(record_dict, dict) for key in record_dict.keys(): setattr(record, key, record_dict[key]) session.add(record) session.commit()
def build_finders_table_main(reproc): ''' Calculate the ephemeris information for the subimages. ''' query = session.query(MasterFinders).count() print str(query) + ' total ephemerides' query = session.query(MasterFinders, MasterImages)\ .join(MasterImages, MasterImages.id == MasterFinders.master_images_id)\ .filter(MasterFinders.ephem_x >= 0)\ .filter(MasterFinders.ephem_y >= 0)\ .filter(MasterFinders.ephem_x <= 1725)\ .filter(MasterFinders.ephem_y <= 1300)\ .filter(MasterImages.drz_mode == 'wide')\ .all() print str(len(query)) + ' ephemerides in wide mode image FOVs' count = 0 for record in query: count = counter(count) region_list = get_region_list(record.MasterFinders.ephem_x, record.MasterFinders.ephem_y) for region in region_list: add_new_record(record, region) session.commit() session.close()