def run_ephem_main(reproc=False):
'''
The main controller for the module. It executes the code in ephem_main
and writes the output to the database.
'''
#print 'Processing ' + str(len(file_list)) + ' files.'
count = 0
query_list = session.query(MasterFinders, MasterImages).\
join(MasterImages).\
options(subqueryload('master_images_rel')).\
filter(or_(MasterFinders.jpl_ra != None, MasterFinders.jpl_dec != None)).\
all()
print 'Processing ' + str(len(query_list)) + ' files.'
for record in query_list:
if record.MasterFinders.ephem_x == None \
or record.MasterFinders.ephem_y == None \
or reproc == True:
file_dict = get_header_info(\
os.path.join(\
record.MasterImages.file_location[0:-4],
record.MasterImages.fits_file))
delta_x, delta_y = calc_delta(file_dict, record.MasterFinders)
ephem_x, ephem_y = calc_pixel_position(file_dict, delta_x, delta_y)
update_dict = {}
update_dict['ephem_x'] = int(ephem_x)
update_dict['ephem_y'] = int(ephem_y)
session.query(MasterFinders).\
filter(MasterFinders.id == record.MasterFinders.id).\
update(update_dict)
count = counter(count)
session.commit()
session.close()
def run_ephem_main(reproc=False):
'''
The main controller for the module. It executes the code in ephem_main
and writes the output to the database.
'''
# Build the record list and log the length.
query_all_records = session.query(MasterFinders, MasterImages).\
join(MasterImages).\
options(subqueryload('master_images_rel'))
logging.info('{} total records in master_finders.'.\
format(query_all_records.count()))
if reproc == True:
query_list = query_all_records.all()
logging.info('reproc == True, Reprocessing all records')
else:
logging.info('reproc == False')
query_list = session.query(MasterFinders, MasterImages).\
join(MasterImages).\
options(subqueryload('master_images_rel')).\
filter(or_(MasterFinders.jpl_ra != None, MasterFinders.jpl_dec != None)).\
all()
logging.info('Processing {} files where ephem_x or ephem_y IS NULL.'.\
format(len(query_list)))
for counter, record in enumerate(query_list):
logging.info('Working on {}'.\
format(record.MasterImages.name, record.MasterFinders.object_name))
file_dict = get_header_info(\
os.path.join(\
record.MasterImages.file_location[0:-4],
record.MasterImages.fits_file))
delta_x, delta_y = calc_delta(file_dict, record.MasterFinders)
ephem_x, ephem_y = calc_pixel_position(file_dict, delta_x, delta_y)
update_dict = {}
update_dict['ephem_x'] = int(ephem_x)
update_dict['ephem_y'] = int(ephem_y)
session.query(MasterFinders).\
filter(MasterFinders.id == record.MasterFinders.id).\
update(update_dict)
if counter % 100 == 0:
session.commit()
session.close()
def jpl2db_main(filename, reproc=False):
'''
The main controller.
'''
# Get the unique record from the master_images table.
assert os.path.splitext(filename)[1] == '.fits', \
'Expected .fits got ' + filename
master_images_query = session.query(MasterImages).filter(\
MasterImages.fits_file == os.path.basename(filename)).one()
# Gather some file information and iterate over the moons
file_dict = get_header_info(os.path.abspath(filename))
file_dict = convert_datetime(file_dict)
all_moon_dict = make_all_moon_dict('planets_and_moons.txt', file_dict)
for moon in all_moon_dict.keys():
logging.info('Processing {} for {}'.format(moon, filename))
moon_dict = all_moon_dict[moon]
moon_dict.update(file_dict)
# Check if a record exists. If there's no record insert one.
master_finders_count = session.query(MasterFinders).filter(\
MasterFinders.master_images_id == master_images_query.id).filter(\
MasterFinders.object_name == moon).count()
if master_finders_count == 0:
jpl_dict = get_jpl_data(moon_dict)
moon_dict.update(jpl_dict)
insert_record(moon_dict, master_images_query.id)
# When a record exists, check if it has jpl_ra info. If it
# doesn't then update.
else:
master_finders_count = session.query(MasterFinders).filter(\
MasterFinders.master_images_id == master_images_query.id).filter(\
MasterFinders.object_name == moon).filter(\
MasterFinders.jpl_ra == None).count()
if master_finders_count == 1 or reproc == True:
jpl_dict = get_jpl_data(moon_dict)
moon_dict.update(jpl_dict)
update_record(moon_dict, master_images_query.id)
session.close()
def add_new_record(record, region):
'''
Make a dictionary of all the data for a record.
'''
finders = Finders()
finders.sub_images_id = session.query(SubImages.id)\
.filter(SubImages.master_images_id == record.MasterFinders.master_images_id)\
.filter(SubImages.region == region)\
.one().id
finders.master_finders_id = record.MasterFinders.id
finders.object_name = record.MasterFinders.object_name
finders.x = record.MasterFinders.ephem_x - ((record.MasterFinders.ephem_x // 425) * 425)
finders.y = record.MasterFinders.ephem_y - ((record.MasterFinders.ephem_y // 425) * 425)
session.add(finders)
def build_sub_images_table_main(filename, reproc):
'''
The main controller.
'''
assert os.path.splitext(filename)[1] == '.png', \
'Expected .png got ' + filename
path = os.path.split(filename)[0]
basename = os.path.basename(filename)
master_filename = get_master_filename(basename)
master_images_query = session.query(MasterImages).filter(\
MasterImages.name == master_filename).one()
sub_images_query = session.query(SubImages).filter(\
SubImages.name == basename).count()
image_width, image_height = get_image_size(filename)
# Make the input dict
record_dict = {}
record_dict['master_images_id'] = master_images_query.id
record_dict['master_images_name'] = master_images_query.name
record_dict['name'] = basename
record_dict['file_location'] = path
record_dict['image_width'] = image_width
record_dict['image_height'] = image_height
record_dict['region'] = get_region(filename)
if sub_images_query == 0:
record = SubImages()
insert_record(record_dict, record)
elif sub_images_query == 1:
sub_images_query = session.query(SubImages).filter(\
SubImages.name == basename).update(record_dict)
session.commit()
else:
pass
session.close()
def build_finders_table_main(reproc):
'''
Calculate the ephemeris information for the subimages.
'''
query = session.query(MasterFinders).count()
print str(query) + ' total ephemerides'
query = session.query(MasterFinders, MasterImages)\
.join(MasterImages, MasterImages.id == MasterFinders.master_images_id)\
.filter(MasterFinders.ephem_x >= 0)\
.filter(MasterFinders.ephem_y >= 0)\
.filter(MasterFinders.ephem_x <= 1725)\
.filter(MasterFinders.ephem_y <= 1300)\
.filter(MasterImages.drz_mode == 'wide')\
.all()
print str(len(query)) + ' ephemerides in wide mode image FOVs'
count = 0
for record in query:
count = counter(count)
region_list = get_region_list(record.MasterFinders.ephem_x, record.MasterFinders.ephem_y)
for region in region_list:
add_new_record(record, region)
session.commit()
session.close()
def update_record(moon_dict, master_images_id):
'''
Update a record in the master_finders table.
'''
update_dict = {}
update_dict['object_name'] = moon_dict['object']
update_dict['jpl_ra'] = moon_dict['jpl_ra']
update_dict['jpl_dec'] = moon_dict['jpl_dec']
try:
update_dict['magnitude'] = float(moon_dict['jpl_APmag'])
except Exception as err:
logging.critical('{0} {1} {2}'.format(
type(err), err.message, sys.exc_traceback.tb_lineno))
try:
update_dict['diameter'] = float(moon_dict['jpl_ang_diam'])
except Exception as err:
logging.critical('{0} {1} {2}'.format(
type(err), err.message, sys.exc_traceback.tb_lineno))
update_dict['master_images_id'] = master_images_id
update_dict['version'] = __version__
session.query(MasterFinders).filter(\
MasterFinders.master_images_id == master_images_id,
MasterFinders.object_name == moon_dict['object']).update(update_dict)
session.commit()
def check_database_completeness_main():
"""The main function for the module."""
# Log the hostname and database record counds.
logging.info('Host is {}'.format(socket.gethostname()))
# Log the total number of records in master_images table.
master_images_query = session.query(MasterImages).all()
master_images_count = len(master_images_query)
logging.info('{} records found in master_images'.format(master_images_count))
# Use a wildcard search to log the number of existing records in
# master_images for the 2 different modes.
wide_count = session.query(MasterImages).\
filter(MasterImages.fits_file.like('%wide_single_sci.fits')).\
count()
logging.info('{} of expected {} wide records found in master_finders'.\
format(wide_count, master_images_count / 2))
center_count = session.query(MasterImages).\
filter(MasterImages.fits_file.like('%center_single_sci.fits')).\
count()
logging.info('{} of expected {} center records found in master_finders'.\
format(center_count, master_images_count / 2))
# Use a length count for the fits_file field to infer the number
# of records in the 4 combinations of the 2 cr mode and the 2
# astro drizzle modes.
cr_wide_count = 0
cr_center_count = 0
non_cr_wide_count = 0
non_cr_center_count = 0
unknown_type_count = 0
for record in master_images_query:
if len(record.fits_file) == 37:
cr_wide_count += 1
elif len(record.fits_file) == 39:
cr_center_count += 1
elif len(record.fits_file) == 34:
non_cr_wide_count += 1
elif len(record.fits_file) == 36:
non_cr_center_count += 1
else:
unknown_type_count += 1
logging.info('{} of {} expected cr wide records found in master_images by length.'.\
format(cr_wide_count, master_images_count / 4))
logging.info('{} of {} expected cr center records found in master_images by length.'.\
format(cr_center_count, master_images_count / 4))
logging.info('{} of {} expected non-cr wide records found in master_images by length.'.\
format(non_cr_wide_count, master_images_count / 4))
logging.info('{} of {} expected non-cr center records found in master_images by length.'.\
format(non_cr_center_count, master_images_count / 4))
logging.info('{} of 0 expected unknown type records found in master_images by length.'.\
format(unknown_type_count))
# Do the record counts for the master_finders records
logging.info('{} records found in master_finders'.\
format(session.query(MasterFinders).count()))
logging.info('{} records found in master_images LEFT JOIN master_finders'.\
format(session.query(MasterImages, MasterFinders).\
outerjoin(MasterFinders).count()))
logging.info('{} records with object_name IS NULL in master_images LEFT JOIN master_finders'.\
format(session.query(MasterImages, MasterFinders).\
outerjoin(MasterFinders).\
filter(MasterFinders.object_name == None).count()))
# Log the missing master_finders records
master_finders_query = session.query(MasterImages, MasterFinders).\
outerjoin(MasterFinders).\
filter(MasterFinders.object_name == None).all()
if len(master_finders_query) != 0:
for record in master_finders_query:
logging.error('No object_name value for {}'.\
format(os.path.join(record.MasterImages.file_location,
record.MasterImages.name)))
#! /usr/bin/env python
import glob
import os
from database_interface import session
from database_interface import Master
from database_interface import IR_FLT_0
query = session.query(Master, IR_FLT_0).\
join(IR_FLT_0, Master.id == IR_FLT_0.id).\
order_by(IR_FLT_0.EXPSTART)
test = True
if test:
sorted_local_files = []
local_file_list = glob.glob('../data/*/Visit*/*.fits')
for local_file in local_file_list:
local_query = query.filter(Master.filename == os.path.basename(local_file)).one()
sorted_local_files.append((local_file, local_query.IR_FLT_0.EXPSTART))
#for record in query[0:25]:
# print os.path.join(record.Master.dir, record.Master.filename)
sorted_local_files = sorted(sorted_local_files, key=lambda item : item[1])
with open('per_list.txt', 'w') as f:
for line in sorted_local_files:
f.write(line[0] + '\n')
logging.info('Machine: {0}'.format(machine()))
logging.info('Platform: {0}'.format(platform()))
logging.info("Command-line arguments used:")
for arg in args.__dict__:
logging.info(arg + ": " + str(args.__dict__[arg]))
# Create the filelist.
if args.filelist != None:
filelist = glob.glob(args.filelist)
filelist = [x for x in filelist if len(os.path.basename(x)) == 18]
assert isinstance(filelist, list), \
'Expected list for filelist, got ' + str(type(filelist))
assert filelist != [], 'No files found.'
elif args.missing_only == True:
master_finders_query = session.query(MasterImages).\
outerjoin(MasterFinders).\
filter(MasterFinders.object_name == None).all()
filelist = [os.path.join(record.file_location[:-4], record.fits_file) \
for record in master_finders_query]
print 'Processing ' + str(len(filelist)) + ' files.'
logging.info('Processing ' + str(len(filelist)) + ' files.')
# Run with exception handling.
count = 0
for filename in filelist:
logging.info ('Now running for ' + filename)
try:
jpl2db_main(filename, args.reproc)
logging.info ("Completed for : " + filename)
except Exception as err:
logging.critical('{0} {1} {2}'.format(
