png_switch = args_list.png,
reproc_switch = args_list.reproc)
logging.info("Completed: " + filename)
except Exception as err:
logging.critical("{0} {1} {2}".format(type(err), err.message, sys.exc_traceback.tb_lineno))
@email_decorator.email_decorator
def run():
"""
"""
logging.info("Command-line arguments used:")
for arg in args_list.__dict__:
logging.info(arg + ": " + str(args_list.__dict__[arg]))
rootfile_list = glob.glob(args_list.filelist)
rootfile_list = [filename for filename
in rootfile_list
if 'hlsp_' not in filename
and ('_flt.fits' in filename or '_c0m.fits' in filename)]
assert rootfile_list != [], 'empty rootfile_list in mtpipeline.py.'
logging.info("Processing: {} files".format(len(rootfile_list)))
num_cores = SETTINGS['num_cores']
logging.info("Number of Processes: {}".format(num_cores))
pool = mp.Pool(processes=num_cores)
pool.map(run_imaging_pipeline, rootfile_list)
logging.info("Script completed")
if __name__ == '__main__':
setup_logging('run_imaging_pipeline')
args_list = parse_args()
run()
check_missing[key1] = {}
check_missing[key1]['input_file'] = 0
for key2 in check_dict[key1].keys():
if key2 == 'cr_reject_output':
check_missing[key1][key2] = 2*check_dict[key1]['input_file'] - check_dict[key1][key2]
elif key2 == 'drizzle_output':
check_missing[key1][key2] = 4*check_dict[key1]['input_file'] - check_dict[key1][key2]
elif key2 == 'drizzle_weight':
check_missing[key1][key2] = 4*check_dict[key1]['input_file'] - check_dict[key1][key2]
elif key2 == 'png_output':
check_missing[key1][key2] = 28*check_dict[key1]['input_file'] - check_dict[key1][key2]
missing = 0
missing_list = ''
for mis in check_missing.keys():
for key in check_missing[mis].keys():
if check_missing[mis][key] != 0:
missing_list += '{} : {} : {}\n'.format(mis, key, check_missing[mis][key])
missing += check_missing[mis][key]
if len(missing_list) > 0:
logging.info('List of missing files:\n{}'.format(missing_list).strip())
logging.info('Missing: {} files'.format(missing))
logging.info('Found: {} files'.format(found))
if __name__ == '__main__':
setup_logging('check_file_completeness')
t1 = time.time()
check_filesystem_completeness_main()
t2 = time.time()
logging.info('Ran in {} s'.format(int(float(t2 - t1))))
'''
parse the command line arguemnts.
'''
parser = argparse.ArgumentParser(
description = 'Populate the master_images table.')
parser.add_argument(
'-filelist',
required = True,
help = 'Search string for files. Wildcards accepted.')
parser.add_argument(
'-reproc',
required = False,
action='store_true',
default = False,
dest = 'reproc',
help = 'Overwrite existing entries except for the set information.')
parser.add_argument(
'-reproc_sets',
required = False,
action='store_true',
default = False,
dest = 'reproc_sets',
help = 'Overwrite existing set information.')
args = parser.parse_args()
return args
if __name__ == '__main__':
args = parse_args()
setup_logging('build_master_images_table')
build_master_images_table_main(args.filelist, args.reproc, args.reproc_sets)