def test_make_output_file_dict():
"""
Iteratively generating tests.
Six tests are generated per dictionary in expected_entry_list.
Each dictionary represents a particular case, each corresponding
to a particular input filename.
Parameters:
nothing
Returns:
nothing
Output:
nothing
"""
# Go through each test case (different input filenames):
for expected_output, metadata in zip(expected_output_list, metadata_list):
output_dict = make_output_file_dict(expected_output["input_file"], metadata)
expected_keys = expected_output.keys()
output_keys = output_dict.keys()
# Test to make sure all the keys are the same.
yield check_output_keys, output_keys, expected_keys
for key in expected_keys:
expected_entry = expected_output[key]
output_entry = output_dict[key]
# Test to make sure the entries for each key are the
# same.
yield check_output_entry, output_entry, expected_entry
def check_filesystem_completeness_main():
"""
The main function for the check_filesystem_completeness module.
Parameters:
nothing
Returns:
nothing
Output:
output: string
information about files that are missing. Logged in log files.
Example:
'INFO: Checking in: /astro/3/mutchler/mt/drizzled/
INFO: Found 10000 root c0m.fits files
INFO: Found 500000 files
INFO: List of missing files:
11990_comet_hartley2 : cr_reject_output : 108
11990_comet_hartley2 : drizzle_weight : 432
11990_comet_hartley2 : drizzle_output : 432
11990_comet_hartley2 : png_output : 3024
06497_kbo : png_output : 1960
11518_kbo_2003el61 : cr_reject_output : 20
11518_kbo_2003el61 : drizzle_weight : 80
11518_kbo_2003el61 : drizzle_output : 80
11518_kbo_2003el61 : png_output : 560
09585_kbo : png_output : 252
Total: 6948 missing files'
"""
all_files_list = glob.glob(os.path.join(SETTINGS['wfpc2_output_path'], '*_*/*c0m*.fits'))
c0m_file_list = [filename for filename
in all_files_list
if len(filename.split('/')[-1]) == 18
and filename.split('/')[-1].split('_')[-1] == 'c0m.fits']
all_files_list += glob.glob(os.path.join(SETTINGS['wfpc2_output_path'], '*_*/png/*c0m*.png'))
files_set = set(all_files_list)
logging.info('Checking in: {}'.format(SETTINGS['wfpc2_output_path']))
logging.info('Found {} root c0m.fits files'.format(len(c0m_file_list)))
logging.info('Found {} files'.format(len(all_files_list)))
# creating a dictionary to store the quantities of files that were found
check_dict = {}
found = 0
for filename in c0m_file_list:
proposal_folder = filename.split('/')[-2]
if proposal_folder not in check_dict:
check_dict[proposal_folder] = defaultdict(int)
check_dict[proposal_folder]['input_file'] += 1
file_dict = make_output_file_dict(filename)
for key in file_dict.iterkeys():
if key != 'input_file':
for file in file_dict[key]:
if file in files_set:
check_dict[proposal_folder][key] += 1
found += 1
for mis in check_dict.keys():
if 'drizzle_weight' not in check_dict[mis]:
check_dict[mis]['drizzle_weight'] = 0
if 'drizzle_output' not in check_dict[mis]:
check_dict[mis]['drizzle_output'] = 0
if 'cr_reject_output' not in check_dict[mis]:
check_dict[mis]['cr_reject_output'] = 0
if 'png_output' not in check_dict[mis]:
check_dict[mis]['png_output'] = 0
# creating a dictionary to store the quantities of missing files
check_missing = {}
for key1 in check_dict.keys():
check_missing[key1] = {}
check_missing[key1]['input_file'] = 0
for key2 in check_dict[key1].keys():
if key2 == 'cr_reject_output':
check_missing[key1][key2] = 2*check_dict[key1]['input_file'] - check_dict[key1][key2]
elif key2 == 'drizzle_output':
check_missing[key1][key2] = 4*check_dict[key1]['input_file'] - check_dict[key1][key2]
elif key2 == 'drizzle_weight':
check_missing[key1][key2] = 4*check_dict[key1]['input_file'] - check_dict[key1][key2]
elif key2 == 'png_output':
check_missing[key1][key2] = 28*check_dict[key1]['input_file'] - check_dict[key1][key2]
missing = 0
missing_list = ''
for mis in check_missing.keys():
for key in check_missing[mis].keys():
if check_missing[mis][key] != 0:
missing_list += '{} : {} : {}\n'.format(mis, key, check_missing[mis][key])
missing += check_missing[mis][key]
if len(missing_list) > 0:
logging.info('List of missing files:\n{}'.format(missing_list).strip())
logging.info('Missing: {} files'.format(missing))
logging.info('Found: {} files'.format(found))
