def test_pipeline_same_source():
print('\n===== test_pipeline_same_source: BEGIN =====')
# Make the dat list.
with open(PATH_DAT_LIST_FILE, 'w') as fh:
for path_dat in DAT_LIST_ONS:
fh.write('{}\n'.format(path_dat))
# With the list of DAT files, do find_event_pipeline()
df_event = find_event_pipeline(PATH_DAT_LIST_FILE,
filter_threshold=3,
number_in_cadence=3,
user_validation=False,
saving=True,
on_source_complex_cadence='VOYAGER-1',
csv_name=PATH_CSVF)
# df_event should not be nil.
if df_event is None:
raise ValueError(
'test_pipeline_same_source: returned pandas df is None!')
# CSV file created?
if not Path(PATH_CSVF).exists():
raise ValueError('test_pipeline_same_source: No CSV of events created')
# An event CSV was created.
# Validate the hit table file.
utl.validate_hittbl(df_event, PATH_CSVF, 'test_pipeline_same_source')
print('\n===== test_pipeline_same_source: END =====')
def test_pipeline_h5_dat_separated():
print('\n===== test_pipeline_h5_dat_separated: BEGIN =====')
# If there is an old ALT_DAT_DIR, recreate it empty.
rmtree(ALT_DAT_DIR, ignore_errors=True)
os.mkdir(ALT_DAT_DIR)
# Copy the .dat files to ALT_DAT_DIR.
for path_dat in DAT_LIST_ONS:
cmd = 'cp ' + path_dat + " " + ALT_DAT_DIR
os.system(cmd)
# Make the dat list relative to ALT_DAT_DIR.
with open(PATH_DAT_LIST_FILE, 'w') as fh:
for path_dat in ALT_DAT_LIST_ONS:
fh.write('{}\n'.format(path_dat))
# Make the h5 list.
with open(PATH_H5_LIST_FILE, 'w') as fh:
for path_dat in H5_LIST_ONS:
fh.write('{}\n'.format(path_dat))
# With the list of separated .dat and .h5 files, do find_event_pipeline()
df_event = find_event_pipeline(PATH_DAT_LIST_FILE,
PATH_H5_LIST_FILE,
filter_threshold=3,
number_in_cadence=3,
user_validation=False,
saving=True,
on_source_complex_cadence='VOYAGER-1',
csv_name=PATH_CSVF)
# df_event should not be nil.
if df_event is None:
raise ValueError(
'test_pipeline_same_source: returned pandas df is None!')
# CSV file created?
if not Path(PATH_CSVF).exists():
raise ValueError('test_pipeline_same_source: No CSV of events created')
# An event CSV was created.
# Validate the hit table file.
utl.validate_hittbl(df_event, PATH_CSVF, 'test_pipeline_same_source',
N_EVENTS)
# Plots!
plot_event_pipeline(PATH_CSVF,
PATH_H5_LIST_FILE,
filter_spec=3,
user_validation=False)
print('\n===== test_pipeline_h5_dat_separated: END =====')
def try_mixed(arg_list, init_needed=True):
print('\n===== try_mixed: BEGIN =====')
# Make the off-cadence file.
if init_needed:
generate_fil_file(PATH_IRRELEVANT_FIL, -1, -1)
# Make a DAT file.
if init_needed:
make_one_dat_file(PATH_IRRELEVANT_FIL,
max_drift=10.0,
min_snr=20.0,
remove_h5=False)
# Make the dat list.
with open(PATH_DAT_LIST_FILE, 'w') as fh:
for path_dat in arg_list:
fh.write('{}\n'.format(path_dat))
# With the list of DAT files, do find_event_pipeline()
df_event = find_event_pipeline(PATH_DAT_LIST_FILE,
sortby_tstart=False,
filter_threshold=3,
number_in_cadence=len(arg_list),
user_validation=False,
saving=True,
on_source_complex_cadence='VOYAGER-1',
csv_name=PATH_CSVF)
# df_event should not be nil.
if df_event is None:
raise ValueError('try_mixed: returned pandas df is None!')
# CSV file created?
if not Path(PATH_CSVF).exists():
raise ValueError('try_mixed: No CSV of events created')
# An event CSV was created.
# Validate the hit table file.
utl.validate_hittbl(df_event, PATH_CSVF, 'test_pipeline_mixed')
print('\n===== try_mixed: END =====')
def find_plot_pipelines(need_init=True, filter_threshold=FILTER_THRESHOLD):
r'''
Exercise find_event_pipeline() and plot_event_pipeline()
'''
main_time_start = time()
# If configured to do so, initialize temp directory
# and fetch all of the HDF5 files from the Internet.
if need_init:
initialize()
for filename_h5 in H5_FILE_LIST:
wgetter(filename_h5)
# Make all of the DAT files.
make_all_dat_files()
print('find_plot_pipelines: Filter threshold = ', filter_threshold)
number_in_cadence = len(H5_FILE_LIST)
print('find_plot_pipelines: Cadence length = ', number_in_cadence)
print('find_plot_pipelines: find_event_pipeline({}) ...'.format(
PATH_DAT_LIST_FILE))
# With the list of DAT files, do find_event_pipeline()
df_event = find_event_pipeline(PATH_DAT_LIST_FILE,
filter_threshold=filter_threshold,
number_in_cadence=number_in_cadence,
user_validation=False,
saving=True,
csv_name=PATH_CSVF)
# CSV file created?
if not Path(PATH_CSVF).exists():
raise ValueError('find_plot_pipelines: No CSV of events created')
# An event CSV was created.
# Validate the hit table file.
utl.validate_hittbl(df_event, PATH_CSVF, 'test_pipe_lines', N_EVENTS)
# Make a list of the HDF5 files.
print(
'find_plot_pipelines: making a list of HDF5 files in ({}) ...'.format(
PATH_DAT_LIST_FILE))
with open(PATH_H5_LIST_FILE, 'w') as file_handle:
for filename_h5 in H5_FILE_LIST:
file_handle.write('{}\n'.format(TESTDIR + filename_h5))
# Do the plots for all of the HDF5/DAT file pairs.
print('find_plot_pipelines: plot_event_pipeline({}, {}) ...'.format(
PATH_CSVF, PATH_H5_LIST_FILE))
plot_event_pipeline(PATH_CSVF,
PATH_H5_LIST_FILE,
filter_spec='f{}'.format(filter_threshold),
user_validation=False)
# Check that the right number of PNG files were created.
outdir_list = listdir(TESTDIR)
npngs = 0
for cur_file in outdir_list:
if cur_file.split('.')[-1] == 'png':
if imghdr.what(TESTDIR + cur_file) != 'png':
raise ValueError(
'find_plot_pipelines: File {} is not a PNG file'.format(
cur_file))
npngs += 1
if npngs != N_EVENTS:
raise ValueError(
'find_plot_pipelines: Expected to find {} PNG files but observed {}'
.format(N_EVENTS, npngs))
# Stop the clock - we're done.
main_time_stop = time()
print('find_plot_pipelines: End, et = {:.1f} seconds'.format(
main_time_stop - main_time_start))
def find_plot_pipelines(filter_threshold=3,
on_off_first='ON',
plot_dir=None,
on_source_complex_cadence=False):
r'''
Exercise find_event_pipeline() and plot_event_pipeline()
'''
main_time_start = time()
print('find_plot_pipelines_2: Filter threshold = ', filter_threshold)
h5_file_list = sorted(glob.glob(TESTDIR + 'single*.h5'))
dat_file_list = sorted(glob.glob(TESTDIR + 'single*.dat'))
number_in_cadence = len(h5_file_list)
if number_in_cadence != 6:
raise ValueError(
'find_plot_pipelines_2: Expected to find 6 h5 files but observed {}'
.format(number_in_cadence))
number_in_cadence = len(dat_file_list)
if number_in_cadence != 6:
raise ValueError(
'find_plot_pipelines_2: Expected to find 6 dat files but observed {}'
.format(number_in_cadence))
# Re-order the H5 and DAT files into OFF-ON-...
# In the 2 lists, switch 1 and 2, 3 and 4, 5 and 6
for ix in [0, 2, 4]:
temp = h5_file_list[ix]
h5_file_list[ix] = h5_file_list[ix + 1]
h5_file_list[ix + 1] = temp
temp = dat_file_list[ix]
dat_file_list[ix] = dat_file_list[ix + 1]
dat_file_list[ix + 1] = temp
fh_h5 = open(PATH_H5_LIST_FILE, 'w')
fh_dat = open(PATH_DAT_LIST_FILE, 'w')
for ix in range(6):
fh_h5.write(h5_file_list[ix] + '\n')
fh_dat.write(dat_file_list[ix] + '\n')
fh_h5.close()
fh_dat.close()
print('find_plot_pipelines_2: H5/dat cadence length = ', number_in_cadence)
# If CSV exists from a previous execution, remove it.
try:
remove(PATH_CSVF)
except:
pass
# With the list of DAT files, do find_event_pipeline()
print('===== find_event_pipeline BEGIN =====')
df_event = find_event_pipeline(
PATH_DAT_LIST_FILE,
sortby_tstart=False,
filter_threshold=filter_threshold,
number_in_cadence=number_in_cadence,
user_validation=False,
saving=True,
on_off_first=on_off_first,
on_source_complex_cadence=on_source_complex_cadence,
csv_name=PATH_CSVF)
print('===== find_event_pipeline END =====')
# CSV file created?
if not Path(PATH_CSVF).exists():
raise ValueError('find_plot_pipelines_2: No CSV of events created')
# An event CSV was created.
# Validate the hit table file.
utl.validate_hittbl(df_event, PATH_CSVF, 'test_pipe_lines_2')
# Do the plots for all of the HDF5/DAT file pairs.
print(
'===== plot_event_pipeline #1 (plot_dir does not yet exist) BEGIN ====='
)
rmtree(plot_dir, ignore_errors=True)
plot_event_pipeline(PATH_CSVF,
PATH_H5_LIST_FILE,
filter_spec='f{}'.format(filter_threshold),
plot_dir=plot_dir,
user_validation=False)
print('===== plot_event_pipeline #2 (plot_dir already exists) BEGIN =====')
plot_event_pipeline(PATH_CSVF,
PATH_H5_LIST_FILE,
filter_spec='f{}'.format(filter_threshold),
plot_dir=plot_dir,
user_validation=False)
# Check that the right number of PNG files were created.
print('===== plot_event_pipeline END =====')
outdir_list = listdir(plot_dir)
npngs = 0
for cur_file in outdir_list:
if cur_file.split('.')[-1] == 'png':
if imghdr.what(plot_dir + cur_file) != 'png':
raise ValueError(
'find_plot_pipelines_2: File {} is not a PNG file'.format(
cur_file))
npngs += 1
if npngs != 6:
raise ValueError(
'find_plot_pipelines_2: Expected to find 6 PNG files but observed {}'
.format(npngs))
# Stop the clock - we're done.
main_time_stop = time()
print('find_plot_pipelines_2: End, et = {:.1f} seconds'.format(
main_time_stop - main_time_start))