def run_find_event_pipeline(fildir, datdir, datlist, SNR, filter_threshold=np.nan, number_in_cadence=np.nan,
on_source_complex_cadence=False):
assert filter_threshold != np.nan
assert number_in_cadence != np.nan
# write to .lst, as required by find_event_pipeline
lst_dat = datdir + "dat_files.lst"
if not os.path.isfile(lst_dat):
with open(lst_dat, 'w') as f:
for item in datlist:
f.write("%s\n" % item)
# move the h5 files to the dat directory
for file in datlist:
h5_file = fildir + os.path.basename(file).replace('.dat','.h5')
if not os.path.isfile(h5_file):
fil = os.path.basename(h5_file).replace('.h5','.fil')
bp.fil2h5.make_h5_file(fil, out_dir=datdir)
else:
mvh5_in = 'mv ' + h5_file + " " + datdir
call([mvh5_in],shell=True)
# run find_event_pipeline
print("\nRunning find event pipeline...")
find_event_pipeline(lst_dat,
SNR_cut=SNR,
check_zero_drift=False,
filter_threshold=filter_threshold,
number_in_cadence=number_in_cadence,
on_source_complex_cadence=on_source_complex_cadence)
# move the h5 files back to the fil directory to save space
for file in datlist:
mvh5_out = 'mv ' + datdir + os.path.basename(file).replace('.dat','.h5') + " " + fildir
call([mvh5_out],shell=True)
def test_pipeline_same_source():
print('\n===== test_pipeline_same_source: BEGIN =====')
# Make the dat list.
with open(PATH_DAT_LIST_FILE, 'w') as fh:
for path_dat in DAT_LIST_ONS:
fh.write('{}\n'.format(path_dat))
# With the list of DAT files, do find_event_pipeline()
df_event = find_event_pipeline(PATH_DAT_LIST_FILE,
filter_threshold=3,
number_in_cadence=3,
user_validation=False,
saving=True,
on_source_complex_cadence='VOYAGER-1',
csv_name=PATH_CSVF)
# df_event should not be nil.
if df_event is None:
raise ValueError(
'test_pipeline_same_source: returned pandas df is None!')
# CSV file created?
if not Path(PATH_CSVF).exists():
raise ValueError('test_pipeline_same_source: No CSV of events created')
# An event CSV was created.
# Validate the hit table file.
utl.validate_hittbl(df_event, PATH_CSVF, 'test_pipeline_same_source')
print('\n===== test_pipeline_same_source: END =====')
def test_pipeline_h5_dat_separated():
print('\n===== test_pipeline_h5_dat_separated: BEGIN =====')
# If there is an old ALT_DAT_DIR, recreate it empty.
rmtree(ALT_DAT_DIR, ignore_errors=True)
os.mkdir(ALT_DAT_DIR)
# Copy the .dat files to ALT_DAT_DIR.
for path_dat in DAT_LIST_ONS:
cmd = 'cp ' + path_dat + " " + ALT_DAT_DIR
os.system(cmd)
# Make the dat list relative to ALT_DAT_DIR.
with open(PATH_DAT_LIST_FILE, 'w') as fh:
for path_dat in ALT_DAT_LIST_ONS:
fh.write('{}\n'.format(path_dat))
# Make the h5 list.
with open(PATH_H5_LIST_FILE, 'w') as fh:
for path_dat in H5_LIST_ONS:
fh.write('{}\n'.format(path_dat))
# With the list of separated .dat and .h5 files, do find_event_pipeline()
df_event = find_event_pipeline(PATH_DAT_LIST_FILE,
PATH_H5_LIST_FILE,
filter_threshold=3,
number_in_cadence=3,
user_validation=False,
saving=True,
on_source_complex_cadence='VOYAGER-1',
csv_name=PATH_CSVF)
# df_event should not be nil.
if df_event is None:
raise ValueError(
'test_pipeline_same_source: returned pandas df is None!')
# CSV file created?
if not Path(PATH_CSVF).exists():
raise ValueError('test_pipeline_same_source: No CSV of events created')
# An event CSV was created.
# Validate the hit table file.
utl.validate_hittbl(df_event, PATH_CSVF, 'test_pipeline_same_source',
N_EVENTS)
# Plots!
plot_event_pipeline(PATH_CSVF,
PATH_H5_LIST_FILE,
filter_spec=3,
user_validation=False)
print('\n===== test_pipeline_h5_dat_separated: END =====')
def run_find_event_pipeline(datdir,
SNR,
filter_threshold=np.nan,
number_in_cadence=np.nan,
on_source_complex_cadence=False):
assert filter_threshold != np.nan
assert number_in_cadence != np.nan
#get list of files and sort by file ID
dat_list = glob.glob(datdir + "*.dat")
dat_list = sort_by_timestamp(dat_list)
# write to .lst, as required by find_event_pipeline
lst_dat = datdir + "dat_files.lst"
if os.path.isfile(lst_dat):
os.remove(lst_dat)
with open(lst_dat, 'w') as f:
for item in dat_list:
f.write("%s\n" % item)
assert os.path.isfile(lst_dat)
# construct csv file name
# csv_name = datdir + "hits_SNR_" + SNR + "_f_" + str(filter_threshold) + ".csv"
# if os.path.isfile(csv_name):
# os.remove(csv_name)
# run find_event_pipeline
print("\nRunning find event pipeline...")
find_event_pipeline(lst_dat,
SNR_cut=SNR,
check_zero_drift=False,
filter_threshold=filter_threshold,
number_in_cadence=number_in_cadence,
on_source_complex_cadence=on_source_complex_cadence)
try:
csv_name = (glob.glob('*f' + str(filter_threshold) + '_snr' +
str(SNR) + '*.csv'))[0]
except:
csv_name = []
print("\n")
return csv_name
def try_mixed(arg_list, init_needed=True):
print('\n===== try_mixed: BEGIN =====')
# Make the off-cadence file.
if init_needed:
generate_fil_file(PATH_IRRELEVANT_FIL, -1, -1)
# Make a DAT file.
if init_needed:
make_one_dat_file(PATH_IRRELEVANT_FIL,
max_drift=10.0,
min_snr=20.0,
remove_h5=False)
# Make the dat list.
with open(PATH_DAT_LIST_FILE, 'w') as fh:
for path_dat in arg_list:
fh.write('{}\n'.format(path_dat))
# With the list of DAT files, do find_event_pipeline()
df_event = find_event_pipeline(PATH_DAT_LIST_FILE,
sortby_tstart=False,
filter_threshold=3,
number_in_cadence=len(arg_list),
user_validation=False,
saving=True,
on_source_complex_cadence='VOYAGER-1',
csv_name=PATH_CSVF)
# df_event should not be nil.
if df_event is None:
raise ValueError('try_mixed: returned pandas df is None!')
# CSV file created?
if not Path(PATH_CSVF).exists():
raise ValueError('try_mixed: No CSV of events created')
# An event CSV was created.
# Validate the hit table file.
utl.validate_hittbl(df_event, PATH_CSVF, 'test_pipeline_mixed')
print('\n===== try_mixed: END =====')
def test_pipeline_wrong_source():
print('\n===== test_pipeline_wrong_source: BEGIN =====')
# Make the dat list.
with open(PATH_DAT_LIST_FILE, 'w') as fh:
for path_dat in DAT_LIST_ONS:
fh.write('{}\n'.format(path_dat))
# With the list of DAT files, do find_event_pipeline()
df_event = find_event_pipeline(PATH_DAT_LIST_FILE,
filter_threshold=3,
number_in_cadence=3,
user_validation=False,
saving=True,
on_source_complex_cadence='VOYAGER-42',
csv_name=PATH_CSVF)
# df_event should be nil
if not df_event is None:
raise ValueError('test_pipeline_wrong_source: returned pandas df has entries but should be nil!')
print('\n===== test_pipeline_wrong_source: END =====')
path = '/home/charlesg/turboSETI/trappist1/'
fil_path = path + 'fil_files/high'
png_path = '/home/charlesg/turboSETI/blc00_Graphs/trappist1/hits'
dat_path = path + 'doppler_output/'
dat_list = [
dat_path + 'blc00_guppi_57807_75725_DIAG_TRAPPIST1_0015.gpuspec.0000.dat',
dat_path +
'blc00_guppi_57807_75805_DIAG_TRAPPIST1_OFF_0016.gpuspec.0000.dat',
dat_path + 'blc00_guppi_57807_75885_DIAG_TRAPPIST1_0017.gpuspec.0000.dat',
dat_path +
'blc00_guppi_57807_75965_DIAG_TRAPPIST1_OFF_0018.gpuspec.0000.dat'
]
check_zero_drift = False
filter_threshold = 3
number_in_cadence = 4
with open(dat_path + 'dat_files.lst', 'w') as f:
for item in dat_list:
f.write("%s\n" % item)
event_dataframe = find_event_pipeline(dat_path + 'dat_files.lst',
number_in_cadence=number_in_cadence,
filter_threshold=filter_threshold)
print(event_dataframe)
#
def find_plot_pipelines(need_init=True, filter_threshold=FILTER_THRESHOLD):
r'''
Exercise find_event_pipeline() and plot_event_pipeline()
'''
main_time_start = time()
# If configured to do so, initialize temp directory
# and fetch all of the HDF5 files from the Internet.
if need_init:
initialize()
for filename_h5 in H5_FILE_LIST:
wgetter(filename_h5)
# Make all of the DAT files.
make_all_dat_files()
print('find_plot_pipelines: Filter threshold = ', filter_threshold)
number_in_cadence = len(H5_FILE_LIST)
print('find_plot_pipelines: Cadence length = ', number_in_cadence)
print('find_plot_pipelines: find_event_pipeline({}) ...'.format(
PATH_DAT_LIST_FILE))
# With the list of DAT files, do find_event_pipeline()
df_event = find_event_pipeline(PATH_DAT_LIST_FILE,
filter_threshold=filter_threshold,
number_in_cadence=number_in_cadence,
user_validation=False,
saving=True,
csv_name=PATH_CSVF)
# CSV file created?
if not Path(PATH_CSVF).exists():
raise ValueError('find_plot_pipelines: No CSV of events created')
# An event CSV was created.
# Validate the hit table file.
utl.validate_hittbl(df_event, PATH_CSVF, 'test_pipe_lines', N_EVENTS)
# Make a list of the HDF5 files.
print(
'find_plot_pipelines: making a list of HDF5 files in ({}) ...'.format(
PATH_DAT_LIST_FILE))
with open(PATH_H5_LIST_FILE, 'w') as file_handle:
for filename_h5 in H5_FILE_LIST:
file_handle.write('{}\n'.format(TESTDIR + filename_h5))
# Do the plots for all of the HDF5/DAT file pairs.
print('find_plot_pipelines: plot_event_pipeline({}, {}) ...'.format(
PATH_CSVF, PATH_H5_LIST_FILE))
plot_event_pipeline(PATH_CSVF,
PATH_H5_LIST_FILE,
filter_spec='f{}'.format(filter_threshold),
user_validation=False)
# Check that the right number of PNG files were created.
outdir_list = listdir(TESTDIR)
npngs = 0
for cur_file in outdir_list:
if cur_file.split('.')[-1] == 'png':
if imghdr.what(TESTDIR + cur_file) != 'png':
raise ValueError(
'find_plot_pipelines: File {} is not a PNG file'.format(
cur_file))
npngs += 1
if npngs != N_EVENTS:
raise ValueError(
'find_plot_pipelines: Expected to find {} PNG files but observed {}'
.format(N_EVENTS, npngs))
# Stop the clock - we're done.
main_time_stop = time()
print('find_plot_pipelines: End, et = {:.1f} seconds'.format(
main_time_stop - main_time_start))
def find_plot_pipelines(filter_threshold=3,
on_off_first='ON',
plot_dir=None,
on_source_complex_cadence=False):
r'''
Exercise find_event_pipeline() and plot_event_pipeline()
'''
main_time_start = time()
print('find_plot_pipelines_2: Filter threshold = ', filter_threshold)
h5_file_list = sorted(glob.glob(TESTDIR + 'single*.h5'))
dat_file_list = sorted(glob.glob(TESTDIR + 'single*.dat'))
number_in_cadence = len(h5_file_list)
if number_in_cadence != 6:
raise ValueError(
'find_plot_pipelines_2: Expected to find 6 h5 files but observed {}'
.format(number_in_cadence))
number_in_cadence = len(dat_file_list)
if number_in_cadence != 6:
raise ValueError(
'find_plot_pipelines_2: Expected to find 6 dat files but observed {}'
.format(number_in_cadence))
# Re-order the H5 and DAT files into OFF-ON-...
# In the 2 lists, switch 1 and 2, 3 and 4, 5 and 6
for ix in [0, 2, 4]:
temp = h5_file_list[ix]
h5_file_list[ix] = h5_file_list[ix + 1]
h5_file_list[ix + 1] = temp
temp = dat_file_list[ix]
dat_file_list[ix] = dat_file_list[ix + 1]
dat_file_list[ix + 1] = temp
fh_h5 = open(PATH_H5_LIST_FILE, 'w')
fh_dat = open(PATH_DAT_LIST_FILE, 'w')
for ix in range(6):
fh_h5.write(h5_file_list[ix] + '\n')
fh_dat.write(dat_file_list[ix] + '\n')
fh_h5.close()
fh_dat.close()
print('find_plot_pipelines_2: H5/dat cadence length = ', number_in_cadence)
# If CSV exists from a previous execution, remove it.
try:
remove(PATH_CSVF)
except:
pass
# With the list of DAT files, do find_event_pipeline()
print('===== find_event_pipeline BEGIN =====')
df_event = find_event_pipeline(
PATH_DAT_LIST_FILE,
sortby_tstart=False,
filter_threshold=filter_threshold,
number_in_cadence=number_in_cadence,
user_validation=False,
saving=True,
on_off_first=on_off_first,
on_source_complex_cadence=on_source_complex_cadence,
csv_name=PATH_CSVF)
print('===== find_event_pipeline END =====')
# CSV file created?
if not Path(PATH_CSVF).exists():
raise ValueError('find_plot_pipelines_2: No CSV of events created')
# An event CSV was created.
# Validate the hit table file.
utl.validate_hittbl(df_event, PATH_CSVF, 'test_pipe_lines_2')
# Do the plots for all of the HDF5/DAT file pairs.
print(
'===== plot_event_pipeline #1 (plot_dir does not yet exist) BEGIN ====='
)
rmtree(plot_dir, ignore_errors=True)
plot_event_pipeline(PATH_CSVF,
PATH_H5_LIST_FILE,
filter_spec='f{}'.format(filter_threshold),
plot_dir=plot_dir,
user_validation=False)
print('===== plot_event_pipeline #2 (plot_dir already exists) BEGIN =====')
plot_event_pipeline(PATH_CSVF,
PATH_H5_LIST_FILE,
filter_spec='f{}'.format(filter_threshold),
plot_dir=plot_dir,
user_validation=False)
# Check that the right number of PNG files were created.
print('===== plot_event_pipeline END =====')
outdir_list = listdir(plot_dir)
npngs = 0
for cur_file in outdir_list:
if cur_file.split('.')[-1] == 'png':
if imghdr.what(plot_dir + cur_file) != 'png':
raise ValueError(
'find_plot_pipelines_2: File {} is not a PNG file'.format(
cur_file))
npngs += 1
if npngs != 6:
raise ValueError(
'find_plot_pipelines_2: Expected to find 6 PNG files but observed {}'
.format(npngs))
# Stop the clock - we're done.
main_time_stop = time()
print('find_plot_pipelines_2: End, et = {:.1f} seconds'.format(
main_time_stop - main_time_start))
def execute_pipelines(args):
r"""
Interface to the pipeline functions, called by main().
Parameters
----------
args : dict
"""
# Setup some parameter values for find_event_pipeline().
if args.cadence == "complex":
complex_cadence = True
if len(args.source_name) < 1:
print(
"\n*** plotSETI: Complex cadence requires a source_name. Bye-bye."
)
sys.exit(86)
else:
complex_cadence = False
if args.cadence == "on":
first_file = "ON"
else:
first_file = "OFF"
from_dir = os.path.abspath(args.input_dir_path) + "/"
dest_dir = os.path.abspath(args.out_dir) + "/"
if not os.path.exists(dest_dir):
os.mkdir(dest_dir)
# Establish output pathnames,
path_h5_list = dest_dir + NAME_H5_LIST
path_dat_list = dest_dir + NAME_DAT_LIST
path_csvf = dest_dir + NAME_CSVF
clean_event_stuff(dest_dir)
make_lists(from_dir, path_h5_list, path_dat_list)
# Run find_event_pipeline()
number_in_cadence = len(open(path_h5_list).readlines())
if complex_cadence:
df_check = find_event_pipeline(
path_dat_list,
filter_threshold=args.filt_thresh,
number_in_cadence=number_in_cadence,
on_source_complex_cadence=args.source_name,
sortby_tstart=True,
check_zero_drift=False,
SNR_cut=args.snr_threshold,
user_validation=False,
csv_name=path_csvf,
saving=True)
else: # not a complex cadence
df_check = find_event_pipeline(path_dat_list,
filter_threshold=args.filt_thresh,
number_in_cadence=number_in_cadence,
on_source_complex_cadence=False,
on_off_first=first_file,
sortby_tstart=True,
check_zero_drift=False,
SNR_cut=args.snr_threshold,
user_validation=False,
csv_name=path_csvf,
saving=True)
if df_check is None:
print(
"\n*** plotSETI: No events produced in find_event_pipeline()! Bye-bye."
)
return 86
# Make the plots for all of the HDF5/DAT file pairs in batch mode.
matplotlib.use("agg", force=True)
plot_event_pipeline(path_csvf,
path_h5_list,
plot_dir=dest_dir,
filter_spec=args.filt_thresh,
offset=0,
user_validation=False)
print("\nplotSETI: Plots are stored in directory {}. Bye-bye.".format(
dest_dir))
return 0
def beginning():
# Initialize output directory
rmtree(OUTDIR, ignore_errors=True)
mkdir(OUTDIR)
# Make the DAT files
make_dat_files()
#=================================================================
print('============== turbo_seti version ===============>', VERSION)
#beginning()
# Generate CSV file from find_event_pipeline()
num_in_cadence = len(h5_files_list)
find_event_pipeline(PATH_DAT_LIST_FILE,
filter_threshold=THRESHOLD,
number_in_cadence=num_in_cadence,
user_validation=False,
csv_name=PATH_CSVF,
saving=True)
print("Produced {}".format(PATH_CSVF))
plot_event_pipeline(PATH_CSVF, H5_LIST_FILE, user_validation=False)
'''file_path = '/home/ellie/research/seti/VoyagerTutorial2/VoyagerData/single_coarse_guppi_59046_80672_DIAG_VOYAGER-1_0013.rawspec.0000.h5'
f_start = 8419.51
f_stop = 8419.58
obs = Waterfall(file_path, f_start, f_stop)
obs.plot_spectrum(f_start=f_start, f_stop=f_stop)'''
def find_plot_pipelines(need_init=True, filter_threshold=2):
'''
Exercise find_event_pipeline() and plot_event_pipeline()
'''
main_time_start = time()
# If configured to do so, initialize temp directory.
if need_init:
initialize()
# If configured to do so, fetch all of the HDF5 files from the Internet.
if need_init:
for filename_h5 in H5_FILE_LIST:
wgetter(filename_h5)
# Make all of the DAT files.
make_all_dat_files()
# Create the CSV file used by plot_event_pipeline.
print('find_plot_pipelines: Filter threshold = ', filter_threshold)
number_in_cadence = len(H5_FILE_LIST)
print('find_plot_pipelines: Cadence length = ', number_in_cadence)
print('find_plot_pipelines: find_event_pipeline({}) ...'.format(
PATH_DAT_LIST_FILE))
# If CSV exists from a previous execution, remove it.
try:
remove(PATH_CSVF)
except:
pass
# With the list of DAT files, do find_event_pipeline()
df_event = find_event_pipeline(PATH_DAT_LIST_FILE,
filter_threshold=filter_threshold,
number_in_cadence=number_in_cadence,
user_validation=False,
saving=True,
csv_name=PATH_CSVF)
# CSV created?
if not Path(PATH_CSVF).exists():
print('*** find_plot_pipelines: No CSV of events created')
return
# An event CSV was created.
print('find_plot_pipelines: find_event_pipeline() returned dataframe:\n{}'.
format(df_event))
# Make a list of the HDF5 files.
print(
'find_plot_pipelines: making a list of HDF5 files in ({}) ...'.format(
PATH_DAT_LIST_FILE))
with open(PATH_H5_LIST_FILE, 'w') as file_handle:
for filename_h5 in H5_FILE_LIST:
file_handle.write('{}\n'.format(TESTDIR + filename_h5))
# Do the plots for all of the HDF5/DAT file pairs.
print('find_plot_pipelines: plot_event_pipeline({}, {}) ...'.format(
PATH_CSVF, PATH_H5_LIST_FILE))
plot_event_pipeline(PATH_CSVF, PATH_H5_LIST_FILE, user_validation=False)
main_time_stop = time()
print('find_plot_pipelines: End, et = {:.1f} seconds'.format(
main_time_stop - main_time_start))
def find_plot_pipelines(need_init=True, filter_threshold=3):
r'''
Exercise find_event_pipeline() and plot_event_pipeline()
'''
main_time_start = time()
# If configured to do so, initialize temp directory.
if need_init:
initialize()
# If configured to do so, fetch all of the HDF5 files from the Internet.
if need_init:
for filename_h5 in H5_FILE_LIST:
wgetter(filename_h5)
# Make all of the DAT files.
make_all_dat_files()
# Create the CSV file used by plot_event_pipeline.
print('find_plot_pipelines: Filter threshold = ', filter_threshold)
number_in_cadence = len(H5_FILE_LIST)
print('find_plot_pipelines: Cadence length = ', number_in_cadence)
print('find_plot_pipelines: find_event_pipeline({}) ...'.format(
PATH_DAT_LIST_FILE))
# If CSV exists from a previous execution, remove it.
try:
remove(PATH_CSVF)
except:
pass
# With the list of DAT files, do find_event_pipeline()
df_event = find_event_pipeline(PATH_DAT_LIST_FILE,
filter_threshold=filter_threshold,
number_in_cadence=number_in_cadence,
user_validation=False,
saving=True,
csv_name=PATH_CSVF)
# CSV file created?
if not Path(PATH_CSVF).exists():
raise ValueError('find_plot_pipelines: No CSV of events created')
# An event CSV was created.
# Validate CSV file.
df_validate = validate_csvf(PATH_CSVF)
if df_validate.equals(df_event):
raise ValueError(
'find_plot_pipelines: Expected df_validate to be identical to df_event but observed\n{}'
.format(df_validate))
# Make a list of the HDF5 files.
print(
'find_plot_pipelines: making a list of HDF5 files in ({}) ...'.format(
PATH_DAT_LIST_FILE))
with open(PATH_H5_LIST_FILE, 'w') as file_handle:
for filename_h5 in H5_FILE_LIST:
file_handle.write('{}\n'.format(TESTDIR + filename_h5))
# Do the plots for all of the HDF5/DAT file pairs.
print('find_plot_pipelines: plot_event_pipeline({}, {}) ...'.format(
PATH_CSVF, PATH_H5_LIST_FILE))
plot_event_pipeline(PATH_CSVF,
PATH_H5_LIST_FILE,
filter_spec='f{}'.format(filter_threshold),
user_validation=False)
# Check that the right number of PNG files were created.
outdir_list = listdir(TESTDIR)
npngs = 0
for cur_file in outdir_list:
if cur_file.split('.')[-1] == 'png':
if imghdr.what(TESTDIR + cur_file) != 'png':
raise ValueError(
'find_plot_pipelines: File {} is not a PNG file'.format(
cur_file))
npngs += 1
if npngs != 6:
raise ValueError(
'find_plot_pipelines: Expected to find 6 PNG files but observed {}'
.format(npngs))
# Stop the clock - we're done.
main_time_stop = time()
print('find_plot_pipelines: End, et = {:.1f} seconds'.format(
main_time_stop - main_time_start))
def execute_pipelines(args):
"""
Interface to the pipeline functions, called by main().
Parameters
----------
args : dict
"""
# Setup some parameter values for find_event_pipeline().
if args.cadence == "complex":
complex_cadence = True
if len(args.source_name) < 1:
print("\n*** plotSETI: Complex cadence requires a source_name.")
sys.exit(RETURN_ERROR)
else:
complex_cadence = False
if args.cadence == "on":
first_file = "ON"
else:
first_file = "OFF"
h5_dir = os.path.abspath(args.h5_dir) + "/"
dat_dir = os.path.abspath(args.dat_dir) + "/"
out_dir = os.path.abspath(args.out_dir) + "/"
if not os.path.exists(out_dir):
os.mkdir(out_dir)
if args.plot_offset:
offset="auto"
else:
offset=0
# Establish output pathnames,
path_csvf = out_dir + NAME_CSVF
clean_event_stuff(out_dir)
# Make the h5 and dat lists.
# Default to auto-generation?
if args.h5dat_lists is None:
SZ_user_list = 0
else:
SZ_user_list = len(args.h5dat_lists)
if args.debug:
print(f"DEBUG h5dats_list: #{SZ_user_list} {args.h5dat_lists}")
if SZ_user_list == 0: # Default to auto-generation.
path_h5_list = out_dir + NAME_H5_LIST
path_dat_list = out_dir + NAME_DAT_LIST
number_in_cadence = make_lists(h5_dir, path_h5_list, dat_dir, path_dat_list)
if number_in_cadence == 0:
return RETURN_ERROR
else: # User-specified lists
if SZ_user_list != 2:
print(f"\n*** plotSETI: h5dat_lists had {SZ_user_list} elements; must be 2 (one for h5 and one for dat)!")
return RETURN_ERROR
if args.h5dat_lists[0] is None or args.h5dat_lists[1] is None:
print(f"\n*** plotSETI: h5dat_lists had {SZ_user_list} elements; must be 2 (one for h5 and one for dat)!")
return RETURN_ERROR
# Check the list of h5 files.
path_h5_list = args.h5dat_lists[0]
if not os.path.exists(path_h5_list):
print(f"\n*** plotSETI: File {path_h5_list} does not exist!")
return RETURN_ERROR
N_h5 = count_text_lines(path_h5_list)
print(f"plotSETI: Found {N_h5} h5 files.")
# Check the list of dat files.
path_dat_list = args.h5dat_lists[1]
if not os.path.exists(path_dat_list):
print(f"\n*** plotSETI: File {path_dat_list} does not exist!")
return RETURN_ERROR
N_dat = count_text_lines(path_dat_list)
print(f"plotSETI: Found {N_dat} dat files.")
# Make sure that the lists are of the same size.
if N_h5 != N_dat:
print("\n*** plotSETI: Count of dat files must = count of h5 files!")
return RETURN_ERROR
number_in_cadence = N_h5
# Run find_event_pipeline()
if complex_cadence:
df_check = find_event_pipeline(path_dat_list,
path_h5_list,
filter_threshold = args.filter_threshold,
number_in_cadence = number_in_cadence,
on_source_complex_cadence=args.source_name,
sortby_tstart=True,
check_zero_drift=False,
SNR_cut=args.snr_threshold,
min_drift_rate=args.min_drift_rate,
max_drift_rate=args.max_drift_rate,
user_validation=False,
csv_name=path_csvf,
saving=True)
else: # not a complex cadence
df_check = find_event_pipeline(path_dat_list,
path_h5_list,
filter_threshold = args.filter_threshold,
number_in_cadence = number_in_cadence,
on_source_complex_cadence=False,
on_off_first=first_file,
sortby_tstart=True,
check_zero_drift=False,
SNR_cut=args.snr_threshold,
min_drift_rate=args.min_drift_rate,
max_drift_rate=args.max_drift_rate,
user_validation=False,
csv_name=path_csvf,
saving=True)
if df_check is None:
print("\n*** plotSETI: No events produced in find_event_pipeline()!")
return RETURN_ERROR
# Make the plots for all of the HDF5/DAT file pairs in batch mode.
matplotlib.use("agg", force=True)
plot_event_pipeline(path_csvf,
path_h5_list,
plot_dir=out_dir,
filter_spec=args.filter_threshold,
offset=offset,
user_validation=False)
print(f"\nplotSETI: Plots are stored in directory {out_dir}.")
return RETURN_NORMAL
