def run_find_event_pipeline(fildir, datdir, datlist, SNR, filter_threshold=np.nan, number_in_cadence=np.nan, on_source_complex_cadence=False): assert filter_threshold != np.nan assert number_in_cadence != np.nan # write to .lst, as required by find_event_pipeline lst_dat = datdir + "dat_files.lst" if not os.path.isfile(lst_dat): with open(lst_dat, 'w') as f: for item in datlist: f.write("%s\n" % item) # move the h5 files to the dat directory for file in datlist: h5_file = fildir + os.path.basename(file).replace('.dat','.h5') if not os.path.isfile(h5_file): fil = os.path.basename(h5_file).replace('.h5','.fil') bp.fil2h5.make_h5_file(fil, out_dir=datdir) else: mvh5_in = 'mv ' + h5_file + " " + datdir call([mvh5_in],shell=True) # run find_event_pipeline print("\nRunning find event pipeline...") find_event_pipeline(lst_dat, SNR_cut=SNR, check_zero_drift=False, filter_threshold=filter_threshold, number_in_cadence=number_in_cadence, on_source_complex_cadence=on_source_complex_cadence) # move the h5 files back to the fil directory to save space for file in datlist: mvh5_out = 'mv ' + datdir + os.path.basename(file).replace('.dat','.h5') + " " + fildir call([mvh5_out],shell=True)
def test_pipeline_same_source(): print('\n===== test_pipeline_same_source: BEGIN =====') # Make the dat list. with open(PATH_DAT_LIST_FILE, 'w') as fh: for path_dat in DAT_LIST_ONS: fh.write('{}\n'.format(path_dat)) # With the list of DAT files, do find_event_pipeline() df_event = find_event_pipeline(PATH_DAT_LIST_FILE, filter_threshold=3, number_in_cadence=3, user_validation=False, saving=True, on_source_complex_cadence='VOYAGER-1', csv_name=PATH_CSVF) # df_event should not be nil. if df_event is None: raise ValueError( 'test_pipeline_same_source: returned pandas df is None!') # CSV file created? if not Path(PATH_CSVF).exists(): raise ValueError('test_pipeline_same_source: No CSV of events created') # An event CSV was created. # Validate the hit table file. utl.validate_hittbl(df_event, PATH_CSVF, 'test_pipeline_same_source') print('\n===== test_pipeline_same_source: END =====')
def test_pipeline_h5_dat_separated(): print('\n===== test_pipeline_h5_dat_separated: BEGIN =====') # If there is an old ALT_DAT_DIR, recreate it empty. rmtree(ALT_DAT_DIR, ignore_errors=True) os.mkdir(ALT_DAT_DIR) # Copy the .dat files to ALT_DAT_DIR. for path_dat in DAT_LIST_ONS: cmd = 'cp ' + path_dat + " " + ALT_DAT_DIR os.system(cmd) # Make the dat list relative to ALT_DAT_DIR. with open(PATH_DAT_LIST_FILE, 'w') as fh: for path_dat in ALT_DAT_LIST_ONS: fh.write('{}\n'.format(path_dat)) # Make the h5 list. with open(PATH_H5_LIST_FILE, 'w') as fh: for path_dat in H5_LIST_ONS: fh.write('{}\n'.format(path_dat)) # With the list of separated .dat and .h5 files, do find_event_pipeline() df_event = find_event_pipeline(PATH_DAT_LIST_FILE, PATH_H5_LIST_FILE, filter_threshold=3, number_in_cadence=3, user_validation=False, saving=True, on_source_complex_cadence='VOYAGER-1', csv_name=PATH_CSVF) # df_event should not be nil. if df_event is None: raise ValueError( 'test_pipeline_same_source: returned pandas df is None!') # CSV file created? if not Path(PATH_CSVF).exists(): raise ValueError('test_pipeline_same_source: No CSV of events created') # An event CSV was created. # Validate the hit table file. utl.validate_hittbl(df_event, PATH_CSVF, 'test_pipeline_same_source', N_EVENTS) # Plots! plot_event_pipeline(PATH_CSVF, PATH_H5_LIST_FILE, filter_spec=3, user_validation=False) print('\n===== test_pipeline_h5_dat_separated: END =====')
def run_find_event_pipeline(datdir, SNR, filter_threshold=np.nan, number_in_cadence=np.nan, on_source_complex_cadence=False): assert filter_threshold != np.nan assert number_in_cadence != np.nan #get list of files and sort by file ID dat_list = glob.glob(datdir + "*.dat") dat_list = sort_by_timestamp(dat_list) # write to .lst, as required by find_event_pipeline lst_dat = datdir + "dat_files.lst" if os.path.isfile(lst_dat): os.remove(lst_dat) with open(lst_dat, 'w') as f: for item in dat_list: f.write("%s\n" % item) assert os.path.isfile(lst_dat) # construct csv file name # csv_name = datdir + "hits_SNR_" + SNR + "_f_" + str(filter_threshold) + ".csv" # if os.path.isfile(csv_name): # os.remove(csv_name) # run find_event_pipeline print("\nRunning find event pipeline...") find_event_pipeline(lst_dat, SNR_cut=SNR, check_zero_drift=False, filter_threshold=filter_threshold, number_in_cadence=number_in_cadence, on_source_complex_cadence=on_source_complex_cadence) try: csv_name = (glob.glob('*f' + str(filter_threshold) + '_snr' + str(SNR) + '*.csv'))[0] except: csv_name = [] print("\n") return csv_name
def try_mixed(arg_list, init_needed=True): print('\n===== try_mixed: BEGIN =====') # Make the off-cadence file. if init_needed: generate_fil_file(PATH_IRRELEVANT_FIL, -1, -1) # Make a DAT file. if init_needed: make_one_dat_file(PATH_IRRELEVANT_FIL, max_drift=10.0, min_snr=20.0, remove_h5=False) # Make the dat list. with open(PATH_DAT_LIST_FILE, 'w') as fh: for path_dat in arg_list: fh.write('{}\n'.format(path_dat)) # With the list of DAT files, do find_event_pipeline() df_event = find_event_pipeline(PATH_DAT_LIST_FILE, sortby_tstart=False, filter_threshold=3, number_in_cadence=len(arg_list), user_validation=False, saving=True, on_source_complex_cadence='VOYAGER-1', csv_name=PATH_CSVF) # df_event should not be nil. if df_event is None: raise ValueError('try_mixed: returned pandas df is None!') # CSV file created? if not Path(PATH_CSVF).exists(): raise ValueError('try_mixed: No CSV of events created') # An event CSV was created. # Validate the hit table file. utl.validate_hittbl(df_event, PATH_CSVF, 'test_pipeline_mixed') print('\n===== try_mixed: END =====')
def test_pipeline_wrong_source(): print('\n===== test_pipeline_wrong_source: BEGIN =====') # Make the dat list. with open(PATH_DAT_LIST_FILE, 'w') as fh: for path_dat in DAT_LIST_ONS: fh.write('{}\n'.format(path_dat)) # With the list of DAT files, do find_event_pipeline() df_event = find_event_pipeline(PATH_DAT_LIST_FILE, filter_threshold=3, number_in_cadence=3, user_validation=False, saving=True, on_source_complex_cadence='VOYAGER-42', csv_name=PATH_CSVF) # df_event should be nil if not df_event is None: raise ValueError('test_pipeline_wrong_source: returned pandas df has entries but should be nil!') print('\n===== test_pipeline_wrong_source: END =====')
path = '/home/charlesg/turboSETI/trappist1/' fil_path = path + 'fil_files/high' png_path = '/home/charlesg/turboSETI/blc00_Graphs/trappist1/hits' dat_path = path + 'doppler_output/' dat_list = [ dat_path + 'blc00_guppi_57807_75725_DIAG_TRAPPIST1_0015.gpuspec.0000.dat', dat_path + 'blc00_guppi_57807_75805_DIAG_TRAPPIST1_OFF_0016.gpuspec.0000.dat', dat_path + 'blc00_guppi_57807_75885_DIAG_TRAPPIST1_0017.gpuspec.0000.dat', dat_path + 'blc00_guppi_57807_75965_DIAG_TRAPPIST1_OFF_0018.gpuspec.0000.dat' ] check_zero_drift = False filter_threshold = 3 number_in_cadence = 4 with open(dat_path + 'dat_files.lst', 'w') as f: for item in dat_list: f.write("%s\n" % item) event_dataframe = find_event_pipeline(dat_path + 'dat_files.lst', number_in_cadence=number_in_cadence, filter_threshold=filter_threshold) print(event_dataframe) #
def find_plot_pipelines(need_init=True, filter_threshold=FILTER_THRESHOLD): r''' Exercise find_event_pipeline() and plot_event_pipeline() ''' main_time_start = time() # If configured to do so, initialize temp directory # and fetch all of the HDF5 files from the Internet. if need_init: initialize() for filename_h5 in H5_FILE_LIST: wgetter(filename_h5) # Make all of the DAT files. make_all_dat_files() print('find_plot_pipelines: Filter threshold = ', filter_threshold) number_in_cadence = len(H5_FILE_LIST) print('find_plot_pipelines: Cadence length = ', number_in_cadence) print('find_plot_pipelines: find_event_pipeline({}) ...'.format( PATH_DAT_LIST_FILE)) # With the list of DAT files, do find_event_pipeline() df_event = find_event_pipeline(PATH_DAT_LIST_FILE, filter_threshold=filter_threshold, number_in_cadence=number_in_cadence, user_validation=False, saving=True, csv_name=PATH_CSVF) # CSV file created? if not Path(PATH_CSVF).exists(): raise ValueError('find_plot_pipelines: No CSV of events created') # An event CSV was created. # Validate the hit table file. utl.validate_hittbl(df_event, PATH_CSVF, 'test_pipe_lines', N_EVENTS) # Make a list of the HDF5 files. print( 'find_plot_pipelines: making a list of HDF5 files in ({}) ...'.format( PATH_DAT_LIST_FILE)) with open(PATH_H5_LIST_FILE, 'w') as file_handle: for filename_h5 in H5_FILE_LIST: file_handle.write('{}\n'.format(TESTDIR + filename_h5)) # Do the plots for all of the HDF5/DAT file pairs. print('find_plot_pipelines: plot_event_pipeline({}, {}) ...'.format( PATH_CSVF, PATH_H5_LIST_FILE)) plot_event_pipeline(PATH_CSVF, PATH_H5_LIST_FILE, filter_spec='f{}'.format(filter_threshold), user_validation=False) # Check that the right number of PNG files were created. outdir_list = listdir(TESTDIR) npngs = 0 for cur_file in outdir_list: if cur_file.split('.')[-1] == 'png': if imghdr.what(TESTDIR + cur_file) != 'png': raise ValueError( 'find_plot_pipelines: File {} is not a PNG file'.format( cur_file)) npngs += 1 if npngs != N_EVENTS: raise ValueError( 'find_plot_pipelines: Expected to find {} PNG files but observed {}' .format(N_EVENTS, npngs)) # Stop the clock - we're done. main_time_stop = time() print('find_plot_pipelines: End, et = {:.1f} seconds'.format( main_time_stop - main_time_start))
def find_plot_pipelines(filter_threshold=3, on_off_first='ON', plot_dir=None, on_source_complex_cadence=False): r''' Exercise find_event_pipeline() and plot_event_pipeline() ''' main_time_start = time() print('find_plot_pipelines_2: Filter threshold = ', filter_threshold) h5_file_list = sorted(glob.glob(TESTDIR + 'single*.h5')) dat_file_list = sorted(glob.glob(TESTDIR + 'single*.dat')) number_in_cadence = len(h5_file_list) if number_in_cadence != 6: raise ValueError( 'find_plot_pipelines_2: Expected to find 6 h5 files but observed {}' .format(number_in_cadence)) number_in_cadence = len(dat_file_list) if number_in_cadence != 6: raise ValueError( 'find_plot_pipelines_2: Expected to find 6 dat files but observed {}' .format(number_in_cadence)) # Re-order the H5 and DAT files into OFF-ON-... # In the 2 lists, switch 1 and 2, 3 and 4, 5 and 6 for ix in [0, 2, 4]: temp = h5_file_list[ix] h5_file_list[ix] = h5_file_list[ix + 1] h5_file_list[ix + 1] = temp temp = dat_file_list[ix] dat_file_list[ix] = dat_file_list[ix + 1] dat_file_list[ix + 1] = temp fh_h5 = open(PATH_H5_LIST_FILE, 'w') fh_dat = open(PATH_DAT_LIST_FILE, 'w') for ix in range(6): fh_h5.write(h5_file_list[ix] + '\n') fh_dat.write(dat_file_list[ix] + '\n') fh_h5.close() fh_dat.close() print('find_plot_pipelines_2: H5/dat cadence length = ', number_in_cadence) # If CSV exists from a previous execution, remove it. try: remove(PATH_CSVF) except: pass # With the list of DAT files, do find_event_pipeline() print('===== find_event_pipeline BEGIN =====') df_event = find_event_pipeline( PATH_DAT_LIST_FILE, sortby_tstart=False, filter_threshold=filter_threshold, number_in_cadence=number_in_cadence, user_validation=False, saving=True, on_off_first=on_off_first, on_source_complex_cadence=on_source_complex_cadence, csv_name=PATH_CSVF) print('===== find_event_pipeline END =====') # CSV file created? if not Path(PATH_CSVF).exists(): raise ValueError('find_plot_pipelines_2: No CSV of events created') # An event CSV was created. # Validate the hit table file. utl.validate_hittbl(df_event, PATH_CSVF, 'test_pipe_lines_2') # Do the plots for all of the HDF5/DAT file pairs. print( '===== plot_event_pipeline #1 (plot_dir does not yet exist) BEGIN =====' ) rmtree(plot_dir, ignore_errors=True) plot_event_pipeline(PATH_CSVF, PATH_H5_LIST_FILE, filter_spec='f{}'.format(filter_threshold), plot_dir=plot_dir, user_validation=False) print('===== plot_event_pipeline #2 (plot_dir already exists) BEGIN =====') plot_event_pipeline(PATH_CSVF, PATH_H5_LIST_FILE, filter_spec='f{}'.format(filter_threshold), plot_dir=plot_dir, user_validation=False) # Check that the right number of PNG files were created. print('===== plot_event_pipeline END =====') outdir_list = listdir(plot_dir) npngs = 0 for cur_file in outdir_list: if cur_file.split('.')[-1] == 'png': if imghdr.what(plot_dir + cur_file) != 'png': raise ValueError( 'find_plot_pipelines_2: File {} is not a PNG file'.format( cur_file)) npngs += 1 if npngs != 6: raise ValueError( 'find_plot_pipelines_2: Expected to find 6 PNG files but observed {}' .format(npngs)) # Stop the clock - we're done. main_time_stop = time() print('find_plot_pipelines_2: End, et = {:.1f} seconds'.format( main_time_stop - main_time_start))
def execute_pipelines(args): r""" Interface to the pipeline functions, called by main(). Parameters ---------- args : dict """ # Setup some parameter values for find_event_pipeline(). if args.cadence == "complex": complex_cadence = True if len(args.source_name) < 1: print( "\n*** plotSETI: Complex cadence requires a source_name. Bye-bye." ) sys.exit(86) else: complex_cadence = False if args.cadence == "on": first_file = "ON" else: first_file = "OFF" from_dir = os.path.abspath(args.input_dir_path) + "/" dest_dir = os.path.abspath(args.out_dir) + "/" if not os.path.exists(dest_dir): os.mkdir(dest_dir) # Establish output pathnames, path_h5_list = dest_dir + NAME_H5_LIST path_dat_list = dest_dir + NAME_DAT_LIST path_csvf = dest_dir + NAME_CSVF clean_event_stuff(dest_dir) make_lists(from_dir, path_h5_list, path_dat_list) # Run find_event_pipeline() number_in_cadence = len(open(path_h5_list).readlines()) if complex_cadence: df_check = find_event_pipeline( path_dat_list, filter_threshold=args.filt_thresh, number_in_cadence=number_in_cadence, on_source_complex_cadence=args.source_name, sortby_tstart=True, check_zero_drift=False, SNR_cut=args.snr_threshold, user_validation=False, csv_name=path_csvf, saving=True) else: # not a complex cadence df_check = find_event_pipeline(path_dat_list, filter_threshold=args.filt_thresh, number_in_cadence=number_in_cadence, on_source_complex_cadence=False, on_off_first=first_file, sortby_tstart=True, check_zero_drift=False, SNR_cut=args.snr_threshold, user_validation=False, csv_name=path_csvf, saving=True) if df_check is None: print( "\n*** plotSETI: No events produced in find_event_pipeline()! Bye-bye." ) return 86 # Make the plots for all of the HDF5/DAT file pairs in batch mode. matplotlib.use("agg", force=True) plot_event_pipeline(path_csvf, path_h5_list, plot_dir=dest_dir, filter_spec=args.filt_thresh, offset=0, user_validation=False) print("\nplotSETI: Plots are stored in directory {}. Bye-bye.".format( dest_dir)) return 0
def beginning(): # Initialize output directory rmtree(OUTDIR, ignore_errors=True) mkdir(OUTDIR) # Make the DAT files make_dat_files() #================================================================= print('============== turbo_seti version ===============>', VERSION) #beginning() # Generate CSV file from find_event_pipeline() num_in_cadence = len(h5_files_list) find_event_pipeline(PATH_DAT_LIST_FILE, filter_threshold=THRESHOLD, number_in_cadence=num_in_cadence, user_validation=False, csv_name=PATH_CSVF, saving=True) print("Produced {}".format(PATH_CSVF)) plot_event_pipeline(PATH_CSVF, H5_LIST_FILE, user_validation=False) '''file_path = '/home/ellie/research/seti/VoyagerTutorial2/VoyagerData/single_coarse_guppi_59046_80672_DIAG_VOYAGER-1_0013.rawspec.0000.h5' f_start = 8419.51 f_stop = 8419.58 obs = Waterfall(file_path, f_start, f_stop) obs.plot_spectrum(f_start=f_start, f_stop=f_stop)'''
def find_plot_pipelines(need_init=True, filter_threshold=2): ''' Exercise find_event_pipeline() and plot_event_pipeline() ''' main_time_start = time() # If configured to do so, initialize temp directory. if need_init: initialize() # If configured to do so, fetch all of the HDF5 files from the Internet. if need_init: for filename_h5 in H5_FILE_LIST: wgetter(filename_h5) # Make all of the DAT files. make_all_dat_files() # Create the CSV file used by plot_event_pipeline. print('find_plot_pipelines: Filter threshold = ', filter_threshold) number_in_cadence = len(H5_FILE_LIST) print('find_plot_pipelines: Cadence length = ', number_in_cadence) print('find_plot_pipelines: find_event_pipeline({}) ...'.format( PATH_DAT_LIST_FILE)) # If CSV exists from a previous execution, remove it. try: remove(PATH_CSVF) except: pass # With the list of DAT files, do find_event_pipeline() df_event = find_event_pipeline(PATH_DAT_LIST_FILE, filter_threshold=filter_threshold, number_in_cadence=number_in_cadence, user_validation=False, saving=True, csv_name=PATH_CSVF) # CSV created? if not Path(PATH_CSVF).exists(): print('*** find_plot_pipelines: No CSV of events created') return # An event CSV was created. print('find_plot_pipelines: find_event_pipeline() returned dataframe:\n{}'. format(df_event)) # Make a list of the HDF5 files. print( 'find_plot_pipelines: making a list of HDF5 files in ({}) ...'.format( PATH_DAT_LIST_FILE)) with open(PATH_H5_LIST_FILE, 'w') as file_handle: for filename_h5 in H5_FILE_LIST: file_handle.write('{}\n'.format(TESTDIR + filename_h5)) # Do the plots for all of the HDF5/DAT file pairs. print('find_plot_pipelines: plot_event_pipeline({}, {}) ...'.format( PATH_CSVF, PATH_H5_LIST_FILE)) plot_event_pipeline(PATH_CSVF, PATH_H5_LIST_FILE, user_validation=False) main_time_stop = time() print('find_plot_pipelines: End, et = {:.1f} seconds'.format( main_time_stop - main_time_start))
def find_plot_pipelines(need_init=True, filter_threshold=3): r''' Exercise find_event_pipeline() and plot_event_pipeline() ''' main_time_start = time() # If configured to do so, initialize temp directory. if need_init: initialize() # If configured to do so, fetch all of the HDF5 files from the Internet. if need_init: for filename_h5 in H5_FILE_LIST: wgetter(filename_h5) # Make all of the DAT files. make_all_dat_files() # Create the CSV file used by plot_event_pipeline. print('find_plot_pipelines: Filter threshold = ', filter_threshold) number_in_cadence = len(H5_FILE_LIST) print('find_plot_pipelines: Cadence length = ', number_in_cadence) print('find_plot_pipelines: find_event_pipeline({}) ...'.format( PATH_DAT_LIST_FILE)) # If CSV exists from a previous execution, remove it. try: remove(PATH_CSVF) except: pass # With the list of DAT files, do find_event_pipeline() df_event = find_event_pipeline(PATH_DAT_LIST_FILE, filter_threshold=filter_threshold, number_in_cadence=number_in_cadence, user_validation=False, saving=True, csv_name=PATH_CSVF) # CSV file created? if not Path(PATH_CSVF).exists(): raise ValueError('find_plot_pipelines: No CSV of events created') # An event CSV was created. # Validate CSV file. df_validate = validate_csvf(PATH_CSVF) if df_validate.equals(df_event): raise ValueError( 'find_plot_pipelines: Expected df_validate to be identical to df_event but observed\n{}' .format(df_validate)) # Make a list of the HDF5 files. print( 'find_plot_pipelines: making a list of HDF5 files in ({}) ...'.format( PATH_DAT_LIST_FILE)) with open(PATH_H5_LIST_FILE, 'w') as file_handle: for filename_h5 in H5_FILE_LIST: file_handle.write('{}\n'.format(TESTDIR + filename_h5)) # Do the plots for all of the HDF5/DAT file pairs. print('find_plot_pipelines: plot_event_pipeline({}, {}) ...'.format( PATH_CSVF, PATH_H5_LIST_FILE)) plot_event_pipeline(PATH_CSVF, PATH_H5_LIST_FILE, filter_spec='f{}'.format(filter_threshold), user_validation=False) # Check that the right number of PNG files were created. outdir_list = listdir(TESTDIR) npngs = 0 for cur_file in outdir_list: if cur_file.split('.')[-1] == 'png': if imghdr.what(TESTDIR + cur_file) != 'png': raise ValueError( 'find_plot_pipelines: File {} is not a PNG file'.format( cur_file)) npngs += 1 if npngs != 6: raise ValueError( 'find_plot_pipelines: Expected to find 6 PNG files but observed {}' .format(npngs)) # Stop the clock - we're done. main_time_stop = time() print('find_plot_pipelines: End, et = {:.1f} seconds'.format( main_time_stop - main_time_start))
def execute_pipelines(args): """ Interface to the pipeline functions, called by main(). Parameters ---------- args : dict """ # Setup some parameter values for find_event_pipeline(). if args.cadence == "complex": complex_cadence = True if len(args.source_name) < 1: print("\n*** plotSETI: Complex cadence requires a source_name.") sys.exit(RETURN_ERROR) else: complex_cadence = False if args.cadence == "on": first_file = "ON" else: first_file = "OFF" h5_dir = os.path.abspath(args.h5_dir) + "/" dat_dir = os.path.abspath(args.dat_dir) + "/" out_dir = os.path.abspath(args.out_dir) + "/" if not os.path.exists(out_dir): os.mkdir(out_dir) if args.plot_offset: offset="auto" else: offset=0 # Establish output pathnames, path_csvf = out_dir + NAME_CSVF clean_event_stuff(out_dir) # Make the h5 and dat lists. # Default to auto-generation? if args.h5dat_lists is None: SZ_user_list = 0 else: SZ_user_list = len(args.h5dat_lists) if args.debug: print(f"DEBUG h5dats_list: #{SZ_user_list} {args.h5dat_lists}") if SZ_user_list == 0: # Default to auto-generation. path_h5_list = out_dir + NAME_H5_LIST path_dat_list = out_dir + NAME_DAT_LIST number_in_cadence = make_lists(h5_dir, path_h5_list, dat_dir, path_dat_list) if number_in_cadence == 0: return RETURN_ERROR else: # User-specified lists if SZ_user_list != 2: print(f"\n*** plotSETI: h5dat_lists had {SZ_user_list} elements; must be 2 (one for h5 and one for dat)!") return RETURN_ERROR if args.h5dat_lists[0] is None or args.h5dat_lists[1] is None: print(f"\n*** plotSETI: h5dat_lists had {SZ_user_list} elements; must be 2 (one for h5 and one for dat)!") return RETURN_ERROR # Check the list of h5 files. path_h5_list = args.h5dat_lists[0] if not os.path.exists(path_h5_list): print(f"\n*** plotSETI: File {path_h5_list} does not exist!") return RETURN_ERROR N_h5 = count_text_lines(path_h5_list) print(f"plotSETI: Found {N_h5} h5 files.") # Check the list of dat files. path_dat_list = args.h5dat_lists[1] if not os.path.exists(path_dat_list): print(f"\n*** plotSETI: File {path_dat_list} does not exist!") return RETURN_ERROR N_dat = count_text_lines(path_dat_list) print(f"plotSETI: Found {N_dat} dat files.") # Make sure that the lists are of the same size. if N_h5 != N_dat: print("\n*** plotSETI: Count of dat files must = count of h5 files!") return RETURN_ERROR number_in_cadence = N_h5 # Run find_event_pipeline() if complex_cadence: df_check = find_event_pipeline(path_dat_list, path_h5_list, filter_threshold = args.filter_threshold, number_in_cadence = number_in_cadence, on_source_complex_cadence=args.source_name, sortby_tstart=True, check_zero_drift=False, SNR_cut=args.snr_threshold, min_drift_rate=args.min_drift_rate, max_drift_rate=args.max_drift_rate, user_validation=False, csv_name=path_csvf, saving=True) else: # not a complex cadence df_check = find_event_pipeline(path_dat_list, path_h5_list, filter_threshold = args.filter_threshold, number_in_cadence = number_in_cadence, on_source_complex_cadence=False, on_off_first=first_file, sortby_tstart=True, check_zero_drift=False, SNR_cut=args.snr_threshold, min_drift_rate=args.min_drift_rate, max_drift_rate=args.max_drift_rate, user_validation=False, csv_name=path_csvf, saving=True) if df_check is None: print("\n*** plotSETI: No events produced in find_event_pipeline()!") return RETURN_ERROR # Make the plots for all of the HDF5/DAT file pairs in batch mode. matplotlib.use("agg", force=True) plot_event_pipeline(path_csvf, path_h5_list, plot_dir=out_dir, filter_spec=args.filter_threshold, offset=offset, user_validation=False) print(f"\nplotSETI: Plots are stored in directory {out_dir}.") return RETURN_NORMAL