def make_one_dat_file(arg_h5_name):
r'''
Make a single DAT file:
* Instantiate the FindDoppler class object.
* With the object, search the H5, creating the DAT file
and a LOG file (not used).
Note that a max drift of 1 assumes a drift rate of +/- 1
SNR threshold = 25
'''
print('make_one_dat_file: Begin FindDoppler({}) .....'.format(arg_h5_name))
h5_path = TESTDIR + arg_h5_name
time_start = time()
doppler = FindDoppler(h5_path,
max_drift=MAX_DRIFT,
snr=MIN_SNR,
out_dir=TESTDIR)
time_stop = time()
print('make_one_dat_file: End FindDoppler({}), et = {:.1f} seconds'.format(
arg_h5_name, time_stop - time_start))
print('make_one_dat_file: Begin Doppler search({}) .....'.format(
arg_h5_name))
# ----------------------------------------------------------------------------
# No more than 1 execution of this program because of dask methodology!
# To do multiple dask partitions would cause initialization & cleanup chaos.
time_start = time()
doppler.search(n_partitions=1)
time_stop = time()
# ----------------------------------------------------------------------------
print('make_one_dat_file: End Doppler search({}), et = {:.1f} seconds'.
format(arg_h5_name, time_stop - time_start))
def find_doppler(fname):
# Do our own coarse channel counting heuristics
fpath = os.path.join(DATA_DIR, fname)
header = H5Reader(fpath, load_data=False).read_header()
nchans = header["nchans"]
if nchans == 64000000 and fname.endswith("0000.h5"):
# According to davidm this means this is Parkes UWL single node, so 1 coarse channel.
# We can't process that much at once, though, so we use a higher number for batching.
n_coarse_chan = 16
else:
# Use default n_coarse_chan heuristics
n_coarse_chan = None
with tempfile.TemporaryDirectory() as tmp_dir:
print("analyzing", fname)
doppler = FindDoppler(
fpath,
min_drift=0.001,
max_drift=4,
snr=10,
out_dir=tmp_dir,
gpu_backend=True,
n_coarse_chan=n_coarse_chan,
)
start = time.time()
doppler.search()
elapsed = time.time() - start
print(f"time to turboseti {fname}: {elapsed:.2f}s")
copy_tree(tmp_dir, OUTPUT_DIR)
def make_one_dat_file(arg_path_fil,
min_drift=0.0,
max_drift=4.0,
min_snr=25.0,
remove_h5=True):
r'''
Make a single DAT file:
* Instantiate the FindDoppler class object.
* With the object, search the H5, creating the DAT file
and a LOG file (not used).
'''
if max_drift is None:
raise ValueError('make_one_dat_file: max_drift not set')
woutdir = dirname(arg_path_fil)
fdop = FindDoppler(datafile=arg_path_fil,
min_drift=min_drift,
max_drift=max_drift,
snr=min_snr,
log_level_int=logging.WARNING,
gpu_backend=using_gpu(),
out_dir=woutdir)
fdop.search()
path_h5_file = arg_path_fil.replace('.fil', '.h5')
if remove_h5:
remove(path_h5_file)
def make_one_dat_file(arg_h5_name):
'''
Make a single DAT file:
* Instantiate the FindDoppler class object.
* With the object, search the H5, creating the DAT file
and a LOG file (not used).
Note that a max drift of 1 assumes a drift rate of +/- 1
SNR threshold = 25
'''
print('test_plotting: Begin FindDoppler({}) .....'.format(arg_h5_name))
h5_path = TESTDIR + arg_h5_name
time_start = time()
doppler = FindDoppler(h5_path,
max_drift=1,
min_drift=-1,
snr=25,
out_dir=TESTDIR)
time_stop = time()
print('test_plotting: End FindDoppler({}), et = {:.1f} seconds'.format(
arg_h5_name, time_stop - time_start))
print('test_plotting: Begin Doppler search({}) .....'.format(arg_h5_name))
time_start = time()
doppler.search()
time_stop = time()
print('test_plotting: End Doppler search({}), et = {:.1f} seconds'.format(
arg_h5_name, time_stop - time_start))
def run_turbo_seti(file,
max_drift=np.nan,
min_drift=0,
min_snr=10.0,
outdir="./",
clobber=False,
gpu_backend=False):
assert max_drift != np.nan #< Make sure the drift rate input is a number
pre, ext = os.path.splitext(os.path.basename(
file)) #< Split the input filename by its file extension
out_file = outdir + pre + ".dat" #< Create the output filename with the new extension .dat
if os.path.isfile(out_file):
if not clobber:
print("\n" + pre + ".dat" + " already exists. Moving on...")
return out_file
else:
print("Removing old .dat to make way for the new one...")
os.remove(out_file)
out_log = outdir + pre + ".log"
os.remove(out_log)
print("%s has been removed successfully\n" % out_file)
# call FindDoppler
log_level_int = logging.WARNING
fdop = FindDoppler(datafile=file,
max_drift=max_drift,
min_drift=min_drift,
snr=min_snr,
out_dir=outdir,
gpu_backend=gpu_backend,
log_level_int=log_level_int)
# search for hits and report elapsed time.
startt = time.time()
fdop.search()
delt, time_label = get_elapsed_time(startt)
print(
f"\nfind_doppler.py populated the corresponding .dat file with hits in %.2f {time_label}."
% delt)
print("\n")
# return the .dat file name
return out_file
def make_dat_files():
ii = 0
file_handle = open(PATH_DAT_LIST_FILE, 'w')
h5_file_handle = open(H5_LIST_FILE, 'w')
for filename in h5_files_list:
path_h5 = filename
doppler = FindDoppler(path_h5,
max_drift=4,
snr=10,
log_level_int=logging.WARNING,
out_dir=H5DIR)
doppler.search()
ii += 1
h5_file_handle.write('{}\n'.format(path_h5))
path_dat = filename.replace('.h5', '.dat')
file_handle.write('{}\n'.format(path_dat))
print("make_dat_files: {} - finished making DAT file for {}".format(
ii, path_h5))
def _turbo_runner(self,
waterfall_itr,
max_drift=5,
num_inserted=2,
drifts=None,
snrs=None,
widths=None,
snrRatio=None,
min_snr=15,
default_snr=40,
default_width=4,
default_drift=0,
default_snr_noise=0.5):
from turbo_seti.find_doppler.find_doppler import FindDoppler
#from .signal_inject import *
def insert_signal(frame, freq, drift, snr, width):
print("Inserting signal", freq, drift, snr, width, frame.df,
frame.dt, frame.fch1, frame.tchans, frame.fchans)
tchans = frame.tchans * u.pixel
dt = frame.dt * u.s
#fexs = freq*u.MHz - math.copysign(1, drift)*width*u.Hz
fexs = freq - math.copysign(1, drift) * width * u.Hz
#fex = (freq*u.mhz + ((drift*u.hz/u.s)*dt*tchans/u.pixel)
# + math.copysign(1, drift)*width*u.hz)/u.hz
fex = (freq + ((drift * u.Hz / u.s) * dt * tchans / u.pixel) +
math.copysign(1, drift) * width * u.Hz) / u.Hz
signal = frame.add_signal(
paths.constant_path(f_start=freq,
drift_rate=drift * u.Hz / u.s),
t_profiles.constant_t_profile(level=frame.get_intensity(
snr=snr)),
f_profiles.gaussian_f_profile(width=width * u.Hz),
bp_profiles.constant_bp_profile(level=1),
bounding_f_range=(min(fexs / u.Hz, fex), max(fexs / u.Hz,
fex)))
def noise_parameters(frame, near=False, drift_max=2):
parameters = [0] * 2
drift = random.uniform(-1, 1) * drift_max
parameters[1] = drift
if near:
parameters[0] = (frame.get_frequency(frame.fchans // 2) +
random.uniform(-1, 1) *
(drift * frame.dt * frame.tchans)
) #*u.MHz#*u.Hz/u.MHz
else:
parameters[0] = frame.get_frequency(
int(random.uniform(0, frame.fchans))) #*u.Hz/u.MHz
return parameters
#signal_capture = {'num_recovered':[], 'num_inserted':[], 'frequency':[], 'drift_rate':[], 'SNR':[], 'width':[], '}
signal_capture = {
'num_recovered': [],
'num_inserted': [],
'capture_ratio': [],
'inserted_frequency': [],
'frequency_cap': [],
'inserted_drift': [],
'drift_cap': [],
'inserted_snr': [],
'snr_cap': [],
'inserted_width': [],
'distance_noise_center': [],
'noise_drift': [],
'noise_snr': []
}
#signal_capture += [[num_recovered, num_inserted, temp_freq/1e6, drift, snr, width, [data], [noise[0]/1e6, noise[1], snr_noise]]]
turbo_vals = []
#signal_capture=[]
for i, waterfall in enumerate(waterfall_itr):
if not hasattr(self, 'temp_added_signals'):
self.temp_added_signals = {}
temp_frame = Frame(waterfall=waterfall)
temp_freq = temp_frame.get_frequency(
temp_frame.fchans // 2) * u.MHz #*u.Hz/u.MHz
noise_recovered = 0
if drifts is None:
drift = default_drift
if snrs is None:
snr = default_snr_noise * default_snr
if widths is None:
width = default_width
else:
width = widths[i]
drift_noise = max_drift
if snrRatio is None:
snr_noise = default_snr_noise
width_noise = width
#insert_signal(temp_frame, temp_freq, drift, snr, width)
self.add_constant_signal(f_start=temp_freq,
drift_rate=drift,
snr=snr,
width=width,
temp=temp_frame)
noise = [0 * u.Hz, 0]
for _ in range(num_inserted - 1):
noise = noise_parameters(temp_frame, False, drift_noise)
#insert_signal(temp_frame, noise[0], noise[1], snr_noise, width_noise)
self.add_constant_signal(f_start=noise[0],
drift_rate=noise[1],
snr=snr_noise,
width=width_noise,
temp=temp_frame)
# save_frame(temp_frame)
#if not hasattr(self, 'temp_frames'):
# self.temp_frames = []
#else:
# self.temp_frames.append(temp_frame)
temp_frame.save_fil(filename=f"frame_{i}.fil")
find_seti_event = FindDoppler(f"frame_{i}.fil",
max_drift=max_drift,
snr=min_snr)
find_seti_event.search()
f = open(f"frame_{i}.dat", 'r')
try:
# Maybe just use 'read_dat' from 'find_events' here?
data = [
dataline.split() for dataline in
[line for line in f.readlines() if line[0] != '#']
]
data = [[float(val) for val in dataline] for dataline in data]
except:
data = []
if len(data) == 0:
data = [[0.0] * 12]
num_recovered = 0
else:
num_recovered = len(data)
turbo_vals += [data]
#signal_capture += [[num_recovered, num_inserted, temp_freq.value/1e6, drift, snr, width, [data], [noise[0].value/1e6, noise[1], snr_noise]]]
acquired_data = [[
num_recovered, num_inserted, temp_freq.value / 1e6, drift, snr,
width, [data], [noise[0] / 1e6, noise[1], snr_noise]
]]
signal_capture['num_recovered'].append(num_recovered)
signal_capture['num_inserted'].append(num_inserted)
signal_capture['capture_ratio'].append(num_recovered /
num_inserted)
signal_capture['inserted_frequency'].append(temp_freq.value / 1e6)
signal_capture['frequency_cap'].append(
acquired_data[0][6][0][0][1])
signal_capture['inserted_drift'].append(acquired_data[0][3])
signal_capture['drift_cap'].append(acquired_data[0][6][0][0][3])
signal_capture['inserted_snr'].append(acquired_data[0][4])
signal_capture['snr_cap'].append(acquired_data[0][6][0][0][2])
signal_capture['inserted_width'].append(acquired_data[0][5])
signal_capture['distance_noise_center'].append(
acquired_data[0][7][0] - acquired_data[0][2])
signal_capture['noise_drift'].append(acquired_data[0][7][1])
signal_capture['noise_snr'].append(acquired_data[0][7][2])
#print(i, signal_capture[i])
#print("")
#try:
# os.remove(f"frame_{i}.h5")
# os.remove(f"frame_{i}.fil")
# os.system(f"frame_{i}.dat")
# os.remove(f"frame_{i}.log")
#except:
# pass
return signal_capture
'/DIAG_PSR_J0953+0755_0003.gpuspec.0000'
]
exts = ['.dat', '.log', '.h5']
max_drift_rate = 1 #Hz/s
signal_to_noise = 25
for file_name in filenames:
for ext in exts:
if os.path.exists(path + file_name + ext):
os.remove(path + file_name + ext)
else:
continue
find_signal = FindDoppler(path + file_name + '.fil',
max_drift=max_drift_rate,
snr=signal_to_noise,
out_dir=path)
fil_file = path + file_name + '.fil'
dat_file = path + file_name + '.dat'
find_signal.search()
df = make_table(dat_file)
df = df.set_index('TopHitNum')
df = df.sort_values('SNR')
print(df)
hits = df.index.values.tolist()
print(len(hits), hits)
fig = plt.figure()
if len(hits) <= 5:
for i in range(len(hits)):
print(i, hits[i])
plt.subplot(len(hits), 1, i + 1)
'blc00_guppi_57807_75805_DIAG_TRAPPIST1_OFF_0016.gpuspec.0000',
'blc00_guppi_57807_75885_DIAG_TRAPPIST1_0017.gpuspec.0000',
'blc00_guppi_57807_75965_DIAG_TRAPPIST1_OFF_0018.gpuspec.0000'
]
max_drift_rate = 1 #Hz/s
SNR = 25
for file_name in filenames:
dat_file = dat_path + file_name + '.dat'
fil_file = fil_path + file_name + '.fil'
#if os.path.exists(dat_file):
# continue
#else:
#print('Searching'+fil_file)
find_signal = FindDoppler(fil_file,
max_drift=max_drift_rate,
snr=SNR,
out_dir=dat_path)
find_signal.search()
print('Generating Dataframe')
df = make_table(dat_file)
df_sorted = df.set_index('TopHitNum')
df.sort_values('SNR')
print(df)
hits = df.index.values.tolist()
print('Top Hits are: ', hits)
def main():
experiment_path = '/datax/scratch/bbrzycki/data/nb-localization/training/'
output_dir = experiment_path + 'turboseti/'
turbo_rmse_dict = {}
wrong_num_signals = 0
timing_array = []
turbo_start = time.time()
for db in range(0, 30, 5):
for j in range(4000):
print(db, j)
fn = '{:02}db_{:06d}.npy'.format(db, j)
npy_fn = '/datax/scratch/bbrzycki/data/nb-localization/1sig/test/{}'.format(
fn)
fil_fn = output_dir + '{:02}db_{:06d}.fil'.format(db, j)
dat_fn = output_dir + '{:02}db_{:06d}.dat'.format(db, j)
frame = stg.Frame(fchans=1024,
tchans=32,
df=1.3969838619232178 * u.Hz,
dt=1.4316557653333333 * u.s,
fch1=6095.214842353016 * u.MHz,
data=np.load(npy_fn))
frame.save_fil(fil_fn)
try:
os.remove(dat_fn)
except FileNotFoundError:
pass
start_time = time.time()
find_seti_event = FindDoppler(fil_fn,
max_drift=31,
snr=10,
out_dir=output_dir)
find_seti_event.search()
end_time = time.time()
timing_array.append(end_time - start_time)
with open(dat_fn, 'r') as f:
data = [
line.split() for line in f.readlines() if line[0] != '#'
]
# Count number of times turboseti predicts wrong number
if len(data) != 1:
wrong_num_signals += 1
estimates = []
snrs = []
for signal in data:
snr = float(signal[2])
drift_rate = float(signal[1])
start_index = 1024 - int(signal[5])
end_index = frame.get_index(
frame.get_frequency(start_index) +
drift_rate * frame.tchans * frame.dt)
estimates.append([start_index, end_index])
snrs.append(snr)
if len(estimates) != 0:
# Get the ground truth positions from the saved dictionaries
true_indices = (
np.load(experiment_path +
'final_1sig_32bs_bright/test_predictions.npy',
allow_pickle=True).item()[fn] * 1024)[0]
# If turboseti found signals, choose the highest SNR one
turbo_rmse_dict[fn] = rmse(true_indices,
estimates[np.argsort(snrs)[-1]])
timing_array = np.array(timing_array)
print('Wrong: {} frames'.format(wrong_num_signals))
print('Total search: {:.2f} seconds'.format(time.time() - turbo_start))
np.save(output_dir + 'timing_array.npy', timing_array)
np.save(output_dir + 'test_predictions.npy', turbo_rmse_dict)
