def generate_signal(sig_type='constant', line_width=0.02**3, level=10, bias_no_drift=0, **kwargs): # bias no drift is fraction of the time drift rate should be set to 0 start_index = np.random.randint(0, fchans) if np.random.uniform(0, 1) < bias_no_drift: drift_rate = 0 else: drift_rate = np.random.uniform(-start_index*df/(tsamp*tchans), (fchans-1-start_index)*df/(tsamp*tchans)) # Placeholder for non rfi types, which would have spread = 0 anyway spread = 0 if sig_type == 'noise': signal = stg.generate(ts, fs, stg.constant_path(f_start = fs[start_index], drift_rate = drift_rate), stg.constant_t_profile(level = 0), stg.gaussian_f_profile(width = line_width), stg.constant_bp_profile(level = 1.0), integrate = False) elif sig_type == 'constant': signal = stg.generate(ts, fs, stg.constant_path(f_start = fs[start_index], drift_rate = drift_rate), stg.constant_t_profile(level = level), stg.gaussian_f_profile(width = line_width), stg.constant_bp_profile(level = 1.0), integrate = True) elif sig_type == 'simple_rfi': spread = kwargs['spread'] # np.random.uniform(0.0002, 0.0003) signal = stg.generate(ts, fs, stg.choppy_rfi_path(f_start = fs[start_index], drift_rate = drift_rate, spread=spread, spread_type='gaussian'), stg.constant_t_profile(level = level), stg.gaussian_f_profile(width = line_width), stg.constant_bp_profile(level = 1.0), integrate = True) elif sig_type == 'scintillated': period = kwargs['period'] # np.random.uniform(1,5) phase = kwargs['phase'] # np.random.uniform(0, period) sigma = kwargs['sigma'] # np.random.uniform(0.1, 2) pulse_dir = kwargs['pulse_dir'] #'rand' width = kwargs['width'] # np.random.uniform(0.1, 2) pnum = kwargs['pnum'] # 10 amplitude = kwargs['amplitude'] # np.random.uniform(level*2/3, level) signal = stg.generate(ts, fs, stg.constant_path(f_start = fs[start_index], drift_rate = drift_rate), stg.periodic_gaussian_t_profile(period, phase, sigma, pulse_dir, width, pnum, amplitude, level), stg.gaussian_f_profile(width = line_width), stg.constant_bp_profile(level = 1.0), integrate = True) return signal, [start_index, drift_rate, line_width, level, spread]
def generate_frame(sig_num=True, sig_db=10, rfi_num=0, rfi_db=25, means_dist=None, stds_dist=None, mins_dist=None, **kwargs): noise_mean, noise_std, noise_min = make_normal(means_dist, stds_dist, mins_dist, 1) noise_frame = np.maximum( np.random.normal(noise_mean, noise_std, [tchans, fchans]), noise_min) frame = noise_frame rfi_indices = [] rfi_widths = [] for i in range(rfi_num): rfi_snr = np.power(10, rfi_db / 10) rfi_level = noise_std * rfi_snr / np.sqrt(tchans) rfi_start_index = np.random.randint(0, fchans) rfi_indices.append(rfi_start_index) rfi_line_width = np.random.uniform(1e-6, 30e-6) rfi_widths.append(rfi_line_width) rfi_signal = stg.generate( ts, fs, stg.constant_path(f_start=fs[rfi_start_index], drift_rate=0), stg.constant_t_profile(level=rfi_level), stg.gaussian_f_profile(width=rfi_line_width), stg.constant_bp_profile(level=1.0)) frame += rfi_signal if sig_num == 1: snr = np.power(10, sig_db / 10) level = noise_std * snr / np.sqrt(tchans) start_index = np.random.randint(0, fchans) drift_rate = np.random.uniform(-start_index * df / (tsamp * tchans), (fchans - 1 - start_index) * df / (tsamp * tchans)) line_width = np.random.uniform(1e-6, 30e-6) signal = stg.generate( ts, fs, stg.constant_path(f_start=fs[start_index], drift_rate=drift_rate), stg.constant_t_profile(level=level), stg.gaussian_f_profile(width=line_width), stg.constant_bp_profile(level=1.0)) frame += signal else: level = 0 start_index = -1 drift_rate = 0 line_width = 0 return frame, [ sig_num, sig_db, start_index, drift_rate, line_width, rfi_num, rfi_db, rfi_indices, rfi_widths ]
def test_setigen(): """ Generate a setigen frame and convert it into a DataArray""" metadata = {'fch1': 6095.214842353016*u.MHz, 'dt': 18.25361108*u.s, 'df': 2.7939677238464355*u.Hz} frame = stg.Frame(fchans=2**12*u.pixel, tchans=128*u.pixel, df=metadata['df'], dt=metadata['dt'], fch1=metadata['fch1']) test_tones = [ {'f_start': frame.get_frequency(index=500), 'drift_rate': 0.01*u.Hz/u.s, 'snr': 100, 'width': 20*u.Hz}, {'f_start': frame.get_frequency(index=800), 'drift_rate': -0.10*u.Hz/u.s, 'snr': 100, 'width': 20*u.Hz}, {'f_start': frame.get_frequency(index=2048), 'drift_rate': 0.00*u.Hz/u.s, 'snr': 20, 'width': 6*u.Hz}, {'f_start': frame.get_frequency(index=3000), 'drift_rate': 0.07*u.Hz/u.s, 'snr': 50, 'width': 3*u.Hz} ] noise = frame.add_noise(x_mean=100, x_std=5, noise_type='gaussian') for tone in test_tones: signal = frame.add_signal(stg.constant_path(f_start=tone['f_start'], drift_rate=tone['drift_rate']), stg.constant_t_profile(level=frame.get_intensity(snr=tone['snr'])), stg.gaussian_f_profile(width=tone['width']), stg.constant_bp_profile(level=1)) d = from_setigen(frame)
def test_normalize_mask(plot=False): """ Test normalization works when data are masked """ n_int, n_ifs, n_chan = 16, 1, 8192 np.random.seed(1234) frame = stg.Frame(fchans=1024 * u.pixel, tchans=32 * u.pixel, df=2.7939677238464355 * u.Hz, dt=18.253611008 * u.s, fch1=6095.214842353016 * u.MHz) noise = frame.add_noise(x_mean=10, noise_type='chi2') signal = frame.add_signal( stg.constant_path(f_start=frame.get_frequency(index=200), drift_rate=1 * u.Hz / u.s), stg.constant_t_profile(level=frame.get_intensity(snr=1000)), stg.gaussian_f_profile(width=20 * u.Hz), stg.constant_bp_profile(level=1)) d = from_setigen(frame) d.data = d.data.astype('float32') if plot: frame.plot() # Over 20% of data will be masked mask = d.data.mean(axis=0).squeeze() > 15 d_norm = normalize(d, mask=mask, return_space='cpu') #print(d_norm.mean(), d_norm.std()) plt.figure() if plot: plt.plot(d_norm.mean(axis=0).squeeze()) mask_cpu = cp.asnumpy(mask) assert np.isclose(d_norm[..., ~mask_cpu].mean(), 0, atol=1e-6) assert np.isclose(d_norm[..., ~mask_cpu].std(), 1.0, atol=1e-6) print("Mask test passed!")
def draw(self, frame, seed, drift = 0.5,noise_toggle = False): if noise_toggle: noise = frame.add_noise(x_mean=5, x_std=2, x_min=0) frame.add_signal(stg.constant_path(f_start=frame.get_frequency(seed), drift_rate=drift*u.Hz/u.s), stg.constant_t_profile(level=30), stg.gaussian_f_profile(width=20*u.Hz), stg.constant_bp_profile(level=1)) return frame
def generate(self, total_num_samples, data, intensity = 0.7, test=False, labels= True): start_time = time.time() num_channels = 32 # Prepare the target set fchans = num_channels tchans = 32 df = 2.7939677238464355*u.Hz dt = 18.25361108*u.s fch1 = 6095.214842353016*u.MHz tune_base = intensity generated_signals = np.zeros((total_num_samples,data.shape[1],1, num_channels), dtype=float) for i in range(0,total_num_samples): base = data[i,:,0,:] start = int(random()*(fchans-15)) artifical_radio = np.zeros((32,32)) period = random() drifrate = (random()-0.5)*10**int(random()*-2) *u.Hz/u.s choose = random() amplitude = random() tune = tune_base * (random()+0.3) for blur in range(0,2): frame = stg.Frame(fchans=fchans, tchans=tchans, df=df, dt=dt, fch1=fch1) signal = frame.add_signal(stg.constant_path(f_start=frame.fs[start+10], drift_rate=drifrate), stg.constant_t_profile(level=tune*blur), stg.gaussian_f_profile(width=(20-5*blur)*u.Hz), stg.constant_bp_profile(level=tune*blur)) artifical_radio = np.add(frame.get_data(), artifical_radio) artifical_radio = np.add(base, artifical_radio) artifical_radio = np.reshape(artifical_radio, (tchans,1,fchans)) generated_signals[i,:,:,:]= artifical_radio if i %int(total_num_samples/2) ==0 and test: fig = plt.figure(figsize=(10, 6)) plt.imshow(generated_signals[i,:,0,:], aspect='auto') plt.colorbar() # Label the dataset supervised_true = np.concatenate((np.ones((generated_signals.shape[0],1),dtype='int64'),np.zeros((generated_signals.shape[0],1),dtype='int64')), axis=1) supervised_false = np.concatenate((np.zeros((total_num_samples,1),dtype='int64'),np.ones((total_num_samples,1),dtype='int64')), axis=1) label = np.concatenate((supervised_true, supervised_false)) supervised_dataset = np.concatenate((generated_signals, data[0:total_num_samples,:,:,:])) print(label.shape) print(supervised_dataset.shape) print("Synethtic Generation Execution Time: "+ str(time.time()-start_time)) X_train_supervised, X_test_supervised, y_train_supervised, y_test_supervised = train_test_split(supervised_dataset, label, test_size=0.2, random_state=2) return X_train_supervised, X_test_supervised, y_train_supervised, y_test_supervised
def test_constant_signal_from_add_signal(frame_setup_no_data, constant_signal_data): frame = copy.deepcopy(frame_setup_no_data) signal = frame.add_signal( stg.constant_path(f_start=frame.get_frequency(200), drift_rate=2 * u.Hz / u.s), stg.constant_t_profile(level=1), stg.gaussian_f_profile(width=50 * u.Hz), stg.constant_bp_profile(level=1)) assert_allclose(signal, frame.get_data(use_db=False)) assert_allclose(frame.get_data(use_db=False), constant_signal_data)
def test_hitsearch_multi(): """ Test hit search routine with multiple signals """ metadata = {'fch1': 6095.214842353016*u.MHz, 'dt': 18.25361108*u.s, 'df': 2.7939677238464355*u.Hz} frame = stg.Frame(fchans=2**12*u.pixel, tchans=32*u.pixel, df=metadata['df'], dt=metadata['dt'], fch1=metadata['fch1']) test_tones = [ {'f_start': frame.get_frequency(index=500), 'drift_rate': 0.50*u.Hz/u.s, 'snr': 100, 'width': 20*u.Hz}, {'f_start': frame.get_frequency(index=800), 'drift_rate': -0.40*u.Hz/u.s, 'snr': 100, 'width': 20*u.Hz}, {'f_start': frame.get_frequency(index=2048), 'drift_rate': 0.00*u.Hz/u.s, 'snr': 20, 'width': 6*u.Hz}, {'f_start': frame.get_frequency(index=3000), 'drift_rate': 0.07*u.Hz/u.s, 'snr': 50, 'width': 3*u.Hz} ] frame.add_noise(x_mean=0, x_std=5, noise_type='gaussian') for tone in test_tones: frame.add_signal(stg.constant_path(f_start=tone['f_start'], drift_rate=tone['drift_rate']), stg.constant_t_profile(level=frame.get_intensity(snr=tone['snr'])), stg.gaussian_f_profile(width=tone['width']), stg.constant_bp_profile(level=1)) frame.save_fil(filename=synthetic_fil) darray = from_fil(synthetic_fil) fig = plt.figure(figsize=(10, 6)) # <============ fig is UNUSED dedopp, md, hits = run_pipeline(darray.data, metadata, max_dd=1.0, min_dd=None, threshold=100, n_boxcar=5, merge_boxcar_trials=True) print(hits.sort_values('snr', ascending=False)) plt.figure(figsize=(10, 4)) plt.subplot(1,2,1) imshow_waterfall(darray.data, md, 'channel', 'timestep') plt.subplot(1,2,2) imshow_dedopp(dedopp, md, 'channel', 'driftrate') overlay_hits(hits, 'channel', 'driftrate') plt.savefig('docs/figs/test_hitsearch_multi.png') plt.show()
(tsamp * tchans)) line_width = np.random.uniform(0.02, 0.03)**3 #drift_rate = 0 level = np.random.uniform(1, 5) level = 5 period = np.random.uniform(1, 5) phase = np.random.uniform(0, period) sigma = np.random.uniform(0.1, 2) pulse_dir = 'rand' width = np.random.uniform(0.1, 2) pnum = 10 amplitude = np.random.uniform(level * 2 / 3, level) signal = stg.generate(ts, fs, stg.constant_path(f_start=fs[start_index], drift_rate=drift_rate), stg.periodic_gaussian_t_profile( period, phase, sigma, pulse_dir, width, pnum, amplitude, level), stg.gaussian_f_profile(width=line_width), stg.constant_bp_profile(level=1.0), integrate=True) signal = stg.normalize(stg.inject_noise(signal), cols=128, exclude=0.2, use_median=False) plt.imsave(output_fn, signal) print('Saved %s of %s scintillated data for %s' % (i + 1, num, name))
def generate_fil_file(outpath, flag_fascending, flag_sign_drift_rate): r''' Using setigen, generate a filterbank file. Parameters: outpath - full path of where to store the resultant filterbank file. flag_fascending - use an ascending (+1) or descending (-1) sequence of frequencies flag_sign_drift_rate - use a positive (+1) or negative (-1) drift rate ''' if DEBUGGING: print('generate_fil_file: flag_fascending={}, flag_sign_drift_rate={}'. format(flag_fascending, flag_sign_drift_rate)) # Set up setigne parameters stg_parms = SetigenParms() if flag_sign_drift_rate < 0: stg_parms.drift_rate_1 = -stg_parms.drift_rate_1 stg_parms.drift_rate_2 = -stg_parms.drift_rate_2 stg_parms.drift_rate_3 = -stg_parms.drift_rate_3 stg_parms.drift_rate_4 = -stg_parms.drift_rate_4 stg_parms.drift_rate_5 = -stg_parms.drift_rate_5 # Instantiate a setigen Frame object frame = stg.Frame(fchans=stg_parms.fchans, tchans=stg_parms.tchans, df=stg_parms.df, dt=stg_parms.dt, fch1=stg_parms.fch1, ascending=(flag_fascending > 0)) # Add noise to stg object. frame.add_noise(x_mean=0, x_std=stg_parms.noise_std, noise_type='gaussian') # Signal 1 will be detected. signal_1_intensity = frame.get_intensity(snr=stg_parms.snr_1) frame.add_constant_signal(f_start=frame.get_frequency( stg_parms.signal_start_1), drift_rate=stg_parms.drift_rate_1, level=signal_1_intensity, width=stg_parms.width_1, f_profile_type='gaussian') # Signal 2 will be detected. signal_2_intensity = frame.get_intensity(snr=stg_parms.snr_2) frame.add_constant_signal(f_start=frame.get_frequency( stg_parms.signal_start_2), drift_rate=stg_parms.drift_rate_2, level=signal_2_intensity, width=stg_parms.width_2, f_profile_type='gaussian') # Signal 3 is a symmetric signal with three Gaussians # that will fall below the SNR requirements. signal_3_intensity = frame.get_intensity(snr=stg_parms.snr_3) frame.add_signal( stg.constant_path(f_start=frame.get_frequency( stg_parms.signal_start_3), drift_rate=stg_parms.drift_rate_3), stg.constant_t_profile(level=1), stg.multiple_gaussian_f_profile(width=stg_parms.width_3), stg.constant_bp_profile(level=signal_3_intensity)) # Signal 4 is a symmetric signal with three Gaussians # that will be drifting too quickly. signal_4_intensity = frame.get_intensity(snr=stg_parms.snr_4) frame.add_signal( stg.constant_path(f_start=frame.get_frequency( stg_parms.signal_start_4), drift_rate=stg_parms.drift_rate_4), stg.constant_t_profile(level=1), stg.multiple_gaussian_f_profile(width=stg_parms.width_4), stg.constant_bp_profile(level=signal_4_intensity)) # Signal 5 is similar to signal 4 but drifting in the opposite direction. signal_5_intensity = frame.get_intensity(snr=stg_parms.snr_5) frame.add_signal( stg.constant_path(f_start=frame.get_frequency( stg_parms.signal_start_5), drift_rate=stg_parms.drift_rate_5), stg.constant_t_profile(level=1), stg.multiple_gaussian_f_profile(width=stg_parms.width_5), stg.constant_bp_profile(level=signal_5_intensity)) # Save the frame as a filterbank file. frame.save_fil(filename=outpath) print("generate_fil_file: generated {}".format(outpath)) del frame
def test_dedoppler(): """ Basic tests of the dedoppler functionality """ # zero drift test, no normalization test_data = np.ones(shape=(32, 1, 1024), dtype='float32') test_data[:, :, 511] = 10 metadata_in = { 'frequency_start': 1000 * u.MHz, 'time_step': 1.0 * u.s, 'frequency_step': 1.0 * u.Hz } dedopp, metadata = dedoppler(test_data, metadata_in, boxcar_size=1, max_dd=1.0) print("type(dedopp):", type(dedopp)) print("dedopp.data:", dedopp.data) print("np.max(dedopp.data):", np.max(dedopp.data), ", np.sum(test_data[:, :, 511]):", np.sum(test_data[:, :, 511])) assert np.max(dedopp.data) == np.sum(test_data[:, :, 511]) for dr_test in (0.0, 0.1, 0.5, -0.25, -0.5): # single drifting tone frame = stg.Frame(fchans=2**10 * u.pixel, tchans=32 * u.pixel, df=metadata_in['frequency_step'], dt=metadata_in['time_step'], fch1=metadata_in['frequency_start']) tone = { 'f_start': frame.get_frequency(index=500), 'drift_rate': dr_test * u.Hz / u.s, 'snr': 500, 'width': metadata_in['frequency_step'] } frame.add_noise(x_mean=1, noise_type='chi2') frame.add_signal( stg.constant_path(f_start=tone['f_start'], drift_rate=tone['drift_rate']), stg.constant_t_profile(level=frame.get_intensity(snr=tone['snr'])), stg.gaussian_f_profile(width=tone['width']), stg.constant_bp_profile(level=1)) frame.save_fil(filename=synthetic_fil) darray = from_fil(synthetic_fil) dedopp, metadata = dedoppler(darray, boxcar_size=1, max_dd=1.0, return_space='cpu') # Manual dedoppler search -- just find max channel (only works if S/N is good) manual_dd_tot = 0 for ii in range(darray.data.shape[0]): manual_dd_tot += np.max(darray.data[ii]) imshow_dedopp(dedopp, show_colorbar=False) maxpixel = np.argmax(dedopp.data) mdrift, mchan = (maxpixel // 1024, maxpixel % 1024) optimal_drift = metadata['drift_rates'][mdrift].value maxpixel_val = np.max(dedopp.data) frac_recovered = (maxpixel_val / manual_dd_tot) print( f"Inserted drift rate: {tone['drift_rate']} \tSUM: {manual_dd_tot:2.2f}" ) print( f"Recovered drift rate: {optimal_drift} Hz / s \tSUM: {maxpixel_val:2.2f}\n" ) # Channel should detected at +/- 1 chan print(mdrift, mchan) #assert np.abs(mchan - 500) <= 1 # Drift rate should be detected +/- 1 drift resolution assert np.abs(optimal_drift - dr_test) <= 1.01 * np.abs( metadata['drift_rate_step'].value) # Recovered signal sum should be close to manual method assert 1.001 >= frac_recovered >= 0.825 # Finish off figure plotting plt.colorbar() plt.savefig(os.path.join(test_fig_dir, 'test_dedoppler.png')) plt.show()
def generate_frame(sig_num=0, sig_db=10, rfi_num=0, rfi_db=25, means_dist=None, stds_dist=None, mins_dist=None, **kwargs): noise_mean, noise_std, noise_min = make_normal(means_dist, stds_dist, mins_dist, 1) noise_frame = np.maximum( np.random.normal(noise_mean, noise_std, [tchans, fchans]), noise_min) frame = noise_frame frame_info = { 'noise_mean': noise_mean, 'noise_std': noise_std, 'noise_min': noise_min, 'signals': [], } rfi_indices = [] rfi_widths = [] for i in range(rfi_num): rfi_snr = np.power(10, rfi_db / 10) rfi_level = noise_std * rfi_snr / np.sqrt(tchans) rfi_start_index = np.random.randint(0, fchans) rfi_end_index = rfi_start_index rfi_drift_rate = 0 rfi_indices.append(rfi_start_index) rfi_line_width = np.random.uniform(1e-6, 30e-6) rfi_widths.append(rfi_line_width) rfi_signal = stg.generate( ts, fs, stg.constant_path(f_start=fs[rfi_start_index], drift_rate=rfi_drift_rate), stg.constant_t_profile(level=rfi_level), stg.gaussian_f_profile(width=rfi_line_width), stg.constant_bp_profile(level=1.0)) sig_info = { 'class': 'rfi', 'start_index': rfi_start_index, 'end_index': rfi_end_index, 'line_width': rfi_line_width, 'snr': rfi_snr, } sig_info = np.array( [rfi_start_index, rfi_end_index, rfi_line_width, rfi_snr, 1]) frame += rfi_signal frame_info['signals'].append(sig_info) for i in range(sig_num): snr = np.power(10, sig_db / 10) level = noise_std * snr / np.sqrt(tchans) start_index = np.random.randint(0, fchans) end_index = np.random.randint(0, fchans) drift_rate = (end_index - start_index) * df / (tsamp * tchans) # drift_rate = np.random.uniform(-start_index*df/(tsamp*tchans), # (fchans-1-start_index)*df/(tsamp*tchans)) line_width = np.random.uniform(1e-6, 30e-6) signal = stg.generate( ts, fs, stg.constant_path(f_start=fs[start_index], drift_rate=drift_rate), stg.constant_t_profile(level=level), stg.gaussian_f_profile(width=line_width), stg.constant_bp_profile(level=1.0)) sig_info = { 'class': 'constant', 'start_index': start_index, 'end_index': end_index, 'line_width': line_width, 'snr': snr, } sig_info = np.array([start_index, end_index, line_width, snr, 0]) frame += signal frame_info['signals'].append(sig_info) else: level = 0 start_index = -1 drift_rate = 0 line_width = 0 return frame, frame_info
def generate_frame(sig_num=0, max_sig_num=10, rfi_frac=0.5, sig_snr_range=(25, 250), width_range=(5, 10), means_dist=None, stds_dist=None, mins_dist=None, **kwargs): noise_mean, noise_std, noise_min = make_normal(means_dist, stds_dist, mins_dist, 1) noise_frame = np.maximum( np.random.normal(noise_mean, noise_std, [tchans, fchans]), noise_min) frame = noise_frame frame_info = { 'noise': [noise_mean[0], noise_std[0], noise_min[0]], 'signals': [], 'frame_params': [ sig_num, rfi_frac, sig_snr_range[0], sig_snr_range[1], width_range[0], width_range[1] ], } for i in range(sig_num): snr = np.random.uniform(sig_snr_range[0], sig_snr_range[1]) level = noise_std * snr / np.sqrt(tchans) start_index = np.random.randint(0, fchans) if np.random.rand() > rfi_frac: end_index = np.random.randint(0, fchans) else: end_index = start_index drift_rate = (end_index - start_index) / tchans * (df / tsamp) line_width = np.random.uniform(width_range[0], width_range[1]) * np.abs(df) signal = stg.generate( ts, fs, stg.constant_path(f_start=fs[start_index], drift_rate=drift_rate), stg.constant_t_profile(level=level), stg.gaussian_f_profile(width=line_width), stg.constant_bp_profile(level=1.0)) # sig_info = { # 'class': 'constant', # 'start_index': start_index, # 'end_index': end_index, # 'line_width': line_width, # 'snr': snr, # } sig_info = np.array([ start_index / fchans, end_index / fchans, (end_index - start_index) / tchans, line_width / np.abs(df), snr ]) frame += signal frame_info['signals'].append(sig_info) for i in range(max_sig_num - sig_num): frame_info['signals'].append([-1, -1, 0, 0, -1]) return frame, frame_info