def gen_injfrb_pulse(self,
upchan_factor=1,
upsamp_factor=1,
conv_dmsmear=False):
""" Generate pulse dynamic spectrum
with injectfrb.simulate_frb
"""
data_bg = np.zeros(
[upchan_factor * self.nfreq, upsamp_factor * self.ntime])
data_injfrb, p = simulate_frb.gen_simulated_frb(
NFREQ=upchan_factor * self.nfreq,
NTIME=upsamp_factor * self.ntime,
sim=True,
fluence=self.fluence,
spec_ind=self.spec_ind,
width=self.width,
dm=self.dm,
background_noise=data_bg,
delta_t=self.dt / upsamp_factor,
plot_burst=False,
freq=(self.freq_hi_MHz, self.freq_lo_MHz),
FREQ_REF=self.freq_ref,
scintillate=False,
scat_tau_ref=self.scat_tau_ref,
disp_ind=2.0,
conv_dmsmear=conv_dmsmear)
data_injfrb = data_injfrb.reshape(self.nfreq, upchan_factor,
self.ntime, upsamp_factor)
data_injfrb = data_injfrb.mean(1).mean(-1)
return data_injfrb
def inject_in_filterbank_gaussian(data_fil_obj,
header,
fn_fil_out,
N_FRB,
chunksize=100000,
simfrb=True):
NFREQ = header['nchans']
for ii in range(N_FRB):
if ii == 0:
fn_rfi_clean = reader.write_to_fil(np.zeros([NFREQ, 0]), header,
fn_fil_out)
print("%d gaussian chunks" % ii)
#data = data_fil_obj.data*0.0
data = (np.random.normal(120, 10,
NFREQ * chunksize)) #.astype(np.uint8)
data = data.reshape(NFREQ, chunksize)
if simfrb is True:
delta_t = header['tsamp'] # delta_t in seconds
fch1 = header['fch1']
foff = header['foff']
fch_f = fch1 + NFREQ * foff
freq_arr = np.linspace(fch1, fch_f, NFREQ)
dm = 50 + ii
freq_ref = 1400.
print("Adding FRB to Gaussian data")
data_chunk, params = simulate_frb.gen_simulated_frb(
NFREQ=NFREQ,
NTIME=chunksize,
sim=True,
fluence=1000,
spec_ind=0,
width=(10 * delta_t, 1),
dm=dm,
scat_factor=(-5., -4),
background_noise=data,
delta_t=delta_t,
plot_burst=False,
freq=(freq_arr[0], freq_arr[-1]),
FREQ_REF=freq_ref,
scintillate=True)
if ii < 0:
fn_rfi_clean = reader.write_to_fil(data_chunk.transpose(), header,
fn_fil_out)
elif ii >= 0:
fil_obj = reader.filterbank.FilterbankFile(fn_fil_out,
mode='readwrite')
fil_obj.append_spectra(data.transpose())
continue
sp = simpulse.single_pulse(pulse_nt, nfreq, freq_lo_MHz, freq_hi_MHz, dm, sm,
intrinsic_width, fluence, spectral_index,
undispersed_arrival_time)
data_simpulse = np.zeros([nfreq, pulse_nt])
sp.add_to_timestream(data_simpulse, 0.0, pulse_nt * dt)
data_simpulse = data_simpulse[::-1]
data_injfrb, p = simulate_frb.gen_simulated_frb(
NFREQ=nfreq,
NTIME=pulse_nt,
sim=True,
fluence=fluence,
spec_ind=0.0,
width=intrinsic_width,
dm=dm,
background_noise=np.zeros([nfreq, pulse_nt]),
delta_t=dt,
plot_burst=False,
freq=(freq_hi_MHz, freq_lo_MHz),
FREQ_REF=freq_ref,
scintillate=False,
scat_tau_ref=0.0,
disp_ind=2.0)
data_simpulse /= np.max(data_simpulse)
data_injfrb /= np.max(data_injfrb)
fig = plt.figure()
plt.subplot(121)
plt.plot(np.roll(data_simpulse[0],
def plot_four_frbs():
fig = plt.figure(figsize=(10, 8))
cmap = 'Greys'
print("\nGenerating Apertif-like FRB with blue spectrum")
NTIME = 2**12
NFREQ = 1536
dt = 0.000081
upchan_factor = 2
upsamp_factor = 2
freq = np.linspace(1520., 1220., upchan_factor * NFREQ)
freq_ref = 1400.
dm = 500.
# simulate FRB with Apertif-like observing parameters with no background noise
data_apertif, p = simulate_frb.gen_simulated_frb(
NFREQ=upchan_factor * NFREQ,
NTIME=upsamp_factor * NTIME,
sim=True,
fluence=1.,
spec_ind=4.,
width=5 * dt,
dm=dm,
background_noise=np.zeros(
[upchan_factor * NFREQ, upsamp_factor * NTIME]),
delta_t=dt / upsamp_factor,
plot_burst=False,
freq=(freq[0], freq[-1]),
FREQ_REF=freq_ref,
scintillate=False,
scat_tau_ref=0.0)
# downsample in time/freq
data_apertif = data_apertif.reshape(-1, upchan_factor, NTIME,
upsamp_factor)
data_apertif = data_apertif.mean(1).mean(-1)
ext = [0, NTIME * dt, freq[-1], freq[0]]
plt.subplot(221)
plt.imshow(data_apertif, aspect='auto', cmap=cmap, extent=ext)
plt.ylabel('Frequency [MHz]')
plt.text(NTIME * dt * .6,
freq[NFREQ // 2],
'Apertif\nNo noise \nBlue spectrum\
\nDM=%d' % (dm),
alpha=1.0) # bbox=dict(facecolor='white', alpha=1.0))
print("\nGenerating CHIME-like FRB with scattering")
NTIME = 2**11
NFREQ = 16384
dt = 0.000983
upchan_factor = 2
upsamp_factor = 2
freq = np.linspace(800., 400., upchan_factor * NFREQ)
freq_ref = 600.
dm = 100.
# simulate FRB with CHIME-like observing parameters with no background noise
data_chime, p = simulate_frb.gen_simulated_frb(NFREQ=upchan_factor * NFREQ,
NTIME=upsamp_factor * NTIME,
sim=True,
fluence=1.,
spec_ind=-3.,
width=2 * dt,
dm=dm,
background_noise=np.zeros([
upchan_factor * NFREQ,
upsamp_factor * NTIME
]),
delta_t=dt / upsamp_factor,
plot_burst=False,
freq=(freq[0], freq[-1]),
FREQ_REF=freq_ref,
scintillate=False,
scat_tau_ref=0.001)
# downsample in time/freq
data_chime = data_chime.reshape(-1, upchan_factor, NTIME, upsamp_factor)
data_chime = data_chime.mean(1).mean(-1)
ext = [0, NTIME * dt, freq[-1], freq[0]]
plt.subplot(222)
plt.imshow(data_chime, aspect='auto', cmap=cmap, extent=ext)
plt.text(NTIME * dt * .6,
freq[NFREQ // 2],
'CHIME\nNo noise \nScattered\
\nDM=%d' % (dm),
alpha=1.0) #
# bbox=dict(facecolor='white', alpha=1.0))
print("\nGenerating Breakthrough-listen-like FRB with Gaussian noise")
NTIME = 2**8
NFREQ = 2048
dt = 0.001
upchan_factor = 2
upsamp_factor = 2
freq = np.linspace(8000., 4000., upchan_factor * NFREQ)
freq_ref = 6000.
dm = 1000.
# simulate FRB with Breakthrough-listen-like observing parameters with Gaussian noise
data_noise_breakthrough, p = simulate_frb.gen_simulated_frb(
NFREQ=upchan_factor * NFREQ,
NTIME=upsamp_factor * NTIME,
sim=True,
fluence=20000.,
spec_ind=4.,
width=dt / upsamp_factor,
dm=500.,
background_noise=None,
delta_t=dt,
plot_burst=False,
freq=(freq[0], freq[-1]),
FREQ_REF=freq_ref,
scintillate=True,
scat_tau_ref=0.0)
# downsample in time/freq
data_noise_breakthrough = data_noise_breakthrough.reshape(-1, upchan_factor, \
NTIME, upsamp_factor)
data_noise_breakthrough = data_noise_breakthrough.mean(1).mean(-1)
plt.subplot(223)
ext = [0, NTIME * dt, freq[-1], freq[0]]
plt.ylabel('Frequency [MHz]')
plt.imshow(data_noise_breakthrough, aspect='auto', cmap=cmap, extent=ext)
plt.text(NTIME * dt * .6,
freq[NFREQ // 2],
'Breakthrough Listen\nGaussian noise \nScintillated\
\nDM=%d' % (dm),
alpha=1.0) #, bbox=dict(facecolor='white', alpha=1.0))
plt.xlabel('Time [sec]')
print("\nGenerating Parkes-like FRB with Gaussian noise\n")
NTIME = 2**15
NFREQ = 2048
dt = 0.000064
upchan_factor = 2
upsamp_factor = 2
freq = np.linspace(1520., 1220., upchan_factor * NFREQ)
freq_ref = 1400.
dm = 2000.
# simulate FRB with Breakthrough-listen-like observing parameters with Gaussian noise
data_noise_dm2000, p = simulate_frb.gen_simulated_frb(
NFREQ=upchan_factor * NFREQ,
NTIME=upsamp_factor * NTIME,
sim=True,
fluence=1000000.,
spec_ind=4.,
width=50 * dt,
dm=dm,
background_noise=None,
delta_t=dt / upsamp_factor,
plot_burst=False,
freq=(freq[0], freq[-1]),
FREQ_REF=freq_ref,
scintillate=False,
scat_tau_ref=0.0)
# downsample in time/freq
data_noise_dm2000 = data_noise_dm2000.reshape(-1, upchan_factor, \
NTIME, upsamp_factor)
data_noise_dm2000 = data_noise_dm2000.mean(1).mean(-1)
plt.subplot(224)
ext = [0, NTIME * dt, freq[-1], freq[0]]
plt.imshow(data_noise_dm2000, aspect='auto', cmap=cmap, extent=ext)
plt.xlabel('Time [sec]')
plt.text(NTIME * dt * .6,
freq[NFREQ // 2],
'Parkes\nGaussian noise\nDM=%d' % dm,
alpha=1.0) #, bbox=dict(facecolor='white', alpha=1.0))
fnout = './test_fig_4FRBs.pdf'
print("Saving figure to %s" % fnout)
plt.savefig(fnout)
plt.show()
def inject_in_filterbank(fn_fil,
fn_out_dir,
N_FRB=1,
NFREQ=1536,
NTIME=2**15,
rfi_clean=False,
dm=250.0,
freq=(1550, 1250),
dt=0.00004096,
chunksize=5e4,
calc_snr=True,
start=0,
freq_ref=1400.,
subtract_zero=False,
clipping=None):
""" Inject an FRB in each chunk of data
at random times. Default params are for Apertif data.
Parameters:
-----------
fn_fil : str
name of filterbank file
fn_out_dir : str
directory for output files
N_FRB : int
number of FRBs to inject
NTIME : int
number of time samples per data chunk
rfi_clean : bool
apply rfi filters
dm : float / tuple
dispersion measure(s) to inject FRB with
freq : tuple
(freq_bottom, freq_top)
dt : float
time resolution
chunksize : int
size of data in samples to read in
calc_snr : bool
calculates S/N of injected pulse
start : int
start sample
freq_ref : float
reference frequency for injection code
subtract_zero : bool
subtract zero DM timestream from data
clipping :
zero out bright events in zero-DM timestream
Returns:
--------
None
"""
if type(dm) is not tuple:
max_dm = dm
else:
max_dm = max(dm)
t_delay_max = abs(4.14e3 * max_dm * (freq[0]**-2 - freq[1]**-2))
t_delay_max_pix = int(t_delay_max / dt)
# ensure that dispersion sweep is not too large
# for chunksize
f_edge = 0.3
while chunksize <= t_delay_max_pix / f_edge:
chunksize *= 2
NTIME *= 2
print('Increasing to NTIME:%d, chunksize:%d' % (NTIME, chunksize))
ii = 0
params_full_arr = []
ttot = int(N_FRB * chunksize * dt)
timestr = time.strftime("%Y%m%d-%H%M")
fn_fil_out = '%s/dm%s_nfrb%d_%s_sec_%s.fil' % (fn_out_dir, dm, N_FRB, ttot,
timestr)
fn_params_out = fn_fil_out.strip('.fil') + '.txt'
f_params_out = open(fn_params_out, 'w+')
f_params_out.write(
'# DM Sigma Time (s) Sample Downfact\n')
f_params_out.close()
for ii in xrange(N_FRB):
# drop FRB in random location in data chunk
offset = int(
np.random.uniform(0.1 * chunksize, (1 - f_edge) * chunksize))
data_filobj, freq_arr, delta_t, header = reader.read_fil_data(
fn_fil, start=start + chunksize * ii, stop=chunksize)
if ii == 0:
fn_rfi_clean = reader.write_to_fil(np.zeros([NFREQ, 0]), header,
fn_fil_out)
data = data_filobj.data
# injected pulse time in seconds since start of file
t0_ind = offset + NTIME // 2 + chunksize * ii
t0_ind = start + chunksize * ii + offset # hack because needs to agree with presto
t0 = t0_ind * delta_t
if len(data) == 0:
break
data_event = (data[:, offset:offset + NTIME]).astype(np.float)
data_event, params = simulate_frb.gen_simulated_frb(
NFREQ=NFREQ,
NTIME=NTIME,
sim=True,
# fluence=3000*(1+0.1*ii),
fluence=(200, 500),
spec_ind=0,
width=(delta_t, delta_t * 100),
dm=dm + 10 * ii,
scat_factor=(-4, -3.5),
background_noise=data_event,
delta_t=delta_t,
plot_burst=False,
freq=(freq_arr[0], freq_arr[-1]),
FREQ_REF=freq_ref,
scintillate=False)
dm_ = params[0]
params.append(offset)
print("%d/%d Injecting with DM:%d width: %.2f offset: %d" %
(ii, N_FRB, dm_, params[2], offset))
data[:, offset:offset + NTIME] = data_event
#params_full_arr.append(params)
width = params[2]
downsamp = max(1, int(width / delta_t))
t_delay_mid = 4.15e3 * dm_ * (freq_ref**-2 - freq_arr[0]**-2)
# this is an empirical hack. I do not know why
# the PRESTO arrival times are different from t0
# by the dispersion delay between the reference and
# upper frequency
t0 -= t_delay_mid
if rfi_clean is True:
data = rfi_test.apply_rfi_filters(data.astype(np.float32), delta_t)
if subtract_zero is True:
print("Subtracting zero DM")
data_ts_zerodm = data.mean(0)
data -= data_ts_zerodm[None]
if clipping is not None:
# Find tsamples > 8sigma and replace them with median
assert type(clipping) in (float,
int), 'clipping must be int or float'
data_ts_zerodm = data.mean(0)
stds, med = sigma_from_mad(data_ts_zerodm)
ind = np.where(np.absolute(data_ts_zerodm - med) > 8.0 * stds)[0]
data[:, ind] = np.median(data, axis=-1, keepdims=True)
if ii < 0:
fn_rfi_clean = reader.write_to_fil(data.transpose(), header,
fn_fil_out)
elif ii >= 0:
fil_obj = reader.filterbank.FilterbankFile(fn_fil_out,
mode='readwrite')
fil_obj.append_spectra(data.transpose())
if calc_snr is True:
data_filobj.data = data
data_filobj.dedisperse(dm_)
end_t = abs(4.15e3 * dm_ * (freq[0]**-2 - freq[1]**-2))
end_pix = int(end_t / dt)
end_pix_ds = int(end_t / dt / downsamp)
data_rb = (data_filobj.data).copy()
data_rb = data_rb[:, :-end_pix].mean(0)
data_rb -= np.median(data_rb)
SNRTools = tools.SNR_Tools()
snr_max, width_max = SNRTools.calc_snr_widths(data_rb,
widths=range(100))
# snr_max2, width_max2 = tools.calc_snr_widths(data_rb,
# )
print("S/N: %.2f width_used: %.3f width_tru: %.3f DM: %.1f" %
(snr_max, width_max, width / delta_t, dm_))
else:
snr_max = 10.0
width_max = int(width / dt)
f_params_out = open(fn_params_out, 'a+')
f_params_out.write('%2f %2f %5f %7d %d\n' %
(params[0], snr_max, t0, t0_ind, width_max))
f_params_out.close()
del data, data_event
params_full_arr = np.array(params_full_arr)