def load_tab_data(fname, start_bin, chunksize, out=None, tab=None):
f = filterbank.filterbank(fname)
if out is not None:
out[tab] = f.get_spectra(start_bin, chunksize).data
return
else:
data = f.get_spectra(start_bin, chunksize).data
f.close()
return data
def open_data_file(fn):
if fn.endswith(".fil"):
# Filterbank file
filetype = "filterbank"
rawdatafile = filterbank.filterbank(fn)
elif fn.endswith(".fits"):
# PSRFITS file
filetype = "psrfits"
rawdatafile = psrfits.PsrfitsFile(fn)
else:
raise ValueError("Cannot recognize data file type from "
"extension. (Only '.fits' and '.fil' "
"are supported.)")
return rawdatafile
def open_data_file(fn):
if fn.endswith(".fil"):
# Filterbank file
filetype = "filterbank"
rawdatafile = filterbank.filterbank(fn)
elif fn.endswith(".fits"):
# PSRFITS file
filetype = "psrfits"
rawdatafile = psrfits.PsrfitsFile(fn)
else:
raise ValueError("Cannot recognize data file type from "
"extension. (Only '.fits' and '.fil' "
"are supported.)")
return rawdatafile
def proc_trigger(fn_fil, dm0, t0, sig_cut,
ndm=50, mk_plot=False, downsamp=1,
beamno='', fn_mask=None, nfreq_plot=32,
ntime_plot=250,
cmap='RdBu', cand_no=1, multiproc=False,
rficlean=False, snr_comparison=-1,
outdir='./', sig_thresh_local=5.0,
subtract_zerodm=False,
threshold_time=3.25, threshold_frequency=2.75,
bin_size=32, n_iter_time=3,
n_iter_frequency=3, clean_type='time',
freq=1370.0, sb_generator=None, sb=None,
dumb_mask=True, save_sb_fil=False):
""" Locate data within filterbank file (fn_fi)
at some time t0, and dedisperse to dm0, generating
plots
Parameters:
----------
fn_fil : str
name of filterbank file
dm0 : float
trigger dm found by single pulse search software
t0 : float
time in seconds where trigger was found
sig_cut : np.float
sigma of detected trigger at (t0, dm0)
ndm : int
number of DMs to use in DM transform
mk_plot : bool
make three-panel plots
downsamp : int
factor by which to downsample in time. comes from searchsoft.
beamno : str
beam number, for fig names
nfreq_plot : int
number of frequencies channels to plot
freq : int
central frequency used to find zapped channels file
sb_generator: SBGenerator object
synthesized beam mapper from DARC (None for TAB/IAB)
sb : int
synthesized beam to generate (None for TAB/IAB)
Returns:
-------
full_dm_arr_downsamp : np.array
data array with downsampled dm-transformed intensities
full_freq_arr_downsamp : np.array
data array with downsampled freq-time intensities
"""
if dumb_mask:
try:
fndmask='/home/arts/.controller/amber_conf/zapped_channels_{:.0f}.conf'.format(int(freq))
rfimask = np.loadtxt(fndmask)
rfimask = rfimask.astype(int)
except:
rfimask = np.array([])
logging.warning("Could not load dumb RFIMask")
else:
rfimask = np.array([])
SNRtools = tools.SNR_Tools()
downsamp = min(4096, downsamp)
if downsamp >= 100:
wideclean = int(downsamp)
else:
wideclean = None
# store path to filterbanks
if sb is not None:
prefix_fil = fn_fil
# get first file
fn_fil = prefix_fil + '_00.fil'
if not os.path.exists(fn_fil):
fn_fil = prefix_fil + '00.fil'
rawdatafile = filterbank.filterbank(fn_fil)
dfreq_MHz = rawdatafile.header['foff']
dt = rawdatafile.header['tsamp']
freq_up = rawdatafile.header['fch1']
nfreq = rawdatafile.header['nchans']
# fix RFI mask order
rfimask = nfreq - rfimask
freq_low = freq_up + nfreq*rawdatafile.header['foff']
ntime_fil = (os.path.getsize(fn_fil) - rawdatafile.header_size)/nfreq
tdm = np.abs(8.3*1e-6*dm0*dfreq_MHz*(freq_low/1000.)**-3)
dm_min = max(0, dm0-40)
dm_max = dm0 + 40
dms = np.linspace(dm_min, dm_max, ndm, endpoint=True)
# make sure dm0 is in the array
dm_max_jj = np.argmin(abs(dms-dm0))
dms += (dm0-dms[dm_max_jj])
dms[0] = max(0, dms[0])
global t_min, t_max
# if smearing timescale is < 4*pulse width,
# downsample before dedispersion for speed
downsamp_smear = max(1, int(downsamp*dt/tdm/2.))
# ensure that it's not larger than pulse width
downsamp_smear = int(min(downsamp, downsamp_smear))
downsamp_res = int(downsamp//downsamp_smear)
downsamp = int(downsamp_res*downsamp_smear)
time_res = dt * downsamp
logging.info("Width_full:%d Width_smear:%d Width_res: %d" %
(downsamp, downsamp_smear, downsamp_res))
start_bin = int(t0/dt - ntime_plot*downsamp//2)
width = abs(4.148e3 * dm0 * (freq_up**-2 - freq_low**-2))
chunksize = int(width/dt + ntime_plot*downsamp)
t_min, t_max = 0, ntime_plot*downsamp
if start_bin < 0:
extra = start_bin
start_bin = 0
t_min += extra
t_max += extra
t_min, t_max = int(t_min), int(t_max)
snr_max = 0
# Account for the pre-downsampling to speed up dedispersion
t_min /= downsamp_smear
t_max /= downsamp_smear
ntime = t_max-t_min
if ntime_fil < (start_bin+chunksize):
logging.info("Trigger at end of file, skipping")
return [], [], [], []
# get data of all files (SB) or one file (TAB/IAB)
if sb is not None:
ntab = 12
data = np.zeros((ntab, nfreq, chunksize))
# get list of unique TABs in required SB
sb_map = list(set(sb_generator.get_map(sb)))
logging.info("SB {} consists of TABs {}".format(sb, sb_map))
threads = []
for tab in range(ntab):
# skip if we do not need this TAB
if not tab in sb_map:
continue
fname = prefix_fil + '_{:02d}.fil'.format(tab)
if not os.path.exists(fname):
fname = prefix_fil + '{:02d}.fil'.format(tab)
load_tab_data(fname, start_bin, chunksize, out=data, tab=tab)
for thread in threads:
logging.info("Waiting for loading of {}".format(thread.name))
thread.join()
# generate sb
logging.info("Synthesizing beam {}".format(sb))
data = sb_generator.synthesize_beam(data, sb=sb)
# convert to a spectra object, mimicking filterbank.get_spectra
data = spectra.Spectra(rawdatafile.frequencies,
rawdatafile.tsamp, data,
starttime=start_bin*rawdatafile.tsamp,
dm=0)
else:
data = rawdatafile.get_spectra(start_bin, chunksize)
if save_sb_fil:
fn_sbfil_out = '%s/data/CB%s_snr%d_dm%d_t0%d_sb%d.fil' % \
(outdir, beamno, sig_cut, dm0, t0, sb)
reader.create_new_filterbank(fn_sbfil_out, telescope='Apertif')
reader.write_to_fil(data.data.transpose(), rawdatafile.header, fn_sbfil_out)
# fil_obj = reader.filterbank_.FilterbankFile(fn_sbfil_out, mode='readwrite')
# fil_obj.append_spectra(data.data.transpose())
# np.save(fn_fig_out, data.data)
rawdatafile.close()
# apply dumb mask
if len(rfimask)>0:data.data[rfimask] = 0.
if rficlean is True:
data.data = cleandata(data.data, threshold_time,
threshold_frequency, bin_size,
n_iter_time, n_iter_frequency,
clean_type, wideclean=wideclean)
if subtract_zerodm:
data.data -= np.mean(data.data, axis=0, keepdims=True)
freq_ref = 0.5*(freq_up+freq_low)
# Downsample before dedispersion up to 1/4th
# DM smearing limit
data.downsample(downsamp_smear)
data.data -= np.median(data.data, axis=-1)[:, None]
# full_arr = np.empty([int(ndm), int(ntime)])
if not fn_mask is None:
pass
# rfimask = rfifind.rfifind(fn_mask)
# mask = get_mask(rfimask, start_bin, chunksize)
# data = data.masked(mask, maskval='median-mid80')
if multiproc is True:
pass
else:
logging.info("\nDedispersing Serially\n")
data_copy = copy.deepcopy(data)
data_copy.dedisperse(dm0)
data_dm_max = data_copy.data[:, max(0, t_min):t_max]
snr_max = SNRtools.calc_snr_matchedfilter(data_dm_max.mean(0), widths=[downsamp_res])[0]
if t_min<0:
Z = np.zeros([nfreq, np.abs(t_min)])
data_dm_max = np.concatenate([Z, data_dm_max], axis=1)
# scale max DM by pulse width, 5 units for each ms
dm_max_trans = 10. + 5*time_res/0.001 + 10*dm0/1000.
dm_min_trans = -10. - 5*time_res/0.001 - 10*dm0/1000.
if dm0+dm_min_trans<=0:
dm_min_trans = 0.
dm_max_trans = 2*dm0
dm_center = dm0
else:
dm_center = 0.
full_arr, dms, times = RTproc.dm_transform(data_dm_max,
(freq_up, freq_low),
dt=dt*downsamp_smear,
dm_max=dm_max_trans,
dm_min=dm_min_trans,
freq_ref=freq_ref,ndm=ndm,
dm0=dm_center)
dms += dm0
# bin down to nfreq_plot freq channels
full_freq_arr_downsamp = data_dm_max[:nfreq//nfreq_plot*nfreq_plot, :]
full_freq_arr_downsamp = full_freq_arr_downsamp.reshape(\
nfreq_plot, -1, ntime).mean(1)
# bin down in time by factor of downsamp
full_freq_arr_downsamp = full_freq_arr_downsamp[:,:ntime//downsamp_res*downsamp_res]
full_freq_arr_downsamp = full_freq_arr_downsamp.reshape(-1, ntime//downsamp_res,
downsamp_res).mean(-1)
if snr_max < sig_thresh_local:
logging.info("\nSkipping trigger below local threshold %.2f:" % sig_thresh_local)
logging.info("snr_local=%.2f snr_trigger=%.2f\n" % (snr_max, sig_cut))
return [], [], [], []
times = np.linspace(0,ntime_plot*downsamp*dt,len(full_freq_arr_downsamp[0]))
full_dm_arr_downsamp = full_arr[:, :ntime//downsamp_res*downsamp_res]
full_dm_arr_downsamp = full_dm_arr_downsamp.reshape(-1,
ntime//downsamp_res, downsamp_res).mean(-1)
full_freq_arr_downsamp /= np.std(full_freq_arr_downsamp)
full_dm_arr_downsamp /= np.std(full_dm_arr_downsamp)
suptitle = " CB:%s S/N$_{pipe}$:%.1f S/N$_{presto}$:%.1f\
S/N$_{compare}$:%.1f \nDM:%d t:%.1fs width:%d" %\
(beamno, sig_cut, snr_max, snr_comparison,
dm0, t0, downsamp)
if not os.path.isdir('%s/plots' % outdir):
os.system('mkdir -p %s/plots' % outdir)
if sb is None:
sbname = -1
else:
sbname = sb
fn_fig_out = '%s/plots/CB%s_snr%d_dm%d_t0%d_sb%d.pdf' % \
(outdir, beamno, sig_cut, dm0, t0, sbname)
params = sig_cut, dms[dm_max_jj], downsamp, t0, dt
tmed = np.median(full_freq_arr_downsamp, axis=-1, keepdims=True)
full_freq_arr_downsamp -= tmed
if mk_plot is True:
logging.info(fn_fig_out)
if ndm == 1:
plotter.plot_two_panel(full_freq_arr_downsamp, params,
prob=None,
freq_low=freq_low, freq_up=freq_up,
cand_no=cand_no, times=times,
suptitle=suptitle,
fnout=fn_fig_out)
else:
plotter.plot_three_panel(full_freq_arr_downsamp,
full_dm_arr_downsamp, params, dms,
times=times, freq_low=freq_low,
freq_up=freq_up,
suptitle=suptitle, fnout=fn_fig_out,
cand_no=cand_no)
return full_dm_arr_downsamp, full_freq_arr_downsamp, time_res, params
def proc_trigger(fn_fil,
dm0,
t0,
sig_cut,
ndm=50,
mk_plot=False,
downsamp=1,
beamno='',
fn_mask=None,
nfreq_plot=32,
ntime_plot=250,
cmap='RdBu',
cand_no=1,
multiproc=False):
""" Locate data within filterbank file (fn_fi)
at some time t0, and dedisperse to dm0, generating
plots
Parameters:
----------
fn_fil : str
name of filterbank file
dm0 : float
trigger dm found by single pulse search software
t0 : float
time in seconds where trigger was found
sig_cut : np.float
sigma of detected trigger at (t0, dm0)
ndm : int
number of DMs to use in DM transform
mk_plot : bool
make three-panel plots
downsamp : int
factor by which to downsample in time. comes from searchsoft.
beamno : str
beam number, for fig names
nfreq_plot : int
number of frequencies channels to plot
Returns:
-------
full_dm_arr_downsamp : np.array
data array with downsampled dm-transformed intensities
full_freq_arr_downsamp : np.array
data array with downsampled freq-time intensities
"""
SNRtools = tools.SNR_Tools()
rawdatafile = filterbank.filterbank(fn_fil)
mask = [
] #np.array([ 5, 6, 9, 32, 35, 49, 75, 76, 78, 82, 83, 87, 92,
# 93, 97, 98, 108, 110, 111, 112, 114, 118, 122, 123, 124, 157,
# 160, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 660,
# 661])
dt = rawdatafile.header['tsamp']
freq_up = rawdatafile.header['fch1']
nfreq = rawdatafile.header['nchans']
freq_low = freq_up + nfreq * rawdatafile.header['foff']
time_res = dt * downsamp
ntime_fil = (os.path.getsize(fn_fil) - 467.) / nfreq
dm_min = max(0, dm0 - 20)
dm_max = dm0 + 20
dms = np.linspace(dm_min, dm_max, ndm, endpoint=True)
# make sure dm0 is in the array
dm_max_jj = np.argmin(abs(dms - dm0))
dms += (dm0 - dms[dm_max_jj])
dms[0] = max(0, dms[0])
# Read in 5 disp delays
width = 2 * abs(4.14e3 * dm0 * (freq_up**-2 - freq_low**-2))
tdisp = width / dt
tplot = ntime_plot * downsamp
global t_min, t_max
if tdisp > tplot:
# Need to read in more data than you'll plot
# because of large dispersion time
chunksize = int(tdisp)
t_min = chunksize // 2 - (ntime_plot * downsamp) // 2
t_max = chunksize // 2 + (ntime_plot * downsamp) // 2
else:
# Only need ot read in enough to plot
chunksize = int(tplot)
t_min, t_max = 0, chunksize
start_bin = int(t0 / dt - chunksize / 2.)
if start_bin < 0:
extra = start_bin
start_bin = 0
t_min += extra
t_max += extra
t_min, t_max = int(t_min), int(t_max)
ntime = t_max - t_min
snr_max = 0
if ntime_fil < (start_bin + chunksize):
print("Trigger at end of file, skipping")
return 0, 0, 0
data = rawdatafile.get_spectra(start_bin, chunksize)
data.data -= np.median(data.data, axis=-1)[:, None]
data.data[mask] = 0.
full_arr = np.empty([int(ndm), int(ntime)])
if not fn_mask is None:
rfimask = rfifind.rfifind(fn_mask)
mask = get_mask(rfimask, start_bin, chunksize)
data = data.masked(mask, maskval='median-mid80')
if multiproc is True:
print("\nDedispersing in Parallel\n")
t0 = time.time()
global datacopy
ndm_ = min(10, ndm)
for kk in range(ndm // ndm_):
dms_ = dms[10 * kk:10 * (kk + 1)]
datacopy = copy.deepcopy(data)
pool = multiprocessing.Pool(processes=ndm_)
data_tuple = pool.map(multiproc_dedisp, [i for i in dms_])
pool.close()
data_tuple = np.concatenate(data_tuple)
ddm = np.concatenate(data_tuple[0::2]).reshape(ndm_, -1)
df = np.concatenate(data_tuple[1::2]).reshape(ndm_, nfreq, -1)
print(time.time() - t0)
full_arr[10 * kk:10 * (kk + 1)] = ddm[:, t_min:t_max]
ind_kk = range(10 * kk, 10 * (kk + 1))
if dm_max_jj in ind_kk:
print(dms_[ind_kk.index(dm_max_jj)])
data_dm_max = df[ind_kk.index(dm_max_jj)] #dm_max_jj]hack
del ddm, df
else:
print("\nDedispersing Serially\n")
for jj, dm_ in enumerate(dms):
print("Dedispersing to dm=%0.1f at t=%0.1f sec with width=%.2f" %
(dm_, start_bin * dt, downsamp))
data_copy = copy.deepcopy(data)
data_copy.dedisperse(dm_)
dm_arr = data_copy.data[:, t_min:t_max].mean(0)
snr_ = SNRtools.calc_snr(dm_arr)
full_arr[jj] = copy.copy(dm_arr)
if jj == dm_max_jj:
data_dm_max = data_copy.data[:, t_min:t_max]
downsamp = int(downsamp)
# bin down to nfreq_plot freq channels
full_freq_arr_downsamp = data_dm_max[:nfreq//nfreq_plot*nfreq_plot, :].reshape(\
nfreq_plot, -1, ntime).mean(1)
# bin down in time by factor of downsamp
full_freq_arr_downsamp = full_freq_arr_downsamp[:, :ntime//downsamp*downsamp\
].reshape(-1, ntime//downsamp, downsamp).mean(-1)
times = np.linspace(0, ntime * dt, len(full_freq_arr_downsamp[0]))
full_dm_arr_downsamp = full_arr[:, :ntime // downsamp * downsamp]
full_dm_arr_downsamp = full_dm_arr_downsamp.reshape(
-1, ntime // downsamp, downsamp).mean(-1)
full_freq_arr_downsamp /= np.std(full_freq_arr_downsamp)
full_dm_arr_downsamp /= np.std(full_dm_arr_downsamp)
suptitle = "beam%s snr%d dm%d t0%d width%d" %\
(beamno, sig_cut, dms[dm_max_jj], t0, downsamp)
fn_fig_out = './plots/train_data_beam%s_snr%d_dm%d_t0%d.pdf' % \
(beamno, sig_cut, dms[dm_max_jj], t0)
if mk_plot is True:
print(fn_fig_out)
if ndm == 1:
params = snr_, dm_, downsamp, t0
plotter.plot_two_panel(full_freq_arr_downsamp,
params,
prob=None,
freq_low=1250.09765625,
freq_up=1549.90234375,
cand_no=cand_no)
else:
print(fn_fig_out)
plotter.plot_three_panel(full_freq_arr_downsamp,
full_dm_arr_downsamp,
times,
dms,
freq_low=freq_low,
freq_up=freq_up,
suptitle=suptitle,
fnout=fn_fig_out,
cand_no=cand_no)
return full_dm_arr_downsamp, full_freq_arr_downsamp, time_res
def proc_trigger(fn_fil, dm0, t0, sig_cut,
ndm=50, mk_plot=False, downsamp=1,
beamno='', fn_mask=None, nfreq_plot=32,
ntime_plot=250,
cmap='RdBu', cand_no=1, multiproc=False,
rficlean=False, snr_comparison=-1,
outdir='./', sig_thresh_local=5.0,
subtract_zerodm=False,
threshold_time=3.25, threshold_frequency=2.75, bin_size=32,
n_iter_time=3, n_iter_frequency=3, clean_type='time'):
""" Locate data within filterbank file (fn_fi)
at some time t0, and dedisperse to dm0, generating
plots
Parameters:
----------
fn_fil : str
name of filterbank file
dm0 : float
trigger dm found by single pulse search software
t0 : float
time in seconds where trigger was found
sig_cut : np.float
sigma of detected trigger at (t0, dm0)
ndm : int
number of DMs to use in DM transform
mk_plot : bool
make three-panel plots
downsamp : int
factor by which to downsample in time. comes from searchsoft.
beamno : str
beam number, for fig names
nfreq_plot : int
number of frequencies channels to plot
Returns:
-------
full_dm_arr_downsamp : np.array
data array with downsampled dm-transformed intensities
full_freq_arr_downsamp : np.array
data array with downsampled freq-time intensities
"""
try:
rfimask = np.loadtxt('/home/arts/ARTS-obs/amber_conf/zapped_channels_1400.conf')
rfimask = rfimask.astype(int)
except:
rfimask = []
logging.warning("Could not load dumb RFIMask")
SNRtools = tools.SNR_Tools()
downsamp = min(4096, downsamp)
rawdatafile = filterbank.filterbank(fn_fil)
dfreq_MHz = rawdatafile.header['foff']
mask = []
dt = rawdatafile.header['tsamp']
freq_up = rawdatafile.header['fch1']
nfreq = rawdatafile.header['nchans']
# fix RFI mask order
rfimask = nfreq - rfimask
freq_low = freq_up + nfreq*rawdatafile.header['foff']
ntime_fil = (os.path.getsize(fn_fil) - 467.)/nfreq
tdm = np.abs(8.3*1e-6*dm0*dfreq_MHz*(freq_low/1000.)**-3)
dm_min = max(0, dm0-40)
dm_max = dm0 + 40
dms = np.linspace(dm_min, dm_max, ndm, endpoint=True)
# make sure dm0 is in the array
dm_max_jj = np.argmin(abs(dms-dm0))
dms += (dm0-dms[dm_max_jj])
dms[0] = max(0, dms[0])
global t_min, t_max
# if smearing timescale is < 4*pulse width,
# downsample before dedispersion for speed
downsamp_smear = max(1, int(downsamp*dt/tdm/2.))
# ensure that it's not larger than pulse width
downsamp_smear = int(min(downsamp, downsamp_smear))
downsamp_res = int(downsamp//downsamp_smear)
downsamp = int(downsamp_res*downsamp_smear)
time_res = dt * downsamp
tplot = ntime_plot * downsamp
logging.info("Width_full:%d Width_smear:%d Width_res: %d" %
(downsamp, downsamp_smear, downsamp_res))
# print("Width_full:%d Width_smear:%d Width_res: %d" %
# (downsamp, downsamp_smear, downsamp_res))
start_bin = int(t0/dt - ntime_plot*downsamp//2)
width = abs(4.148e3 * dm0 * (freq_up**-2 - freq_low**-2))
chunksize = int(width/dt + ntime_plot*downsamp)
t_min, t_max = 0, ntime_plot*downsamp
if start_bin < 0:
extra = start_bin
start_bin = 0
t_min += extra
t_max += extra
t_min, t_max = int(t_min), int(t_max)
snr_max = 0
# Account for the pre-downsampling to speed up dedispersion
t_min /= downsamp_smear
t_max /= downsamp_smear
ntime = t_max-t_min
if ntime_fil < (start_bin+chunksize):
logging.info("Trigger at end of file, skipping")
# print("Trigger at end of file, skipping")
return [],[],[],[]
data = rawdatafile.get_spectra(start_bin, chunksize)
# apply dumb mask
data.data[rfimask] = 0.
if rficlean is True:
data = cleandata(data, threshold_time, threshold_frequency, bin_size, \
n_iter_time, n_iter_frequency, clean_type)
if subtract_zerodm:
data.data -= np.mean(data.data, axis=0)[None]
# Downsample before dedispersion up to 1/4th
# DM smearing limit
data.downsample(downsamp_smear)
data.data -= np.median(data.data, axis=-1)[:, None]
full_arr = np.empty([int(ndm), int(ntime)])
if not fn_mask is None:
pass
# rfimask = rfifind.rfifind(fn_mask)
# mask = get_mask(rfimask, start_bin, chunksize)
# data = data.masked(mask, maskval='median-mid80')
if multiproc is True:
tbeg=time.time()
global datacopy
size_arr = sys.getsizeof(data.data)
nproc = int(32.0e9/size_arr)
ndm_ = min(min(nproc, ndm), 10)
for kk in range(ndm//ndm_):
dms_ = dms[ndm_*kk:ndm_*(kk+1)]
datacopy = copy.deepcopy(data)
pool = multiprocessing.Pool(processes=ndm_)
data_tuple = pool.map(multiproc_dedisp, [i for i in dms_])
pool.close()
data_tuple = np.concatenate(data_tuple)
ddm = np.concatenate(data_tuple[0::2]).reshape(ndm_, -1)
df = np.concatenate(data_tuple[1::2]).reshape(ndm_, nfreq, -1)
print(time.time()-tbeg)
full_arr[ndm_*kk:ndm_*(kk+1)] = ddm[:, t_min:t_max]
ind_kk = range(ndm_*kk, ndm_*(kk+1))
if dm_max_jj in ind_kk:
data_dm_max = df[ind_kk.index(dm_max_jj)]
del ddm, df
else:
logging.info("\nDedispersing Serially\n")
#print("\nDedispersing Serially\n")
for jj, dm_ in enumerate(dms):
tcopy = time.time()
data_copy = copy.deepcopy(data)
t0_dm = time.time()
data_copy.dedisperse(dm_)
dm_arr = data_copy.data[:, max(0, t_min):t_max].mean(0)
full_arr[jj, np.abs(min(0, t_min)):] = copy.copy(dm_arr)
logging.info("Dedispersing to dm=%0.1f at t=%0.1fsec with width=%.1f S/N=%.1f" %
(dm_, t0, downsamp, sig_cut))
# print("Dedispersing to dm=%0.1f at t=%0.1fsec with width=%.1f S/N=%.1f" %
# (dm_, t0, downsamp, sig_cut))
if jj==dm_max_jj:
data_dm_max = data_copy.data[:, max(0, t_min):t_max]
snr_max = SNRtools.calc_snr_matchedfilter(data_dm_max.mean(0), widths=[downsamp_res])[0]
if t_min<0:
Z = np.zeros([nfreq, np.abs(t_min)])
data_dm_max = np.concatenate([Z, data_dm_max], axis=1)
# bin down to nfreq_plot freq channels
full_freq_arr_downsamp = data_dm_max[:nfreq//nfreq_plot*nfreq_plot, :].reshape(\
nfreq_plot, -1, ntime).mean(1)
# bin down in time by factor of downsamp
full_freq_arr_downsamp = full_freq_arr_downsamp[:, :ntime//downsamp_res*downsamp_res\
].reshape(-1, ntime//downsamp_res, downsamp_res).mean(-1)
# snr_max = SNRtools.calc_snr_mad(full_freq_arr_downsamp.mean(0))
if snr_max < sig_thresh_local:
logging.info("\nSkipping trigger below local threshold %.2f:" % sig_thresh_local)
logging.info("snr_local=%.2f snr_trigger=%.2f\n" % (snr_max, sig_cut))
return [],[],[],[]
times = np.linspace(0, ntime_plot*downsamp*dt, len(full_freq_arr_downsamp[0]))
full_dm_arr_downsamp = full_arr[:, :ntime//downsamp_res*downsamp_res]
full_dm_arr_downsamp = full_dm_arr_downsamp.reshape(-1,
ntime//downsamp_res, downsamp_res).mean(-1)
full_freq_arr_downsamp /= np.std(full_freq_arr_downsamp)
full_dm_arr_downsamp /= np.std(full_dm_arr_downsamp)
suptitle = " CB:%s S/N$_{pipe}$:%.1f S/N$_{presto}$:%.1f\
S/N$_{compare}$:%.1f \nDM:%d t:%.1fs width:%d" %\
(beamno, sig_cut, snr_max, snr_comparison, \
dms[dm_max_jj], t0, downsamp)
if not os.path.isdir('%s/plots' % outdir):
os.system('mkdir -p %s/plots' % outdir)
fn_fig_out = '%s/plots/CB%s_snr%d_dm%d_t0%d.pdf' % \
(outdir, beamno, sig_cut, dms[dm_max_jj], t0)
params = sig_cut, dms[dm_max_jj], downsamp, t0, dt
tmed = np.median(full_freq_arr_downsamp, axis=-1, keepdims=True)
full_freq_arr_downsamp -= tmed
if mk_plot is True:
logging.info(fn_fig_out)
if ndm==1:
plotter.plot_two_panel(full_freq_arr_downsamp, params, prob=None,
freq_low=freq_low, freq_up=freq_up,
cand_no=cand_no, times=times, suptitle=suptitle,
fnout=fn_fig_out)
else:
plotter.plot_three_panel(full_freq_arr_downsamp,
full_dm_arr_downsamp, params, dms,
times=times, freq_low=freq_low,
freq_up=freq_up,
suptitle=suptitle, fnout=fn_fig_out,
cand_no=cand_no)
return full_dm_arr_downsamp, full_freq_arr_downsamp, time_res, params
def fil_trigger(fn_fil, dm0, t0, sig_cut,
ndm=50, mk_plot=False, downsamp=1,
beamno='', fn_mask=None, nfreq_plot=32,
ntime_plot=250,
cmap='RdBu', cand_no=1, multiproc=False,
rficlean=False, snr_comparison=-1,
outdir='./', sig_thresh_local=5.0,
threshold_time=3.25, threshold_frequency=2.75, bin_size=32,
n_iter_time=3, n_iter_frequency=3, clean_type='time'):
try:
rfimask = np.loadtxt('/home/arts/ARTS-obs/amber_conf/zapped_channels_1400.conf')
rfimask = rfimask.astype(int)
except:
rfimask = []
logging.info("Could not load dumb RFIMask")
SNRtools = tools.SNR_Tools()
downsamp = min(4096, downsamp)
rawdatafile = filterbank.filterbank(fn_fil)
dfreq_MHz = rawdatafile.header['foff']
mask = []
dt = rawdatafile.header['tsamp']
freq_up = rawdatafile.header['fch1']
nfreq = rawdatafile.header['nchans']
freq_low = freq_up + nfreq*rawdatafile.header['foff']
ntime_fil = (os.path.getsize(fn_fil) - 467.)/nfreq
tdm = np.abs(8.3*1e-6*dm0*dfreq_MHz*(freq_low/1000.)**-3)
dm_min = max(0, dm0-40)
dm_max = dm0 + 40
dms = np.linspace(dm_min, dm_max, ndm, endpoint=True)
# make sure dm0 is in the array
dm_max_jj = np.argmin(abs(dms-dm0))
dms += (dm0-dms[dm_max_jj])
dms[0] = max(0, dms[0])
global t_min, t_max
# if smearing timescale is < 4*pulse width,
# downsample before dedispersion for speed
downsamp_smear = max(1, int(downsamp*dt/tdm/2.))
# ensure that it's not larger than pulse width
downsamp_smear = int(min(downsamp, downsamp_smear))
downsamp_res = int(downsamp//downsamp_smear)
downsamp = int(downsamp_res*downsamp_smear)
time_res = dt * downsamp
tplot = ntime_plot * downsamp
# print("Width_full:%d Width_smear:%d Width_res: %d" %
# (downsamp, downsamp_smear, downsamp_res))
start_bin = int(t0/dt - ntime_plot*downsamp//2)
width = abs(4.148e3 * dm0 * (freq_up**-2 - freq_low**-2))
chunksize = int(width/dt + ntime_plot*downsamp)
t_min, t_max = 0, ntime_plot*downsamp
if start_bin < 0:
extra = start_bin
start_bin = 0
t_min += extra
t_max += extra
t_min, t_max = int(t_min), int(t_max)
snr_max = 0
# Account for the pre-downsampling to speed up dedispersion
t_min /= downsamp_smear
t_max /= downsamp_smear
ntime = t_max-t_min
data = rawdatafile.get_spectra(start_bin, chunksize)
if rficlean is True:
data = cleandata(data, threshold_time, threshold_frequency, \
bin_size, n_iter_time, n_iter_frequency, clean_type)
return data, downsamp, downsamp_smear
def main():
fn = args[0]
if fn.endswith(".fil"):
# Filterbank file
filetype = "filterbank"
rawdatafile = filterbank.filterbank(fn)
elif fn.endswith(".fits"):
# PSRFITS file
filetype = "psrfits"
rawdatafile = psrfits.PsrfitsFile(fn)
else:
raise ValueError("Cannot recognize data file type from "
"extension. (Only '.fits' and '.fil' "
"are supported.)")
# Read data
start_bin = np.round(options.start/rawdatafile.tsamp).astype('int')
#dmfac = 4.15e3 * np.abs(1./rawdatafile.fch1**2 - 1./(rawdatafile.frequencies[-1])**2)
dmfac = 4.15e3 * np.abs(1./rawdatafile.frequencies[0]**2 - 1./rawdatafile.frequencies[-1]**2)
if options.nbins is None:
nbins = np.round(options.duration/rawdatafile.tsamp).astype('int')
else:
nbins = options.nbins
binratio = 50
if options.dm:
nbinsextra = np.round((options.duration + dmfac * options.dm)/rawdatafile.tsamp).astype('int')
else:
nbinsextra = nbins
data = rawdatafile.get_spectra(start_bin, nbinsextra)
data = maskfile(data, start_bin, nbinsextra)
data, bins = waterfall(start_bin, dmfac, options.duration, nbins, options.zerodm, options.nsub, options.subdm, options.dm, options.integrate_dm, options.downsamp, options.scaleindep, options.width_bins, rawdatafile, binratio, data)
# Ploting it up
fig = plt.figure()
fig.canvas.set_window_title("Frequency vs. Time")
ax = plt.axes((0.15, 0.15, 0.8, 0.60))
ragfac = float(nbins)/bins
dmrange, trange = data.data.shape
nbinlim = np.int(trange * ragfac)
print data.dt, rawdatafile.tsamp
#np.save('data',data.data[..., :nbinlim])
plt.imshow(data.data[..., :nbinlim], aspect='auto', \
cmap=matplotlib.cm.cmap_d[options.cmap], \
interpolation='nearest', origin='upper', \
extent=(data.starttime, data.starttime+ nbinlim*data.dt, \
data.freqs.min(), data.freqs.max()))
if options.show_cb:
cb = plt.colorbar()
cb.set_label("Scaled signal intensity (arbitrary units)")
#plt.axis('tight')
# Sweeping it up
for ii, sweep_dm in enumerate(options.sweep_dms):
ddm = sweep_dm-data.dm
delays = psr_utils.delay_from_DM(ddm, data.freqs)
delays -= delays.min()
if options.sweep_posns is None:
sweep_posn = 0.0
elif len(options.sweep_posns) == 1:
sweep_posn = options.sweep_posns[0]
else:
sweep_posn = options.sweep_posns[ii]
sweepstart = data.dt*data.numspectra*sweep_posn+data.starttime
sty = SWEEP_STYLES[ii%len(SWEEP_STYLES)]
plt.plot(delays+sweepstart, data.freqs, sty, lw=4, alpha=0.5)
# Dressing it up
plt.xlabel("Time")
plt.ylabel("Observing frequency (MHz)")
plt.suptitle("Frequency vs. Time")
# Plot Time series
if options.integrate_dm is not None:
Data = np.array(data.data[..., :nbinlim])
Dedisp_ts = Data.sum(axis=0)
times = (np.arange(data.numspectra)*data.dt + options.start)[..., :nbinlim]
ax = plt.axes((0.15, 0.75, 0.8, 0.2))
plt.plot(times, Dedisp_ts)
plt.setp(ax.get_xticklabels(), visible = False)
plt.setp(ax.get_yticklabels(), visible = False)
fig.canvas.mpl_connect('key_press_event', \
lambda ev: (ev.key in ('q','Q') and plt.close(fig)))
plt.show()
