def test_invertFreq(self, filfile):
myFil = FilReader(filfile)
data = myFil.readBlock(0, 100)
outfile = myFil.invertFreq()
mynewFil = FilReader(outfile)
newdata = mynewFil.readBlock(0, 100)
np.testing.assert_equal(mynewFil.header.dtype, myFil.header.dtype)
np.testing.assert_equal(mynewFil.header.nsamples,
myFil.header.nsamples)
np.testing.assert_equal(mynewFil.header.foff, -1 * myFil.header.foff)
np.testing.assert_array_equal(data, np.flip(newdata, axis=0))
os.remove(outfile)
def main():
out_name = 'fake_pulsar_train'
png_path = os.path.join('./fake_images', out_name)
if not os.path.exists(png_path):
create_folder(png_path)
fil_path = os.path.join('./fake_fils', out_name)
if not os.path.exists(fil_path):
create_folder(fil_path)
ann_path = './annotations'
if not os.path.exists(ann_path):
create_folder(ann_path)
nopulse_files = glob.glob('./nopulse_fils/*.fil')
fake_anns = multiprocessing.Manager().dict()
count = 0
pool = multiprocessing.Pool(12) # option: 使用多线程加速
for f in tqdm.tqdm(nopulse_files):
fil = FilReader(f)
creator = PulseCreator(fil)
for i in range(20): #每个背景数据生成20个不同参数的fake pulsar文件
pool.apply_async(run,
args=(creator, fil_path, png_path, fake_anns))
# run(creator, fil_path, png_path, fake_anns)
# count += 1
# if count == 10:
# break
pool.close()
pool.join()
# 保存标注文件
fake_anns = dict(fake_anns)
with open('./annotations/{}_ann.json'.format(out_name), 'w') as jf:
json.dump(fake_anns, jf, indent=4)
def get_spectral_stats(fname, chunksize=10000, show=False):
log.info("Getting spectral statistics...")
fil = FilReader(fname)
header = fil.header
nsamples = header['nsamples']
spectrum = 0.
spectrsq = 0.
for istart in tqdm.tqdm(range(0, nsamples, chunksize)):
size = min(chunksize, nsamples - istart)
array = fil.readBlock(istart, size, as_filterbankBlock=False)
local_spec = array.astype(float).sum(1)
local_sq = ((array).astype(float)**2).sum(1)
spectrum += local_spec
spectrsq += local_sq
if show:
plt.figure("Spectrum")
plt.plot(local_spec, lw=0.5, alpha=0.5, color='k')
plt.figure("Var spectrum")
plt.plot(np.sqrt(local_sq / size - (local_spec / size)**2),
lw=0.5,
alpha=0.5,
color='k')
mean_spec = spectrum / nsamples
mean_spectrsq = spectrsq / nsamples
std_spec = np.sqrt(mean_spectrsq - mean_spec**2)
if show:
plt.show()
return mean_spec, std_spec
def fb_avg(fname, tgoal=1000, fgoal=128, fscr=None):
""" Downsample a filterbank file to appropriate resolution for plotting.
"""
fr = FilReader(fname)
if fscr is None:
fscr = int(fr.header['nchans'] / fgoal)
tscr = 2**int(np.log2(fr.header['nsamples'] / tgoal))
gulp = tscr
nchan = fr.header['nchans']
nblock = fr.header['nsamples'] / tscr
nsub = nchan / fscr
output = np.empty((nsub, nblock))
for nsamps, i, data in fr.readPlan(gulp, verbose=False):
if nsamps != gulp:
#print 'breaking at %d'%i
break
t = data.reshape(
(gulp, nchan)).transpose().astype(np.float32).mean(axis=1)
output[:, i] = np.average(t.reshape(nchan / fscr, fscr), axis=1)
freqs = np.arange(
fr.header['nchans']) * fr.header['foff'] + fr.header['fch1']
freqs = freqs.reshape(len(freqs) / fscr, fscr).mean(axis=1)
times = np.arange(nblock) * fr.header['tsamp'] * gulp
return output, freqs, times
def main():
parser = argparse.ArgumentParser(description='Compute SEFD using the '+
'input filterbank files')
parser.add_argument('-on', dest='on_files', required=True,
help = 'on-source filterbank files', nargs='+', type=str)
parser.add_argument('-off', dest='off_files', required=True,
help = 'off-source filterbank files', nargs='+', type=str)
parser.add_argument('-s', dest='source_name', type=str, required=True,
help = 'source name')
parser.add_argument('-p', dest='plot', action='store_true',
help = 'plot on/off power')
parser.add_argument('-v', dest='verbose', action='store_true',
help = 'be verbose')
args = parser.parse_args()
# make sure we have the same number of filterbank files
assert len(args.on_files) == len(args.off_files),\
"Expecting same number of on and off files"
# prepare to read data
on_fils = [FilReader(eachFil) for eachFil in args.on_files]
off_fils = [FilReader(eachFil) for eachFil in args.off_files]
# some sanity checks
assert len(set([fil.header.fch1 for fil in on_fils+off_fils])) == 1,\
"Filterbanks do not have the same start frequency"
assert len(set([fil.header.bandwidth for fil in on_fils+off_fils])) == 1,\
"Filterbanks do not have the same bandwidth"
assert len(set([fil.header.tsamp for fil in on_fils+off_fils])) == 1,\
"Filterbanks do not have the same sampling time"
SEFD, SEFD_var, powOn, powOff, maskedArrays, flxSEFD =\
calcSEFDFils(on_fils, off_fils, args.source_name, args.verbose)
print ("-"*79)
print ("SEFD: ", SEFD)
print ("var: ", SEFD_var)
print ("-"*79)
if args.plot:
for i in range(len(powOn)):
plt.figure(i)
plt.plot(powOn[i], label="On")
plt.plot(powOff[i], label="Off")
plt.legend()
plt.show()
def openfile(filename):
files = FilReader(filename)
nsamples = files.header['nsamples']
data_arr = files.readBlock(0, nsamples)
chans = np.array(range(files.header['nchans']))
bandwidth = files.header['fch1'] - files.header['fbottom']
freqs = files.header['fbottom'] + chans / float(
files.header['nchans']) * bandwidth
def get_scaling(dest_value):
f = FilReader('gain.fil')
bandpass = f.bandpass() / f.header.nsamples
maxsample = np.amax(bandpass[np.nonzero(bandpass)])
avgsample = np.average(bandpass[np.nonzero(bandpass)])
print "Maximum sample value: {:.2f}".format(maxsample)
print "Average sample value: {:.2f}".format(avgsample)
scale = float(dest_value) / maxsample
return scale
def plot_sigproc(cand):
utc = utc_input.value
beam_str = str(int(cand['beam'])).zfill(BeamZfill)
fil_path = DataPathTemplate.substitute(BEAM=beam_str, UTC=utc)
#fil_path = os.path.join(data_path,".fil")
sys.stderr.write("%s\n" % fil_path)
fil = FilReader(fil_path)
disp = fil.readBlock(int(cand['sample']), 5000)
dedisp = disp.dedisperse(cand['dm'])
sigproc_source.data = dict(disp=[disp], dedisp=[dedisp], conv=[dedisp])
def split_filterbank(opts, info):
filename = info['filename']
segments = info["segments"]
output_location = info["basename"]
segment_lengths = map(float, segments.split(','))
for segment_length in segment_lengths:
obs_length_req = segment_length * 60.0
input_file = FilReader(filename)
number_samples = input_file.header.nsamples
obs_time_seconds = input_file.header.tobs
if obs_length_req > obs_time_seconds:
log.info(
"Segment length exceeds observation file length. Skipping the split.."
)
continue
log.info("calculate acceleration range for %d min" % segment_length)
pb = 10.0 * segment_length / (60.0) # 10 percent rule
amax = calculate_amax(pb, info["mc"])
info['start_acc'] = -1 * amax
info['end_acc'] = amax
sampling_time = input_file.header.tsamp
NTOREAD = int(obs_length_req / sampling_time)
number_of_fils = int(obs_time_seconds / obs_length_req)
for i in range(number_of_fils + 1):
START = i * NTOREAD
path = output_location + '_segment_%d_length_%dmin' % (
i, segment_length) + '.fil'
output_name = os.path.basename(path)[:-4]
if i == number_of_fils:
log.info("Last split")
input_file.split(number_samples - NTOREAD,
NTOREAD,
filename=path)
info["output_name"] = output_name
info["input_name"] = path
call_peasoup(info)
log.info("Segmented search done!")
else:
log.info("Split %d" % i)
log.info('START at sample: %d' % START)
input_file.split(START, NTOREAD, filename=path)
info["output_name"] = output_name
info["input_name"] = path
call_peasoup(info)
def get_power(utc, ant):
filfile = glob.glob(os.path.join(OBSDIR, utc, ant, "*x.fil"))[0]
fil = FilReader(filfile)
chans = np.linspace(fil.header.ftop, fil.header.fbottom, fil.header.nchans)
block = fil.readBlock(0, fil.header.nsamples)
bp = block.mean(axis=1)
args = (chans > 1678) & (chans < 1696)
power_inband = (bp[args]).sum()
return power_inband
def find_0329():
fnames = glob.glob('/data/kerrm/vlite_fb/*.fil')
good_fnames = []
for fname in fnames:
try:
fr = FilReader(fname)
#if '0329' in fr.header.source_name:
if fr.header.source_name == 'J0303+4716':
good_fnames.append(fname)
except:
print 'failed on %s.'%fname
continue
return sorted(good_fnames)
def fil_splitter(fil_fns, outfn, nsub, tqdm_desc=None, verbose=True):
"""
Split filterbank data to different subband files
"""
start_time = timeit.default_timer()
if isinstance(fil_fns, str):
fil_fns = [fil_fns]
if len(fil_fns) > 1:
raise TypeError(f'Input files: "{len(fil_fns)}". Not supported yet!.')
if nsub == 1:
raise TypeError(f'Nsub: {nsub}. No need for conversion!.')
myFil = FilReader(fil_fns[0])
out_files = []
hdr_changes = {}
chanpersub = myFil.header.nchans // nsub
for isub in range(nsub):
hdr_changes["nchans"] = chanpersub
hdr_changes[
"fch1"] = myFil.header.fch1 + isub * chanpersub * myFil.header.foff
outfil = f'{outfn.split(".fil")[0]}_sub{str(isub).zfill(2)}.fil'
out_files.append(
myFil.header.prepOutfile(outfil,
hdr_changes,
nbits=myFil.header.nbits))
if verbose:
print("Splitting Filterbank:")
print("---------------------------------")
gulpsize = 32768
for nsamps, ii, data in tqdm(myFil.readPlan(gulp=gulpsize, verbose=False),
total=np.ceil(myFil.header.nsamples /
gulpsize).astype('int'),
desc=tqdm_desc):
for isub, out_file in enumerate(out_files):
data = data.reshape(nsamps, myFil.header.nchans)
subint_tofil = data[:, chanpersub * isub:chanpersub * (isub + 1)]
out_file.cwrite(subint_tofil.ravel())
# Now close each of the filterbank file.
for out_file in out_files:
out_file.close()
end_time = timeit.default_timer()
if verbose:
print(f'Done')
print(f'Execution time: {(end_time-start_time):.3f} seconds\n')
def fb_avg(fname,tbingoal=1000,fgoal=128,fscr=None,last_minute=False):
""" Downsample a filterbank file to appropriate resolution for plotting.
"""
fr = FilReader(fname)
if fscr is None:
fscr = int(fr.header['nchans']/fgoal)
nsamp = fr.header['nsamples']
if (nsamp*fr.header['tsamp']) < 60:
last_minute = False
if last_minute:
nsamp = int(60./fr.header['tsamp'])+1
try:
tscr = 2**int(np.log2(nsamp/tbingoal))
except OverflowError:
tscr = 1
gulp = tscr
nchan = fr.header['nchans']
nblock = nsamp/tscr
nsub = nchan/fscr
output = np.empty( (nsub, nblock) )
start = 0
if last_minute:
start = max(0,int(fr.header['nsamples']-nsamp)-1)
for nsamps,i,data in fr.readPlan(gulp,start=start,verbose=False):
if nsamps != gulp:
#print 'breaking at %d'%i
break
t = data.reshape((gulp,nchan)).transpose().astype(np.float32).mean(axis=1)
output[:,i] = np.average(t.reshape(nchan/fscr,fscr),axis=1)
freqs = np.arange(fr.header['nchans'])*fr.header['foff']+fr.header['fch1']
freqs = freqs.reshape(len(freqs)/fscr,fscr).mean(axis=1)
times = (start + np.arange(nblock)*gulp)*fr.header['tsamp']
return output,freqs,times
#!/usr/bin/python
from sigpyproc.Readers import FilReader
import sigpyproc.Filterbank
import matplotlib.pyplot as plt
import numpy as np
file = FilReader("new.fil")
file.downsample(tfactor=500, ffactor=1, filename="test500.fil")
options, args = parser.parse_args()
if (not options.fil_file):
print('Input file required.')
print(parser.print_help())
sys.exit(1)
nodsamp = options.nodsamp
fil_file = os.path.abspath(options.fil_file)
if options.csv_file: csv_file = os.path.abspath(options.csv_file)
else: csv_file = ""
if options.model: ml_model = options.model
else: ml_model = ""
origf = FilReader(fil_file)
inchans = origf.header['nchans']
inbits = origf.header['nbits']
# For 32-bit BL, downsample to 8-bit and create new file
if (nodsamp is not True):
if (inchans > 8192 or inbits > 8):
print("Running sum_fil")
fil_file = downsample(fil_file, inbits, inchans)
fname = fil_file.split("/")[-1]
print(fil_file)
time = options.time
timesig = options.timesig
freqsig = options.freqsig
chanfrac = options.chanfrac
hd_cands = np.genfromtxt('/home/cbochenek/heimdall.cand', dtype=str)
boxcar_filters = np.array([1, 2, 3, 5, 10])
SNR = hd_cands[:, 0].astype(float)
inds = hd_cands[:, 1].astype(int)
cand_time = hd_cands[:, 2].astype(float)
cand_sample_width = boxcar_filters[hd_cands[:, 3].astype(int)]
cand_DM_trial_n = hd_cands[:, 4].astype(int)
cand_DM = hd_cands[:, 5].astype(float)
cand_members = hd_cands[:, 6].astype(int)
cand_data = hd_cands[:, 7]
good_inds = np.where(SNR >= 8.7)[0]
with PdfPages("candidates.pdf") as pdf:
for ind in good_inds:
files = FilReader("%s" % (cand_data[ind]))
nsamples = files.header['nsamples']
data_arr = files.readBlock(inds[ind] - 150, 300)
tsamp = files.header['tsamp']
chans = np.array(range(files.header['nchans']))
bandwidth = files.header['fch1'] - files.header['fbottom']
freqs = files.header['fbottom'] + chans / float(
files.header['nchans']) * bandwidth
smooth_bins = int(0.066 / tsamp)
plt.figure(ind, figsize=(20, 10))
gs = gridspec.GridSpec(nrows=6, ncols=1)
ax0 = plt.subplot(gs[0])
ax1 = plt.subplot(gs[1])
ax2 = plt.subplot(gs[2])
ax3 = plt.subplot(gs[3])
ax4 = plt.subplot(gs[4])
int(num_file * (BeamID - 1) +
n_file)] #"*M%02d_%00004d.fil"%(BeamID,n_file)
filename_last = pathlist[
index_last] #"*M%02d_%00004d.fil"%(BeamID,n_file-1)
filename_next = pathlist[
index_next] #"*M%02d_%00004d.fil"%(BeamID,n_file+1)
filename = filename.split('/')[-1]
filename_next = filename_next.split('/')[-1]
filename_last = filename_last.split('/')[-1]
#print "next filterbank:\n",filename_next
#print "Last filterbank:\n",filename_last
#print "Right Now:\n",filename
print "File Path:", fil_path + project + date_f + filename
f = FilReader(fil_path + project + date_f + '/' + filename)
MJD_Start = f.header.tstart
t_rsl = f.header.tsamp # Unit (s)
f_rsl = abs(f.header.foff)
f_top = f.header.ftop
f_bot = f.header.fbottom
mjd_burst = sample_ID * t_rsl / 3600 / 24. + MJD_Start
sample_delay = np.int(
((f_bot**-2 - f_top**-2) * C * DM) / (t_rsl * 1000))
if sample_delay <= 15000:
sample_delay = 15000
filterbank_new_dir = "../Data/Pick_Filterbank_Data/" + project + date_f + cand_f.split(
'/')[-1][:-5] + '/'
# filterbank_new_dir = cand_f[:-5]+"/Pick_Filterbank_Data/"
if not os.path.exists(filterbank_new_dir):
import numpy as np
import scipy as sp
import sys
from sigpyproc.Readers import FilReader
import matplotlib.pyplot as plt
if __name__ == "__main__":
if (len(sys.argv) < 4):
print "USAGE : <.fil file> <Start samp> <Total samp> "
print "Plot Filterbank file starting from the Start samples with total samples."
sys.exit(0)
f = FilReader(sys.argv[1])
f_top = f.header['ftop']
f_bott = f.header['fbottom']
t_rsl = f.header['tsamp']
d = f.readBlock(int(sys.argv[2]), int(sys.argv[3]))
freq = np.linspace(f_top, f_bott, len(d))
np.save(sys.argv[1][:-4], d)
np.save(sys.argv[1][:-4] + '_freq', freq)
plt.pcolormesh(d)
plt.colorbar()
# plt.imshow(d,aspect='auto')
plt.savefig('DM_sweep.png')
plt.show()
#bandpass=downsampled_file.bandpass()
#bandpass.toFile("
from sigpyproc.Readers import FilReader
import matplotlib.pyplot as plt
import numpy as np
#compute the bandpass and frequency of each channel
my_bpass = FilReader("100D.32.fil").bandpass()
freqs = np.linspace(my_bpass.header.ftop, my_bpass.header.fbottom,
my_bpass.size)
#plot the result
plt.plot(freqs, my_bpass)
plt.title("Bandpass")
plt.xlabel("Observing frequency (MHz)")
plt.ylabel("Power (Arbitrary units)")
plt.show()
plt.savefig("new_bandpass.png")
import numpy as np
import sys
import glob
from sigpyproc.Readers import FilReader
listx = []
listy = []
obsdir = "2021*"
#for i, filename in enumerate(sys.argv[1:]):
for filename in glob.glob(obsdir):
#print(filename)
filx = FilReader(filename + "/" + "2kB/" + filename + "_x.fil")
#print(filename+"/"+"2kB/"+filename+"_x.fil")
blockx = filx.readBlock(0, filx.header.nsamples)[1660:1740]
fily = FilReader(filename + "/" + "2kB/" + filename + "_y.fil")
blocky = fily.readBlock(0, filx.header.nsamples)[1660:1740]
xmean = blockx.mean()
ymean = blocky.mean()
#print(xmean)
listx.append(xmean)
listy.append(ymean)
from sigpyproc.Readers import FilReader
import numpy as np
import matplotlib.pyplot as plt
f = FilReader('../data/2011-02-20-01:52:19.fil')
data = f.readBlock(2 * 1024 * 20, 1024 * 20)
d_l = []
fft_l = []
for i in range(10):
d_l.append(data[:, i * 2048:(i + 1) * 2048])
d_l = np.array(d_l)
for i in range(10):
fft = np.fft.fft2(d_l[i])
fft = np.fft.fftshift(fft) / fft.size
fft = fft[1:fft.shape[0] / 2 + 1,
fft.shape[1] / 2:fft.shape[1] / 2 + fft.shape[0] / 2]
fft[-1, :] = 0
fft[:, 0] = 0
fft_l.append(abs(fft))
for i in range(10):
plt.pcolormesh(fft_l[i])
plt.colorbar()
plt.show()
def cleanup_data(fname, outname):
mask = get_bad_chans(fname)
fil = FilReader(fname)
def search_by_chunks(fname, chunk_length=None, new_sample_time=None, tmin=0, dmmin=200, dmmax=800, surelybad=[]):
log.info(f"Opening file {fname}")
fname_root = os.path.basename(fname).split('.')[0]
mask = get_bad_chans(fname)
for bad_chan in surelybad:
mask[bad_chan] = True
fil = FilReader(fname)
header = fil.header
nsamples = header['nsamples']
sample_time = header['tsamp']
start_freq = header['fbottom']
stop_freq = header['ftop']
bandwidth = header['bandwidth']
sample_time = header['tsamp']
nchan = header['nchans']
foff = header['foff']
date = header['tstart']
delta = delta_delay(dmmax, start_freq, stop_freq)
log.info(f"Expected delay at DM {dmmax} between {start_freq}--{stop_freq} MHz: {delta:2} s")
if chunk_length is None:
chunk_length = delta
step = max(int(chunk_length / sample_time) * 2, 128)
# Get the _minimum_ DM broadening to set the automatic rebinning
dm_dt = dm_broadening(dmmin, start_freq, np.abs(foff))
if new_sample_time is None:
new_sample_time = max(dm_dt / 10, sample_time)
log.info(f"Expected broadening in spectral bin at DM {dmmin} at {start_freq} MHz: {dm_dt * 1e6:2} us")
sampl_ratio = new_sample_time / sample_time
N = 1
if sampl_ratio >= 2:
N = int(np.rint(sampl_ratio))
new_sample_time = N * sample_time
log.info(f"Data will be resampled to {new_sample_time} s")
for istart in tqdm.tqdm(range(0, nsamples, step // 2)):
chunk_size = min(step, nsamples - istart)
t0 = istart * sample_time
if t0 < tmin:
continue
info = PulseInfo()
if chunk_size < step // 2:
continue
iend = istart + chunk_size
array = fil.readBlock(istart, chunk_size, as_filterbankBlock=False)
array = renormalize_data(array, badchans_mask=mask)
print(not np.any(array[mask, :]))
# array = (array - array.min()) / np.std(array)
if foff < 0:
array = array[::-1]
if N > 1:
array = quick_resample(array, N)
info.allprofs = array
info.start_freq = start_freq
info.bandwidth = bandwidth
info.nbin = array.shape[1]
info.nchan = array.shape[0]
info.date = date
info.pulse_freq = 1 / (info.nbin * new_sample_time)
table = fast_dedispersion_search(info.allprofs, dmmin, dmmax, start_freq, bandwidth, new_sample_time)
plot_diagnostics(info, outname=f'{fname_root}_{istart}-{iend}.jpg', dmmin=dmmin, dmmax=dmmax, t0=t0, show=True)
if np.any(table['snr'] > 6):
plot_diagnostics(info, outname=f'{fname_root}_{istart}-{iend}.jpg', dmmin=dmmin, dmmax=dmmax, t0=t0)
pickle.dump(info, open(f'{fname_root}_{istart}-{iend}.pkl', 'wb'))
import numpy as np
import matplotlib.pyplot as plt
from sigpyproc.Readers import FilReader
import time
t1 = time.time()
f_dir = '../data/'
f_name = 'out.fil'
f = FilReader(f_dir + f_name)
#dm = np.linspace(1,2000,1000)
DM = 100
snr = []
dd = f.dedisperse(DM)
snr = (dd.max() - dd.mean()) / dd.std()
t2 = time.time()
t_tol = t2 - t1
print "Time Total:", t_tol, " s = ", t_tol / 60., " m"
plt.title("SNR:" + str(snr))
plt.xlabel("SRC:" + f.header['source_name'])
plt.plot(dd)
plt.show()
def main(args):
inputlist = args.inlist
outdir = args.outdir
neig = args.neig
nbeams = len(inputlist)
outnames = [
os.path.join(outdir, "cleaned_" + os.path.basename(i))
for i in inputlist
]
if len(set(outnames)) < nbeams:
sys.stderr.write(
"ERR: please do not use same basename for input filterbank files\n"
)
sys.exit()
inputfils = []
for beam in inputlist:
inputfils.append(FilReader(beam))
nsamples = inputfils[0].header.nsamples
nchans = inputfils[0].header.nchans
samp_per_block = args.samps
work = np.zeros((nbeams, nchans, samp_per_block), dtype=np.float32)
if inputfils[0].header.nbits == 8:
rescale = True
for inputfil in inputfils:
inputfil.header.nbits = 32
else:
rescale = False
outfiles = [
inputfils[ifile].header.prepOutfile(i)
for ifile, i in enumerate(outnames)
]
total_n_blocks = nsamples / samp_per_block
statfile = args.statfile
if not statfile.endswith(".eig"):
statfile += ".eig"
eig_file = open(args.statfile, "w")
atexit.register(eig_file.close)
if USEPBAR:
pbar = Bar('Processing filterbanks',
max=total_n_blocks,
suffix='%(percent)d%%')
atexit.register(pbar.finish)
for iblock in range(total_n_blocks):
nzaps = []
# Extract data
for ibeam in range(nbeams):
work[ibeam] = inputfils[ibeam].readBlock(iblock * samp_per_block,
samp_per_block)
if rescale:
work[ibeam] = (work[ibeam].T -
np.median(work[ibeam], axis=1)).T
std = 1.4826 * get_mad(work[ibeam])
mask = np.abs(std) > 1e-12
work[ibeam][mask] = (work[ibeam][mask].T / std[mask]).T
# Do SVD for each data block
for ichan in range(nchans):
try:
U, W, Vt = np.linalg.svd(work[:, ichan, :],
full_matrices=False)
except Exception as e:
np.save("debug.npy", work[:, ichan, :])
sys.stderr.write("Exception caught: saving ./debug.npy\n")
raise e
neig = neigs_to_zap(W)
nzaps.append(neig)
if neig > 0:
Y = np.dot(work[:, ichan, :], Vt.T)
Vt.T[:, :neig] = 0.0
work[:, ichan, :] = np.dot(Y, Vt)
# Write output nzap file
nzaps = np.array(nzaps, dtype=np.int32)
nzaps.tofile(eig_file)
# Write output
for ibeam, outfile in enumerate(outfiles):
if args.resc_out:
work[ibeam] = (work[ibeam].T -
np.median(work[ibeam], axis=1)).T
std = 1.4826 * get_mad(work[ibeam])
mask = ((np.abs(std) > 1e-12) & (nzaps > 0))
work[ibeam][mask] = (work[ibeam][mask].T / std[mask]).T
work[ibeam].T.astype("float32").tofile(outfile)
if USEPBAR:
pbar.next()
if USEPBAR:
pbar.finish()
cands_snr = cands[:, 0]
cands_dm = cands[:, 5]
cands_ind = cands[:, 1]
cands_width = cands[:, 3]
cands2 = np.genfromtxt("/home/user/cand_times_sync_od/heimdall_2.cand")
cands2_snr = cands2[:, 0]
cands2_dm = cands2[:, 5]
cands2_ind = cands2[:, 1]
cands2_width = cands2[:, 3]
#ovro_int = 20355632#112793376
#gdscc_int = 103012504 #20355632
#ovro = FilReader("/home/user/coincidences/candidate_%s.fil" %ovro_int)
#gdscc = FilReader("/home/user/coincidences/candidate_%s.fil" %gdscc_int)
ovro = FilReader("candidate_ovro_200428.fil")
gdscc = FilReader("candidate_gdscc_200428.fil")
gdscc_arr = gdscc.readBlock(0, gdscc.header['nsamples'])
ovro_arr = ovro.readBlock(0, ovro.header['nsamples'])
ovro_ind = np.where(cands_ind == ovro_int)[0][0].astype(int)
gdscc_ind = np.where(cands2_ind == gdscc_int)[0][0].astype(int)
av_window = int(np.min([2**cands2_width[gdscc_ind], 2**cands_width[ovro_ind]]))
#av_window = 26
dm = 333.3 #cands_dm[ovro_ind]/2. + cands2_dm[gdscc_ind]/2.
gdscc_arr_conv = snd.uniform_filter1d(gdscc_arr, av_window, axis=1)
ovro_arr_conv = snd.uniform_filter1d(ovro_arr, av_window, axis=1)
fs = np.linspace(1280, 1530, 2048)
gdscc_dedisp = dedisperse(gdscc_arr_conv, dm, 0.000065536, fs)
ovro_dedisp = dedisperse(ovro_arr_conv, dm, 0.000065536, fs)
'names': ('snr', 'time', 'samp_idx', 'dm',
'filter', 'prim_beam'),
'formats': ('f4', 'f4', 'i4', 'f4', 'i4', 'i4')
})
#uGMRT
#fl = 300
#fh = 500
#FAST
fl = 1100
fh = 1500
noplot = 0
tint = 0.000163
Ttot = 80 # Total length of the file
kill_time_range = []
kill_chans = []
nchan = 2048
source_name = "Fake"
f = FilReader(fil_file)
nchan = f.header['nchans']
fch1 = f.header['fch1']
foff = f.header['foff']
tint = f.header['tsamp']
Ttot = f.header['tobs']
fh = fch1
fl = fch1 + (foff * nchan)
source_name = f.header['source_name']
extractPlotCand(fil_file, frb_cands, noplot, fl, fh, tint, Ttot,
kill_time_range, kill_chans, source_name, nchan)
#PWR = [-3, 0, 3, 6, 9 ,12]
PWR = [10]
BASEDIR = "/mnt/buf0/obs"
outdir = "/home/sonata/RF_signal_generator/june24/same_RF_freq/"
for iutc, utc in enumerate(UTCs):
print (utc)
for ant in os.listdir(os.path.join(BASEDIR, utc)):
if len(ant) != 2:
continue
print(ant)
filx = glob.glob(os.path.join(BASEDIR, utc, ant, "*_x.fil"))[0]
fil = FilReader(filx)
block = fil.readBlock(0, fil.header.nsamples)
N = 200000
fig, ax = plt.subplots(nrows=3, sharex=True, figsize=(13,10))
tsx = 10*np.log10(block[:-1].sum(axis=0))
m = np.median(tsx)
up = np.median(tsx[tsx > m])
down = np.median(tsx[tsx < m])
ax[0].plot(np.arange(block.shape[1])*fil.header.tsamp, tsx,
label="XX")
ax[0].hlines([up,down], 0, (block.shape[1])*fil.header.tsamp, ls="--",
label="XX compression: %.1f dB" %(up - down))
def get_readers(fil_files, nbeams=16):
"""Load blimpy Waterfall objects for filterbank file reading"""
fils = []
for f in sorted(fil_files)[:nbeams]:
fils.append(FilReader(f))
return fils
def main():
parser = argparse.ArgumentParser(description='Sum filterbank'\
'files')
parser.add_argument('-c',
dest='candfile',
type=str,
help="heimdall's candidate file")
parser.add_argument('-f', dest='filfile', type=str, help="filterbank file")
parser.add_argument('-s',
dest='show',
action='store_true',
help='iteratively show, rather than save, plots')
parser.add_argument('-u',
dest='utc',
type=str,
default="",
help='utc name (to append to png name')
parser.add_argument(
'-n',
dest='maxcand',
type=int,
help='Maximum number of candidates to plot [default: %i]' % (MAXN),
default=MAXN)
parser.add_argument('-z',
'--zap_chans',
dest='zap_chans',
action='append',
nargs=2,
type=int,
help='channels to zap')
args = parser.parse_args()
all_candidates = pd.read_table(args.candfile,
index_col=False,
names=COL_NAMES)
# hack to get the sampling time
tsamp = (all_candidates.tsec / all_candidates.tsample).mean()
box_thresh = int(np.log2(W_THRESH / tsamp))
all_candidates = all_candidates[(all_candidates.SNR > SNR_THRESH)
& (all_candidates.boxcar <= box_thresh)
& (all_candidates.DM > DM_THRESH)]
basedir = os.path.dirname(args.candfile)
candsdir = os.path.join(basedir, "candidates")
if not args.show:
if not os.path.isdir(candsdir):
os.mkdir(candsdir)
fil = FilReader(args.filfile)
nskip = int(0.3 / fil.header.tsamp)
print("ncands: %i" % all_candidates.shape[0])
pbar = tqdm(total=all_candidates.shape[0])
if all_candidates.shape[0] > args.maxcand:
print ("WARNING: number of candidates (%i) is greater than"\
"the maximum allowed (%i)" %(all_candidates.shape[0],
args.maxcand))
if not os.path.exists("cands.warning"):
open("cands.warning", 'a').close()
icand = 0
for ind, cand in all_candidates.iterrows():
dm_smear_sec = (8.3 * fil.header.bandwidth*1e-3 *\
cand.DM * (fil.header.fcenter*1e-3)**(-3))*1e-3
dm_smear_nsamp = int(dm_smear_sec / fil.header.tsamp)
istart = int(cand.tsample) - nskip
istart = 0 if istart < 0 else istart
block = fil.readBlock(istart, dm_smear_nsamp + 2 * nskip)
# zap unwanted channels
if args.zap_chans:
for low_freq, high_freq in args.zap_chans:
block[low_freq:high_freq] = 0. #np.nan
disp_org = block.dedisperse(cand.DM)
if cand.SNR < 10:
ffactor = 32
elif cand.SNR < 15:
ffactor = 16
elif cand.SNR < 20:
ffactor = 8
else:
ffactor = 1
tfactor = min(8, int(2**cand.boxcar))
disp = block_reduce(disp_org, (ffactor, tfactor),
func=np.sum,
cval=np.mean(disp_org))
matplotlib.use("Agg")
fig = plt.figure(0)
fig.suptitle("SNR = %.2f, DM = %.2f, boxcar width = %i" %
(cand.SNR, cand.DM, cand.boxcar))
gs = gridspec.GridSpec(2, 1)
gs.update(hspace=0)
ax1 = fig.add_subplot(gs[0])
ax2 = fig.add_subplot(gs[1], sharex=ax1)
ts = disp.sum(axis=0)
x = np.arange(
int(cand.tsample) - nskip,
int(cand.tsample) - nskip + len(ts) * tfactor, tfactor)
ax1.plot(x * fil.header.tsamp, ts)
ax1.get_xaxis().set_visible(False)
ylim = ax1.get_ylim()
ax1.vlines(np.array([cand.tsample - (2**cand.boxcar/2),
cand.tsample + (2**cand.boxcar/2)])*fil.header.tsamp -\
tfactor/2*fil.header.tsamp,
ymin=ylim[0], ymax=ylim[1], lw=1, ls='--')
if args.zap_chans:
for low_freq, high_freq in args.zap_chans:
disp_org[low_freq:high_freq] = np.nan
disp_org = block_reduce(disp_org, (ffactor, tfactor),
func=np.sum,
cval=np.mean(disp_org))
ax2.imshow(disp_org,
interpolation='nearest',
aspect='auto',
extent=[
x[0] * fil.header.tsamp, x[-1] * fil.header.tsamp,
fil.header.fbottom, fil.header.ftop
])
ax2.set_ylabel("Frequency (MHz)")
ax2.set_xlabel("Time since obs_start")
if args.show:
plt.show()
else:
basefigname = "tsamp_%i_dm_%.2f.png" % (cand.tsample, cand.DM)
if args.utc:
basefigname = args.utc + "_" + basefigname
figname = os.path.join(candsdir, basefigname)
plt.savefig(figname, fmt='png', bbox_inches='tight')
plt.close(fig)
pbar.update(1)
if icand > args.maxcand:
sys.exit(-1)
icand += 1
pbar.close()
