def main():
fn = args.infile
if args.inplace:
fil = filterbank.FilterbankFile(fn, mode='readwrite')
else:
fil = filterbank.FilterbankFile(fn, mode='read')
if args.inprof is not None:
warnings.warn("Saved profiles already may be tuned to a particular " \
"DM, period and filterbank file (freq, nchans, " \
"tsamp, etc).")
prof = load_profile(args.inprof)
else:
prof = make_profile(args.vonmises)
prof = apply_dm(prof, args.period, args.dm, \
fil.foff, fil.frequencies, fil.tsamp)
scale_profile(prof, args.scale_name, args.scale_cfgstrs, fil)
if args.outprof is not None:
save_profile(prof, args.outprof)
outfn = args.outname % fil.header
print "Showing plot of profile to be injected..."
plt.figure()
plt.clf()
prof.plot(dedisp=True)
plt.xlim(0,1)
plt.savefig(outfn+".ps")
if args.dryrun:
sys.exit()
inject(fil, outfn, prof, args.period, args.dm, \
nbitsout=args.output_nbits, block_size=args.block_size, \
pulsar_only=args.pulsar_only, inplace=args.inplace)
def main():
fn = args[0]
if fn.endswith(".fil"):
# Filterbank file
filetype = "filterbank"
rawdatafile = filterbank.FilterbankFile(fn)
elif fn.endswith(".fits"):
# PSRFITS file
filetype = "psrfits"
rawdatafile = psrfits.PsrfitsFile(fn)
else:
raise ValueError("Cannot recognize data file type from "
"extension. (Only '.fits' and '.fil' "
"are supported.)")
data, bins, nbins, start, source_name = waterfall(rawdatafile, options.start, \
options.duration, dm=options.dm,\
nbins=options.nbins, nsub=options.nsub,\
subdm=options.subdm, zerodm=options.zerodm, \
downsamp=options.downsamp, \
scaleindep=options.scaleindep, \
width_bins=options.width_bins, mask=options.mask, \
maskfn=options.maskfile, \
bandpass_corr=options.bandpass_corr)
plot_waterfall(data, start, source_name, options.duration, \
dm=options.dm,ofile=options.ofile, integrate_ts=options.integrate_ts, \
integrate_spec=options.integrate_spec, show_cb=options.show_cb,
cmap_str=options.cmap, sweep_dms=options.sweep_dms, \
sweep_posns=options.sweep_posns, downsamp=options.downsamp,width=options.width,snr=options.snr)
def read_fil_data(fn, start=0, stop=1):
""" Read in filterbank data object starting
at sample start and reading in chunksize stop.
"""
print("Reading filterbank file %s \n" % fn)
fil_obj = filterbank_.FilterbankFile(fn)
header = fil_obj.header
header_size = fil_obj.header_size
header['hdr'] = header_size
delta_t = fil_obj.header['tsamp'] # delta_t in seconds
fch1 = header['fch1']
nchans = header['nchans']
foff = header['foff']
fch_f = fch1 + nchans * foff
freq = np.linspace(fch1, fch_f, nchans)
try:
data = fil_obj.get_spectra(start, stop)
except (ValueError):
data = 0
# turn array into time-major, for preprocess
# data = data.transpose()
return data, freq, delta_t, header
def h5_to_fil(fnh5, fn_fil_out, nchunk='all', rficlean=False):
if rficlean:
print("RFI Cleaning")
f = h5py.File(fnh5,'r')
chunksize = int(1e4)
if nchunk=='all':
nchunk=int(1e6)
for ii in range(int(nchunk)):
startsample, endsample = ii*chunksize, (ii+1)*chunksize
data = f["/SUB_ARRAY_POINTING_000/BEAM_000/STOKES_0"][startsample:endsample,:]
if rficlean:
data = data.T
data, ind_use, mask = dumb_clean(data, plot_clean=False)
data[np.isnan(data)] = 0.0
data = data.T
data = data.astype('f4')
if len(data)==0:
continue
if ii==0:
header = read_fil_data(fn_fil_out, start=0, stop=1)[-1]
fn_rfi_clean = write_to_fil(data, header, fn_fil_out)
elif ii>0:
print("Writing chunk %d" % ii)
fil_obj = filterbank.FilterbankFile(fn_fil_out, mode='readwrite')
fil_obj.append_spectra(data)
def main():
fn = args[0]
if fn.endswith(".fil"):
# Filterbank file
filetype = "filterbank"
rawdatafile = filterbank.FilterbankFile(fn)
elif fn.endswith(".fits"):
# PSRFITS file
filetype = "psrfits"
rawdatafile = psrfits.PsrfitsFile(fn)
else:
raise ValueError("Cannot recognize data file type from "
"extension. (Only '.fits' and '.fil' "
"are supported.)")
data, bins, nbins, start, source_name = waterfall(rawdatafile, options.start, \
options.duration, dm=options.dm,\
nbins=options.nbins, nsub=options.nsub,\
subdm=options.subdm, zerodm=options.zerodm, \
downsamp=options.downsamp, \
scaleindep=options.scaleindep, \
width_bins=options.width_bins, mask=options.mask, \
maskfn=options.maskfile, \
csv_file=options.csv_file,\
bandpass_corr=options.bandpass_corr)
ofile = plot_waterfall(data, start, source_name, options.duration, \
dm=options.dm,ofile=options.ofile, integrate_ts=options.integrate_ts, \
integrate_spec=options.integrate_spec, show_cb=options.show_cb,
cmap_str=options.cmap, sweep_dms=options.sweep_dms, \
sweep_posns=options.sweep_posns, downsamp=options.downsamp,width=options.width,snr=options.snr,csv_file=options.csv_file)
# Update CSV file if file is provided
if csv_file:
sourcename=rawdatafile.header['source_name']
src_ra=rawdatafile.header['src_raj']
src_dec=rawdatafile.header['src_dej']
tstart=rawdatafile.header['tstart']
fch1=rawdatafile.header['fch1']
nchans=rawdatafile.header['nchans']
bw=int(rawdatafile.header['nchans'])*rawdatafile.header['foff']
cat=ofile.split("_")[0]
snr=options.snr
width=options.width
dm=options.dm
df = pd.DataFrame({'PNGFILE':[ofile],'Category':[cat],'SNR':[snr],'WIDTH':[width],'DM':[dm],'SourceName':[sourcename],'RA':[src_ra],'DEC':[src_dec],'MJD':[tstart],'Hfreq':[fch1],'NCHANS':[nchans],'BANDWIDTH':[bw],'filename':[fn]})
#Column order coming out irregular, so fixing it here
col=['PNGFILE','Category','SNR','WIDTH','DM','SourceName','RA','DEC','MJD','Hfreq','NCHANS','BANDWIDTH','filename']
df = df.reindex(columns=col)
if os.path.exists(csv_file) is False:
with open(csv_file,'w') as f:
df.to_csv(f,header=True,index=False)
else:
with open(csv_file,'a') as f:
df.to_csv(f,header=False,index=False)
def read_fil_data(fn, start=0, stop=1e7):
print "Reading filterbank file %s \n" % fn
fil_obj = filterbank.FilterbankFile(fn)
header = fil_obj.header
delta_t = fil_obj.header['tsamp'] # delta_t in milliseconds
freq = get_freqs(fil_obj)
data = fil_obj.get_spectra(start, stop)
# turn array into time-major, for preprocess
# data = data.transpose()
return data, freq, delta_t, header
def main():
fn = args[0]
if fn.endswith(".fil"):
# Filterbank file
filetype = "filterbank"
print_debug("Reading filterbank file..")
rawdatafile = filterbank.FilterbankFile(fn)
basename = fn[:-4]
elif fn.endswith(".fits"):
# PSRFITS file
filetype = "psrfits"
print_debug("Reading PSRFITS file..")
rawdatafile = psrfits.PsrfitsFile(fn)
basename = fn[:-5]
else:
raise ValueError("Cannot recognize data file type from "
"extension. (Only '.fits' and '.fil' "
"are supported.)")
if options.outbasenm:
basename = options.outbasenm
spdcand = spcand.params()
if not options.man_params:
print_debug('Maximum number of candidates to plot: %i' %
options.maxnumcands)
make_spd_from_file(spdcand, rawdatafile, \
options.txtfile, options.maskfile, \
options.min_rank, options.group_rank, \
options.plot, options.just_waterfall, \
options.integrate_ts, options.integrate_spec, options.disp_pulse, \
options.loc_pulse, options.nsub, \
options.maxnumcands, \
basename, \
mask=options.mask, barytime=options.barytime, \
bandpass_corr=options.bandpass_corr)
else:
print_debug("Making spd files based on mannual parameters. I suggest" \
"reading in parameters from the groups.txt file.")
make_spd_from_man_params(spdcand, rawdatafile, \
options.txtfile, options.maskfile, \
options.plot, options.just_waterfall, \
options.subdm, options.dm, options.sweep_dms, \
options.sigma, \
options.start, options.duration, \
options.width_bins, options.nbins, options.downsamp, \
options.nsub, \
options.scaleindep, \
options.spec_width, options.loc_pulse, \
options.integrate_ts, options.integrate_spec, options.disp_pulse, \
basename, \
options.mask, options.bandpass_corr, options.barytime, \
options.man_params)
def read_fil_data(fn, start=0, stop=1e7):
print("Reading filterbank file %s \n" % fn)
fil_obj = filterbank.FilterbankFile(fn)
header = fil_obj.header
delta_t = fil_obj.header['tsamp'] # delta_t in milliseconds
fch1 = header['fch1']
nchans = header['nchans']
foff = header['foff']
fch_f = fch1 + nchans * foff
freq = np.linspace(fch1, fch_f, nchans)
data = fil_obj.get_spectra(start, stop)
# turn array into time-major, for preprocess
# data = data.transpose()
return data, freq, delta_t, header
def process_full_fil(fn):
ii = 0
while True:
start, stop = 5e5 * ii, 5e5 * (ii + 1)
data, freq, delta_t, header = read_fil_data(fn, start=start, stop=stop)
if len(data[0]) == 0:
break
print data.shape, start
# data = dataapply_rfi_filters(data, delta_t)
if ii == 0:
fn_rfi_clean = write_to_fil(data.transpose(), header, fn)
elif ii > 0:
fil_obj = filterbank.FilterbankFile(fn_rfi_clean, mode='readwrite')
fil_obj.append_spectra(data.transpose())
ii += 1
def main():
fn = args[0]
if fn.endswith(".fil"):
# Filterbank file
filetype = "filterbank"
rawdatafile = filterbank.FilterbankFile(fn)
elif fn.endswith(".fits"):
# PSRFITS file
filetype = "psrfits"
rawdatafile = psrfits.PsrfitsFile(fn)
else:
raise ValueError("Cannot recognize data file type from "
"extension. (Only '.fits' and '.fil' "
"are supported.)")
data, bins, nbins, start = waterfall(rawdatafile, options.start, \
options.duration, dm=options.dm,\
nbins=options.nbins, nsub=options.nsub,\
subdm=options.subdm, zerodm=options.zerodm, \
downsamp=options.downsamp, \
scaleindep=options.scaleindep, \
width_bins=options.width_bins, mask=options.mask, \
maskfn=options.maskfile, \
bandpass_corr=options.bandpass_corr)
#### PuMA edition starts:
#Get the data:
nbinlim = np.int(options.duration / data.dt)
data_out = np.array(data.data[..., :nbinlim])
Dedisp_ts = data_out.sum(axis=0)
times = (np.arange(data.numspectra) * data.dt + start)[..., :nbinlim]
# we write the original time and intensity vectors into csv files
np.savetxt('times.csv', times, delimiter=',')
np.savetxt('original.csv', Dedisp_ts, delimiter=',')
def inject(infile, outfn, prof, period, dm, nbitsout=None,
block_size=BLOCKSIZE, pulsar_only=False, inplace=False):
if isinstance(infile, filterbank.FilterbankFile):
fil = infile
elif inplace:
fil = filterbank.FilterbankFile(infile, 'readwrite')
else:
fil = filterbank.FilterbankFile(infile, 'read')
print "Injecting pulsar signal into: %s" % fil.filename
if False:
delays = psr_utils.delay_from_DM(dm, fil.frequencies)
delays -= delays[np.argmax(fil.frequencies)]
get_phases = lambda times: (times-delays)/period % 1
else:
get_phases = lambda times: times/period % 1
# Create the output filterbank file
if nbitsout is None:
nbitsout = fil.nbits
if inplace:
warnings.warn("Injecting pulsar signal *in-place*")
outfil = fil
else:
# Start an output file
print "Creating out file: %s" % outfn
outfil = filterbank.create_filterbank_file(outfn, fil.header, \
nbits=nbitsout, mode='append')
if outfil.nbits == 8:
raise NotImplementedError("This code is out of date. 'delays' is not " \
"done in this way anymore..")
# Read the first second of data to get the global scaling to use
onesec = fil.get_timeslice(0, 1).copy()
onesec_nspec = onesec.shape[0]
times = np.atleast_2d(np.arange(onesec_nspec)*fil.tsamp).T+delays
phases = times/period % 1
onesec += prof(phases)
minimum = np.min(onesec)
median = np.median(onesec)
# Set median to 1/3 of dynamic range
global_scale = (256.0/3.0) / median
del onesec
else:
# No scaling to be performed
# These values will cause scaling to keep data unchanged
minimum = 0
global_scale = 1
sys.stdout.write(" %3.0f %%\r" % 0)
sys.stdout.flush()
oldprogress = -1
# Loop over data
lobin = 0
spectra = fil.get_spectra(0, block_size)
numread = spectra.shape[0]
while numread:
if pulsar_only:
# Do not write out data from input file
# zero it out
spectra *= 0
hibin = lobin+numread
# Sample at middle of time bin
times = (np.arange(lobin, hibin, 1.0/NINTEG_PER_BIN)+0.5/NINTEG_PER_BIN)*fil.dt
#times = (np.arange(lobin, hibin)+0.5)*fil.dt
phases = get_phases(times)
profvals = prof(phases)
shape = list(profvals.shape)
shape[1:1] = [NINTEG_PER_BIN]
shape[0] /= NINTEG_PER_BIN
profvals.shape = shape
toinject = profvals.mean(axis=1)
#toinject = profvals
if np.ndim(toinject) > 1:
injected = spectra+toinject
else:
injected = spectra+toinject[:,np.newaxis]
scaled = (injected-minimum)*global_scale
if inplace:
outfil.write_spectra(scaled, lobin)
else:
outfil.append_spectra(scaled)
# Print progress to screen
progress = int(100.0*hibin/fil.nspec)
if progress > oldprogress:
sys.stdout.write(" %3.0f %%\r" % progress)
sys.stdout.flush()
oldprogress = progress
# Prepare for next iteration
lobin = hibin
spectra = fil.get_spectra(lobin, block_size)
numread = spectra.shape[0]
sys.stdout.write("Done \n")
sys.stdout.flush()
lrows, 100, 25, 4, 7)
hdu.close()
bank_freq_index += 1
print "writing rows : ", nrow_list[rows], "to", nrow_list[rows + 1]
# write the 7 beams
# band flip if needed
# ignore chan in needed
if ignorechannels:
if last_pass_flag:
last_pass[:, :, ignorechan, :, :] = 0
else:
band_pass[:, :, ignorechan, :, :] = 0
for file_num in range(7):
file = fb.FilterbankFile(out_file_names[file_num], mode='append')
if last_pass_flag:
if flip:
file.append_spectra((last_pass[:, :, :, 1, file_num] +
last_pass[:, :, :, 0, file_num]).reshape(
100 * lrows, 500)[:, ::-1])
else:
file.append_spectra((last_pass[:, :, :, 1, file_num] +
last_pass[:, :, :, 0, file_num]).reshape(
100 * lrows, 500))
else:
if flip:
file.append_spectra((band_pass[:, :, :, 1, file_num] +
band_pass[:, :, :, 0, file_num]).reshape(
100 * nrows, 500)[:, ::-1])
else:
import copy
import filterbank
import imp
import numpy as np
from matplotlib import pyplot as plt
"""Takes in a filterbank file, dedisperses the Spectra object inside, and displays its dedispersed image."""
wt = imp.load_source('waterfaller', '/home/vgajjar/SpS/sps/src/waterfaller/waterfaller.py')
rawdatafile = filterbank.FilterbankFile("/datax/scratch/vgajjar/Test_pipeline/fake.fil")
spectra_data, bins, nbins, start_time = wt.waterfall(rawdatafile, 4.4, 0.3, 100, 1024, 128)
plt.imshow(spectra_data.data, aspect='auto')
plt.show()
print(spectra_data.data)
lodm, hidm, dmstep = 600, 800, 1
dmvstm_array = []
print(nbins)
datacopy = copy.deepcopy(spectra_data)
for ii in np.arange(lodm,hidm,dmstep):
spectra_data.dedisperse(0,padval='rotate')
spectra_data.dedisperse(ii,padval='rotate')
Data = np.array(spectra_data.data[..., :nbins])
Dedisp_ts = Data.sum(axis=0)
dmvstm_array.append(Dedisp_ts)
dmvstm_array=np.array(dmvstm_array)
plt.imshow(dmvstm_array, aspect='auto')
plt.show()
plt.imshow(spectra_data.data, aspect='auto')
plt.show()
def main():
infn = args[0]
print("Reading filterbank file (%s)" % infn)
fil = filterbank.FilterbankFile(infn)
if options.start_time is None:
startbin = 0
else:
startbin = int(np.round(options.start_time / fil.tsamp))
if options.end_time is None:
endbin = fil.nspec
else:
endbin = int(np.round(options.end_time / fil.tsamp)) + 1
new_nspec = endbin - startbin
if new_nspec <= 0:
raise ValueError("Bad number of spectra to be written (%d). " \
"Check start/end times." % new_nspec)
# Determine lo/hi channels to write to file
# If high frequencies come first in spectra 'hichan' refers to
# the lo-freq cutoff and 'lochan' refers to the hi-freq cutoff.
if options.lo_freq is None:
if fil.foff > 0:
lochan = 0
else:
hichan = fil.nchans
else:
ichan = int(np.round((options.lo_freq - fil.fch1) / fil.foff))
if fil.foff > 0:
lochan = ichan
else:
hichan = ichan + 1
if options.hi_freq is None:
if fil.foff > 0:
hichan = fil.nchans
else:
lochan = 0
else:
ichan = int(np.round((options.hi_freq - fil.fch1) / fil.foff))
if fil.foff > 0:
hichan = ichan + 1
else:
lochan = ichan
new_nchans = hichan - lochan
if new_nchans <= 0:
raise ValueError("Bad number of channels to be written (%d). " \
"Check lo/hi frequencies." % new_nchans)
print("Will extract")
print(" %d bins (%d to %d incl.)" % (new_nspec, startbin, endbin - 1))
print(" (Original num bins: %d)" % fil.nspec)
print(" %d channels (%d to %d incl.)" %
(new_nchans, lochan, hichan - 1))
print(" (Original num chans: %d)" % fil.nchans)
# Create output file
outfn = options.outname % fil.header
print("Creating out file: %s" % outfn)
outhdr = copy.deepcopy(fil.header)
outhdr['nchans'] = new_nchans
outhdr['fch1'] = fil.frequencies[lochan]
filterbank.create_filterbank_file(outfn, outhdr, nbits=fil.nbits)
outfil = filterbank.FilterbankFile(outfn, mode='write')
# Write data
sys.stdout.write(" %3.0f %%\r" % 0)
sys.stdout.flush()
nblocks = int(new_nspec / options.block_size)
remainder = new_nspec % options.block_size
oldprogress = -1
for iblock in np.arange(nblocks):
lobin = iblock * options.block_size + startbin
hibin = lobin + options.block_size
spectra = fil.get_spectra(lobin, hibin)
spectra = spectra[:, lochan:hichan] # restrict channels
outfil.append_spectra(spectra)
progress = int(100.0 * ((hibin - startbin) / new_nspec))
if progress > oldprogress:
sys.stdout.write(" %3.0f %%\r" % progress)
sys.stdout.flush()
oldprogress = progress
# Read all remaining spectra
if remainder:
spectra = fil.get_spectra(endbin - remainder, endbin)
spectra = spectra[:, lochan:hichan] # restrict channels
outfil.append_spectra(spectra)
sys.stdout.write("Done \n")
sys.stdout.flush()
def readFilterbank(inputFilename, logFile=""):
if (logFile == ""):
print("Reading filterbank file (%s)...\n" % inputFilename)
else:
logFile.write("Reading filterbank file (%s)...\n\n" % inputFilename)
fb = filterbank.FilterbankFile(inputFilename)
inputHeader = copy.deepcopy(fb.header)
inputNbits = fb.nbits
totalChans = fb.nchans
nchans = np.arange(0, fb.nchans - 1, 1) # Top of the band is index 0.
freqs = fb.frequencies
startbin = 0
endbin = fb.nspec
nspec = np.subtract(endbin, startbin)
nblocks = int(np.divide(nspec, BLOCKSIZE))
remainder = nspec % BLOCKSIZE
totalBlocks = nblocks
if (remainder):
totalBlocks = nblocks + 1
h5pyFile = h5py.File("%s.hdf5" % inputFilename, "w")
spectraData = h5pyFile.create_dataset("data", (totalChans, nspec),
dtype="float32")
iblock = 0
for iblock in np.arange(0, nblocks, 1):
progress = np.multiply(np.divide(iblock + 1.0, totalBlocks), 100.0)
if (logFile == ""):
sys.stdout.write("Reading... [%3.2f%%]\r" % progress)
sys.stdout.flush()
else:
logFile.write("Reading... [%3.2f%%]\n" % progress)
lobin = int(np.add(np.multiply(iblock, BLOCKSIZE), startbin))
hibin = int(np.add(lobin, BLOCKSIZE))
spectra = fb.get_spectra(lobin, hibin)
for ichan in np.arange(0, totalChans, 1):
spectraData[ichan, lobin:hibin] = spectra[:, ichan]
if (remainder):
progress = np.multiply(np.divide(iblock + 2.0, totalBlocks), 100.0)
if (logFile == ""):
sys.stdout.write("Reading... [%3.2f%%]\r" % progress)
sys.stdout.flush()
else:
logFile.write("Reading... [%3.2f%%]\n" % progress)
lobin = int(np.subtract(endbin, remainder))
hibin = int(endbin)
spectra = fb.get_spectra(lobin, hibin)
for ichan in np.arange(0, totalChans, 1):
spectraData[ichan, lobin:hibin] = spectra[:, ichan]
if (logFile == ""):
print("\n")
else:
logFile.write("\n")
return spectraData, inputHeader, inputNbits, h5pyFile
def writeFilterbank(outputFilename,
spectraData,
inputHeader,
inputNbits,
logFile=""):
if (logFile == ""):
print("Writing filterbank file (%s)...\n" % outputFilename)
else:
logFile.write("Writing filterbank file (%s)...\n\n" % outputFilename)
filterbank.create_filterbank_file(outputFilename,
inputHeader,
nbits=inputNbits)
outfil = filterbank.FilterbankFile(outputFilename, mode='write')
startbin = 0
endbin = np.shape(spectraData)[1]
nblocks = int(np.divide(endbin, BLOCKSIZE))
remainder = endbin % BLOCKSIZE
totalBlocks = nblocks
if (remainder):
totalBlocks = nblocks + 1
iblock = 0
for iblock in np.arange(0, nblocks, 1):
progress = np.multiply(np.divide(iblock + 1.0, totalBlocks), 100.0)
if (logFile == ""):
sys.stdout.write("Writing... [%3.2f%%]\r" % progress)
sys.stdout.flush()
else:
logFile.write("Writing... [%3.2f%%]\n" % progress)
lobin = int(np.add(np.multiply(iblock, BLOCKSIZE), startbin))
hibin = int(np.add(lobin, BLOCKSIZE))
spectra = spectraData[:, lobin:hibin].T
outfil.append_spectra(spectra)
if (remainder):
progress = np.multiply(np.divide(iblock + 2.0, totalBlocks), 100.0)
if (logFile == ""):
sys.stdout.write("Writing... [%3.2f%%]\r" % progress)
sys.stdout.flush()
else:
logFile.write("Writing... [%3.2f%%]\n" % progress)
lobin = int(np.subtract(endbin, remainder))
hibin = int(endbin)
spectra = spectraData[:, lobin:hibin].T
outfil.append_spectra(spectra)
if (logFile == ""):
print("\n")
else:
logFile.write("\n")
return
