def get_spectral_stats(fname, chunksize=10000, show=False):
log.info("Getting spectral statistics...")
fil = FilReader(fname)
header = fil.header
nsamples = header['nsamples']
spectrum = 0.
spectrsq = 0.
for istart in tqdm.tqdm(range(0, nsamples, chunksize)):
size = min(chunksize, nsamples - istart)
array = fil.readBlock(istart, size, as_filterbankBlock=False)
local_spec = array.astype(float).sum(1)
local_sq = ((array).astype(float)**2).sum(1)
spectrum += local_spec
spectrsq += local_sq
if show:
plt.figure("Spectrum")
plt.plot(local_spec, lw=0.5, alpha=0.5, color='k')
plt.figure("Var spectrum")
plt.plot(np.sqrt(local_sq / size - (local_spec / size)**2),
lw=0.5,
alpha=0.5,
color='k')
mean_spec = spectrum / nsamples
mean_spectrsq = spectrsq / nsamples
std_spec = np.sqrt(mean_spectrsq - mean_spec**2)
if show:
plt.show()
return mean_spec, std_spec
def fb_avg(fname, tgoal=1000, fgoal=128, fscr=None):
""" Downsample a filterbank file to appropriate resolution for plotting.
"""
fr = FilReader(fname)
if fscr is None:
fscr = int(fr.header['nchans'] / fgoal)
tscr = 2**int(np.log2(fr.header['nsamples'] / tgoal))
gulp = tscr
nchan = fr.header['nchans']
nblock = fr.header['nsamples'] / tscr
nsub = nchan / fscr
output = np.empty((nsub, nblock))
for nsamps, i, data in fr.readPlan(gulp, verbose=False):
if nsamps != gulp:
#print 'breaking at %d'%i
break
t = data.reshape(
(gulp, nchan)).transpose().astype(np.float32).mean(axis=1)
output[:, i] = np.average(t.reshape(nchan / fscr, fscr), axis=1)
freqs = np.arange(
fr.header['nchans']) * fr.header['foff'] + fr.header['fch1']
freqs = freqs.reshape(len(freqs) / fscr, fscr).mean(axis=1)
times = np.arange(nblock) * fr.header['tsamp'] * gulp
return output, freqs, times
def openfile(filename):
files = FilReader(filename)
nsamples = files.header['nsamples']
data_arr = files.readBlock(0, nsamples)
chans = np.array(range(files.header['nchans']))
bandwidth = files.header['fch1'] - files.header['fbottom']
freqs = files.header['fbottom'] + chans / float(
files.header['nchans']) * bandwidth
def get_scaling(dest_value):
f = FilReader('gain.fil')
bandpass = f.bandpass() / f.header.nsamples
maxsample = np.amax(bandpass[np.nonzero(bandpass)])
avgsample = np.average(bandpass[np.nonzero(bandpass)])
print "Maximum sample value: {:.2f}".format(maxsample)
print "Average sample value: {:.2f}".format(avgsample)
scale = float(dest_value) / maxsample
return scale
def plot_sigproc(cand):
utc = utc_input.value
beam_str = str(int(cand['beam'])).zfill(BeamZfill)
fil_path = DataPathTemplate.substitute(BEAM=beam_str, UTC=utc)
#fil_path = os.path.join(data_path,".fil")
sys.stderr.write("%s\n" % fil_path)
fil = FilReader(fil_path)
disp = fil.readBlock(int(cand['sample']), 5000)
dedisp = disp.dedisperse(cand['dm'])
sigproc_source.data = dict(disp=[disp], dedisp=[dedisp], conv=[dedisp])
def get_power(utc, ant):
filfile = glob.glob(os.path.join(OBSDIR, utc, ant, "*x.fil"))[0]
fil = FilReader(filfile)
chans = np.linspace(fil.header.ftop, fil.header.fbottom, fil.header.nchans)
block = fil.readBlock(0, fil.header.nsamples)
bp = block.mean(axis=1)
args = (chans > 1678) & (chans < 1696)
power_inband = (bp[args]).sum()
return power_inband
def main():
out_name = 'fake_pulsar_train'
png_path = os.path.join('./fake_images', out_name)
if not os.path.exists(png_path):
create_folder(png_path)
fil_path = os.path.join('./fake_fils', out_name)
if not os.path.exists(fil_path):
create_folder(fil_path)
ann_path = './annotations'
if not os.path.exists(ann_path):
create_folder(ann_path)
nopulse_files = glob.glob('./nopulse_fils/*.fil')
fake_anns = multiprocessing.Manager().dict()
count = 0
pool = multiprocessing.Pool(12) # option: 使用多线程加速
for f in tqdm.tqdm(nopulse_files):
fil = FilReader(f)
creator = PulseCreator(fil)
for i in range(20): #每个背景数据生成20个不同参数的fake pulsar文件
pool.apply_async(run,
args=(creator, fil_path, png_path, fake_anns))
# run(creator, fil_path, png_path, fake_anns)
# count += 1
# if count == 10:
# break
pool.close()
pool.join()
# 保存标注文件
fake_anns = dict(fake_anns)
with open('./annotations/{}_ann.json'.format(out_name), 'w') as jf:
json.dump(fake_anns, jf, indent=4)
def fil_splitter(fil_fns, outfn, nsub, tqdm_desc=None, verbose=True):
"""
Split filterbank data to different subband files
"""
start_time = timeit.default_timer()
if isinstance(fil_fns, str):
fil_fns = [fil_fns]
if len(fil_fns) > 1:
raise TypeError(f'Input files: "{len(fil_fns)}". Not supported yet!.')
if nsub == 1:
raise TypeError(f'Nsub: {nsub}. No need for conversion!.')
myFil = FilReader(fil_fns[0])
out_files = []
hdr_changes = {}
chanpersub = myFil.header.nchans // nsub
for isub in range(nsub):
hdr_changes["nchans"] = chanpersub
hdr_changes[
"fch1"] = myFil.header.fch1 + isub * chanpersub * myFil.header.foff
outfil = f'{outfn.split(".fil")[0]}_sub{str(isub).zfill(2)}.fil'
out_files.append(
myFil.header.prepOutfile(outfil,
hdr_changes,
nbits=myFil.header.nbits))
if verbose:
print("Splitting Filterbank:")
print("---------------------------------")
gulpsize = 32768
for nsamps, ii, data in tqdm(myFil.readPlan(gulp=gulpsize, verbose=False),
total=np.ceil(myFil.header.nsamples /
gulpsize).astype('int'),
desc=tqdm_desc):
for isub, out_file in enumerate(out_files):
data = data.reshape(nsamps, myFil.header.nchans)
subint_tofil = data[:, chanpersub * isub:chanpersub * (isub + 1)]
out_file.cwrite(subint_tofil.ravel())
# Now close each of the filterbank file.
for out_file in out_files:
out_file.close()
end_time = timeit.default_timer()
if verbose:
print(f'Done')
print(f'Execution time: {(end_time-start_time):.3f} seconds\n')
def main():
parser = argparse.ArgumentParser(description='Compute SEFD using the '+
'input filterbank files')
parser.add_argument('-on', dest='on_files', required=True,
help = 'on-source filterbank files', nargs='+', type=str)
parser.add_argument('-off', dest='off_files', required=True,
help = 'off-source filterbank files', nargs='+', type=str)
parser.add_argument('-s', dest='source_name', type=str, required=True,
help = 'source name')
parser.add_argument('-p', dest='plot', action='store_true',
help = 'plot on/off power')
parser.add_argument('-v', dest='verbose', action='store_true',
help = 'be verbose')
args = parser.parse_args()
# make sure we have the same number of filterbank files
assert len(args.on_files) == len(args.off_files),\
"Expecting same number of on and off files"
# prepare to read data
on_fils = [FilReader(eachFil) for eachFil in args.on_files]
off_fils = [FilReader(eachFil) for eachFil in args.off_files]
# some sanity checks
assert len(set([fil.header.fch1 for fil in on_fils+off_fils])) == 1,\
"Filterbanks do not have the same start frequency"
assert len(set([fil.header.bandwidth for fil in on_fils+off_fils])) == 1,\
"Filterbanks do not have the same bandwidth"
assert len(set([fil.header.tsamp for fil in on_fils+off_fils])) == 1,\
"Filterbanks do not have the same sampling time"
SEFD, SEFD_var, powOn, powOff, maskedArrays, flxSEFD =\
calcSEFDFils(on_fils, off_fils, args.source_name, args.verbose)
print ("-"*79)
print ("SEFD: ", SEFD)
print ("var: ", SEFD_var)
print ("-"*79)
if args.plot:
for i in range(len(powOn)):
plt.figure(i)
plt.plot(powOn[i], label="On")
plt.plot(powOff[i], label="Off")
plt.legend()
plt.show()
def process_candidate(in_queue,out_queue):
print "%s Initiated" %os.getpid()
while True:
candidate = in_queue.get()
if len(candidate) == 2:
utc = candidate[1]
candidate = candidate[0]
H_sn , H_w, H_dm = candidate[0],candidate[3],candidate[5]
sample_bin, H_t, beam = int(candidate[1]), float(candidate[2]), int(candidate[12])
direc="/data/mopsr/archives/"+utc+"/BEAM_"+str(int(beam)).zfill(3)+"/"+utc+".fil"
fil_file=FilReader(direc)
dispersion_delay=(31.25*8.3*H_dm)/(0.840)**3
dispersion_delay=np.ceil(dispersion_delay/655.36)
w=2**H_w
min_afterEvent=300
time_extract=np.max([min_afterEvent,(dispersion_delay+w)*2])
fil_block=fil_file.readBlock(sample_bin-backstep,int(backstep+time_extract))
def load_and_save_infodata(inputdir, utc, sampr, savenpy, loadnpy):
'''Load infodata (either .npy or .fil) and decide if "infodata.npy" should be saved. Return infodata. Here, "infodata" contains some parameters about the observations.'''
#Load and save information about this observation.
npydir='../data/input/'+utc+'/'
samprname=('%.3e' %sampr).replace('+','').replace('-','').replace('.','p')
samprdir=npydir+'sampr'+samprname+'/'
npydatafile=samprdir+'INFO'
if loadnpy and os.path.isfile(npydatafile+'.npy'):
infodata=np.load(npydatafile+'.npy')[()]
else:
obsfilename=inputdir+utc+'/obs.info'
obsfile=open(obsfilename,'r')
for line in obsfile:
if line[0:6]=='CONFIG':
maptypeinfo=str(line).split(' ')[-1][:-1]
obsfile.close()
if maptypeinfo=='FAN_BEAM':
print 'This is a scan-drift observation.'
print
maptype='drift'
elif maptypeinfo=='TIED_ARRAY_FAN_BEAM':
print 'This is a tracking observation.'
print
maptype='track'
else:
print 'Type of observation not recognised! Missing "obs.info" file?'
print
exit()
break
fildatafile=inputdir+utc+'/'+'BEAM_001/'+utc+'.fil'
fildata=FilReader(fildatafile)
fildata.setNthreads(1)
infodata={'ra_rad':fildata.header['ra_rad'], 'dec_rad':fildata.header['dec_rad'], 'tobs':fildata.header['tobs'], 'nsamples':fildata.header['nsamples'], 'tsamp':fildata.header['tsamp'], 'maptype':maptype}
if savenpy:
np.save(npydatafile,infodata)
return infodata
def find_0329():
fnames = glob.glob('/data/kerrm/vlite_fb/*.fil')
good_fnames = []
for fname in fnames:
try:
fr = FilReader(fname)
#if '0329' in fr.header.source_name:
if fr.header.source_name == 'J0303+4716':
good_fnames.append(fname)
except:
print 'failed on %s.'%fname
continue
return sorted(good_fnames)
def test_invertFreq(self, filfile):
myFil = FilReader(filfile)
data = myFil.readBlock(0, 100)
outfile = myFil.invertFreq()
mynewFil = FilReader(outfile)
newdata = mynewFil.readBlock(0, 100)
np.testing.assert_equal(mynewFil.header.dtype, myFil.header.dtype)
np.testing.assert_equal(mynewFil.header.nsamples,
myFil.header.nsamples)
np.testing.assert_equal(mynewFil.header.foff, -1 * myFil.header.foff)
np.testing.assert_array_equal(data, np.flip(newdata, axis=0))
os.remove(outfile)
def fb_avg(fname,tbingoal=1000,fgoal=128,fscr=None,last_minute=False):
""" Downsample a filterbank file to appropriate resolution for plotting.
"""
fr = FilReader(fname)
if fscr is None:
fscr = int(fr.header['nchans']/fgoal)
nsamp = fr.header['nsamples']
if (nsamp*fr.header['tsamp']) < 60:
last_minute = False
if last_minute:
nsamp = int(60./fr.header['tsamp'])+1
try:
tscr = 2**int(np.log2(nsamp/tbingoal))
except OverflowError:
tscr = 1
gulp = tscr
nchan = fr.header['nchans']
nblock = nsamp/tscr
nsub = nchan/fscr
output = np.empty( (nsub, nblock) )
start = 0
if last_minute:
start = max(0,int(fr.header['nsamples']-nsamp)-1)
for nsamps,i,data in fr.readPlan(gulp,start=start,verbose=False):
if nsamps != gulp:
#print 'breaking at %d'%i
break
t = data.reshape((gulp,nchan)).transpose().astype(np.float32).mean(axis=1)
output[:,i] = np.average(t.reshape(nchan/fscr,fscr),axis=1)
freqs = np.arange(fr.header['nchans'])*fr.header['foff']+fr.header['fch1']
freqs = freqs.reshape(len(freqs)/fscr,fscr).mean(axis=1)
times = (start + np.arange(nblock)*gulp)*fr.header['tsamp']
return output,freqs,times
def split_filterbank(opts, info):
filename = info['filename']
segments = info["segments"]
output_location = info["basename"]
segment_lengths = map(float, segments.split(','))
for segment_length in segment_lengths:
obs_length_req = segment_length * 60.0
input_file = FilReader(filename)
number_samples = input_file.header.nsamples
obs_time_seconds = input_file.header.tobs
if obs_length_req > obs_time_seconds:
log.info(
"Segment length exceeds observation file length. Skipping the split.."
)
continue
log.info("calculate acceleration range for %d min" % segment_length)
pb = 10.0 * segment_length / (60.0) # 10 percent rule
amax = calculate_amax(pb, info["mc"])
info['start_acc'] = -1 * amax
info['end_acc'] = amax
sampling_time = input_file.header.tsamp
NTOREAD = int(obs_length_req / sampling_time)
number_of_fils = int(obs_time_seconds / obs_length_req)
for i in range(number_of_fils + 1):
START = i * NTOREAD
path = output_location + '_segment_%d_length_%dmin' % (
i, segment_length) + '.fil'
output_name = os.path.basename(path)[:-4]
if i == number_of_fils:
log.info("Last split")
input_file.split(number_samples - NTOREAD,
NTOREAD,
filename=path)
info["output_name"] = output_name
info["input_name"] = path
call_peasoup(info)
log.info("Segmented search done!")
else:
log.info("Split %d" % i)
log.info('START at sample: %d' % START)
input_file.split(START, NTOREAD, filename=path)
info["output_name"] = output_name
info["input_name"] = path
call_peasoup(info)
#!/usr/bin/python
from sigpyproc.Readers import FilReader
import sigpyproc.Filterbank
import matplotlib.pyplot as plt
import numpy as np
file = FilReader("new.fil")
file.downsample(tfactor=500, ffactor=1, filename="test500.fil")
def main(sampr=samprdef,inputdir=inputdirdef,utc=utcdef,numbeams=numbeamsdef,maxnsamp=maxnsampdef):
'''Main operation within the script.'''
#Some parameters about the beam.
hcres=4.4 #Angular resolution of the horizontal coordinate of the fan beam.
vcres=2. #Angular resolution of the vertical coordinate of the fan beam.
hclen=10. #Angular size (horizontal coordinate) of the sky matrix in degrees.
vclen=10. #Angular size (vertical coordinate) of the sky matrix in degrees.
hpix=100 #Number of pixels on the horizontal axis. WHAT WOULD BE THE CORRECT ONE?
vpix=100 #Number of pixels on the vertical axis.
#Load information about this observation (for example from the first beam).
datafile=inputdir+'/'+utc+'/'+'BEAM_001/'+utc+'.fil'
data=FilReader(datafile)
ra=data.header['ra_rad'] #Right ascension in rad.
dec=data.header['dec_rad'] #Declination in rad.
tobs=data.header['tobs'] #Observing time in sec.
nsamp_full=data.header['nsamples'] #Number of samples of original time series.
#Derive some quantities that will be needed.
nsamp_down=int(tobs*sampr) #Number of samples after downsampling.
fsamp=nsamp_down*1./nsamp_full #Fraction of samples before and after downsampling.
print 'Number of samples: %i ' %nsamp_down
print 'Observing time: %.4e h' %(tobs*1./3600.)
print
#raw_input('Enter to proceed')
#Define a matrix of initial sky position of objects in the sky, centered at the tracked object.
hct=ra #The horizontal coordinate of the tracked object will be the RA.
vct=dec #The vertical coordinate of the tracked object will be the DEC.
hcmin=hct-hclen*0.5 #Left border of the sky matrix.
hcmax=hct+hclen*0.5 #Right border of the sky matrix.
vcmin=vct-vclen*0.5 #Bottom border of the sky matrix.
vcmax=vct+vclen*0.5 #Top border of the sky matrix.
hc0mat=np.tile(np.linspace(hcmin,hcmax,hpix),(vpix,1)) #Vector of borders of the horizontal pixels.
vc0mat=np.tile(np.linspace(vcmin,vcmax,vpix)[::-1],(hpix,1)).T #Vector of borders of the vertical pixels. I want it to start from up to bottom.
#If hc0vec and vc0vec are not of the same size, I could create a matrix with them, so that the 'where' condition below works.
#Define a matrix that will contain the final image (in principle it can be of the size of the sky matrix).
image=np.zeros((hpix,vpix))
hcb_ini=hct-hcres*0.5 #Left border of the primary beam.
vcb_ini=vct+vcres*0.5 #Upper border of the primary beam.
hcb_width=hcres*1./numbeams #Width of each of the beams.
vcb_height=vcres #Height of each of the beams.
t=time_estimate(min(nsamp_down,maxnsamp)) #A class that prints estimated computation time.
#Load time series in small chunks (to allow for real time analysis).
for ti in xrange(min(nsamp_down,maxnsamp)):
t.display() #Just to calculate the remaining computation time.
t.increase()
lapse=ti*1./sampr #Lapse of time between samples (in sec).
gulpi=int(1./fsamp) #Number of samples (of the full data) to extract.
starti=gulpi*ti #Number of the sample to start.
hctmat,vctmat=move_oned(hc0mat,vc0mat,lapse)
#Load data from each beam.
for beami in xrange(numbeams):
datafile=inputdir+'/'+utc+'/'+'BEAM_%.3i/' %int(beami+1) + utc + '.fil' #Name of the data file.
data=FilReader(datafile) #This is defined in class Filterbank, in file Filterbank.py.
ts=data.makeTS(start=starti,gulp=gulpi,nsamps=gulpi) #Time series.
value=np.mean(ts) #Mean value of the data acquired with this beam in this lapse of time.
beam_left=hcb_ini+beami*hcb_width #Left border of this beam.
beam_right=hcb_ini+(beami+1)*hcb_width #Right border of this beam.
beam_upper=vcb_ini #Upper border of this beam.
beam_lower=vcb_ini-vcb_height #Lower border of this beam.
#Find the indices of the initial sky matrix such that after this lapse of time would fall in the current beam.
indices=np.where((beam_left<=hctmat) & (hctmat<=beam_right) & (beam_lower<=vctmat) & (vctmat<=beam_upper))
image[indices]+=value
#print image
print
py.ion()
py.matshow(image,cmap='bone',extent=[hcb_ini,hcb_ini+hclen,vcb_ini-vclen,vcb_ini])
#py.imshow(image,interpolation='nearest',origin='lower',aspect='auto',cmap='bone',extent=[hcb_ini,hcb_ini+hclen,vcb_ini-vclen,vcb_ini])
py.xlabel('Horizontal coordinate')
py.ylabel('Vertical coordinate')
#py.imshow(image.T,origin='lower')
raw_input('Next time sample')
options, args = parser.parse_args()
if (not options.fil_file):
print('Input file required.')
print(parser.print_help())
sys.exit(1)
nodsamp = options.nodsamp
fil_file = os.path.abspath(options.fil_file)
if options.csv_file: csv_file = os.path.abspath(options.csv_file)
else: csv_file = ""
if options.model: ml_model = options.model
else: ml_model = ""
origf = FilReader(fil_file)
inchans = origf.header['nchans']
inbits = origf.header['nbits']
# For 32-bit BL, downsample to 8-bit and create new file
if (nodsamp is not True):
if (inchans > 8192 or inbits > 8):
print("Running sum_fil")
fil_file = downsample(fil_file, inbits, inchans)
fname = fil_file.split("/")[-1]
print(fil_file)
time = options.time
timesig = options.timesig
freqsig = options.freqsig
chanfrac = options.chanfrac
import numpy as np
import scipy as sp
import sys
from sigpyproc.Readers import FilReader
import matplotlib.pyplot as plt
if __name__ == "__main__":
if (len(sys.argv) < 4):
print "USAGE : <.fil file> <Start samp> <Total samp> "
print "Plot Filterbank file starting from the Start samples with total samples."
sys.exit(0)
f = FilReader(sys.argv[1])
f_top = f.header['ftop']
f_bott = f.header['fbottom']
t_rsl = f.header['tsamp']
d = f.readBlock(int(sys.argv[2]), int(sys.argv[3]))
freq = np.linspace(f_top, f_bott, len(d))
np.save(sys.argv[1][:-4], d)
np.save(sys.argv[1][:-4] + '_freq', freq)
plt.pcolormesh(d)
plt.colorbar()
# plt.imshow(d,aspect='auto')
plt.savefig('DM_sweep.png')
plt.show()
def main(maptype=maptypedef, sampr=samprdef, inputdir=inputdirdef, utc=utcdef, numbeams=numbeamsdef, maxnsamp=maxnsampdef, savenpy=savenpydef, loadnpy=loadnpydef, saveonly=saveonlydef, liveplot=liveplotdef):
'''Main operation within the script.'''
#Decide if .npy input data should be loaded/saved, and create output dir for .npy data (if desired) and plots.
savenpy, samprdir, outputdir=FN.load_and_save_check(inputdir, utc, sampr, numbeams, savenpy, saveonly, maxnsamp)
#Obtain some parameters about the observation ("infodata", either as .npy or .fil file) and save it.
infodata=FN.load_and_save_infodata(inputdir, utc, sampr, savenpy, loadnpy)
ra, dec, tobs, nsamp_full, tsamp=infodata['ra_rad'], infodata['dec_rad'], infodata['tobs'], infodata['nsamples'], infodata['tsamp'] #A bit redundant, since nsamp_full=tobs/tsamp.+++
old_sampr=1./tsamp #Old number of samples per second.
ra=ra*180./np.pi
dec=dec*180./np.pi
lst_hms=commands.getstatusoutput('mopsr_getlst '+utc)[1].split(' ')[1].split(':') #Local sidereal time in HH:MM:SS.SSS.
lst=(int(lst_hms[0])*3600.+int(lst_hms[1])*60.+float(lst_hms[2]))*360./sidday #+++TRIVIAL QUESTION, IS THIS CONVERSION CORRECT?
if maptype=='drift':
#In the drift scan mode, the RA from the heather is always zero, so I calculate it from the LST.
ra=lst
#Derive or define some quantities that will be needed.
global hpix #This line is needed because to avoid an UnboundLocalError.
if hpix==0:
hpix=tobs
hsize=hres*1./numbeams #Horizontal angular size of each beam, in degrees.
vsize=vres #Vertical angular size of each beam, in degrees.
frac_sampr=int(round(old_sampr*1./sampr)) #Number of old samples per new sample.
nsamp_down=int(round(sampr*tobs))
#tvec=np.arange(0., tobs, 1./sampr) #Vector of time (in seconds).
#tvec=np.linspace(0.,tobs, nsamp_down)
tvec=np.linspace(0.,tobs, nsamp_down-1)#THE -1 IS BECAUSE DATA WILL LOSE ONE SAMPLE!!+++USE THIS FOR DRIFT
#tvec=np.linspace(0.,tobs, nsamp_down-2)#THE -2 IS BECAUSE DATA WILL LOSE TWO SAMPLES!! WHY?+++ USE THIS FOR TRACK
#NOT YET IMPLEMENTED.-----------
if maxnsamp!=True:
tvec=tvec[0:maxnsamp] #Vector of time (in seconds).
#NOT YET IMPLEMENTED.-----------
#Define the size of the final map.
ra_max=ra+hres*0.5 #Maximum RAS observed (initial RAS plus half of the fan beam size).
if maptype=='drift':
ra_min=ra-hres*0.5-(tobs*360./sidday) #Minimum RAS observed (left border of the initial fan beam minus the hour angle described after the observation). +++IS THIS CONVERSION CORRECT?
elif maptype=='track':
ra_min=ra-hres*0.5
map_ra=np.linspace(ra_min, ra_max, hpix) #Vector of RAS.
dec_max=dec+vres*0.5 #I impose the same maximum size as the horizontal coordinate.
dec_min=dec-vres*0.5
ra_vec=np.linspace(ra_min, ra_max, hpix) #Vector of RAS.
dec_vec=np.linspace(dec_min, dec_max, vpix) #Vector of DEC.
mapi=np.zeros((vpix, hpix)) #Matrix of values of the map.
f_t_map=np.zeros((numbeams, len(tvec))) #Matrix of fan beam versus time (raw data from the beams).
ras_cor=(ra_vec%360.) #Restrict RAS to the interval [0,360). This will be used for plotting and saving.
ras_mat=np.tile(ras_cor, (len(tvec), 1)).T
dec_mat=np.tile(dec_vec, (len(tvec), 1)).T
#Load data beam by beam.
if not loadnpy:
print 'Reading filterbank files (it will take longer than reading numpy files)...'
print
else:
print 'Reading numpy files if already existing...'
print
if savenpy:
print 'The input data will be saved as numpy files.'
print
t=time_estimate(numbeams) #A class that prints estimated computation time.
for beami in xrange(numbeams):
t.display() #Shows the remaining computation time.
t.increase() #Needed to calculate the remaining computation time.
#Load/save data.
npydatafile=samprdir+'BEAM_%.3i' %int(beami+1) #Name of the .npy file (to be either loaded or saved).
if loadnpy and os.path.isfile(npydatafile+'.npy'): #Load .npy data.
if saveonly: #If the .npy file already exists, move on to the next file.
continue
data=np.load(npydatafile+'.npy')
else: #Load .fil data.
datafile=inputdir+utc+'/'+'BEAM_%.3i/' %int(beami+1) + utc + '.fil' #Name of the data file.
data_raw=FilReader(datafile)
data=(data_raw.collapse()/data_raw.header.nchans).downsample(frac_sampr)*1./frac_sampr
if savenpy: #Save input data as numpy files, if necessary and if desired.
np.save(npydatafile,data)
if saveonly: #Simply load .fil data and save .npy data.
continue
if maxnsamp!=True:
data=data[0:maxnsamp]
f_t_map[beami,:]=data #Stores raw fan beam versus time data.
ras_ini=ra-hres*0.5+hsize*beami #Initial RAS of this particular beam. It will be the RAS of the left-most border of the fan beam.
for dec_i in xrange(len(dec_vec)):
dec_ini=dec_vec[dec_i] #Initial DEC of this particular beam.
print dec_ini
print ras_ini
print
if maptype=='drift':
rvec, dvec=FN.move_drift(ras_ini, dec_ini, tvec) #RAS, DEC of this beam over time.
elif maptype=='track':
rvec, dvec=FN.move_track(lst, Mol_lat, ras_ini, dec_ini, tvec) #RAS, DEC of this beam over time.
rvec_cor=rvec%360.
rvec_cor=rvec###################
print rvec
print dvec
RA_mat=np.tile(rvec_cor, (hpix, 1))
DEC_mat=np.tile(dvec, (vpix, 1))
ra_indices=np.argmin(abs(RA_mat-ras_mat), axis=0)
dec_indices=np.argmin(abs((DEC_mat+90.)-(dec_mat+90.)), axis=0) #I add 90 to assure that DEC is positive, in order to avoid negative DEC to be identified with positive DEC and viceversa.
py.ion()
py.plot(ras_ini, dec_ini, 'o')
py.plot(rvec, dvec,'.')
raw_input('next')
#sorting=rvec.argsort() #In order to use interp, the vectors must be increasing.
#mapi[dec_i, :]+=np.interp(ra_vec, rvec[sorting], data[sorting])
mapi[dec_indices,ra_indices]+=data
if liveplot:
if beami==numbeams or (beami+1)%plotevery==0:
twod_live(ras_cor, dec_vec, tvec, mapi, beami+1, f_t_map) #Plot maps as they are produced.
if not saveonly:
print 'Saving numpy output data.'
print
#RAS,DEC=np.meshgrid(ras_cor,dec_vec)
#hammer_x, hammer_y=hammer(ras_cor, dec_vec) #Hammer projected x, y coordinates.
#hammer_x, hammer_y=hammer(RAS, DEC)
#datadict={'RAS':RAS, 'DEC':DEC, 'MAP': mapi, 'Hammer_x':hammer_x, 'Hammer_y':hammer_y}
datadict={'RAS':ras_cor, 'DEC':dec_vec, 'MAP': mapi}
outputfile=samprdir.replace('input','output')[:-1]
np.save(outputfile,datadict) #Save final map vector as numpy file.
#Save 2D output plot.
print 'Saving and preparing a 2D plot.'
print
outputplotfile=samprdir.replace('data/input','plots/twod')[:-1]
inputplotfile=outputfile+'.npy' #The input file to produce the plots is the numpy file that was just created.
twod_save(ras_cor, dec_vec, mapi, tvec, f_t_map, outputplotfile)
def main(args):
inputlist = args.inlist
outdir = args.outdir
neig = args.neig
nbeams = len(inputlist)
outnames = [
os.path.join(outdir, "cleaned_" + os.path.basename(i))
for i in inputlist
]
if len(set(outnames)) < nbeams:
sys.stderr.write(
"ERR: please do not use same basename for input filterbank files\n"
)
sys.exit()
inputfils = []
for beam in inputlist:
inputfils.append(FilReader(beam))
nsamples = inputfils[0].header.nsamples
nchans = inputfils[0].header.nchans
samp_per_block = args.samps
work = np.zeros((nbeams, nchans, samp_per_block), dtype=np.float32)
if inputfils[0].header.nbits == 8:
rescale = True
for inputfil in inputfils:
inputfil.header.nbits = 32
else:
rescale = False
outfiles = [
inputfils[ifile].header.prepOutfile(i)
for ifile, i in enumerate(outnames)
]
total_n_blocks = nsamples / samp_per_block
statfile = args.statfile
if not statfile.endswith(".eig"):
statfile += ".eig"
eig_file = open(args.statfile, "w")
atexit.register(eig_file.close)
if USEPBAR:
pbar = Bar('Processing filterbanks',
max=total_n_blocks,
suffix='%(percent)d%%')
atexit.register(pbar.finish)
for iblock in range(total_n_blocks):
nzaps = []
# Extract data
for ibeam in range(nbeams):
work[ibeam] = inputfils[ibeam].readBlock(iblock * samp_per_block,
samp_per_block)
if rescale:
work[ibeam] = (work[ibeam].T -
np.median(work[ibeam], axis=1)).T
std = 1.4826 * get_mad(work[ibeam])
mask = np.abs(std) > 1e-12
work[ibeam][mask] = (work[ibeam][mask].T / std[mask]).T
# Do SVD for each data block
for ichan in range(nchans):
try:
U, W, Vt = np.linalg.svd(work[:, ichan, :],
full_matrices=False)
except Exception as e:
np.save("debug.npy", work[:, ichan, :])
sys.stderr.write("Exception caught: saving ./debug.npy\n")
raise e
neig = neigs_to_zap(W)
nzaps.append(neig)
if neig > 0:
Y = np.dot(work[:, ichan, :], Vt.T)
Vt.T[:, :neig] = 0.0
work[:, ichan, :] = np.dot(Y, Vt)
# Write output nzap file
nzaps = np.array(nzaps, dtype=np.int32)
nzaps.tofile(eig_file)
# Write output
for ibeam, outfile in enumerate(outfiles):
if args.resc_out:
work[ibeam] = (work[ibeam].T -
np.median(work[ibeam], axis=1)).T
std = 1.4826 * get_mad(work[ibeam])
mask = ((np.abs(std) > 1e-12) & (nzaps > 0))
work[ibeam][mask] = (work[ibeam][mask].T / std[mask]).T
work[ibeam].T.astype("float32").tofile(outfile)
if USEPBAR:
pbar.next()
if USEPBAR:
pbar.finish()
'names': ('snr', 'time', 'samp_idx', 'dm',
'filter', 'prim_beam'),
'formats': ('f4', 'f4', 'i4', 'f4', 'i4', 'i4')
})
#uGMRT
#fl = 300
#fh = 500
#FAST
fl = 1100
fh = 1500
noplot = 0
tint = 0.000163
Ttot = 80 # Total length of the file
kill_time_range = []
kill_chans = []
nchan = 2048
source_name = "Fake"
f = FilReader(fil_file)
nchan = f.header['nchans']
fch1 = f.header['fch1']
foff = f.header['foff']
tint = f.header['tsamp']
Ttot = f.header['tobs']
fh = fch1
fl = fch1 + (foff * nchan)
source_name = f.header['source_name']
extractPlotCand(fil_file, frb_cands, noplot, fl, fh, tint, Ttot,
kill_time_range, kill_chans, source_name, nchan)
import numpy as np
import sys
import glob
from sigpyproc.Readers import FilReader
listx = []
listy = []
obsdir = "2021*"
#for i, filename in enumerate(sys.argv[1:]):
for filename in glob.glob(obsdir):
#print(filename)
filx = FilReader(filename + "/" + "2kB/" + filename + "_x.fil")
#print(filename+"/"+"2kB/"+filename+"_x.fil")
blockx = filx.readBlock(0, filx.header.nsamples)[1660:1740]
fily = FilReader(filename + "/" + "2kB/" + filename + "_y.fil")
blocky = fily.readBlock(0, filx.header.nsamples)[1660:1740]
xmean = blockx.mean()
ymean = blocky.mean()
#print(xmean)
listx.append(xmean)
listy.append(ymean)
cands_snr = cands[:, 0]
cands_dm = cands[:, 5]
cands_ind = cands[:, 1]
cands_width = cands[:, 3]
cands2 = np.genfromtxt("/home/user/cand_times_sync_od/heimdall_2.cand")
cands2_snr = cands2[:, 0]
cands2_dm = cands2[:, 5]
cands2_ind = cands2[:, 1]
cands2_width = cands2[:, 3]
#ovro_int = 20355632#112793376
#gdscc_int = 103012504 #20355632
#ovro = FilReader("/home/user/coincidences/candidate_%s.fil" %ovro_int)
#gdscc = FilReader("/home/user/coincidences/candidate_%s.fil" %gdscc_int)
ovro = FilReader("candidate_ovro_200428.fil")
gdscc = FilReader("candidate_gdscc_200428.fil")
gdscc_arr = gdscc.readBlock(0, gdscc.header['nsamples'])
ovro_arr = ovro.readBlock(0, ovro.header['nsamples'])
ovro_ind = np.where(cands_ind == ovro_int)[0][0].astype(int)
gdscc_ind = np.where(cands2_ind == gdscc_int)[0][0].astype(int)
av_window = int(np.min([2**cands2_width[gdscc_ind], 2**cands_width[ovro_ind]]))
#av_window = 26
dm = 333.3 #cands_dm[ovro_ind]/2. + cands2_dm[gdscc_ind]/2.
gdscc_arr_conv = snd.uniform_filter1d(gdscc_arr, av_window, axis=1)
ovro_arr_conv = snd.uniform_filter1d(ovro_arr, av_window, axis=1)
fs = np.linspace(1280, 1530, 2048)
gdscc_dedisp = dedisperse(gdscc_arr_conv, dm, 0.000065536, fs)
ovro_dedisp = dedisperse(ovro_arr_conv, dm, 0.000065536, fs)
def main(sampr=samprdef, inputdir=inputdirdef, utc=utcdef, numbeams=numbeamsdef, maxnsamp=maxnsampdef, savenpy=savenpydef, loadnpy=loadnpydef, saveonly=saveonlydef, liveplot=liveplotdef):
'''Main operation within the script.'''
#Decide if .npy input data should be loaded/saved, and create output dir for .npy data (if desired) and plots.
savenpy, samprdir, outputdir=FN.load_and_save_check(inputdir, utc, sampr, numbeams, savenpy, saveonly, maxnsamp)
#Obtain some parameters about the observation ("infodata", either as .npy or .fil file) and save it.
infodata=FN.load_and_save_infodata(inputdir, utc, sampr, savenpy, loadnpy)
ra, dec, tobs, nsamp_full, tsamp, maptype=infodata['ra_rad'], infodata['dec_rad'], infodata['tobs'], infodata['nsamples'], infodata['tsamp'], infodata['maptype'] #A bit redundant, since nsamp_full=tobs/tsamp.+++
old_sampr=1./tsamp #Old number of samples per second.
ra=ra*180./np.pi
dec=dec*180./np.pi
lst_hms=commands.getstatusoutput('mopsr_getlst '+utc)[1].split(' ')[1].split(':') #Local sidereal time in HH:MM:SS.SSS.
lst=(int(lst_hms[0])*3600.+int(lst_hms[1])*60.+float(lst_hms[2]))*360./sidday
if maptype=='drift':
#In the drift scan mode, the RA from the heather is always zero, so I calculate it from the LST.
ra=lst
#Derive or define some quantities that will be needed.
global hpix #This line is needed because to avoid an UnboundLocalError.
if hpix==0:
hpix=tobs
hsize=hres*1./numbeams #Horizontal angular size of each beam, in degrees.
vsize=vres #Vertical angular size of each beam, in degrees.
frac_sampr=int(round(old_sampr*1./sampr)) #Number of old samples per new sample.
nsamp_down=int(round(sampr*tobs))
#tvec=np.arange(0., tobs, 1./sampr) #Vector of time (in seconds).
#tvec=np.linspace(0.,tobs, nsamp_down)
tvec=np.linspace(0.,tobs, nsamp_down-1)#THE -1 IS BECAUSE DATA WILL LOSE ONE SAMPLE!!+++USE THIS FOR DRIFT
#tvec=np.linspace(0.,tobs, nsamp_down-2)#THE -2 IS BECAUSE DATA WILL LOSE TWO SAMPLES!! WHY?+++ USE THIS FOR TRACK
if maxnsamp!=True:
tvec=tvec[0:maxnsamp] #Vector of time (in seconds).
#Define the size of the final map.
ra_max=ra+hres*0.5 #Maximum RAS observed (initial RAS plus half of the fan beam size).
if maptype=='drift':
ra_min=ra-hres*0.5-(tobs*360./sidday) #Minimum RAS observed (left border of the initial fan beam minus the hour angle described after the observation). +++IS THIS CONVERSION CORRECT?
elif maptype=='track':
ra_min=ra-hres*0.5
map_ra=np.linspace(ra_min, ra_max, hpix) #Vector of RAS.
dec_max=dec+vres*0.5 #I impose the same maximum size as the horizontal coordinate.
dec_min=dec-vres*0.5
ra_vec=np.linspace(ra_min, ra_max, hpix) #Vector of RAS.
dec_vec=np.linspace(dec_min, dec_max, vpix) #Vector of DEC.
mapi=np.zeros((vpix, hpix)) #Matrix of values of the map.
f_t_map=np.zeros((numbeams, len(tvec))) #Matrix of fan beam versus time (raw data from the beams).
ras_cor=(ra_vec%360.) #Restrict RAS to the interval [0,360). This will be used for plotting and saving.
ras_mat=np.tile(ras_cor, (len(tvec), 1)).T
dec_mat=np.tile(dec_vec, (len(tvec), 1)).T
#Load data beam by beam.
if not loadnpy:
print 'Reading filterbank files (it will take longer than reading numpy files)...'
print
else:
print 'Reading numpy files if already existing...'
print
if savenpy:
print 'The input data will be saved as numpy files.'
print
t=time_estimate(numbeams) #A class that prints estimated computation time.
for beami in xrange(numbeams):
t.display() #Shows the remaining computation time.
t.increase() #Needed to calculate the remaining computation time.
#Load/save data.
npydatafile=samprdir+'BEAM_%.3i' %int(beami+1) #Name of the .npy file (to be either loaded or saved).
if loadnpy and os.path.isfile(npydatafile+'.npy'): #Load .npy data.
if saveonly: #If the .npy file already exists, move on to the next file.
continue
data=np.load(npydatafile+'.npy')
else: #Load .fil data.
datafile=inputdir+utc+'/'+'BEAM_%.3i/' %int(beami+1) + utc + '.fil' #Name of the data file.
data_raw=FilReader(datafile)
data=(data_raw.collapse()/data_raw.header.nchans).downsample(frac_sampr)*1./frac_sampr
if savenpy: #Save input data as numpy files, if necessary and if desired.
np.save(npydatafile,data)
if saveonly: #Simply load .fil data and save .npy data.
continue
if maxnsamp!=True:
data=data[0:maxnsamp]
f_t_map[beami,:]=data #Stores raw fan beam versus time data.
ras_ini=ra-hres*0.5+hsize*beami #Initial RAS of this particular beam. It will be the RAS of the left-most border of the fan beam.
'''
print np.shape(f_t_map)
mapi=f_t_map.copy()
lefti=np.array([])
righti=np.zeros(len(tvec))
newmapi=np.zeros((numbeams, len(tvec)+len(righti)))
shifti=1
for fi in xrange(numbeams):
lefti=np.zeros(fi)
righti=np.zeros()
newmapi[fi]=np.hstack((lefti,mapi[fi],righti))
lefti=np.concatentae
'''
return tvec, f_t_map
'''
def get_readers(fil_files, nbeams=16):
"""Load blimpy Waterfall objects for filterbank file reading"""
fils = []
for f in sorted(fil_files)[:nbeams]:
fils.append(FilReader(f))
return fils
namedir='/data/mopsr/archives/'+utc+'/'
#--------------------------
#Define a function that describes the evolution of objects in the sky.
def move(hc0,vc0,t):
'''Returns the horizontal and verical coordinates hc,vc (whatever coordinates they are) in deg, of an object in the sky with initial coordinates hc0,vc0, after a time t (in seconds). For the moment I just assume that objects move in circles around the tracked object with a period of 1 day (as if we were tracking the pole).'''
ang=2.*np.pi*t*1./(24*3600.)
hc=hc0*np.cos(ang)-vc0*np.sin(ang)
vc=hc0*np.sin(ang)+vc0*np.cos(ang)
return hc,vc
#Load information about this observation (for example from the first beam).
datafile=namedir+'BEAM_001/'+utc+'.fil'
data=FilReader(datafile)
ra=data.header['ra_rad'] #Right ascension in rad.
dec=data.header['dec_rad'] #Declination in rad.
tobs=data.header['tobs'] #Observing time in sec.
nsamp_full=data.header['nsamples'] #Number of samples of original time series.
#Derive some quantities that will be needed.
nsamp_down=int(tobs*sampr) #Number of samples after downsampling.
fsamp=nsamp_down*1./nsamp_full #Fraction of samples before and after downsampling.
print 'Number of samples: %i ' %nsamp_down
print 'Observing time: %.4e h' %(tobs*1./3600.)
print
#raw_input('Enter to proceed')
#Define a matrix of initial sky position of objects in the sky, centered at the tracked object.
hct=ra #The horizontal coordinate of the tracked object will be the RA.
def get_ftmap(sampr=samprdef, inputdir=inputdirdef, utc=utcdef, first_beam=first_beam_def, last_beam=last_beam_def, first_tsamp=first_tsamp_def, last_tsamp=last_tsamp_def, savenpy=savenpydef, loadnpy=loadnpydef, saveonly=saveonlydef, liveplot=liveplotdef):
'''Main operation within the script.'''
#Decide if .npy input data should be loaded/saved, and create output dir for .npy data (if desired) and plots.
numbeams=last_beam-first_beam+1
beamsvec=np.arange(first_beam, last_beam+1)
savenpy, samprdir, outputdir=load_and_save_check(inputdir, utc, sampr, savenpy, saveonly, first_tsamp, last_tsamp)
#Obtain some parameters about the observation ("infodata", either as .npy or .fil file) and save it.
infodata=load_and_save_infodata(inputdir, utc, sampr, savenpy, loadnpy)
ra, dec, tobs, nsamp_full, tsamp, maptype=infodata['ra_rad'], infodata['dec_rad'], infodata['tobs'], infodata['nsamples'], infodata['tsamp'], infodata['maptype'] #A bit redundant, since nsamp_full=tobs/tsamp.+++
old_sampr=1./tsamp #Old number of samples per second.
ra=ra*180./np.pi
dec=dec*180./np.pi
lst_hms=commands.getstatusoutput('mopsr_getlst '+utc)[1].split(' ')[1].split(':') #Local sidereal time in HH:MM:SS.SSS.
lst=(int(lst_hms[0])*3600.+int(lst_hms[1])*60.+float(lst_hms[2]))*360./sidday
if maptype=='drift':
#In the drift scan mode, the RA from the heather is always zero, so I calculate it from the LST.
ra=lst
#Derive or define some quantities that will be needed.
frac_sampr=int(round(old_sampr*1./sampr)) #Number of old samples per new sample.
nsamp_down=int(round(sampr*tobs)) #This is the number of samples that the data should have (if no samples were lost after downsampling).
#tvec=np.arange(0., tobs, 1./sampr) #Vector of time (in seconds).
#tvec=np.linspace(0.,tobs, nsamp_down)
#Load data beam by beam.
if not loadnpy:
print 'Reading filterbank files (it will take longer than reading numpy files)...'
print
else:
print 'Reading numpy files if already existing...'
print
if savenpy:
print 'The input data will be saved as numpy files.'
print
t=time_estimate(numbeams) #A class that prints estimated computation time.
for beami in beamsvec:
t.display() #Shows the remaining computation time.
t.increase() #Needed to calculate the remaining computation time.
#Load/save data.
npydatafile=samprdir+'BEAM_%.3i' %int(beami) #Name of the .npy file (to be either loaded or saved).
if loadnpy and os.path.isfile(npydatafile+'.npy'): #Load .npy data.
if saveonly: #If the .npy file already exists, move on to the next file.
continue
data=np.load(npydatafile+'.npy')
else: #Load .fil data.
datafile=inputdir+utc+'/'+'BEAM_%.3i/' %int(beami) + utc + '.fil' #Name of the data file.
data_raw=FilReader(datafile)
data_raw.setNthreads(1)
data=(data_raw.collapse()/data_raw.header.nchans).downsample(frac_sampr)*1./frac_sampr
if savenpy: #Save input data as numpy files, if necessary and if desired.
np.save(npydatafile,data)
if saveonly: #Simply load .fil data and save .npy data.
continue
if beami==beamsvec[0]:
#if abs(len(data)-nsamp_down)>2:
# print 'Too many samples are missing when downsampling! Why?'
# print
# exit()
tvec=np.linspace(0., tobs, nsamp_down)[0:len(data)]
ftmap=np.zeros((totalnumbeams, len(tvec))) #Matrix of fan beam versus time (raw data from the beams).
if first_tsamp>1 and last_tsamp==True:
tvec=tvec[first_tsamp-1:]
ftmap=ftmap[:,first_tsamp-1:]
elif first_tsamp==1 and last_tsamp!=True:
tvec=tvec[:(last_tsamp-1)]
ftmap=ftmap[:,:(last_tsamp-1)]
elif first_tsamp==1 and last_tsamp==True:
pass
else:
tvec=tvec[(first_tsamp-1):(last_tsamp-1)]
ftmap=ftmap[:,(first_tsamp-1):(last_tsamp-1)]
if first_tsamp>1 and last_tsamp==True:
data=data[first_tsamp-1:]
elif first_tsamp==1 and last_tsamp!=True:
data=data[:(last_tsamp-1)]
elif first_tsamp==1 and last_tsamp==True:
pass
else:
data=data[(first_tsamp-1):(last_tsamp-1)]
ftmap[beami-1,:]=data #Stores raw fan beam versus time data.
return tvec, ftmap, dec
cmdline.add_option('--block', type = 'string', dest = 'block', help = 'Select multiple spectra to average and plot.')
cmdline.add_option('--dynamic', type = 'string', dest = 'dynamic', help = 'Select multiple spectra to plot dynamic spectrum.')
cmdline.add_option('--timeseries', type = 'string', dest = 'timeseries', help = 'Select single channel and range of spectra to plot the timeseries.')
cmdline.add_option('--bandpass', dest = 'bandpass', action = "store_true", help = 'Plot total bandpass.')
cmdline.add_option('--save', dest = 'savePlot', action = "store_true", help = 'Save plot to .png file instead of interacive plotting.')
# reading cmd options
(opts, args) = cmdline.parse_args()
if not opts.file:
cmdline.print_usage()
sys.exit(0)
# Define constans and variables.
filterbankFilename = opts.file
filterbankFile = FilReader(filterbankFilename) # read filterbank file
#print "File %s opened." % (filterbankFilename)
numberChannels = filterbankFile.header.nchans # get number of spectral channels
numberSpectra = filterbankFile.header.nsamples # get number of spectra/samples
# Only one option for plotting is allowed.
if (opts.spectrum and opts.dynamic) or\
(opts.spectrum and opts.block) or\
(opts.spectrum and opts.bandpass) or\
(opts.spectrum and opts.timeseries) or\
(opts.dynamic and opts.block) or\
(opts.dynamic and opts.bandpass) or\
(opts.dynamic and opts.timeseries) or\
(opts.block and opts.bandpass) or\
(opts.block and opts.timeseries):
print "Only one option allowed."
fbot: float, bottom frequency
ftop: float, top frequency
'''
C = 4.148908e6 # (ms)
t_delay = C * DM_range[1] * (fbot**-2 - ftop**-2)
#Time delay betweent top and bottom frequency with max DM value. (ms)
return t_delay
if __name__ == '__main__':
ang = 0.3357 #np.float(sys.argv[1])
DM_range = [50, 500] #np.float(sys.argv[1])
f_dir = '/data0/FRB_data/parameter_test/'
f_name = 'Benchmark_2048_1ms_5s_snr_5_dm_500.fil'
f = FilReader(f_dir + f_name)
hdr = f.header
ftop = hdr['ftop']
fbot = hdr['fbottom']
nch = hdr['nchans']
t_rsl = hdr['tsamp'] * 1000. # unit (ms)
N_s_chunck = nch #* 4
# t_len = 250/1000.
freq = np.linspace(ftop, fbot, nch)
#np.save(f_dir+'freq_s.npy',freq)
fy = freq**-2
nbin = nch
T = N_s_chunck * t_rsl
Nsamp = time_delay(DM_range, fbot, ftop) / t_rsl
Nchunck = time_delay(DM_range, fbot, ftop) / t_rsl / N_s_chunck
dm = DM_calculate(fy, ang, nbin, T)
#PWR = [-3, 0, 3, 6, 9 ,12]
PWR = [10]
BASEDIR = "/mnt/buf0/obs"
outdir = "/home/sonata/RF_signal_generator/june24/same_RF_freq/"
for iutc, utc in enumerate(UTCs):
print (utc)
for ant in os.listdir(os.path.join(BASEDIR, utc)):
if len(ant) != 2:
continue
print(ant)
filx = glob.glob(os.path.join(BASEDIR, utc, ant, "*_x.fil"))[0]
fil = FilReader(filx)
block = fil.readBlock(0, fil.header.nsamples)
N = 200000
fig, ax = plt.subplots(nrows=3, sharex=True, figsize=(13,10))
tsx = 10*np.log10(block[:-1].sum(axis=0))
m = np.median(tsx)
up = np.median(tsx[tsx > m])
down = np.median(tsx[tsx < m])
ax[0].plot(np.arange(block.shape[1])*fil.header.tsamp, tsx,
label="XX")
ax[0].hlines([up,down], 0, (block.shape[1])*fil.header.tsamp, ls="--",
label="XX compression: %.1f dB" %(up - down))
import numpy as np
import matplotlib.pyplot as plt
from sigpyproc.Readers import FilReader
import time
t1 = time.time()
f_dir = '../data/'
f_name = 'out.fil'
f = FilReader(f_dir + f_name)
#dm = np.linspace(1,2000,1000)
DM = 100
snr = []
dd = f.dedisperse(DM)
snr = (dd.max() - dd.mean()) / dd.std()
t2 = time.time()
t_tol = t2 - t1
print "Time Total:", t_tol, " s = ", t_tol / 60., " m"
plt.title("SNR:" + str(snr))
plt.xlabel("SRC:" + f.header['source_name'])
plt.plot(dd)
plt.show()
import numpy as np
from sigpyproc.Readers import FilReader
import sys
from DM_calculate import time_delay
import matplotlib.pyplot as plt
if __name__ == '__main__':
DM_range = [50, 100]
f_dir = '../data/'
f_name = 'noise_level_test.fil'
f = FilReader(f_dir + f_name)
data = f.dedisperse(100)
std = data[:2000].std()
mean = data[:2000].mean()
# snr = (data.max()-mean)/std
snr = (data.max() - data.mean()) / data.std()
lo = np.where(data == data.max())
flag = 0
for i in range(len(lo[0])):
if 2490 < lo[i][0] < 2505:
flag = 1
print 'True'
else:
print 'False'
print snr
plt.plot(data)
plt.show()
exit(1)
hdr = f.header
ftop = hdr['ftop']
nsample = hdr['nsamples']
#bandpass=downsampled_file.bandpass()
#bandpass.toFile("
from sigpyproc.Readers import FilReader
import matplotlib.pyplot as plt
import numpy as np
#compute the bandpass and frequency of each channel
my_bpass = FilReader("100D.32.fil").bandpass()
freqs = np.linspace(my_bpass.header.ftop, my_bpass.header.fbottom,
my_bpass.size)
#plot the result
plt.plot(freqs, my_bpass)
plt.title("Bandpass")
plt.xlabel("Observing frequency (MHz)")
plt.ylabel("Power (Arbitrary units)")
plt.show()
plt.savefig("new_bandpass.png")
def main():
parser = argparse.ArgumentParser(description='Sum filterbank'\
'files')
parser.add_argument('-c',
dest='candfile',
type=str,
help="heimdall's candidate file")
parser.add_argument('-f', dest='filfile', type=str, help="filterbank file")
parser.add_argument('-s',
dest='show',
action='store_true',
help='iteratively show, rather than save, plots')
parser.add_argument('-u',
dest='utc',
type=str,
default="",
help='utc name (to append to png name')
parser.add_argument(
'-n',
dest='maxcand',
type=int,
help='Maximum number of candidates to plot [default: %i]' % (MAXN),
default=MAXN)
parser.add_argument('-z',
'--zap_chans',
dest='zap_chans',
action='append',
nargs=2,
type=int,
help='channels to zap')
args = parser.parse_args()
all_candidates = pd.read_table(args.candfile,
index_col=False,
names=COL_NAMES)
# hack to get the sampling time
tsamp = (all_candidates.tsec / all_candidates.tsample).mean()
box_thresh = int(np.log2(W_THRESH / tsamp))
all_candidates = all_candidates[(all_candidates.SNR > SNR_THRESH)
& (all_candidates.boxcar <= box_thresh)
& (all_candidates.DM > DM_THRESH)]
basedir = os.path.dirname(args.candfile)
candsdir = os.path.join(basedir, "candidates")
if not args.show:
if not os.path.isdir(candsdir):
os.mkdir(candsdir)
fil = FilReader(args.filfile)
nskip = int(0.3 / fil.header.tsamp)
print("ncands: %i" % all_candidates.shape[0])
pbar = tqdm(total=all_candidates.shape[0])
if all_candidates.shape[0] > args.maxcand:
print ("WARNING: number of candidates (%i) is greater than"\
"the maximum allowed (%i)" %(all_candidates.shape[0],
args.maxcand))
if not os.path.exists("cands.warning"):
open("cands.warning", 'a').close()
icand = 0
for ind, cand in all_candidates.iterrows():
dm_smear_sec = (8.3 * fil.header.bandwidth*1e-3 *\
cand.DM * (fil.header.fcenter*1e-3)**(-3))*1e-3
dm_smear_nsamp = int(dm_smear_sec / fil.header.tsamp)
istart = int(cand.tsample) - nskip
istart = 0 if istart < 0 else istart
block = fil.readBlock(istart, dm_smear_nsamp + 2 * nskip)
# zap unwanted channels
if args.zap_chans:
for low_freq, high_freq in args.zap_chans:
block[low_freq:high_freq] = 0. #np.nan
disp_org = block.dedisperse(cand.DM)
if cand.SNR < 10:
ffactor = 32
elif cand.SNR < 15:
ffactor = 16
elif cand.SNR < 20:
ffactor = 8
else:
ffactor = 1
tfactor = min(8, int(2**cand.boxcar))
disp = block_reduce(disp_org, (ffactor, tfactor),
func=np.sum,
cval=np.mean(disp_org))
matplotlib.use("Agg")
fig = plt.figure(0)
fig.suptitle("SNR = %.2f, DM = %.2f, boxcar width = %i" %
(cand.SNR, cand.DM, cand.boxcar))
gs = gridspec.GridSpec(2, 1)
gs.update(hspace=0)
ax1 = fig.add_subplot(gs[0])
ax2 = fig.add_subplot(gs[1], sharex=ax1)
ts = disp.sum(axis=0)
x = np.arange(
int(cand.tsample) - nskip,
int(cand.tsample) - nskip + len(ts) * tfactor, tfactor)
ax1.plot(x * fil.header.tsamp, ts)
ax1.get_xaxis().set_visible(False)
ylim = ax1.get_ylim()
ax1.vlines(np.array([cand.tsample - (2**cand.boxcar/2),
cand.tsample + (2**cand.boxcar/2)])*fil.header.tsamp -\
tfactor/2*fil.header.tsamp,
ymin=ylim[0], ymax=ylim[1], lw=1, ls='--')
if args.zap_chans:
for low_freq, high_freq in args.zap_chans:
disp_org[low_freq:high_freq] = np.nan
disp_org = block_reduce(disp_org, (ffactor, tfactor),
func=np.sum,
cval=np.mean(disp_org))
ax2.imshow(disp_org,
interpolation='nearest',
aspect='auto',
extent=[
x[0] * fil.header.tsamp, x[-1] * fil.header.tsamp,
fil.header.fbottom, fil.header.ftop
])
ax2.set_ylabel("Frequency (MHz)")
ax2.set_xlabel("Time since obs_start")
if args.show:
plt.show()
else:
basefigname = "tsamp_%i_dm_%.2f.png" % (cand.tsample, cand.DM)
if args.utc:
basefigname = args.utc + "_" + basefigname
figname = os.path.join(candsdir, basefigname)
plt.savefig(figname, fmt='png', bbox_inches='tight')
plt.close(fig)
pbar.update(1)
if icand > args.maxcand:
sys.exit(-1)
icand += 1
pbar.close()
from scipy import fftpack
from sigpyproc.Readers import FilReader
import matplotlib.pyplot as plt
import numpy as np
import sys
fil = FilReader(sys.argv[1])
nsamps2read = min(100000, fil.header.nsamples)
block = fil.readBlock(0, nsamps2read)
ts = np.flipud(block)[700:1800].sum(axis=0)
f_s = 1/fil.header.tsamp
X = fftpack.fft(ts)
freqs = fftpack.fftfreq(len(ts))*f_s
plt.figure()
plt.imshow(10*np.log10(block), interpolation='nearest', aspect='auto')
plt.figure()
plt.plot(10*np.log10(block.sum(axis=1)), label="Mean")
plt.plot(10*np.log10(block.std(axis=1)), label="STD")
plt.legend()
plt.figure()
plt.plot(np.linspace(0, nsamps2read*fil.header.tsamp, nsamps2read),
10*np.log10(ts))
plt.xlabel("time (seconds)")
plt.ylabel("dB")
plt.figure()
plt.plot(freqs, np.abs(X)/np.max(np.abs(X)))
hd_cands = np.genfromtxt('/home/cbochenek/heimdall.cand', dtype=str)
boxcar_filters = np.array([1, 2, 3, 5, 10])
SNR = hd_cands[:, 0].astype(float)
inds = hd_cands[:, 1].astype(int)
cand_time = hd_cands[:, 2].astype(float)
cand_sample_width = boxcar_filters[hd_cands[:, 3].astype(int)]
cand_DM_trial_n = hd_cands[:, 4].astype(int)
cand_DM = hd_cands[:, 5].astype(float)
cand_members = hd_cands[:, 6].astype(int)
cand_data = hd_cands[:, 7]
good_inds = np.where(SNR >= 8.7)[0]
with PdfPages("candidates.pdf") as pdf:
for ind in good_inds:
files = FilReader("%s" % (cand_data[ind]))
nsamples = files.header['nsamples']
data_arr = files.readBlock(inds[ind] - 150, 300)
tsamp = files.header['tsamp']
chans = np.array(range(files.header['nchans']))
bandwidth = files.header['fch1'] - files.header['fbottom']
freqs = files.header['fbottom'] + chans / float(
files.header['nchans']) * bandwidth
smooth_bins = int(0.066 / tsamp)
plt.figure(ind, figsize=(20, 10))
gs = gridspec.GridSpec(nrows=6, ncols=1)
ax0 = plt.subplot(gs[0])
ax1 = plt.subplot(gs[1])
ax2 = plt.subplot(gs[2])
ax3 = plt.subplot(gs[3])
ax4 = plt.subplot(gs[4])
from sigpyproc.Readers import FilReader
import numpy as np
import matplotlib.pyplot as plt
f = FilReader('../data/2011-02-20-01:52:19.fil')
data = f.readBlock(2 * 1024 * 20, 1024 * 20)
d_l = []
fft_l = []
for i in range(10):
d_l.append(data[:, i * 2048:(i + 1) * 2048])
d_l = np.array(d_l)
for i in range(10):
fft = np.fft.fft2(d_l[i])
fft = np.fft.fftshift(fft) / fft.size
fft = fft[1:fft.shape[0] / 2 + 1,
fft.shape[1] / 2:fft.shape[1] / 2 + fft.shape[0] / 2]
fft[-1, :] = 0
fft[:, 0] = 0
fft_l.append(abs(fft))
for i in range(10):
plt.pcolormesh(fft_l[i])
plt.colorbar()
plt.show()
from sys import argv
from sigpyproc.Readers import FilReader
fil = FilReader(argv[1])
tim = fil.dedisperse(float(argv[2])).toFile("%s_DM%s.tim"%(fil.header.basename,argv[2]))
