def begin_main(values):
format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
if values.verbose:
logging.basicConfig(level=logging.DEBUG, format=format)
else:
logging.basicConfig(level=logging.INFO, format=format)
try:
filterbank = glob.glob(values.file)[0]
fil_obj = pysigproc.SigprocFile(filterbank)
freqs = fil_obj.chan_freqs
df = mjd2influx(fil_obj.tstart)
if df is not None:
all_data_valid = df['DATA_VALID'].sum()
if all_data_valid < 33:
logging.info(
'Less than 33s of data is valid, skipping this file')
_cmdline(f'rm {filterbank}')
return None
else:
es = Elasticsearch([{'host': 'localhost', 'port': 9200}])
tel_df_to_es(es, df, filterbank)
logging.info(f'{100*all_data_valid/len(df)}% data valid')
bandpass = fil_obj.bandpass
chan_nos = np.arange(0, bandpass.shape[0])
mask = mask_finder(bandpass,
values.sigma) #chan_nos,values.nchans,values.sigma)
bad_chans = chan_nos[mask]
out_chans = []
for chans in bad_chans:
out_chans.append('-zap_chans')
out_chans.append(chans)
out_chans.append(chans + 1)
filterbank_name = filterbank.split('/')[-1].split('.')[0]
out_dir = '/ldata/trunk/{}/'.format(filterbank_name)
_cmdline('mkdir -p {}'.format(out_dir))
_cmdline(f'mv {filterbank} {out_dir}/')
new_fil_path = f'{out_dir}{filterbank_name}.fil'
heimdall_command='heimdall -nsamps_gulp 524288 -dm 10 10000 -boxcar_max 128 -cand_sep_dm_trial 200 -cand_sep_time 128 -cand_sep_filter 3 ' \
+ ''.join(str(x)+' ' for x in out_chans) + ' -output_dir {}'.format(out_dir) + ' -f {}'.format(new_fil_path)
logging.info(f'Running {heimdall_command}')
p1 = Process(target=send2gpuQ, args=[heimdall_command])
p1.start()
p2 = Process(target=write_and_plot,
args=[mask, chan_nos, freqs, bandpass, out_dir])
p2.start()
p1.join()
p2.join()
send2Q('stage02_queue', f'/ldata/trunk/{filterbank_name}')
except IndexError:
pass
return None
def filwriter(Q, Q2, Qinit):
'''
Innocent filterbank writer.
'''
while True:
global f
global filename
try:
spectra_dump = Q.pop()
spectra_time = Q2.pop()
counter_val = next(pop_counter)
if counter_val == 0:
mjd = Time(spectra_time).mjd
# create the filterbank
filobj = pysigproc.SigprocFile()
filename = '/sdata/filterbank/' + str(
spectra_time.strftime("%Y_%m_%d_%H_%M_%S")) + '.fil'
logging.info(f'writing in {filename}')
filobj = fu.make_header(filobj)
filobj.tstart = mjd
fu.write_header(filobj, filename)
f = open(filename, 'r+b')
fu.append_spectra(f, spectra_dump)
elif counter_val == 15:
Q.appendleft(spectra_dump)
Q2.appendleft(spectra_time)
fu.append_spectra(f, spectra_dump)
f.close()
Qinit.appendleft(filename)
else:
fu.append_spectra(f, spectra_dump)
except IndexError:
time.sleep(11)
def begin_main(values):
cand_lists = []
for files in glob.glob(values.files):
try:
cand_lists.append(pd.read_csv(files, header=None,\
comment='#',delim_whitespace=True,\
names=['snr','sample','tcand','width','dm_trial','dm',\
'members','start_sample','end_sample']))
except EmptyDataError:
pass
if len(cand_lists) != 0:
cand_df = pd.concat(cand_lists, ignore_index=True)
mask_thresholds = (cand_df['snr']>= values.snr) & (cand_df['members']>= values.members) & (cand_df['tcand'] <= 504) &\
(cand_df['dm'] > values.dm) & (cand_df['width'] <= values.width)
base_work_dir = '/ldata/trunk'
folder = files.split('/')[-2]
fil_file = glob.glob(f'{base_work_dir}/{folder}/*.fil')[0]
fil_obj = pysigproc.SigprocFile(fp=fil_file)
mjd = fil_obj.tstart
if len(cand_df[mask_thresholds]) > 0:
influx_df = mjd2influx(mjd)
cand_df.loc[:,
'cand_mjd'] = mjd + (cand_df['tcand'] / (60 * 60 * 24))
extended_df = extend_df(influx_df, cand_df[mask_thresholds])
mask = extended_df['cand_valid'] == 1
cand_df_masked = extended_df[mask]
else:
mask = mask_thresholds
cand_df_masked = cand_df[mask]
logging.info(f'Got {len(cand_df)} cands pre-filtering')
send_msg_2_slack(
f'Got {len(cand_df)} cands pre-filtering at MJD {mjd}')
logging.info(f'Got {len(cand_df_masked)} cands post-filtering')
send_msg_2_slack(
f'Got {len(cand_df_masked)} cands post-filtering at MJD {mjd}')
if len(cand_df) != 0:
plt.title(folder)
ax=cand_df.plot('tcand','dm',kind='scatter',\
marker='.',c='none', edgecolors='k',s=cand_df.snr/1.0,alpha=0.5)
if len(cand_df_masked) != 0:
kill_mask = glob.glob(f'{base_work_dir}/{folder}/*.flag')
cand_df_masked.loc[:, 'fil_file'] = fil_file
cand_df_masked.loc[:, 'kill_mask'] = kill_mask[0]
cand_df_masked.plot('tcand','dm',kind='scatter',\
marker='D',c='none', edgecolors='r',s=cand_df_masked.snr/1.0,ax=ax)
cand_df_masked.to_csv(
f'{base_work_dir}/{folder}/{folder}.csv',
index=False,
header=True,
columns=[
'fil_file', 'snr', 'tcand', 'dm', 'width', 'kill_mask',
'cand_mjd', 'ATAZIND', 'ATELIND', 'AZCORR', 'ELCORR',
'RA_deg', 'DEC_deg', 'RA_drift', 'DEC_drift',
'SCPROJID', 'WEBCNTRL', 'IFV1TNCI', 'ATRXOCTA',
'cand_gl', 'cand_gb', 'cand_ne2001', 'cand_ymw16',
'cand_valid'
])
plt.xlabel('Time (s)')
plt.ylabel('DM (pc/cc)')
plt.savefig(f'{base_work_dir}/{folder}/{folder}.png',
bbox_inches='tight')
plt.close()
if len(cand_df_masked) != 0:
cmd = f'mkdir -p {base_work_dir}/{folder}/cands'
subprocess.run(cmd.split(), stdout=subprocess.PIPE)
send2gpuQ(
f'candmaker.py -n 10 -c {base_work_dir}/{folder}/{folder}.csv -o {base_work_dir}/{folder}/cands/'
)
send2gpuQ(
f'predict.py -n 5 -c {base_work_dir}/{folder}/cands/ -m a')
results = pd.read_csv(
f'{base_work_dir}/{folder}/cands/results_a.csv')
label_mask = (results['label'] == 1)
send_msg_2_slack(
f'Got {label_mask.sum()} cands post-FETCH at MJD {mjd}')
if label_mask.sum() >= 1:
files = results[label_mask]['candidate']
for file in files:
send2Q("stage03_queue", file)
else:
delete_cmd = "put delete cmd here"
#subprocess.run(cmd.split(), stdout=subprocess.PIPE)
else:
logging.info('No cands here!')
return None
time_spent = len(_df) #24*3600*(np.sum(np.diff(np.sort(_df['MJD'])))) #24*3600*(np.max(_df['MJD']) - np.min((_df['MJD'][0])))
logging.info(f'Spent {time_spent}s at {_gl} gl and {_gb} gb')
dump_dict['type'] = "Obs"
dump_dict['Filename'] = filterbank
dump_dict['Eq'] = {"lon" : _df['RA_deg'].mean()-180, "lat": _df['DEC_deg'].mean()}
dump_dict['Gal'] = {"lon" :_df['GL_deg'].mean()-180, "lat": _df ['GB_deg'].mean()}
dump_dict['MJD'] = _df['MJD'][0]
dump_dict['UTC'] = time.isot
dump_dict['Receiver'] = _df['IFV1TNCI'][0]
dump_dict['Project_ID'] = _df['SCPROJID'][0]
dump_dict['Time_spent'] = time_spent
result = es.index(index='greenburst',doc_type='_doc',id=time.isot,body=json.dumps(dump_dict))
logging.info(result)
if __name__ == "__main__":
logging_format = '%(asctime)s - %(funcName)s -%(name)s - %(levelname)s - %(message)s'
logging.basicConfig(level=logging.INFO, format=logging_format)
es=Elasticsearch([{'host':'localhost','port':9200}])
for filterbank in glob.glob('/ldata/trunk/data_2019-04*/*.fil'):
fil_obj = pysigproc.SigprocFile(filterbank)
mjd = fil_obj.tstart
influx_df = mjd2influx(mjd)
try:
tel_df_to_es(es, influx_df, filterbank)
except TypeError:
logging.error(filterbank)
def plot_coarse_group(group,
clobber=False,
plot_cands=True,
use_kurto=False,
fignum=2):
""" Produce the waterfall plot for the beams at very coarse resolution.
TODO -- make version without candidates.
"""
fnames = sorted(glob.glob(mkroot + 'fil/%s_ea*.fil' % group))
if use_kurto:
fnames = filter(lambda f: f.endswith('_kur.fil'), fnames)
else:
fnames = filter(lambda f: not f.endswith('_kur.fil'), fnames)
if len(fnames) == 0:
if use_kurto:
print 'No kurtosis for %s.' % (group)
else:
print 'No non-kurtosis for %s.' % (group)
return
base = os.path.split(fnames[0])[-1].split('_ea')[0]
suffix = '_kur' if use_kurto else ''
if plot_cands:
suffix += '_cands'
plot_name = '/data/kerrm/plots/%s_coarse%s.png' % (base, suffix)
if (not clobber) and os.path.isfile(plot_name):
return
sncut = 10
if plot_cands:
l, r = fnames[0].replace("vlite_fb",
"cands").replace('fil', 'cand').split("ea")
cand_fnames = sorted(glob.glob('*'.join((l, r[2:]))))
else:
cand_fnames = []
# make full list -- for case we have more candidates than fils
fil_antennas = [os.path.basename(x).split('_')[3] for x in fnames]
cand_antennas = [os.path.basename(x).split('_')[3] for x in cand_fnames]
all_antennas = sorted(set(fil_antennas).union(cand_antennas))
all_fnames = []
for antenna in all_antennas:
fname = [x for x in fnames if antenna in x]
fil_fname = None if len(fname) == 0 else fname[0]
fname = [x for x in cand_fnames if antenna in x]
cand_fname = None if len(fname) == 0 else fname[0]
all_fnames.append((fil_fname, cand_fname))
fil_fnames = [x[0] for x in all_fnames]
cand_fnames = [x[1] for x in all_fnames]
has_cands = sum((x is not None for x in cand_fnames)) > 0
all_cands = [load_cands(fname, sncut=sncut) for fname in cand_fnames]
if plot_cands and has_cands:
coincidence(all_cands)
msncuts = [1, 10, 15, 20, 30, 50]
msizes = [2, 3, 7, 11, 14, 18]
mwidths = [3, 10, 30, 100, 300, 1000]
colors = get_tableau20(only_dark=True)
width_colors = [
colors[5], colors[3], colors[4], colors[1], colors[0], colors[2]
]
figx = 14
figy = 3.0 * len(all_antennas)
if pl.fignum_exists(fignum):
x, y = pl.gcf().get_size_inches()
if x != figx or y != figy:
pl.close(fignum)
pl.figure(fignum, (figx, figy))
else:
pl.clf()
else:
pl.figure(fignum, (figx, figy))
pl.subplots_adjust(top=0.99,
bottom=0.16 if len(fnames) == 1 else 0.08,
left=0.05,
right=0.98,
hspace=0.02)
grid_shape = 2 * len(all_antennas), 5
pl.clf()
if plot_cands:
fig2_axes = pl.subplot2grid(grid_shape, (0, 4),
rowspan=2 * len(fnames))
fig2_axes.set_xscale('log')
fig2_axes.set_yscale('log')
fig2_axes.yaxis.set_label_position('right')
import pysigproc
from analyze_fb import fb_avg
tmin = np.inf
tmax = 0
fil_axes = []
cand_axes = []
for i, antenna in enumerate(all_antennas):
fname, cand_fname = all_fnames[i]
print i, antenna, fname, cand_fname
if fname is not None:
spf = pysigproc.SigprocFile(fname)
data, freqs, times = fb_avg(fname)
if tmax < times[-1] + (times[1] - times[0]):
tmax = times[-1] + (times[1] - times[0])
if times[0] < tmin:
tmin = times[0]
if plot_cands:
ax = pl.subplot2grid(grid_shape, (2 * i, 0), colspan=4)
else:
ax = pl.subplot(len(all_antennas), 1, i + 1)
fil_axes.append(ax)
if fname is not None:
ax.imshow(data,
extent=[times[0], times[-1], freqs[0], freqs[-1]],
vmin=0.9,
vmax=1.1,
cmap='Greys',
aspect='auto',
origin='lower')
ax.set_ylabel('Freq (MHz)')
ax.text(times[int(len(times) * 0.95)],
328,
'ea%02d' % spf.telescope_id,
size='large',
color='white')
if not plot_cands and i == len(all_antennas) - 1:
ax.set_xlabel('Elapsed Time (s)')
else:
ax.set_xticks([])
ax.set_yticks([330, 340, 350])
if not plot_cands:
continue
axcand = pl.subplot2grid(grid_shape, (2 * i + 1, 0), colspan=4)
cand_axes.append(axcand)
cands = all_cands[i]
cand_times = np.asarray([cand.peak_time for cand in cands])
dms = np.asarray([cand.dm for cand in cands])
sns = np.asarray([cand.sn for cand in cands])
sizes = np.ones_like(sns) * 1
for msn, msz in zip(msncuts, msizes):
sizes[sns > msn] = msz**2
widths = np.asarray([cand.width for cand in cands]) * 1e3 # in ms
c = np.empty((len(widths), 3))
for imw in xrange(len(mwidths)):
c[widths < mwidths[-(imw + 1)]] = width_colors[len(mwidths) -
(imw + 1)]
fat_pulse_count = np.sum(widths >= mwidths[-1])
if fat_pulse_count > 0:
print 'Have %d fat pulses for %s.' % (fat_pulse_count, group)
c[widths >= mwidths[-1]] = width_colors[-1]
axcand.set_yscale('log')
cm = coincidence_mask = np.asarray(
[cand.beam_mask.sum() > 1 for cand in cands])
if len(cm) > 0:
cm &= widths < mwidths[-1]
axcand.scatter(cand_times[~cm],
dms[~cm],
c=c[~cm],
marker='o',
s=sizes[~cm],
alpha=0.5)
axcand.scatter(cand_times[cm],
dms[cm],
c=c[cm],
marker='*',
s=sizes[cm],
alpha=0.8)
axcand.set_ylabel('DM')
if i == len(fnames) - 1:
axcand.set_xlabel('Elapsed Time (s)')
else:
axcand.set_xticks([])
if len(cm) > 0:
fig2_axes.scatter(dms[~cm],
widths[~cm],
marker='o',
s=sizes[~cm],
label=all_antennas[i],
alpha=0.5,
color=colors[i % len(colors)])
fig2_axes.scatter(dms[cm],
widths[cm],
marker='*',
s=sizes[cm],
label=all_antennas[i],
alpha=0.8,
color=colors[i % len(colors)])
#pl.figtext(0.40,0.02,'Elapsed Time (s)',size='large',rotation='horizontal')
if plot_cands:
dms = np.logspace(np.log10(2), 3, 100)
dm_widths = 4.15e-3 * (0.320**-2 -
(0.320 + 0.064 / 6250)**-2) * dms * 1e3
fig2_axes.plot(dms, dm_widths, alpha=0.5)
fig2_axes.plot([1e2, 1e2], [1, 10], color='k', ls='-', lw=1, alpha=0.5)
fig2_axes.plot([1e2, 1e3], [10, 10],
color='k',
ls='-',
lw=1,
alpha=0.5)
fig2_axes.set_ylabel('Width (ms)')
fig2_axes.set_xlabel('DM')
fig2_axes.axis([3, 1001, 1, 1000])
for axcand in cand_axes:
axcand.axis([tmin, tmax, 3, 1001])
#pl.figtext(0.40,0.02,'Dispersion Measure',size='large',rotation='horizontal')
"""
legend_lines = [mlines.Line2D([],[],color=colors[i],marker='o',markersize=msizes[2],alpha=0.5,label=antennas[i],ls=' ') for i in xrange(len(antennas))]
axlegend = pl.axes([0.10,0.96,0.88,0.05])
axlegend.axis('off')
pl.legend(handles=legend_lines,mode='expand',ncol=len(antennas),frameon=False,loc='upper right')
"""
pl.savefig(plot_name)
def plot_group(group,clobber=False,use_kurto=False,first_panel=False,
fignum=2,sncut=1,tspan=2.):
""" Produce the waterfall plot for the beams."""
if (not clobber):
if use_kurto:
pngs = glob.glob(mkroot+'plots/%s*_kur.png'%group)
else:
pngs = filter(lambda l: 'kur' not in l,glob.glob(mkroot+'plots/%s*png'%group))
pngs = [x for x in pngs if 'cand' not in x]
pngs = [x for x in pngs if 'coarse' not in x]
if len(pngs) > 0:
return
# /home/shining/study/MS/vLITE/mkerr
fnames = sorted(glob.glob(mkroot+'fil/%s_ea*.fil'%group))
if use_kurto:
fnames = filter(lambda f: f.endswith('_kur.fil'),fnames)
else:
fnames = filter(lambda f: not f.endswith('_kur.fil'),fnames)
#if len(fnames) != 5:
#raise ValueError("Expected 5 antennas!")
if len(fnames)==0:
if use_kurto:
print 'No kurtosis for %s.'%(group)
else:
print 'No non-kurtosis for %s.'%(group)
return
base = os.path.split(fnames[0])[-1].split('_ea')[0]
spfs = [pysigproc.SigprocFile(fname) for fname in fnames]
# print '%d channels, %f tsamp (ms)'%(spfs[0].nchans,spfs[0].tsamp*1e3)
cands = []
for fname in tqdm(fnames,ascii=True,unit='Cands/s',name='Reading Cands'):
cand_fname = fname.replace('.fil','.cand').replace('fil','cands')
# print cand_fname
try:
cands.append(sorted([Candidate(line) for line in filter(lambda x: len(x.strip())>0, file(cand_fname).readlines())],key=lambda c:c.i0))
cands[-1] = filter(lambda c: c.sn >= sncut,cands[-1])
except IOError:
print "[!!] Error at ",cand_fname
cands.append([])
# get `tspan` of data
nspans = np.asarray([int(spf.nspectra*spfs[0].tsamp/tspan) for spf in spfs])
if not np.all(nspans==nspans[0]):
print 'Warning, files differ in length!!!'
# TEMPORARY
# print 'nspans',nspans
# end TEMPORARY
nspan = nspans.min()
# nspan = min((int(spf.nspectra*spfs[0].tsamp/tspan) for spf in spfs))
pl.figure(fignum, (16,8)); pl.clf()
pl.subplots_adjust(top=0.99,bottom=0.06,left=0.05,right=0.98)
freqs = spfs[0].chan_freqs
delays = 4.15e-3*((freqs*1e-3)**-2-(freqs[0]*1e-3)**-2)
sel_freqs = freqs[np.asarray([spfs[0].nchans/20,spfs[0].nchans/2,19*(spfs[0].nchans/20)])]
sel_dels = delays[np.asarray([spfs[0].nchans/200,spfs[0].nchans/2,19*(spfs[0].nchans/20)])]
colors = ['white','violet','blue','green','yellow','orange','red']
colors = ['blue']*len(colors)
# do a kluge to plot 5 time segments in one go
one_ant_counter = 0
for j in tqdm(range(nspan),name='Plot',units='Plots/s',ascii=True):
if first_panel and (j > 0) and (one_ant_counter==0):
return
if one_ant_counter == 0:
pl.clf()
tstart = j*tspan
ds = [load_data(spf,tstart,tspan) for spf in spfs]
for i,spf in enumerate(spfs):
ax = pl.subplot(5 if len(spfs)==1 else len(spfs),1,i+1+one_ant_counter)
ax.imshow(ds[i].transpose(),extent=[tstart,tstart+tspan,spf.fch1,spf.fch1+spf.foff*spf.nchans],aspect='auto',interpolation='nearest',origin='lower',cmap='Greys',vmin=0.0,vmax=3.0)
ax.set_xticks(np.arange(tstart,tstart+tspan+0.01,0.1))
if j == nspan//2:
ax.set_ylabel('Freq (MHz)')
if j == nspan -1:
ax.set_xlabel('Time (s)')
ax.text(tstart+0.965*tspan,328,'ea%02d'%spf.telescope_id,size='large',color='black')
# process candidates
#plot_cands = filter(lambda c:c.peak_time > tstart and c.peak_time <= tstart+tspan,cands[i])
plot_cands = cands[i]
for cand in plot_cands:
# compute start time in seconds
cand_tstart = cand.i0 * spf.tsamp
cand_tstop = cand.i1 * spf.tsamp
cand_tpeak = cand.peak_time
times = cand_tstart + cand.dm*delays
#times = cand_tpeak + cand.dm*delays
if (times[-1] < tstart) or (times[0] > (tstart+tspan)):
continue
sel_times = cand_tstart + cand.dm*sel_dels
dt = cand_tstop - cand_tstart
color = colors[cand.tfilt-1]
for i in xrange(3):
if i == 1:
#ax.plot([cand.peak_time + cand.dm*sel_dels[i]],[sel_freqs[i]],marker='*',color=color)
ax.text(cand.peak_time + cand.dm*sel_dels[i],sel_freqs[i],'%d'%(int(round(cand.sn))),color=color,fontsize=12)
#ax.plot([cand_tstart + cand.dm*sel_dels[i],cand_tstop + cand.dm*sel_dels[i]],[sel_freqs[i]]*2,color=color)
else:
#ax.plot([sel_times[i],sel_times[i]+dt],[sel_freqs[i]]*2,color=color)
pass
ax.fill_betweenx(freqs,times,times+dt,alpha=0.05,color=color)
#ax.plot(times,freqs,lw=1,color=color,alpha=0.5,ls='-')
#ax.plot(times+dt,freqs,lw=1,color=color,alpha=0.5,ls='-')
ax.axis([tstart,tstart+tspan,freqs[0],freqs[-1]])
suffix = '_kur' if use_kurto else ''
if len(spfs) == 1:
if (one_ant_counter == 4) or j == nspan-1:
pl.savefig('/home/shining/study/MS/vLITE/plots/%s_%03d%s.png'%(base,j/5,suffix))
one_ant_counter = 0
else:
one_ant_counter += 1
else:
pl.savefig('/home/shining/study/MS/vLITE/plots/%s_%03d%s.png'%(base,j,suffix))