def inspect_cand(mergepkl, candnum=-1, scan=0, **kwargs):
""" Create detailed plot of a single candidate.
Thresholds (as minimum), then provides list of candidates to select with candnum.
scan can be used to define scan number with pre-merge pkl file.
kwargs passed to rt.set_pipeline
"""
d = pickle.load(open(mergepkl, "r"))
loc, prop = read_candidates(mergepkl)
if not os.path.dirname(d["filename"]):
d["filename"] = os.path.join(d["workdir"], d["filename"])
# feature columns
if "snr2" in d["features"]:
snrcol = d["features"].index("snr2")
elif "snr1" in d["features"]:
snrcol = d["features"].index("snr1")
if "l2" in d["features"]:
lcol = d["features"].index("l2")
elif "l1" in d["features"]:
lcol = d["features"].index("l1")
if "m2" in d["features"]:
mcol = d["features"].index("m2")
elif "m1" in d["features"]:
mcol = d["features"].index("m1")
if not scan:
scancol = d["featureind"].index("scan")
segmentcol = d["featureind"].index("segment")
intcol = d["featureind"].index("int")
dtindcol = d["featureind"].index("dtind")
dmindcol = d["featureind"].index("dmind")
# sort and prep candidate list
snrs = n.array([prop[i][snrcol] for i in range(len(prop))])
# if isinstance(snrmin, type(None)):
# snrmin = min(snrs)
# sortord = snrs.argsort()
# snrinds = n.where(snrs[sortord] > snrmin)[0]
# loc = loc[sortord][snrinds]
# prop = n.array([prop[i][snrcol] for i in range(len(prop))][sortord][snrinds]) # total hack to get prop as list to sort
if candnum < 0:
for i in range(len(loc)):
logger.info("%d %s %s" % (i, str(loc[i]), str(prop[i][snrcol])))
logger.info("Returning candidate (loc, snr) ...")
return (loc, n.array([prop[i][snrcol] for i in range(len(prop))]))
else:
logger.info(
"Reproducing and visualizing candidate %d at %s with properties %s."
% (candnum, loc[candnum], prop[candnum])
)
if not scan:
scan = loc[candnum, scancol]
nsegments = len(d["segmenttimesdict"][scan])
else:
nsegments = len(d["segmenttimes"])
segment = loc[candnum, segmentcol]
dmind = loc[candnum, dmindcol]
dtind = loc[candnum, dtindcol]
candint = loc[candnum, intcol]
dmarrorig = d["dmarr"]
dtarrorig = d["dtarr"]
d2 = rt.set_pipeline(
d["filename"],
scan,
fileroot=d["fileroot"],
paramfile="rtparams.py",
savecands=False,
savenoise=False,
nsegments=nsegments,
**kwargs
)
im, data = rt.pipeline_reproduce(
d2, segment, (candint, dmind, dtind)
) # with candnum, pipeline will return cand image and data
# calc source location
peakl, peakm = n.where(im == im.max())
xpix, ypix = im.shape
l1 = (xpix / 2.0 - peakl[0]) / (xpix * d["uvres"])
m1 = (ypix / 2.0 - peakm[0]) / (ypix * d["uvres"])
pt_ra, pt_dec = d["radec"]
src_ra, src_dec = source_location(pt_ra, pt_dec, l1, m1)
logger.info("Peak (RA, Dec): %.3f, %.3f" % (src_ra, src_dec))
logger.info("Returning candidate d, im, data")
return d2, im, data
def plot_cand(candsfile,
candloc=[],
candnum=-1,
threshold=0,
savefile=True,
returndata=False,
outname='',
**kwargs):
""" Reproduce detection of a single candidate for plotting or inspection.
candsfile can be merge or single-scan cands pkl file. Difference defined by presence of scan in d['featureind'].
candloc reproduces candidate at given location (scan, segment, integration, dmind, dtind, beamnum).
candnum selects one to reproduce from ordered list
threshold is min of sbs(SNR) used to filter candidates to select with candnum.
savefile/outname define if/how to save png of candidate
if returndata, (im, data) returned.
kwargs passed to rt.set_pipeline
"""
# get candidate info
loc, prop = pc.read_candidates(candsfile)
# define state dict and overload with user prefs
d0 = pickle.load(open(candsfile, 'r'))
for key in kwargs:
logger.info('Setting %s to %s' % (key, kwargs[key]))
d0[key] = kwargs[key]
d0['logfile'] = False # no need to save log
# feature columns
if 'snr2' in d0['features']:
snrcol = d0['features'].index('snr2')
elif 'snr1' in d0['features']:
snrcol = d0['features'].index('snr1')
if 'l2' in d0['features']:
lcol = d0['features'].index('l2')
elif 'l1' in d0['features']:
lcol = d0['features'].index('l1')
if 'm2' in d0['features']:
mcol = d0['features'].index('m2')
elif 'm1' in d0['features']:
mcol = d0['features'].index('m1')
try:
scancol = d0['featureind'].index('scan') # if merged pkl
except ValueError:
scancol = -1 # if single-scan pkl
segmentcol = d0['featureind'].index('segment')
intcol = d0['featureind'].index('int')
dtindcol = d0['featureind'].index('dtind')
dmindcol = d0['featureind'].index('dmind')
# sort and prep candidate list
snrs = prop[:, snrcol]
select = np.where(np.abs(snrs) > threshold)[0]
loc = loc[select]
prop = prop[select]
times = pc.int2mjd(d0, loc)
times = times - times[0]
# default case will print cand info
if (candnum < 0) and (not len(candloc)):
logger.info('Getting candidates...')
logger.info('candnum: loc, SNR, DM (pc/cm3), time (s; rel)')
for i in range(len(loc)):
logger.info("%d: %s, %.1f, %.1f, %.1f" %
(i, str(loc[i]), prop[i, snrcol], np.array(
d0['dmarr'])[loc[i, dmindcol]], times[i]))
else: # if candnum or candloc provided, try to reproduce
if (candnum >= 0) and not len(candloc):
logger.info(
'Reproducing and visualizing candidate %d at %s with properties %s.'
% (candnum, loc[candnum], prop[candnum]))
dmarrorig = d0['dmarr']
dtarrorig = d0['dtarr']
if scancol >= 0: # here we have a merge pkl
scan = loc[candnum, scancol]
else: # a scan-based cands pkl
scan = d0['scan']
segment = loc[candnum, segmentcol]
candint = loc[candnum, intcol]
dmind = loc[candnum, dmindcol]
dtind = loc[candnum, dtindcol]
beamnum = 0
candloc = (scan, segment, candint, dmind, dtind, beamnum)
elif len(candloc) and (candnum < 0):
assert len(
candloc
) == 6, 'candloc should be length 6 ( scan, segment, candint, dmind, dtind, beamnum ).'
logger.info('Reproducing and visualizing candidate %d at %s' %
(candnum, candloc))
dmarrorig = d0['dmarr']
dtarrorig = d0['dtarr']
scan, segment, candint, dmind, dtind, beamnum = candloc
else:
raise Exception, 'Provide candnum or candloc, not both'
# if working locally, set workdir appropriately. Can also be used in queue system with full path given.
if not os.path.dirname(candsfile):
d0['workdir'] = os.getcwd()
else:
d0['workdir'] = os.path.dirname(candsfile)
filename = os.path.join(d0['workdir'],
os.path.basename(d0['filename']))
# clean up d0 of superfluous keys
params = pp.Params() # will be used as input to rt.set_pipeline
for key in d0.keys():
if not hasattr(params, key) and 'memory_limit' not in key:
_ = d0.pop(key)
d0['npix'] = 0
d0['uvres'] = 0
d0['nsegments'] = 0
d0['logfile'] = False
# get cand data
d = rt.set_pipeline(filename, scan, **d0)
im, data = rt.pipeline_reproduce(
d, candloc, product='imdata') # removed loc[candnum]
# optionally plot
if savefile:
loclabel = scan, segment, candint, dmind, dtind, beamnum
make_cand_plot(d, im, data, loclabel, outname=outname)
# optionally return data
if returndata:
return (im, data)
def plot_cand(mergepkl, candnum=-1, outname="", **kwargs):
""" Create detailed plot of a single candidate.
Thresholds (as minimum), then provides list of candidates to select with candnum.
kwargs passed to rt.set_pipeline
"""
# if isinstance(pkllist, list):
# outroot = '_'.join(pkllist[0].split('_')[1:3])
# merge_cands(pkllist, outroot=outroot)
# mergepkl = 'cands_' + outroot + '_merge.pkl'
# elif isinstance(pkllist, str):
# logger.info('Assuming input is mergepkl')
# mergepkl = pkllist
d = pickle.load(open(mergepkl, "r"))
loc, prop = read_candidates(mergepkl)
if not os.path.dirname(d["filename"]):
d["filename"] = os.path.join(d["workdir"], d["filename"])
# feature columns
if "snr2" in d["features"]:
snrcol = d["features"].index("snr2")
elif "snr1" in d["features"]:
snrcol = d["features"].index("snr1")
if "l2" in d["features"]:
lcol = d["features"].index("l2")
elif "l1" in d["features"]:
lcol = d["features"].index("l1")
if "m2" in d["features"]:
mcol = d["features"].index("m2")
elif "m1" in d["features"]:
mcol = d["features"].index("m1")
scancol = d["featureind"].index("scan")
segmentcol = d["featureind"].index("segment")
intcol = d["featureind"].index("int")
dtindcol = d["featureind"].index("dtind")
dmindcol = d["featureind"].index("dmind")
# sort and prep candidate list
snrs = n.array([prop[i][snrcol] for i in range(len(prop))])
# if isinstance(snrmin, type(None)):
# snrmin = min(snrs)
# sortord = snrs.argsort()
# snrinds = n.where(snrs[sortord] > snrmin)[0]
# loc = loc[sortord][snrinds]
# prop = n.array([prop[i][snrcol] for i in range(len(prop))][sortord][snrinds]) # total hack to get prop as list to sort
# prop = prop[sortord][snrinds] # original
if candnum < 0:
logger.info("Getting candidates...")
for i in range(len(loc)):
logger.info("%d %s %s" % (i, str(loc[i]), str(prop[i][snrcol])))
return (loc, n.array([prop[i][snrcol] for i in range(len(prop))]))
else:
logger.info(
"Reproducing and visualizing candidate %d at %s with properties %s."
% (candnum, loc[candnum], prop[candnum])
)
scan = loc[candnum, scancol]
segment = loc[candnum, segmentcol]
dmind = loc[candnum, dmindcol]
dtind = loc[candnum, dtindcol]
candint = loc[candnum, intcol]
dmarrorig = d["dmarr"]
dtarrorig = d["dtarr"]
nsegments = len(d["segmenttimesdict"][scan])
d2 = rt.set_pipeline(
d["filename"],
scan,
fileroot=d["fileroot"],
paramfile="rtparams.py",
savecands=False,
savenoise=False,
nsegments=nsegments,
**kwargs
)
im, data = rt.pipeline_reproduce(
d2, segment, (candint, dmind, dtind)
) # with candnum, pipeline will return cand image and data
# calc source location
peakl, peakm = n.where(im == im.max())
xpix, ypix = im.shape
l1 = (xpix / 2.0 - peakl[0]) / (xpix * d["uvres"])
m1 = (ypix / 2.0 - peakm[0]) / (ypix * d["uvres"])
pt_ra, pt_dec = d["radec"]
src_ra, src_dec = source_location(pt_ra, pt_dec, l1, m1)
logger.info("Peak (RA, Dec): %.3f, %.3f" % (src_ra, src_dec))
# plot it
logger.info("Plotting...")
fig = plt.Figure(figsize=(8.5, 8))
ax = fig.add_subplot(221, axisbg="white")
# # first plot dm-t distribution beneath
times0 = int2mjd(d, loc)
times0 = times0 - times0[0]
times = times0[candnum]
dms0 = n.array(dmarrorig)[list(loc[:, dmindcol])]
dms = dmarrorig[loc[candnum, dmindcol]]
snr0 = prop[:][snrcol]
snr = prop[candnum][snrcol]
snrmin = 0.8 * min(d["sigma_image1"], d["sigma_image2"])
# plot positive
good = n.where(snr0 > 0)
ax.scatter(
times0[good], dms0[good], s=(snr0[good] - snrmin) ** 5, facecolor="none", linewidth=0.2, clip_on=False
)
ax.scatter(times, dms, s=(snr - snrmin) ** 5, facecolor="none", linewidth=2, clip_on=False)
# plot negative
good = n.where(snr0 < 0)
ax.scatter(
times0[good],
dms0[good],
s=(n.abs(snr0)[good] - snrmin) ** 5,
marker="x",
edgecolors="k",
linewidth=0.2,
clip_on=False,
)
ax.set_ylim(dmarrorig[0], dmarrorig[-1])
ax.set_xlim(times0.min(), times0.max())
ax.set_xlabel("Time (s)")
ax.set_ylabel("DM (pc/cm3)")
# # then add annotating info
ax.text(0.1, 0.9, d["fileroot"] + "_sc" + str(scan), fontname="sans-serif", transform=ax.transAxes)
ax.text(
0.1,
0.8,
"seg %d, int %d, DM %.1f, dt %d"
% (segment, loc[candnum, intcol], dmarrorig[loc[candnum, dmindcol]], dtarrorig[loc[candnum, dtindcol]]),
fontname="sans-serif",
transform=ax.transAxes,
)
ax.text(
0.1,
0.7,
"Peak: (" + str(n.round(peakx, 1)) + "' ," + str(n.round(peaky, 1)) + "'), SNR: " + str(n.round(snr, 1)),
fontname="sans-serif",
transform=ax.transAxes,
)
# plot dynamic spectra
left, width = 0.6, 0.2
bottom, height = 0.2, 0.7
rect_dynsp = [left, bottom, width, height]
rect_lc = [left, bottom - 0.1, width, 0.1]
rect_sp = [left + width, bottom, 0.1, height]
ax_dynsp = fig.add_axes(rect_dynsp)
ax_lc = fig.add_axes(rect_lc)
ax_sp = fig.add_axes(rect_sp)
spectra = n.swapaxes(data.real, 0, 1) # seems that latest pickle actually contains complex values in spectra...
dd = n.concatenate(
(spectra[..., 0], n.zeros_like(spectra[..., 0]), spectra[..., 1]), axis=1
) # make array for display with white space between two pols
impl = ax_dynsp.imshow(dd, origin="lower", interpolation="nearest", aspect="auto", cmap=plt.get_cmap("Greys"))
ax_dynsp.text(
0.5,
0.95,
"RR LL",
horizontalalignment="center",
verticalalignment="center",
fontsize=16,
color="w",
transform=ax_dynsp.transAxes,
)
ax_dynsp.set_yticks(range(0, len(d["freq"]), 30))
ax_dynsp.set_yticklabels(d["freq"][::30])
ax_dynsp.set_ylabel("Freq (GHz)")
ax_dynsp.set_xlabel("Integration (rel)")
spectrum = spectra[:, len(spectra[0]) / 2].mean(
axis=1
) # assume pulse in middle bin. get stokes I spectrum. **this is wrong in a minority of cases.**
ax_sp.plot(spectrum, range(len(spectrum)), "k.")
ax_sp.plot(n.zeros(len(spectrum)), range(len(spectrum)), "k:")
ax_sp.set_ylim(0, len(spectrum))
ax_sp.set_yticklabels([])
xmin, xmax = ax_sp.get_xlim()
ax_sp.set_xticks(n.linspace(xmin, xmax, 3).round(2))
ax_sp.set_xlabel("Flux (Jy)")
lc = dd.mean(axis=0)
lenlc = n.where(lc == 0)[0][0]
ax_lc.plot(range(0, lenlc) + range(2 * lenlc, 3 * lenlc), list(lc)[:lenlc] + list(lc)[-lenlc:], "k.")
ax_lc.plot(range(0, lenlc) + range(2 * lenlc, 3 * lenlc), list(n.zeros(lenlc)) + list(n.zeros(lenlc)), "k:")
ax_lc.set_xlabel("Integration")
ax_lc.set_ylabel("Flux (Jy)")
ax_lc.set_xticks([0, 0.5 * lenlc, lenlc, 1.5 * lenlc, 2 * lenlc, 2.5 * lenlc, 3 * lenlc])
ax_lc.set_xticklabels(["0", str(lenlc / 2), str(lenlc), "", "0", str(lenlc / 2), str(lenlc)])
ymin, ymax = ax_lc.get_ylim()
ax_lc.set_yticks(n.linspace(ymin, ymax, 3).round(2))
# image
ax = fig.add_subplot(223)
fov = n.degrees(1.0 / d["uvres"]) * 60.0
ax.scatter(
((xpix / 2 - srcra[0]) - 0.05 * xpix) * fov / xpix,
(ypix / 2 - srcdec[0]) * fov / ypix,
s=80,
marker="<",
facecolor="none",
)
ax.scatter(
((xpix / 2 - srcra[0]) + 0.05 * xpix) * fov / xpix,
(ypix / 2 - srcdec[0]) * fov / ypix,
s=80,
marker=">",
facecolor="none",
)
impl = ax.imshow(
im.transpose(),
aspect="equal",
origin="upper",
interpolation="nearest",
extent=[fov / 2, -fov / 2, -fov / 2, fov / 2],
cmap=plt.get_cmap("Greys"),
vmin=0,
vmax=0.5 * im.max(),
)
ax.set_xlabel("RA Offset (arcmin)")
ax.set_ylabel("Dec Offset (arcmin)")
if not outname:
outname = os.path.join(
d["workdir"],
"cands_%s_sc%dseg%di%ddm%ddt%d.png"
% (d["fileroot"], scan, segment, loc[candnum, intcol], dmind, dtind),
)
canvas = FigureCanvasAgg(fig)
canvas.print_figure(outname)
return ([], [])
def plot_cand(candsfile,
candloc=[],
candnum=-1,
threshold=0,
savefile=True,
returndata=False,
outname='',
newplot=True,
returnstate=False,
**kwargs):
""" Reproduce detection of a single candidate for plotting or inspection.
candsfile can be merge or single-scan cands pkl file. Difference defined by presence of scan in d['featureind'].
candloc reproduces candidate at given location (scan, segment, integration, dmind, dtind, beamnum).
candnum selects one to reproduce from ordered list
threshold is min of sbs(SNR) used to filter candidates to select with candnum.
savefile/outname define if/how to save png of candidate
if returndata, (im, data) returned.
kwargs passed to rt.set_pipeline
if newplot, then plot with the new candidate plot using bridget's version
"""
# get candidate info
loc, prop, d0 = pc.read_candidates(candsfile, returnstate=True)
# define state dict and overload with user prefs
for key in kwargs:
logger.info('Setting %s to %s' % (key, kwargs[key]))
d0[key] = kwargs[key]
d0['logfile'] = False # no need to save log
# feature columns
if 'snr2' in d0['features']:
snrcol = d0['features'].index('snr2')
elif 'snr1' in d0['features']:
snrcol = d0['features'].index('snr1')
if 'l2' in d0['features']:
lcol = d0['features'].index('l2')
elif 'l1' in d0['features']:
lcol = d0['features'].index('l1')
if 'm2' in d0['features']:
mcol = d0['features'].index('m2')
elif 'm1' in d0['features']:
mcol = d0['features'].index('m1')
scancol = d0['featureind'].index('scan')
segmentcol = d0['featureind'].index('segment')
intcol = d0['featureind'].index('int')
dtindcol = d0['featureind'].index('dtind')
dmindcol = d0['featureind'].index('dmind')
# sort and prep candidate list
snrs = prop[:, snrcol]
select = np.where(np.abs(snrs) > threshold)[0]
loc = loc[select]
prop = prop[select]
if candnum >= 0 or len(candloc):
if (candnum >= 0) and not len(candloc):
logger.info(
'Reproducing and visualizing candidate %d at %s with properties %s.'
% (candnum, loc[candnum], prop[candnum]))
dmarrorig = d0['dmarr']
dtarrorig = d0['dtarr']
scan = loc[candnum, scancol]
segment = loc[candnum, segmentcol]
candint = loc[candnum, intcol]
dmind = loc[candnum, dmindcol]
dtind = loc[candnum, dtindcol]
beamnum = 0
candloc = (scan, segment, candint, dmind, dtind, beamnum)
elif len(candloc) and (candnum < 0):
assert len(
candloc
) == 6, 'candloc should be length 6 ( scan, segment, candint, dmind, dtind, beamnum ).'
logger.info('Reproducing and visualizing candidate %d at %s' %
(candnum, candloc))
dmarrorig = d0['dmarr']
dtarrorig = d0['dtarr']
scan, segment, candint, dmind, dtind, beamnum = candloc
else:
raise Exception, 'Provide candnum or candloc, not both'
# if working locally, set workdir appropriately. Can also be used in queue system with full path given.
if not os.path.dirname(candsfile):
d0['workdir'] = os.getcwd()
else:
d0['workdir'] = os.path.dirname(candsfile)
filename = os.path.join(d0['workdir'],
os.path.basename(d0['filename']))
if d0.has_key('segmenttimesdict'): # using merged pkl
segmenttimes = d0['segmenttimesdict'][scan]
else:
segmenttimes = d0['segmenttimes']
# clean up d0 of superfluous keys
params = pp.Params() # will be used as input to rt.set_pipeline
for key in d0.keys():
if not hasattr(params, key): # and 'memory_limit' not in key:
_ = d0.pop(key)
d0['npix'] = 0
d0['uvres'] = 0
d0['logfile'] = False
d0['savenoise'] = False
d0['savecands'] = False
# this triggers redefinition of segment boundaries. memory optimization changed, so this is a problem.
# d0['nsegments'] = 0
# d0['scale_nsegments'] = 1.
d0['segmenttimes'] = segmenttimes
d0['nsegments'] = len(segmenttimes)
# get cand data
d = rt.set_pipeline(filename, scan, **d0)
(vismem, immem) = rt.calc_memory_footprint(d)
if 'memory_limit' in d:
assert vismem + immem < d[
'memory_limit'], 'memory_limit defined, but nsegments must (for now) be set to initial values to properly reproduce candidate'
im, data = rt.pipeline_reproduce(
d, candloc, product='imdata') # removed loc[candnum]
# optionally plot
if savefile:
loclabel = scan, segment, candint, dmind, dtind, beamnum
if newplot:
make_cand_plot(d,
im,
data,
loclabel,
version=2,
snrs=snrs,
outname=outname)
else:
make_cand_plot(d,
im,
data,
loclabel,
version=1,
outname=outname)
# optionally return data
if returndata:
return (im, data)
elif returnstate:
return d
def plot_cand(candsfile, candloc=[], candnum=-1, threshold=0, savefile=True, returndata=False, outname='', **kwargs):
""" Reproduce detection of a single candidate for plotting or inspection.
candsfile can be merge or single-scan cands pkl file. Difference defined by presence of scan in d['featureind'].
candloc reproduces candidate at given location (scan, segment, integration, dmind, dtind, beamnum).
candnum selects one to reproduce from ordered list
threshold is min of sbs(SNR) used to filter candidates to select with candnum.
savefile/outname define if/how to save png of candidate
if returndata, (im, data) returned.
kwargs passed to rt.set_pipeline
"""
# get candidate info
loc, prop = pc.read_candidates(candsfile)
# define state dict and overload with user prefs
d0 = pickle.load(open(candsfile, 'r'))
for key in kwargs:
logger.info('Setting %s to %s' % (key, kwargs[key]))
d0[key] = kwargs[key]
d0['logfile'] = False # no need to save log
# feature columns
if 'snr2' in d0['features']:
snrcol = d0['features'].index('snr2')
elif 'snr1' in d0['features']:
snrcol = d0['features'].index('snr1')
if 'l2' in d0['features']:
lcol = d0['features'].index('l2')
elif 'l1' in d0['features']:
lcol = d0['features'].index('l1')
if 'm2' in d0['features']:
mcol = d0['features'].index('m2')
elif 'm1' in d0['features']:
mcol = d0['features'].index('m1')
try:
scancol = d0['featureind'].index('scan') # if merged pkl
except ValueError:
scancol = -1 # if single-scan pkl
segmentcol = d0['featureind'].index('segment')
intcol = d0['featureind'].index('int')
dtindcol = d0['featureind'].index('dtind')
dmindcol = d0['featureind'].index('dmind')
# sort and prep candidate list
snrs = prop[:, snrcol]
select = np.where(np.abs(snrs) > threshold)[0]
loc = loc[select]
prop = prop[select]
times = pc.int2mjd(d0, loc)
times = times - times[0]
# default case will print cand info
if (candnum < 0) and (not len(candloc)):
logger.info('Getting candidates...')
logger.info('candnum: loc, SNR, DM (pc/cm3), time (s; rel)')
for i in range(len(loc)):
logger.info("%d: %s, %.1f, %.1f, %.1f" % (i, str(loc[i]), prop[i, snrcol], np.array(d0['dmarr'])[loc[i,dmindcol]], times[i]))
else: # if candnum or candloc provided, try to reproduce
if (candnum >= 0) and not len(candloc):
logger.info('Reproducing and visualizing candidate %d at %s with properties %s.' % (candnum, loc[candnum], prop[candnum]))
dmarrorig = d0['dmarr']
dtarrorig = d0['dtarr']
if scancol >= 0: # here we have a merge pkl
scan = loc[candnum, scancol]
else: # a scan-based cands pkl
scan = d0['scan']
segment = loc[candnum, segmentcol]
candint = loc[candnum, intcol]
dmind = loc[candnum, dmindcol]
dtind = loc[candnum, dtindcol]
beamnum = 0
candloc = (scan, segment, candint, dmind, dtind, beamnum)
elif len(candloc) and (candnum < 0):
assert len(candloc) == 6, 'candloc should be length 6 ( scan, segment, candint, dmind, dtind, beamnum ).'
logger.info('Reproducing and visualizing candidate %d at %s' % (candnum, candloc))
dmarrorig = d0['dmarr']
dtarrorig = d0['dtarr']
scan, segment, candint, dmind, dtind, beamnum = candloc
else:
raise Exception, 'Provide candnum or candloc, not both'
# if working locally, set workdir appropriately. Can also be used in queue system with full path given.
if not os.path.dirname(candsfile):
d0['workdir'] = os.getcwd()
else:
d0['workdir'] = os.path.dirname(candsfile)
filename = os.path.join(d0['workdir'], os.path.basename(d0['filename']))
# clean up d0 of superfluous keys
params = pp.Params() # will be used as input to rt.set_pipeline
for key in d0.keys():
if not hasattr(params, key) and 'memory_limit' not in key:
_ = d0.pop(key)
d0['npix'] = 0
d0['uvres'] = 0
d0['nsegments'] = 0
d0['logfile'] = False
# get cand data
d = rt.set_pipeline(filename, scan, **d0)
im, data = rt.pipeline_reproduce(d, candloc, product='imdata') # removed loc[candnum]
# optionally plot
if savefile:
loclabel = scan, segment, candint, dmind, dtind, beamnum
make_cand_plot(d, im, data, loclabel, outname=outname)
# optionally return data
if returndata:
return (im, data)
def inspect_cand(mergepkl, candnum=-1, scan=0, **kwargs):
""" Create detailed plot of a single candidate.
Thresholds (as minimum), then provides list of candidates to select with candnum.
scan can be used to define scan number with pre-merge pkl file.
kwargs passed to rt.set_pipeline
"""
d = pickle.load(open(mergepkl, 'r'))
loc, prop = read_candidates(mergepkl)
if not os.path.dirname(d['filename']):
d['filename'] = os.path.join(d['workdir'], d['filename'])
# feature columns
if 'snr2' in d['features']:
snrcol = d['features'].index('snr2')
elif 'snr1' in d['features']:
snrcol = d['features'].index('snr1')
if 'l2' in d['features']:
lcol = d['features'].index('l2')
elif 'l1' in d['features']:
lcol = d['features'].index('l1')
if 'm2' in d['features']:
mcol = d['features'].index('m2')
elif 'm1' in d['features']:
mcol = d['features'].index('m1')
if not scan:
scancol = d['featureind'].index('scan')
segmentcol = d['featureind'].index('segment')
intcol = d['featureind'].index('int')
dtindcol = d['featureind'].index('dtind')
dmindcol = d['featureind'].index('dmind')
# sort and prep candidate list
snrs = n.array([prop[i][snrcol] for i in range(len(prop))])
# if isinstance(snrmin, type(None)):
# snrmin = min(snrs)
# sortord = snrs.argsort()
# snrinds = n.where(snrs[sortord] > snrmin)[0]
# loc = loc[sortord][snrinds]
# prop = n.array([prop[i][snrcol] for i in range(len(prop))][sortord][snrinds]) # total hack to get prop as list to sort
if candnum < 0:
for i in range(len(loc)):
logger.info("%d %s %s" % (i, str(loc[i]), str(prop[i][snrcol])))
logger.info('Returning candidate (loc, snr) ...')
return (loc, n.array([prop[i][snrcol] for i in range(len(prop))]))
else:
logger.info('Reproducing and visualizing candidate %d at %s with properties %s.' % (candnum, loc[candnum], prop[candnum]))
if not scan:
scan = loc[candnum, scancol]
nsegments = len(d['segmenttimesdict'][scan])
else:
nsegments = len(d['segmenttimes'])
segment = loc[candnum, segmentcol]
dmind = loc[candnum, dmindcol]
dtind = loc[candnum, dtindcol]
candint = loc[candnum, intcol]
dmarrorig = d['dmarr']
dtarrorig = d['dtarr']
d2 = rt.set_pipeline(d['filename'], scan, fileroot=d['fileroot'], paramfile='rtparams.py', savecands=False, savenoise=False, nsegments=nsegments, **kwargs)
im, data = rt.pipeline_reproduce(d2, segment, (candint, dmind, dtind)) # with candnum, pipeline will return cand image and data
# calc source location
if snrmax > -1*snrmin:
l1, m1 = calc_lm(d, im, minmax='max')
else:
l1, m1 = calc_lm(d, im, minmax='min')
pt_ra, pt_dec = d['radec']
src_ra, src_dec = source_location(pt_ra, pt_dec, l1, m1)
logger.info('Peak (RA, Dec): %s, %s' % (src_ra, src_dec))
logger.info('Returning candidate d, im, data')
return d2, im, data
def plot_cand(mergepkl, candnum=-1, outname='', threshold=0, **kwargs):
""" Create detailed plot of a single candidate.
threshold is min of sbs(SNR) used to filter candidates to select with candnum.
kwargs passed to rt.set_pipeline
"""
d = pickle.load(open(mergepkl, 'r'))
loc, prop = read_candidates(mergepkl)
if not os.path.dirname(d['filename']):
d['filename'] = os.path.join(d['workdir'], d['filename'])
# feature columns
if 'snr2' in d['features']:
snrcol = d['features'].index('snr2')
elif 'snr1' in d['features']:
snrcol = d['features'].index('snr1')
if 'l2' in d['features']:
lcol = d['features'].index('l2')
elif 'l1' in d['features']:
lcol = d['features'].index('l1')
if 'm2' in d['features']:
mcol = d['features'].index('m2')
elif 'm1' in d['features']:
mcol = d['features'].index('m1')
scancol = d['featureind'].index('scan')
segmentcol = d['featureind'].index('segment')
intcol = d['featureind'].index('int')
dtindcol = d['featureind'].index('dtind')
dmindcol = d['featureind'].index('dmind')
# sort and prep candidate list
snrs = n.array([prop[i][snrcol] for i in range(len(prop))])
select = n.where(n.abs(snrs) > threshold)[0]
loc = loc[select]
prop = [prop[i] for i in select]
if candnum < 0:
logger.info('Getting candidates...')
for i in range(len(loc)):
logger.info("%d %s %s" % (i, str(loc[i]), str(prop[i][snrcol])))
return (loc, n.array([prop[i][snrcol] for i in range(len(prop))]))
else:
logger.info('Reproducing and visualizing candidate %d at %s with properties %s.' % (candnum, loc[candnum], prop[candnum]))
# get cand info
scan = loc[candnum, scancol]
segment = loc[candnum, segmentcol]
dmind = loc[candnum, dmindcol]
dtind = loc[candnum, dtindcol]
candint = loc[candnum, intcol]
dmarrorig = d['dmarr']
dtarrorig = d['dtarr']
# set up state dict
d['starttime_mjd'] = d['starttime_mjddict'][scan]
d['scan'] = scan
d['nsegments'] = len(d['segmenttimesdict'][scan])
# d2 = rt.set_pipeline(d['filename'], scan, fileroot=d['fileroot'], savecands=False, savenoise=False, nsegments=nsegments, **kwargs)
d['segmenttimes'] = d['segmenttimesdict'][scan]
d['savecands'] = False; d['savenoise'] = False
for key in kwargs.keys():
logger.info('Setting %s to %s' % (key, kwargs[key]))
d[key] = kwargs[key]
# reproduce
im, data = rt.pipeline_reproduce(d, segment, (candint, dmind, dtind)) # with candnum, pipeline will return cand image and data
# calc source location
snrmin = im.min()/im.std()
snrmax = im.max()/im.std()
if snrmax > -1*snrmin:
l1, m1 = rt.calc_lm(d, im, minmax='max')
else:
l1, m1 = rt.calc_lm(d, im, minmax='min')
pt_ra, pt_dec = d['radec']
src_ra, src_dec = source_location(pt_ra, pt_dec, l1, m1)
logger.info('Peak (RA, Dec): %s, %s' % (src_ra, src_dec))
# plot it
logger.info('Plotting...')
fig = plt.Figure(figsize=(8.5,8))
ax = fig.add_subplot(221, axisbg='white')
# # first plot dm-t distribution beneath
times0 = int2mjd(d, loc)
times0 = times0 - times0[0]
times = times0[candnum]
dms0 = n.array(dmarrorig)[list(loc[:,dmindcol])]
dms = dmarrorig[loc[candnum,dmindcol]]
snr0 = [prop[i][snrcol] for i in range(len(prop))]
snr = snr0[candnum]
snrmin = 0.8 * min(d['sigma_image1'], d['sigma_image2'])
# plot positive
good = n.where(snr0 > 0)[0]
ax.scatter(times0[good], dms0[good], s=(snr0[good]-snrmin)**5, facecolor='none', linewidth=0.2, clip_on=False)
ax.scatter(times, dms, s=(snr-snrmin)**5, facecolor='none', linewidth=2, clip_on=False)
# plot negative
good = n.where(snr0 < 0)
ax.scatter(times0[good], dms0[good], s=(n.abs(snr0)[good]-snrmin)**5, marker='x', edgecolors='k', linewidth=0.2, clip_on=False)
ax.set_ylim(dmarrorig[0], dmarrorig[-1])
ax.set_xlim(times0.min(), times0.max())
ax.set_xlabel('Time (s)')
ax.set_ylabel('DM (pc/cm3)')
# # then add annotating info
ax.text(0.1, 0.9, d['fileroot']+'_sc'+str(scan), fontname='sans-serif', transform = ax.transAxes)
ax.text(0.1, 0.8, 'seg %d, int %d, DM %.1f, dt %d' % (segment, loc[candnum, intcol], dmarrorig[loc[candnum, dmindcol]], dtarrorig[loc[candnum,dtindcol]]), fontname='sans-serif', transform = ax.transAxes)
ax.text(0.1, 0.7, 'Peak: (' + str(n.round(l1, 3)) + '\' ,' + str(n.round(m1, 3)) + '\'), SNR: ' + str(n.round(snr, 1)), fontname='sans-serif', transform = ax.transAxes)
# plot dynamic spectra
left, width = 0.6, 0.2
bottom, height = 0.2, 0.7
rect_dynsp = [left, bottom, width, height]
rect_lc = [left, bottom-0.1, width, 0.1]
rect_sp = [left+width, bottom, 0.1, height]
ax_dynsp = fig.add_axes(rect_dynsp)
ax_lc = fig.add_axes(rect_lc)
ax_sp = fig.add_axes(rect_sp)
spectra = n.swapaxes(data.real,0,1) # seems that latest pickle actually contains complex values in spectra...
dd = n.concatenate( (spectra[...,0], n.zeros_like(spectra[...,0]), spectra[...,1]), axis=1) # make array for display with white space between two pols
impl = ax_dynsp.imshow(dd, origin='lower', interpolation='nearest', aspect='auto', cmap=plt.get_cmap('Greys'))
ax_dynsp.text(0.5, 0.95, 'RR LL', horizontalalignment='center', verticalalignment='center', fontsize=16, color='w', transform = ax_dynsp.transAxes)
ax_dynsp.set_yticks(range(0,len(d['freq']),30))
ax_dynsp.set_yticklabels(d['freq'][::30])
ax_dynsp.set_ylabel('Freq (GHz)')
ax_dynsp.set_xlabel('Integration (rel)')
spectrum = spectra[:,len(spectra[0])/2].mean(axis=1) # assume pulse in middle bin. get stokes I spectrum. **this is wrong in a minority of cases.**
ax_sp.plot(spectrum, range(len(spectrum)), 'k.')
ax_sp.plot(n.zeros(len(spectrum)), range(len(spectrum)), 'k:')
ax_sp.set_ylim(0, len(spectrum))
ax_sp.set_yticklabels([])
xmin,xmax = ax_sp.get_xlim()
ax_sp.set_xticks(n.linspace(xmin,xmax,3).round(2))
ax_sp.set_xlabel('Flux (Jy)')
lc = dd.mean(axis=0)
lenlc = n.where(lc == 0)[0][0]
ax_lc.plot(range(0,lenlc)+range(2*lenlc,3*lenlc), list(lc)[:lenlc] + list(lc)[-lenlc:], 'k.')
ax_lc.plot(range(0,lenlc)+range(2*lenlc,3*lenlc), list(n.zeros(lenlc)) + list(n.zeros(lenlc)), 'k:')
ax_lc.set_xlabel('Integration')
ax_lc.set_ylabel('Flux (Jy)')
ax_lc.set_xticks([0,0.5*lenlc,lenlc,1.5*lenlc,2*lenlc,2.5*lenlc,3*lenlc])
ax_lc.set_xticklabels(['0',str(lenlc/2),str(lenlc),'','0',str(lenlc/2),str(lenlc)])
ymin,ymax = ax_lc.get_ylim()
ax_lc.set_yticks(n.linspace(ymin,ymax,3).round(2))
# image
ax = fig.add_subplot(223)
fov = n.degrees(1./d['uvres'])*60.
# ax.scatter(((xpix/2-srcra[0])-0.05*xpix)*fov/xpix, (ypix/2-srcdec[0])*fov/ypix, s=80, marker='<', facecolor='none')
# ax.scatter(((xpix/2-srcra[0])+0.05*xpix)*fov/xpix, (ypix/2-srcdec[0])*fov/ypix, s=80, marker='>', facecolor='none')
impl = ax.imshow(im.transpose(), aspect='equal', origin='upper', interpolation='nearest', extent=[fov/2, -fov/2, -fov/2, fov/2], cmap=plt.get_cmap('Greys'), vmin=0, vmax=0.5*im.max())
ax.set_xlabel('RA Offset (arcmin)')
ax.set_ylabel('Dec Offset (arcmin)')
if not outname:
outname = os.path.join(d['workdir'], 'cands_%s_sc%dseg%di%ddm%ddt%d.png' % (d['fileroot'], scan, segment, loc[candnum, intcol], dmind, dtind))
canvas = FigureCanvasAgg(fig)
canvas.print_figure(outname)
return ([],[])
