def refine_cands(candsfile,
threshold=0,
scaledm=2.1,
scalepix=2,
scaleuv=1.0,
chans=[],
savepkl=True):
""" Runs refine_cand on all positive SNR candidates above threshold. Any detected at higher SNR are highlighted. """
# get snrs above threshold
locs, props, d = pc.read_candidates(candsfile,
snrmin=threshold,
returnstate=True)
if 'snr2' in d['features']:
snrcol = d['features'].index('snr2')
elif 'snr1' in d['features']:
snrcol = d['features'].index('snr1')
scancol = d['featureind'].index('scan')
segmentcol = d['featureind'].index('segment')
intcol = d['featureind'].index('int')
dtindcol = d['featureind'].index('dtind')
dmindcol = d['featureind'].index('dmind')
snrs = props[:, snrcol]
for (i, snr) in enumerate(snrs):
if snr > 0:
d, cands = refine_cand(candsfile,
threshold=threshold,
candnum=i,
scaledm=scaledm,
scalepix=scalepix,
scaleuv=scaleuv,
chans=chans)
if cands:
candlocs = np.array(cands.keys())
candprops = np.array(cands.values())
scan = locs[i, scancol]
segment = locs[i, segmentcol]
candint = locs[i, intcol]
dmind = locs[i, dmindcol]
dtind = locs[i, dtindcol]
candfile = 'cands_{0}_sc{1}-seg{2}-i{3}-dm{4}-dt{5}.pkl'.format(
d['fileroot'], scan, segment, candint, dmind, dtind)
if any([candsnr > snr for candsnr in candprops[:, snrcol]]):
logger.info(
'Cand {0} had SNR {1} and refinement found a higher SNR in new ones: {2}.'
.format(i, snr, candprops[:, snrcol]))
logger.info('Saving to {0}: {1}'.format(candfile, cands))
with open(candfile, 'w') as pkl:
pickle.dump(d, pkl, protocol=2)
pickle.dump((candlocs, candprops), pkl, protocol=2)
else:
logger.info(
'Cand {0} had SNR {1}, but refinement found no improvement: {2}'
.format(i, snr, candprops[:, snrcol]))
def readcandsfile(candsfile, plotdir='/users/claw/public_html/plots', tag=None, copyplots=True):
""" Read candidates from pickle file and format as list of dictionaries
plotdir is path to png plot files which are required in order to keep in datalist
optionally copies png files into plotdir
"""
if tag:
assert isintance(tag, str)
loc, prop, state = read_candidates(candsfile, returnstate=True)
fileroot = state['fileroot']
if plotdir:
logging.info('Filtering data based on presence of png files in {0}'.format(plotdir))
else:
logging.info('Appending all data to datalist.')
datalist = []
for i in range(len(loc)):
data = {}
data['obs'] = fileroot
for feat in state['featureind']:
col = state['featureind'].index(feat)
data[feat] = loc[i][col]
for feat in state['features']:
col = state['features'].index(feat)
if isnan(prop[i][col]):
data[feat] = nan_to_num(prop[i][col])
else:
data[feat] = prop[i][col]
uniqueid = dataid(data)
data['candidate_png'] = 'cands_{0}.png'.format(uniqueid)
data['labeled'] = '0' # has this cand been labeled by active learning loop?
if tag:
data['tag'] = tag # care to add comma-delimited string to this cand?
else:
data['tag'] = ''
# copy plot over and add path to datalist
if plotdir:
if copyplots and os.path.exists(data['candidate_png']) and not os.path.exists(os.path.join(plotdir, data['candidate_png'])):
copy(data['candidate_png'], plotdir)
if os.path.exists(os.path.join(plotdir, data['candidate_png'])):
datalist.append(data)
else:
datalist.append(data)
return datalist
def refine_cand(candsfile,
candloc=[],
candnum=-1,
threshold=0,
scaledm=2.1,
scalepix=2,
scaleuv=1.0,
chans=[],
returndata=False):
""" Helper function to interact with merged cands file and refine analysis
candsfile is merged pkl file
candloc (scan, segment, candint, dmind, dtind, beamnum) is as above.
if no candloc, then it prints out cands above threshold.
"""
if candnum >= 0:
candlocs, candprops, d0 = pc.read_candidates(candsfile,
snrmin=threshold,
returnstate=True)
candloc = candlocs[candnum]
candprop = candprops[candnum]
logger.info('Refining cand {0} with features {1}'.format(
candloc, candprop))
values = rt.pipeline_refine(d0,
candloc,
scaledm=scaledm,
scalepix=scalepix,
scaleuv=scaleuv,
chans=chans,
returndata=returndata)
return values
elif candloc:
logger.info('Refining cand {0}'.format(candloc))
d0 = pickle.load(open(candsfile, 'r'))
values = rt.pipeline_refine(d0,
candloc,
scaledm=scaledm,
scalepix=scalepix,
scaleuv=scaleuv,
chans=chans,
returndata=returndata)
return d, cands
else:
return None
def readcandsfile(candsfile, plotdir='/users/claw/public_html/plots', tag=None):
""" Read candidates from pickle file and format as list of dictionaries
plotdir is path to png plot files which are required in order to keep in datalist
"""
if tag:
assert isintance(tag, str)
loc, prop, state = read_candidates(candsfile, returnstate=True)
fileroot = state['fileroot']
if plotdir:
logging.info('Filtering data based on presence of png files in {0}'.format(plotdir))
else:
logging.info('Appending all data to datalist.')
datalist = []
for i in range(len(loc)):
data = {}
data['obs'] = fileroot
for feat in state['featureind']:
col = state['featureind'].index(feat)
data[feat] = loc[i][col]
for feat in state['features']:
col = state['features'].index(feat)
data[feat] = prop[i][col]
uniqueid = dataid(data)
data['candidate_png'] = 'cands_{0}.png'.format(uniqueid)
data['labeled'] = '0'
if tag:
data['tag'] = tag
else:
data['tag'] = ''
if plotdir:
if os.path.exists(os.path.join(plotdir, data['candidate_png'])):
datalist.append(data)
else:
datalist.append(data)
return datalist
def list_cands(candsfile, threshold=0.):
""" Prints candidate info in time order above some threshold """
loc, prop, d0 = pc.read_candidates(candsfile,
snrmin=threshold,
returnstate=True)
if 'snr2' in d0['features']:
snrcol = d0['features'].index('snr2')
elif 'snr1' in d0['features']:
snrcol = d0['features'].index('snr1')
dmindcol = d0['featureind'].index('dmind')
if len(loc):
snrs = prop[:, snrcol]
times = pc.int2mjd(d0, loc)
times = times - times[0]
logger.info('Getting candidates...')
logger.info('candnum: loc, SNR, DM (pc/cm3), time (s; rel)')
for i in range(len(loc)):
logger.info("%d: %s, %.1f, %.1f, %.1f" %
(i, str(loc[i]), prop[i, snrcol], np.array(
d0['dmarr'])[loc[i, dmindcol]], times[i]))
def plot_cand(candsfile,
candloc=[],
candnum=-1,
threshold=0,
savefile=True,
returndata=False,
outname='',
**kwargs):
""" Reproduce detection of a single candidate for plotting or inspection.
candsfile can be merge or single-scan cands pkl file. Difference defined by presence of scan in d['featureind'].
candloc reproduces candidate at given location (scan, segment, integration, dmind, dtind, beamnum).
candnum selects one to reproduce from ordered list
threshold is min of sbs(SNR) used to filter candidates to select with candnum.
savefile/outname define if/how to save png of candidate
if returndata, (im, data) returned.
kwargs passed to rt.set_pipeline
"""
# get candidate info
loc, prop = pc.read_candidates(candsfile)
# define state dict and overload with user prefs
d0 = pickle.load(open(candsfile, 'r'))
for key in kwargs:
logger.info('Setting %s to %s' % (key, kwargs[key]))
d0[key] = kwargs[key]
d0['logfile'] = False # no need to save log
# feature columns
if 'snr2' in d0['features']:
snrcol = d0['features'].index('snr2')
elif 'snr1' in d0['features']:
snrcol = d0['features'].index('snr1')
if 'l2' in d0['features']:
lcol = d0['features'].index('l2')
elif 'l1' in d0['features']:
lcol = d0['features'].index('l1')
if 'm2' in d0['features']:
mcol = d0['features'].index('m2')
elif 'm1' in d0['features']:
mcol = d0['features'].index('m1')
try:
scancol = d0['featureind'].index('scan') # if merged pkl
except ValueError:
scancol = -1 # if single-scan pkl
segmentcol = d0['featureind'].index('segment')
intcol = d0['featureind'].index('int')
dtindcol = d0['featureind'].index('dtind')
dmindcol = d0['featureind'].index('dmind')
# sort and prep candidate list
snrs = prop[:, snrcol]
select = np.where(np.abs(snrs) > threshold)[0]
loc = loc[select]
prop = prop[select]
times = pc.int2mjd(d0, loc)
times = times - times[0]
# default case will print cand info
if (candnum < 0) and (not len(candloc)):
logger.info('Getting candidates...')
logger.info('candnum: loc, SNR, DM (pc/cm3), time (s; rel)')
for i in range(len(loc)):
logger.info("%d: %s, %.1f, %.1f, %.1f" %
(i, str(loc[i]), prop[i, snrcol], np.array(
d0['dmarr'])[loc[i, dmindcol]], times[i]))
else: # if candnum or candloc provided, try to reproduce
if (candnum >= 0) and not len(candloc):
logger.info(
'Reproducing and visualizing candidate %d at %s with properties %s.'
% (candnum, loc[candnum], prop[candnum]))
dmarrorig = d0['dmarr']
dtarrorig = d0['dtarr']
if scancol >= 0: # here we have a merge pkl
scan = loc[candnum, scancol]
else: # a scan-based cands pkl
scan = d0['scan']
segment = loc[candnum, segmentcol]
candint = loc[candnum, intcol]
dmind = loc[candnum, dmindcol]
dtind = loc[candnum, dtindcol]
beamnum = 0
candloc = (scan, segment, candint, dmind, dtind, beamnum)
elif len(candloc) and (candnum < 0):
assert len(
candloc
) == 6, 'candloc should be length 6 ( scan, segment, candint, dmind, dtind, beamnum ).'
logger.info('Reproducing and visualizing candidate %d at %s' %
(candnum, candloc))
dmarrorig = d0['dmarr']
dtarrorig = d0['dtarr']
scan, segment, candint, dmind, dtind, beamnum = candloc
else:
raise Exception, 'Provide candnum or candloc, not both'
# if working locally, set workdir appropriately. Can also be used in queue system with full path given.
if not os.path.dirname(candsfile):
d0['workdir'] = os.getcwd()
else:
d0['workdir'] = os.path.dirname(candsfile)
filename = os.path.join(d0['workdir'],
os.path.basename(d0['filename']))
# clean up d0 of superfluous keys
params = pp.Params() # will be used as input to rt.set_pipeline
for key in d0.keys():
if not hasattr(params, key) and 'memory_limit' not in key:
_ = d0.pop(key)
d0['npix'] = 0
d0['uvres'] = 0
d0['nsegments'] = 0
d0['logfile'] = False
# get cand data
d = rt.set_pipeline(filename, scan, **d0)
im, data = rt.pipeline_reproduce(
d, candloc, product='imdata') # removed loc[candnum]
# optionally plot
if savefile:
loclabel = scan, segment, candint, dmind, dtind, beamnum
make_cand_plot(d, im, data, loclabel, outname=outname)
# optionally return data
if returndata:
return (im, data)
def readdata(mergepkl=None, d=None, cands=None, sizerange=(2,70)):
""" Converts candidate data to dictionary for bokeh
Can take merged pkl file or d/cands as read separately.
cands is an optional (loc, prop) tuple of numpy arrays.
"""
# get cands from pkl
if mergepkl:
logger.info('Reading {0}'.format(mergepkl))
loc, prop, d = read_candidates(mergepkl, returnstate=True)
elif d and cands:
logger.info('Using provided d/cands')
loc, prop = cands
# define columns to extract
if 'snr2' in d['features']:
snrcol = d['features'].index('snr2')
elif 'snr1' in d['features']:
snrcol = d['features'].index('snr1')
l1col = d['features'].index('l1')
m1col = d['features'].index('m1')
specstdcol = d['features'].index('specstd')
imkurcol = d['features'].index('imkurtosis')
dtindcol = d['featureind'].index('dtind')
dmindcol = d['featureind'].index('dmind')
intcol = d['featureind'].index('int')
segmentcol = d['featureind'].index('segment')
scancol = d['featureind'].index('scan')
# define data to plot
key = ['sc{0}-seg{1}-i{2}-dm{3}-dt{4}'.format(ll[scancol], ll[segmentcol], ll[intcol], ll[dmindcol], ll[dtindcol]) for ll in loc]
# key = [tuple(ll) for ll in loc]
scan = loc[:, scancol]
seg = loc[:, segmentcol]
candint = loc[:, 2]
dmind = loc[:, 3]
dtind = loc[:, 4]
beamnum = loc[:, 5]
logger.info('Setting columns...')
snrs = prop[:, snrcol]
abssnr = np.abs(prop[:, snrcol])
dm = np.array(d['dmarr'])[loc[:, dmindcol]]
l1 = prop[:, l1col]
m1 = prop[:, m1col]
time = np.array([24*3600*d['segmenttimesdict'][scan[i]][seg[i], 0] + d['inttime']*candint[i] for i in range(len(loc))])
# time.append(24*3600*d['segmenttimesdict'][k[scancol]][k[segmentcol],0] + d['inttime']*k[intcol])
specstd = prop[:, specstdcol]
imkur = prop[:, imkurcol]
logger.info('Calculating sizes, colors, normprob...')
time = time - min(time)
sizes = calcsize(snrs)
colors = colorsat(l1, m1)
zs = normprob(d, snrs)
# if pandas is available use dataframe to allow datashader feature
# data = DataFrame(data={'snrs': snrs, 'dm': dm, 'l1': l1, 'm1': m1, 'time': time, 'specstd': specstd,
# 'imkur': imkur, 'scan': scan, 'seg': seg, 'candint': candint, 'dmind': dmind,
# 'dtind': dtind, 'sizes': sizes, 'colors': colors, 'key': key, 'zs': zs, 'abssnr': abssnr})
# logger.info('Returning a pandas dataframe')
data = dict(snrs=snrs, dm=dm, l1=l1, m1=m1, time=time, specstd=specstd, scan=scan,
imkur=imkur, sizes=sizes, colors=colors, key=key, zs=zs, abssnr=abssnr)
# dtind=dtind, scan=scan, seg=seg, candint=candint, dmind=dmind,
return data
def readcandsfile(candsfile,
plotdir='/users/claw/public_html/plots',
tag=None,
copyplots=True):
""" Read candidates from pickle file and format as list of dictionaries
plotdir is path to png plot files which are required in order to keep in datalist
optionally copies png files into plotdir
"""
if tag:
assert isintance(tag, str)
loc, prop, state = read_candidates(candsfile, returnstate=True)
fileroot = state['fileroot']
if plotdir:
logging.info(
'Filtering data based on presence of png files in {0}'.format(
plotdir))
else:
logging.info('Appending all data to datalist.')
datalist = []
for i in range(len(loc)):
data = {}
data['obs'] = fileroot
for feat in state['featureind']:
col = state['featureind'].index(feat)
data[feat] = loc[i][col]
for feat in state['features']:
col = state['features'].index(feat)
if isnan(prop[i][col]):
data[feat] = nan_to_num(prop[i][col])
else:
data[feat] = prop[i][col]
uniqueid = dataid(data)
data['candidate_png'] = 'cands_{0}.png'.format(uniqueid)
data[
'labeled'] = '0' # has this cand been labeled by active learning loop?
if tag:
data[
'tag'] = tag # care to add comma-delimited string to this cand?
else:
data['tag'] = ''
# copy plot over and add path to datalist
if plotdir:
if copyplots and os.path.exists(
data['candidate_png']) and not os.path.exists(
os.path.join(plotdir, data['candidate_png'])):
copy(data['candidate_png'], plotdir)
if os.path.exists(os.path.join(plotdir, data['candidate_png'])):
datalist.append(data)
else:
datalist.append(data)
return datalist
def plot_cand(candsfile,
candloc=[],
candnum=-1,
threshold=0,
savefile=True,
returndata=False,
outname='',
newplot=True,
returnstate=False,
**kwargs):
""" Reproduce detection of a single candidate for plotting or inspection.
candsfile can be merge or single-scan cands pkl file. Difference defined by presence of scan in d['featureind'].
candloc reproduces candidate at given location (scan, segment, integration, dmind, dtind, beamnum).
candnum selects one to reproduce from ordered list
threshold is min of sbs(SNR) used to filter candidates to select with candnum.
savefile/outname define if/how to save png of candidate
if returndata, (im, data) returned.
kwargs passed to rt.set_pipeline
if newplot, then plot with the new candidate plot using bridget's version
"""
# get candidate info
loc, prop, d0 = pc.read_candidates(candsfile, returnstate=True)
# define state dict and overload with user prefs
for key in kwargs:
logger.info('Setting %s to %s' % (key, kwargs[key]))
d0[key] = kwargs[key]
d0['logfile'] = False # no need to save log
# feature columns
if 'snr2' in d0['features']:
snrcol = d0['features'].index('snr2')
elif 'snr1' in d0['features']:
snrcol = d0['features'].index('snr1')
if 'l2' in d0['features']:
lcol = d0['features'].index('l2')
elif 'l1' in d0['features']:
lcol = d0['features'].index('l1')
if 'm2' in d0['features']:
mcol = d0['features'].index('m2')
elif 'm1' in d0['features']:
mcol = d0['features'].index('m1')
scancol = d0['featureind'].index('scan')
segmentcol = d0['featureind'].index('segment')
intcol = d0['featureind'].index('int')
dtindcol = d0['featureind'].index('dtind')
dmindcol = d0['featureind'].index('dmind')
# sort and prep candidate list
snrs = prop[:, snrcol]
select = np.where(np.abs(snrs) > threshold)[0]
loc = loc[select]
prop = prop[select]
if candnum >= 0 or len(candloc):
if (candnum >= 0) and not len(candloc):
logger.info(
'Reproducing and visualizing candidate %d at %s with properties %s.'
% (candnum, loc[candnum], prop[candnum]))
dmarrorig = d0['dmarr']
dtarrorig = d0['dtarr']
scan = loc[candnum, scancol]
segment = loc[candnum, segmentcol]
candint = loc[candnum, intcol]
dmind = loc[candnum, dmindcol]
dtind = loc[candnum, dtindcol]
beamnum = 0
candloc = (scan, segment, candint, dmind, dtind, beamnum)
elif len(candloc) and (candnum < 0):
assert len(
candloc
) == 6, 'candloc should be length 6 ( scan, segment, candint, dmind, dtind, beamnum ).'
logger.info('Reproducing and visualizing candidate %d at %s' %
(candnum, candloc))
dmarrorig = d0['dmarr']
dtarrorig = d0['dtarr']
scan, segment, candint, dmind, dtind, beamnum = candloc
else:
raise Exception, 'Provide candnum or candloc, not both'
# if working locally, set workdir appropriately. Can also be used in queue system with full path given.
if not os.path.dirname(candsfile):
d0['workdir'] = os.getcwd()
else:
d0['workdir'] = os.path.dirname(candsfile)
filename = os.path.join(d0['workdir'],
os.path.basename(d0['filename']))
if d0.has_key('segmenttimesdict'): # using merged pkl
segmenttimes = d0['segmenttimesdict'][scan]
else:
segmenttimes = d0['segmenttimes']
# clean up d0 of superfluous keys
params = pp.Params() # will be used as input to rt.set_pipeline
for key in d0.keys():
if not hasattr(params, key): # and 'memory_limit' not in key:
_ = d0.pop(key)
d0['npix'] = 0
d0['uvres'] = 0
d0['logfile'] = False
d0['savenoise'] = False
d0['savecands'] = False
# this triggers redefinition of segment boundaries. memory optimization changed, so this is a problem.
# d0['nsegments'] = 0
# d0['scale_nsegments'] = 1.
d0['segmenttimes'] = segmenttimes
d0['nsegments'] = len(segmenttimes)
# get cand data
d = rt.set_pipeline(filename, scan, **d0)
(vismem, immem) = rt.calc_memory_footprint(d)
if 'memory_limit' in d:
assert vismem + immem < d[
'memory_limit'], 'memory_limit defined, but nsegments must (for now) be set to initial values to properly reproduce candidate'
im, data = rt.pipeline_reproduce(
d, candloc, product='imdata') # removed loc[candnum]
# optionally plot
if savefile:
loclabel = scan, segment, candint, dmind, dtind, beamnum
if newplot:
make_cand_plot(d,
im,
data,
loclabel,
version=2,
snrs=snrs,
outname=outname)
else:
make_cand_plot(d,
im,
data,
loclabel,
version=1,
outname=outname)
# optionally return data
if returndata:
return (im, data)
elif returnstate:
return d
def readdata(mergepkl=None, d=None, cands=None, sizerange=(2, 70)):
""" Converts candidate data to dictionary for bokeh
Can take merged pkl file or d/cands as read separately.
cands is an optional (loc, prop) tuple of numpy arrays.
"""
# get cands from pkl
if mergepkl:
logger.info('Reading {0}'.format(mergepkl))
loc, prop, d = read_candidates(mergepkl, returnstate=True)
elif d and cands:
logger.info('Using provided d/cands')
loc, prop = cands
# define columns to extract
if 'snr2' in d['features']:
snrcol = d['features'].index('snr2')
elif 'snr1' in d['features']:
snrcol = d['features'].index('snr1')
l1col = d['features'].index('l1')
m1col = d['features'].index('m1')
specstdcol = d['features'].index('specstd')
imkurcol = d['features'].index('imkurtosis')
dtindcol = d['featureind'].index('dtind')
dmindcol = d['featureind'].index('dmind')
intcol = d['featureind'].index('int')
segmentcol = d['featureind'].index('segment')
scancol = d['featureind'].index('scan')
# define data to plot
key = [
'sc{0}-seg{1}-i{2}-dm{3}-dt{4}'.format(ll[scancol], ll[segmentcol],
ll[intcol], ll[dmindcol],
ll[dtindcol]) for ll in loc
]
# key = [tuple(ll) for ll in loc]
scan = loc[:, scancol]
seg = loc[:, segmentcol]
candint = loc[:, 2]
dmind = loc[:, 3]
dtind = loc[:, 4]
beamnum = loc[:, 5]
logger.info('Setting columns...')
snrs = prop[:, snrcol]
abssnr = np.abs(prop[:, snrcol])
dm = np.array(d['dmarr'])[loc[:, dmindcol]]
l1 = prop[:, l1col]
m1 = prop[:, m1col]
time = np.array([
24 * 3600 * d['segmenttimesdict'][scan[i]][seg[i], 0] +
d['inttime'] * candint[i] for i in range(len(loc))
])
# time.append(24*3600*d['segmenttimesdict'][k[scancol]][k[segmentcol],0] + d['inttime']*k[intcol])
specstd = prop[:, specstdcol]
imkur = prop[:, imkurcol]
logger.info('Calculating sizes, colors, normprob...')
time = time - min(time)
sizes = calcsize(snrs)
colors = colorsat(l1, m1)
zs = normprob(d, snrs)
# if pandas is available use dataframe to allow datashader feature
# data = DataFrame(data={'snrs': snrs, 'dm': dm, 'l1': l1, 'm1': m1, 'time': time, 'specstd': specstd,
# 'imkur': imkur, 'scan': scan, 'seg': seg, 'candint': candint, 'dmind': dmind,
# 'dtind': dtind, 'sizes': sizes, 'colors': colors, 'key': key, 'zs': zs, 'abssnr': abssnr})
# logger.info('Returning a pandas dataframe')
data = dict(snrs=snrs,
dm=dm,
l1=l1,
m1=m1,
time=time,
specstd=specstd,
scan=scan,
imkur=imkur,
sizes=sizes,
colors=colors,
key=key,
zs=zs,
abssnr=abssnr)
# dtind=dtind, scan=scan, seg=seg, candint=candint, dmind=dmind,
return data
def plot_cand(candsfile, candloc=[], candnum=-1, threshold=0, savefile=True, returndata=False, outname='', **kwargs):
""" Reproduce detection of a single candidate for plotting or inspection.
candsfile can be merge or single-scan cands pkl file. Difference defined by presence of scan in d['featureind'].
candloc reproduces candidate at given location (scan, segment, integration, dmind, dtind, beamnum).
candnum selects one to reproduce from ordered list
threshold is min of sbs(SNR) used to filter candidates to select with candnum.
savefile/outname define if/how to save png of candidate
if returndata, (im, data) returned.
kwargs passed to rt.set_pipeline
"""
# get candidate info
loc, prop = pc.read_candidates(candsfile)
# define state dict and overload with user prefs
d0 = pickle.load(open(candsfile, 'r'))
for key in kwargs:
logger.info('Setting %s to %s' % (key, kwargs[key]))
d0[key] = kwargs[key]
d0['logfile'] = False # no need to save log
# feature columns
if 'snr2' in d0['features']:
snrcol = d0['features'].index('snr2')
elif 'snr1' in d0['features']:
snrcol = d0['features'].index('snr1')
if 'l2' in d0['features']:
lcol = d0['features'].index('l2')
elif 'l1' in d0['features']:
lcol = d0['features'].index('l1')
if 'm2' in d0['features']:
mcol = d0['features'].index('m2')
elif 'm1' in d0['features']:
mcol = d0['features'].index('m1')
try:
scancol = d0['featureind'].index('scan') # if merged pkl
except ValueError:
scancol = -1 # if single-scan pkl
segmentcol = d0['featureind'].index('segment')
intcol = d0['featureind'].index('int')
dtindcol = d0['featureind'].index('dtind')
dmindcol = d0['featureind'].index('dmind')
# sort and prep candidate list
snrs = prop[:, snrcol]
select = np.where(np.abs(snrs) > threshold)[0]
loc = loc[select]
prop = prop[select]
times = pc.int2mjd(d0, loc)
times = times - times[0]
# default case will print cand info
if (candnum < 0) and (not len(candloc)):
logger.info('Getting candidates...')
logger.info('candnum: loc, SNR, DM (pc/cm3), time (s; rel)')
for i in range(len(loc)):
logger.info("%d: %s, %.1f, %.1f, %.1f" % (i, str(loc[i]), prop[i, snrcol], np.array(d0['dmarr'])[loc[i,dmindcol]], times[i]))
else: # if candnum or candloc provided, try to reproduce
if (candnum >= 0) and not len(candloc):
logger.info('Reproducing and visualizing candidate %d at %s with properties %s.' % (candnum, loc[candnum], prop[candnum]))
dmarrorig = d0['dmarr']
dtarrorig = d0['dtarr']
if scancol >= 0: # here we have a merge pkl
scan = loc[candnum, scancol]
else: # a scan-based cands pkl
scan = d0['scan']
segment = loc[candnum, segmentcol]
candint = loc[candnum, intcol]
dmind = loc[candnum, dmindcol]
dtind = loc[candnum, dtindcol]
beamnum = 0
candloc = (scan, segment, candint, dmind, dtind, beamnum)
elif len(candloc) and (candnum < 0):
assert len(candloc) == 6, 'candloc should be length 6 ( scan, segment, candint, dmind, dtind, beamnum ).'
logger.info('Reproducing and visualizing candidate %d at %s' % (candnum, candloc))
dmarrorig = d0['dmarr']
dtarrorig = d0['dtarr']
scan, segment, candint, dmind, dtind, beamnum = candloc
else:
raise Exception, 'Provide candnum or candloc, not both'
# if working locally, set workdir appropriately. Can also be used in queue system with full path given.
if not os.path.dirname(candsfile):
d0['workdir'] = os.getcwd()
else:
d0['workdir'] = os.path.dirname(candsfile)
filename = os.path.join(d0['workdir'], os.path.basename(d0['filename']))
# clean up d0 of superfluous keys
params = pp.Params() # will be used as input to rt.set_pipeline
for key in d0.keys():
if not hasattr(params, key) and 'memory_limit' not in key:
_ = d0.pop(key)
d0['npix'] = 0
d0['uvres'] = 0
d0['nsegments'] = 0
d0['logfile'] = False
# get cand data
d = rt.set_pipeline(filename, scan, **d0)
im, data = rt.pipeline_reproduce(d, candloc, product='imdata') # removed loc[candnum]
# optionally plot
if savefile:
loclabel = scan, segment, candint, dmind, dtind, beamnum
make_cand_plot(d, im, data, loclabel, outname=outname)
# optionally return data
if returndata:
return (im, data)