def run_rf_pipelines(filename_list, output_dir, output_acq_path):
params = ch_frb_rfi.transform_parameters(
plot_type='web_viewer',
make_plots=False,
bonsai_output_plot_stem=None,
maskpath=None,
two_pass=True,
clip_nt=1024,
eq_clip_nt=True,
detrend_nt=1024,
rfi_level=-1,
aux_clip_first=True,
aux_clip_last=True,
aux_detrend_first=False,
detrender_niter=1,
clipper_niter=6,
plot_nypix=1024,
plot_nxpix=256,
plot_downsample_nt=1,
plot_nzoom=2,
spline=True,
bonsai_use_analytic_normalization=False,
bonsai_hdf5_output_filename=None,
bonsai_nt_per_hdf5_file=None,
bonsai_fill_rfi_mask=False,
var_est=False,
mask_filler=False,
bonsai_dynamic_plotter=False,
bonsai_plot_all_trees=False)
t1k = ch_frb_rfi.transform_chain(params)
p1k = rf_pipelines.pipeline(t1k)
t16k = [rf_pipelines.wi_sub_pipeline(p1k, nfreq_out=1024, nds_out=1)]
params.detrend_last = True
t16k += ch_frb_rfi.chains.detrender_chain(params, ix=1, jx=0)
t16k += [ch_frb_rfi.WriteWeights(basename=output_acq_path + '/data')]
p16k = rf_pipelines.pipeline(t16k)
s = rf_pipelines.chime_frb_stream_from_filename_list(filename_list,
nt_chunk=1024,
noise_source_align=0)
ch_frb_rfi.utils.run_in_scratch_dir(output_acq_path, output_dir, s, p16k)
def run_pipeline(pipeline_json, intensity_arr, weights_arr):
# Just for fun, randomize 'nt_chunk'.
p0 = initial_stream(intensity_arr, weights_arr)
p1 = rf_pipelines.pipeline_object.from_json(pipeline_json)
p2 = final_transform()
p = rf_pipelines.pipeline([p0,p1,p2])
p.run(outdir=None, verbosity=0, debug=True)
(intensity, weights) = p2.get_results()
return (intensity, weights)
def init_search_params(self, sparams):
"""Overrides dedisperser_base.init_search_params(). The 'sparams' argument is an instance of 'class search_params'."""
if hasattr(self, 'search_params'):
raise RuntimeError('double call to frb_olympics.bonsai_dedisperser.init_search_params()')
# FIXME I would like to add sanity checks here on the following search_params:
# nfreq, freq_lo_MHz, freq_hi_MHz, dt_sample, max_dm.
s = frb_olympics_stream(sparams)
p = rf_pipelines.pipeline([s, self.base_pipeline])
p.bind(outdir=None, verbosity=0)
self.search_params = sparams
self.full_pipeline = p
self.stream = s
def make_pipeline(*args):
"""
Helper function which combines arguments into a pipeline.
The syntax is flexible: arguments can be either
- pipeline_objects
- filenames ending in .json
- lists of either of these
"""
p = rf_pipelines.pipeline()
_add_to_pipeline(p, *args)
if p.size == 0:
raise RuntimeError(
'ch_frb_rfi: in either run_for_web_viewer() or run_in_scratch_dir(), no pipeline_objects were specified'
)
return p
def run_test():
Df = 2**rand.randint(0,5)
nfreq = Df * 8 * rand.randint(10, 20)
nt_tot = 8 * rand.randint(150, 500)
input_intensity = rand.standard_normal(size=(nfreq,nt_tot))
input_weights = rand.uniform(0.5, 1.0, size=(nfreq,nt_tot))
p0_json = make_random_pipeline_json()
p1_json = make_random_pipeline_json()
p2_json = make_random_pipeline_json()
# First run
(i0,w0) = run_pipeline(p0_json, input_intensity, input_weights)
(i0,w0) = (i0[:,:nt_tot], w0[:,:nt_tot])
(i1,w1) = rf_pipelines.wi_downsample(i0, w0, Df, 1)
(i2,w2) = run_pipeline(p1_json, i1, w1)
(i2,w2) = (i2[:,:nt_tot], w2[:,:nt_tot])
rf_pipelines.weight_upsample(w0, w2)
(i3,w3) = run_pipeline(p2_json, i0, w0)
# Second run
si = initial_stream(input_intensity, input_weights)
p0 = rf_pipelines.pipeline_object.from_json(p0_json)
p1 = rf_pipelines.pipeline_object.from_json(p1_json)
ps = rf_pipelines.wi_sub_pipeline(p1, Df=Df, Dt=1)
p2 = rf_pipelines.pipeline_object.from_json(p2_json)
tf = final_transform()
p = rf_pipelines.pipeline([ si, p0, ps, p2, tf ])
p.run(outdir=None, verbosity=0, debug=True)
(i4,w4) = tf.get_results()
eps_i = maxdiff((i3*w3)[:,:nt_tot],(i4*w4)[:,:nt_tot])
eps_w = maxdiff(w3[:,:nt_tot], w4[:,:nt_tot])
assert eps_i < 1.0e-5
assert eps_w < 1.0e-5
assert np.all(w3[:,nt_tot:] == 0.0)
assert np.all(w4[:,nt_tot:] == 0.0)
def run_test():
s = initial_stream()
u = final_transform()
tj = make_random_transform_list(s.nfreq, 1, nelements=rand.randint(10,20))
t = [ rf_pipelines.pipeline_object.from_json(j) for j in tj ]
p = rf_pipelines.pipeline([s] + t + [u])
p.bind(outdir=None, verbosity=0, debug=True)
# Check jsonization (test is slightly stronger if bind() comes first)
tj2 = [ x.jsonize() for x in t ]
rf_pipelines.utils.json_assert_equal(tj, tj2, name1='reference_json', name2='pipeline_json')
# First run
p.run(outdir=None, verbosity=0, debug=True)
(i0,w0) = u.get_results()
nt0 = i0.shape[1]
assert nt0 >= s.nt_tot
assert i0.shape == w0.shape == (s.nfreq, nt0)
assert np.all(w0[:,s.nt_tot:] == 0.0)
# Second run
pj = { 'class_name': 'pipeline', 'elements': tj }
(i1,w1) = emulate_pipeline(pj, s.intensity, s.weights)
nt1 = i1.shape[1]
assert nt1 >= s.nt_tot
assert i1.shape == w1.shape == (s.nfreq, nt1)
assert np.all(w1[:,s.nt_tot:] == 0.0)
# Compare
eps_i = maxdiff((i0*w0)[:,:s.nt_tot], (i1*w1)[:,:s.nt_tot])
eps_w = maxdiff(w0[:,:s.nt_tot], w1[:,:s.nt_tot])
assert eps_i < 1.0e-5
assert eps_w < 1.0e-5
detrender_niter = 2,
clipper_niter = 6,
spline = True,
bonsai_use_analytic_normalization = False,
bonsai_hdf5_output_filename = None,
bonsai_nt_per_hdf5_file = None,
bonsai_fill_rfi_mask = True,
var_est = False,
mask_filler = False,
mask_filler_w_cutoff = 0.5,
bonsai_plot_threshold1 = 7,
bonsai_plot_threshold2 = 10,
bonsai_dynamic_plotter = False,
bonsai_plot_all_trees = make_plots)
t1k = ch_frb_rfi.transform_chain(params)
p1k = rf_pipelines.pipeline(t1k)
params.detrend_last = False
_t1k = ch_frb_rfi.transform_chain(params)
_p1k = rf_pipelines.pipeline(_t1k)
t16k = [ rf_pipelines.wi_sub_pipeline(_p1k, nfreq_out=1024, nds_out=1) ]
t16k += ch_frb_rfi.chains.detrender_chain(params, ix=0, jx=1)
p16k = rf_pipelines.pipeline(t16k)
for (pobj, suffix) in [ (p1k,'1k'), (p16k,'16k') ]:
suffix2 = '' if make_plots else '-noplot'
filename = '../../json_files/rfi_%s/17-12-01-two-pass%s.json' % (suffix, suffix2)
rf_pipelines.utils.json_write(filename, pobj, clobber=clobber)
print >>sys.stderr, "Fatal: source file '%s' must end in .json" % srcfile
if not os.path.exists(srcfile):
print >>sys.stderr, "Fatal: source file '%s' does not exist" % srcfile
sys.exit(1)
if os.path.exists(dstdir):
print >>sys.stderr, "Fatal: destination directory '%s' already exists, this is treated as an error to avoid accidentally overwriting data" % dstdir
sys.exit(1)
####################################################################################################
import rf_pipelines
j = rf_pipelines.json_read(srcfile)
stream = rf_pipelines.pipeline_object.from_json(j)
dstfile = os.path.join(dstdir, '00000000.h5')
writer = rf_pipelines.chime_file_writer(dstfile)
writer = rf_pipelines.wi_sub_pipeline(writer, Df=16, Dt=1)
pipeline = rf_pipelines.pipeline([stream, writer])
print 'Creating directory', dstdir
os.makedirs(dstdir)
print 'Running pipeline'
pipeline.run(outdir=None)
def make_random_pipeline():
n = rand.randint(1, 5)
return rf_pipelines.pipeline([ make_random_transform() for i in xrange(n) ])
# ripple effect. We compensate for this by using a _negative_ correction, i.e. deripple_fudge_factor = -0.2.
deripple_fudge_factor = -0.2
nparts = 7
missing_1k_flag = False
for part in xrange(nparts):
# Make 16k stream.
acqdir = '/data/17-10-01-16k-data/17-04-25-utkarsh-26m-part%d' % part
stream_16k = rf_pipelines.chime_stream_from_acqdir(acqdir)
# Append derippler.
derippler = rf_pipelines.chime_16k_derippler(
fudge_factor=deripple_fudge_factor)
stream_16k = rf_pipelines.pipeline([stream_16k, derippler])
# Write json file for the 16k acq
json_filename_16k = '../../json_files/acqs/17-04-25-utkarsh-26m-16k/part%d.json' % part
rf_pipelines.utils.json_write(json_filename_16k,
stream_16k,
clobber=clobber)
# Does the 1k acq exist? If not, print instructions for generating it.
acqdir_1k = '/data/17-10-01-16k-to-1k/17-04-25-utkarsh-26m-part%d' % part
acqfile_1k = os.path.join(acqdir_1k, '00000000.h5')
if not os.path.exists(acqfile_1k):
print "Note: 1K-channelized acq '%s' does not exist, you need to run this command:" % acqfile_1k
print " ../../scripts/downsample-16k-acq.py %s /data/17-10-01-16k-to-1k/17-04-25-utkarsh-26m-part4" % json_filename_16k
missing_1k_flag = True
#!/usr/bin/env python
#
# Creates 'rfp_bonsai.json' and 'rfp_detrender.json'.
#
# These are rf_pipelines json files (not frb_olympics json files)
# which represent serialized rf_pipelines.pipeline_objects.
import rf_pipelines
d = rf_pipelines.bonsai_dedisperser(
config_filename='bonsai_ntree4096_nups2.hdf5',
fill_rfi_mask=False,
use_analytic_normalization=True,
track_global_max=True)
rf_pipelines.json_write('rfp_bonsai.json', d, clobber=True)
d1 = rf_pipelines.polynomial_detrender(nt_chunk=1024, axis='time', polydeg=4)
d2 = rf_pipelines.spline_detrender(nt_chunk=1024, axis='freq', nbins=6)
p = rf_pipelines.pipeline([d1, d2])
rf_pipelines.json_write('rfp_detrender.json', p, clobber=True)
class masklist_capturing_transform(rf_pipelines.wi_transform):
def __init__(self):
rf_pipelines.wi_transform.__init__(self,
'masklist_capturing_transform')
def _process_chunk(self, intensity, weights, pos):
if hasattr(self, 'masklist'):
return
self.masklist = []
for ifreq in xrange(self.nfreq):
if weights[ifreq, 0] == 0.0:
self.masklist.append(ifreq)
s = rf_pipelines.gaussian_noise_stream(nfreq=1024,
nt_tot=1024,
freq_lo_MHz=400.0,
freq_hi_MHz=800.0,
dt_sample=1.0e-3)
t1 = rf_pipelines.badchannel_mask(mask_filename)
t2 = masklist_capturing_transform()
p = rf_pipelines.pipeline([s, t1, t2])
p.run(outdir=None, verbosity=1)
print 'The following channels were masked:', t2.masklist
#!/usr/bin/env python
import rf_pipelines as rfp
p1 = rfp.polynomial_detrender(nt_chunk=1024, axis='freq', polydeg=2)
p2 = rfp.polynomial_detrender(nt_chunk=1024, axis='time', polydeg=2)
p3 = rfp.mask_counter(nt_chunk=1024, where="after_rfi")
p = rfp.pipeline([p1, p2, p3])
q = rfp.wi_sub_pipeline(p, nfreq_out=1024, nds_out=1)
rfp.utils.json_write('rfi_placeholder.json', q, clobber=True)
bonsai_use_analytic_normalization = False,
bonsai_hdf5_output_filename = None,
bonsai_nt_per_hdf5_file = None,
bonsai_fill_rfi_mask = True,
var_est = False,
mask_filler = False,
mask_filler_w_cutoff = 0.5,
bonsai_plot_threshold1 = 7,
bonsai_plot_threshold2 = 10,
bonsai_dynamic_plotter = False,
bonsai_plot_all_trees = make_plots,
detrend_last = not detrend_16k,
mask_counter = True)
t1k = ch_frb_rfi.transform_chain(params)
p1k = rf_pipelines.pipeline(t1k)
t16k = [ rf_pipelines.wi_sub_pipeline(p1k, nfreq_out=1024, nds_out=1) ]
if detrend_16k:
params.detrend_last = True
params.mask_counter = False
t16k += ch_frb_rfi.chains.detrender_chain(params, ix=1, jx=0)
params.append_plotter_transform(t16k, 'dc_out_last')
if write_json:
assert isinstance(output_path, str) and output_path.endswith('.json')
p16k = rf_pipelines.pipeline(t16k)
rf_pipelines.utils.json_write(output_path, p16k, clobber=True)
#rf_pipelines.utils.json_write('design-rfi-config_acq.json', s, clobber=True)
# example_gaussian_stream_with_pulse.json \ # stream
# example_bonsai_transform.json # see make-jsonized-bonsai-transform.py
import rf_pipelines
s = rf_pipelines.gaussian_noise_stream(
nfreq = 16384,
nt_tot = 512*1024,
freq_lo_MHz = 400.0,
freq_hi_MHz = 800.0,
dt_sample = 0.98304e-3, # matches value in bonsai dedisperser (from CHIME)
sample_rms = 1.0,
nt_chunk = 1024
)
t = rf_pipelines.frb_injector_transform(
snr = 100.0,
undispersed_arrival_time = 200.0,
dm = 30.0,
variance = 1.0**2, # should be square of 'sample_rms' above
intrinsic_width = 0.005,
)
# The output of this script is logically a "stream" consisting of Gaussian noise plus a
# single pulse, but is implemented as a two-stage mini-pipeline: a stream object which
# produces the Gaussian noise, plus a transform to add the pulse.
p = rf_pipelines.pipeline([s,t])
rf_pipelines.utils.json_write('example_gaussian_stream_with_pulse.json', p, clobber=True)
variance_estimator_v1_chunk = v1_chunk,
variance_estimator_v2_chunk = v2_chunk,
mask_filler_w_cutoff = 0.5,
bonsai_plot_threshold1 = 7,
bonsai_plot_threshold2 = 10,
bonsai_dynamic_plotter = False,
bonsai_event_outfile = './events_example2.dat',
L1Grouper_thr = 10,
L1Grouper_beam = 0,
L1Grouper_addr = None)
# Using the specified parameters make a chain of transforms for estimating the variance.
t = ch_frb_rfi.transform_chain(p)
# Combine stream and transforms into a pipeline.
pipeline = rf_pipelines.pipeline([s]+t)
# The purpose of the first pipeline run is to create the h5 file containing variance
# estimates (p.var_filename = './var_example2.h5'). We do this pipeline run using the
# wrapper function run_in_scratch_dir(), which does not index the run with the web viewer.
ch_frb_rfi.run_in_scratch_dir('example2', pipeline)
# In the v16 API, need to "unbind" the pipeline after running, before its constituent
# pipeline_objects can be reused in another pipeline run.
pipeline.unbind()
# Remove the variance_estimator, append the mask_filler and plotter transforms.
p.var_est = False
p.mask_filler = True
p.make_plots = True
