def run_rf_pipelines(filename_list, output_dir, output_acq_path):
params = ch_frb_rfi.transform_parameters(
plot_type='web_viewer',
make_plots=False,
bonsai_output_plot_stem=None,
maskpath=None,
two_pass=True,
clip_nt=1024,
eq_clip_nt=True,
detrend_nt=1024,
rfi_level=-1,
aux_clip_first=True,
aux_clip_last=True,
aux_detrend_first=False,
detrender_niter=1,
clipper_niter=6,
plot_nypix=1024,
plot_nxpix=256,
plot_downsample_nt=1,
plot_nzoom=2,
spline=True,
bonsai_use_analytic_normalization=False,
bonsai_hdf5_output_filename=None,
bonsai_nt_per_hdf5_file=None,
bonsai_fill_rfi_mask=False,
var_est=False,
mask_filler=False,
bonsai_dynamic_plotter=False,
bonsai_plot_all_trees=False)
t1k = ch_frb_rfi.transform_chain(params)
p1k = rf_pipelines.pipeline(t1k)
t16k = [rf_pipelines.wi_sub_pipeline(p1k, nfreq_out=1024, nds_out=1)]
params.detrend_last = True
t16k += ch_frb_rfi.chains.detrender_chain(params, ix=1, jx=0)
t16k += [ch_frb_rfi.WriteWeights(basename=output_acq_path + '/data')]
p16k = rf_pipelines.pipeline(t16k)
s = rf_pipelines.chime_frb_stream_from_filename_list(filename_list,
nt_chunk=1024,
noise_source_align=0)
ch_frb_rfi.utils.run_in_scratch_dir(output_acq_path, output_dir, s, p16k)
def run_test():
Df = 2**rand.randint(0,5)
nfreq = Df * 8 * rand.randint(10, 20)
nt_tot = 8 * rand.randint(150, 500)
input_intensity = rand.standard_normal(size=(nfreq,nt_tot))
input_weights = rand.uniform(0.5, 1.0, size=(nfreq,nt_tot))
p0_json = make_random_pipeline_json()
p1_json = make_random_pipeline_json()
p2_json = make_random_pipeline_json()
# First run
(i0,w0) = run_pipeline(p0_json, input_intensity, input_weights)
(i0,w0) = (i0[:,:nt_tot], w0[:,:nt_tot])
(i1,w1) = rf_pipelines.wi_downsample(i0, w0, Df, 1)
(i2,w2) = run_pipeline(p1_json, i1, w1)
(i2,w2) = (i2[:,:nt_tot], w2[:,:nt_tot])
rf_pipelines.weight_upsample(w0, w2)
(i3,w3) = run_pipeline(p2_json, i0, w0)
# Second run
si = initial_stream(input_intensity, input_weights)
p0 = rf_pipelines.pipeline_object.from_json(p0_json)
p1 = rf_pipelines.pipeline_object.from_json(p1_json)
ps = rf_pipelines.wi_sub_pipeline(p1, Df=Df, Dt=1)
p2 = rf_pipelines.pipeline_object.from_json(p2_json)
tf = final_transform()
p = rf_pipelines.pipeline([ si, p0, ps, p2, tf ])
p.run(outdir=None, verbosity=0, debug=True)
(i4,w4) = tf.get_results()
eps_i = maxdiff((i3*w3)[:,:nt_tot],(i4*w4)[:,:nt_tot])
eps_w = maxdiff(w3[:,:nt_tot], w4[:,:nt_tot])
assert eps_i < 1.0e-5
assert eps_w < 1.0e-5
assert np.all(w3[:,nt_tot:] == 0.0)
assert np.all(w4[:,nt_tot:] == 0.0)
detrender_niter = 2,
clipper_niter = 6,
spline = True,
bonsai_use_analytic_normalization = False,
bonsai_hdf5_output_filename = None,
bonsai_nt_per_hdf5_file = None,
bonsai_fill_rfi_mask = True,
var_est = False,
mask_filler = False,
mask_filler_w_cutoff = 0.5,
bonsai_plot_threshold1 = 7,
bonsai_plot_threshold2 = 10,
bonsai_dynamic_plotter = False,
bonsai_plot_all_trees = make_plots)
t1k = ch_frb_rfi.transform_chain(params)
p1k = rf_pipelines.pipeline(t1k)
params.detrend_last = False
_t1k = ch_frb_rfi.transform_chain(params)
_p1k = rf_pipelines.pipeline(_t1k)
t16k = [ rf_pipelines.wi_sub_pipeline(_p1k, nfreq_out=1024, nds_out=1) ]
t16k += ch_frb_rfi.chains.detrender_chain(params, ix=0, jx=1)
p16k = rf_pipelines.pipeline(t16k)
for (pobj, suffix) in [ (p1k,'1k'), (p16k,'16k') ]:
suffix2 = '' if make_plots else '-noplot'
filename = '../../json_files/rfi_%s/17-12-01-two-pass%s.json' % (suffix, suffix2)
rf_pipelines.utils.json_write(filename, pobj, clobber=clobber)
print >>sys.stderr, "Fatal: source file '%s' must end in .json" % srcfile
if not os.path.exists(srcfile):
print >>sys.stderr, "Fatal: source file '%s' does not exist" % srcfile
sys.exit(1)
if os.path.exists(dstdir):
print >>sys.stderr, "Fatal: destination directory '%s' already exists, this is treated as an error to avoid accidentally overwriting data" % dstdir
sys.exit(1)
####################################################################################################
import rf_pipelines
j = rf_pipelines.json_read(srcfile)
stream = rf_pipelines.pipeline_object.from_json(j)
dstfile = os.path.join(dstdir, '00000000.h5')
writer = rf_pipelines.chime_file_writer(dstfile)
writer = rf_pipelines.wi_sub_pipeline(writer, Df=16, Dt=1)
pipeline = rf_pipelines.pipeline([stream, writer])
print 'Creating directory', dstdir
os.makedirs(dstdir)
print 'Running pipeline'
pipeline.run(outdir=None)
#!/usr/bin/env python
import rf_pipelines as rfp
p1 = rfp.polynomial_detrender(nt_chunk=1024, axis='freq', polydeg=2)
p2 = rfp.polynomial_detrender(nt_chunk=1024, axis='time', polydeg=2)
p3 = rfp.mask_counter(nt_chunk=1024, where="after_rfi")
p = rfp.pipeline([p1, p2, p3])
q = rfp.wi_sub_pipeline(p, nfreq_out=1024, nds_out=1)
rfp.utils.json_write('rfi_placeholder.json', q, clobber=True)