def run_rf_pipelines(filename_list, output_dir, output_acq_path): params = ch_frb_rfi.transform_parameters( plot_type='web_viewer', make_plots=False, bonsai_output_plot_stem=None, maskpath=None, two_pass=True, clip_nt=1024, eq_clip_nt=True, detrend_nt=1024, rfi_level=-1, aux_clip_first=True, aux_clip_last=True, aux_detrend_first=False, detrender_niter=1, clipper_niter=6, plot_nypix=1024, plot_nxpix=256, plot_downsample_nt=1, plot_nzoom=2, spline=True, bonsai_use_analytic_normalization=False, bonsai_hdf5_output_filename=None, bonsai_nt_per_hdf5_file=None, bonsai_fill_rfi_mask=False, var_est=False, mask_filler=False, bonsai_dynamic_plotter=False, bonsai_plot_all_trees=False) t1k = ch_frb_rfi.transform_chain(params) p1k = rf_pipelines.pipeline(t1k) t16k = [rf_pipelines.wi_sub_pipeline(p1k, nfreq_out=1024, nds_out=1)] params.detrend_last = True t16k += ch_frb_rfi.chains.detrender_chain(params, ix=1, jx=0) t16k += [ch_frb_rfi.WriteWeights(basename=output_acq_path + '/data')] p16k = rf_pipelines.pipeline(t16k) s = rf_pipelines.chime_frb_stream_from_filename_list(filename_list, nt_chunk=1024, noise_source_align=0) ch_frb_rfi.utils.run_in_scratch_dir(output_acq_path, output_dir, s, p16k)
def run_test(): Df = 2**rand.randint(0,5) nfreq = Df * 8 * rand.randint(10, 20) nt_tot = 8 * rand.randint(150, 500) input_intensity = rand.standard_normal(size=(nfreq,nt_tot)) input_weights = rand.uniform(0.5, 1.0, size=(nfreq,nt_tot)) p0_json = make_random_pipeline_json() p1_json = make_random_pipeline_json() p2_json = make_random_pipeline_json() # First run (i0,w0) = run_pipeline(p0_json, input_intensity, input_weights) (i0,w0) = (i0[:,:nt_tot], w0[:,:nt_tot]) (i1,w1) = rf_pipelines.wi_downsample(i0, w0, Df, 1) (i2,w2) = run_pipeline(p1_json, i1, w1) (i2,w2) = (i2[:,:nt_tot], w2[:,:nt_tot]) rf_pipelines.weight_upsample(w0, w2) (i3,w3) = run_pipeline(p2_json, i0, w0) # Second run si = initial_stream(input_intensity, input_weights) p0 = rf_pipelines.pipeline_object.from_json(p0_json) p1 = rf_pipelines.pipeline_object.from_json(p1_json) ps = rf_pipelines.wi_sub_pipeline(p1, Df=Df, Dt=1) p2 = rf_pipelines.pipeline_object.from_json(p2_json) tf = final_transform() p = rf_pipelines.pipeline([ si, p0, ps, p2, tf ]) p.run(outdir=None, verbosity=0, debug=True) (i4,w4) = tf.get_results() eps_i = maxdiff((i3*w3)[:,:nt_tot],(i4*w4)[:,:nt_tot]) eps_w = maxdiff(w3[:,:nt_tot], w4[:,:nt_tot]) assert eps_i < 1.0e-5 assert eps_w < 1.0e-5 assert np.all(w3[:,nt_tot:] == 0.0) assert np.all(w4[:,nt_tot:] == 0.0)
detrender_niter = 2, clipper_niter = 6, spline = True, bonsai_use_analytic_normalization = False, bonsai_hdf5_output_filename = None, bonsai_nt_per_hdf5_file = None, bonsai_fill_rfi_mask = True, var_est = False, mask_filler = False, mask_filler_w_cutoff = 0.5, bonsai_plot_threshold1 = 7, bonsai_plot_threshold2 = 10, bonsai_dynamic_plotter = False, bonsai_plot_all_trees = make_plots) t1k = ch_frb_rfi.transform_chain(params) p1k = rf_pipelines.pipeline(t1k) params.detrend_last = False _t1k = ch_frb_rfi.transform_chain(params) _p1k = rf_pipelines.pipeline(_t1k) t16k = [ rf_pipelines.wi_sub_pipeline(_p1k, nfreq_out=1024, nds_out=1) ] t16k += ch_frb_rfi.chains.detrender_chain(params, ix=0, jx=1) p16k = rf_pipelines.pipeline(t16k) for (pobj, suffix) in [ (p1k,'1k'), (p16k,'16k') ]: suffix2 = '' if make_plots else '-noplot' filename = '../../json_files/rfi_%s/17-12-01-two-pass%s.json' % (suffix, suffix2) rf_pipelines.utils.json_write(filename, pobj, clobber=clobber)
print >>sys.stderr, "Fatal: source file '%s' must end in .json" % srcfile if not os.path.exists(srcfile): print >>sys.stderr, "Fatal: source file '%s' does not exist" % srcfile sys.exit(1) if os.path.exists(dstdir): print >>sys.stderr, "Fatal: destination directory '%s' already exists, this is treated as an error to avoid accidentally overwriting data" % dstdir sys.exit(1) #################################################################################################### import rf_pipelines j = rf_pipelines.json_read(srcfile) stream = rf_pipelines.pipeline_object.from_json(j) dstfile = os.path.join(dstdir, '00000000.h5') writer = rf_pipelines.chime_file_writer(dstfile) writer = rf_pipelines.wi_sub_pipeline(writer, Df=16, Dt=1) pipeline = rf_pipelines.pipeline([stream, writer]) print 'Creating directory', dstdir os.makedirs(dstdir) print 'Running pipeline' pipeline.run(outdir=None)
#!/usr/bin/env python import rf_pipelines as rfp p1 = rfp.polynomial_detrender(nt_chunk=1024, axis='freq', polydeg=2) p2 = rfp.polynomial_detrender(nt_chunk=1024, axis='time', polydeg=2) p3 = rfp.mask_counter(nt_chunk=1024, where="after_rfi") p = rfp.pipeline([p1, p2, p3]) q = rfp.wi_sub_pipeline(p, nfreq_out=1024, nds_out=1) rfp.utils.json_write('rfi_placeholder.json', q, clobber=True)