def run_rf_pipelines(filename_list, output_dir, output_acq_path): params = ch_frb_rfi.transform_parameters( plot_type='web_viewer', make_plots=False, bonsai_output_plot_stem=None, maskpath=None, two_pass=True, clip_nt=1024, eq_clip_nt=True, detrend_nt=1024, rfi_level=-1, aux_clip_first=True, aux_clip_last=True, aux_detrend_first=False, detrender_niter=1, clipper_niter=6, plot_nypix=1024, plot_nxpix=256, plot_downsample_nt=1, plot_nzoom=2, spline=True, bonsai_use_analytic_normalization=False, bonsai_hdf5_output_filename=None, bonsai_nt_per_hdf5_file=None, bonsai_fill_rfi_mask=False, var_est=False, mask_filler=False, bonsai_dynamic_plotter=False, bonsai_plot_all_trees=False) t1k = ch_frb_rfi.transform_chain(params) p1k = rf_pipelines.pipeline(t1k) t16k = [rf_pipelines.wi_sub_pipeline(p1k, nfreq_out=1024, nds_out=1)] params.detrend_last = True t16k += ch_frb_rfi.chains.detrender_chain(params, ix=1, jx=0) t16k += [ch_frb_rfi.WriteWeights(basename=output_acq_path + '/data')] p16k = rf_pipelines.pipeline(t16k) s = rf_pipelines.chime_frb_stream_from_filename_list(filename_list, nt_chunk=1024, noise_source_align=0) ch_frb_rfi.utils.run_in_scratch_dir(output_acq_path, output_dir, s, p16k)
def run_pipeline(pipeline_json, intensity_arr, weights_arr): # Just for fun, randomize 'nt_chunk'. p0 = initial_stream(intensity_arr, weights_arr) p1 = rf_pipelines.pipeline_object.from_json(pipeline_json) p2 = final_transform() p = rf_pipelines.pipeline([p0,p1,p2]) p.run(outdir=None, verbosity=0, debug=True) (intensity, weights) = p2.get_results() return (intensity, weights)
def init_search_params(self, sparams): """Overrides dedisperser_base.init_search_params(). The 'sparams' argument is an instance of 'class search_params'.""" if hasattr(self, 'search_params'): raise RuntimeError('double call to frb_olympics.bonsai_dedisperser.init_search_params()') # FIXME I would like to add sanity checks here on the following search_params: # nfreq, freq_lo_MHz, freq_hi_MHz, dt_sample, max_dm. s = frb_olympics_stream(sparams) p = rf_pipelines.pipeline([s, self.base_pipeline]) p.bind(outdir=None, verbosity=0) self.search_params = sparams self.full_pipeline = p self.stream = s
def make_pipeline(*args): """ Helper function which combines arguments into a pipeline. The syntax is flexible: arguments can be either - pipeline_objects - filenames ending in .json - lists of either of these """ p = rf_pipelines.pipeline() _add_to_pipeline(p, *args) if p.size == 0: raise RuntimeError( 'ch_frb_rfi: in either run_for_web_viewer() or run_in_scratch_dir(), no pipeline_objects were specified' ) return p
def run_test(): Df = 2**rand.randint(0,5) nfreq = Df * 8 * rand.randint(10, 20) nt_tot = 8 * rand.randint(150, 500) input_intensity = rand.standard_normal(size=(nfreq,nt_tot)) input_weights = rand.uniform(0.5, 1.0, size=(nfreq,nt_tot)) p0_json = make_random_pipeline_json() p1_json = make_random_pipeline_json() p2_json = make_random_pipeline_json() # First run (i0,w0) = run_pipeline(p0_json, input_intensity, input_weights) (i0,w0) = (i0[:,:nt_tot], w0[:,:nt_tot]) (i1,w1) = rf_pipelines.wi_downsample(i0, w0, Df, 1) (i2,w2) = run_pipeline(p1_json, i1, w1) (i2,w2) = (i2[:,:nt_tot], w2[:,:nt_tot]) rf_pipelines.weight_upsample(w0, w2) (i3,w3) = run_pipeline(p2_json, i0, w0) # Second run si = initial_stream(input_intensity, input_weights) p0 = rf_pipelines.pipeline_object.from_json(p0_json) p1 = rf_pipelines.pipeline_object.from_json(p1_json) ps = rf_pipelines.wi_sub_pipeline(p1, Df=Df, Dt=1) p2 = rf_pipelines.pipeline_object.from_json(p2_json) tf = final_transform() p = rf_pipelines.pipeline([ si, p0, ps, p2, tf ]) p.run(outdir=None, verbosity=0, debug=True) (i4,w4) = tf.get_results() eps_i = maxdiff((i3*w3)[:,:nt_tot],(i4*w4)[:,:nt_tot]) eps_w = maxdiff(w3[:,:nt_tot], w4[:,:nt_tot]) assert eps_i < 1.0e-5 assert eps_w < 1.0e-5 assert np.all(w3[:,nt_tot:] == 0.0) assert np.all(w4[:,nt_tot:] == 0.0)
def run_test(): s = initial_stream() u = final_transform() tj = make_random_transform_list(s.nfreq, 1, nelements=rand.randint(10,20)) t = [ rf_pipelines.pipeline_object.from_json(j) for j in tj ] p = rf_pipelines.pipeline([s] + t + [u]) p.bind(outdir=None, verbosity=0, debug=True) # Check jsonization (test is slightly stronger if bind() comes first) tj2 = [ x.jsonize() for x in t ] rf_pipelines.utils.json_assert_equal(tj, tj2, name1='reference_json', name2='pipeline_json') # First run p.run(outdir=None, verbosity=0, debug=True) (i0,w0) = u.get_results() nt0 = i0.shape[1] assert nt0 >= s.nt_tot assert i0.shape == w0.shape == (s.nfreq, nt0) assert np.all(w0[:,s.nt_tot:] == 0.0) # Second run pj = { 'class_name': 'pipeline', 'elements': tj } (i1,w1) = emulate_pipeline(pj, s.intensity, s.weights) nt1 = i1.shape[1] assert nt1 >= s.nt_tot assert i1.shape == w1.shape == (s.nfreq, nt1) assert np.all(w1[:,s.nt_tot:] == 0.0) # Compare eps_i = maxdiff((i0*w0)[:,:s.nt_tot], (i1*w1)[:,:s.nt_tot]) eps_w = maxdiff(w0[:,:s.nt_tot], w1[:,:s.nt_tot]) assert eps_i < 1.0e-5 assert eps_w < 1.0e-5
detrender_niter = 2, clipper_niter = 6, spline = True, bonsai_use_analytic_normalization = False, bonsai_hdf5_output_filename = None, bonsai_nt_per_hdf5_file = None, bonsai_fill_rfi_mask = True, var_est = False, mask_filler = False, mask_filler_w_cutoff = 0.5, bonsai_plot_threshold1 = 7, bonsai_plot_threshold2 = 10, bonsai_dynamic_plotter = False, bonsai_plot_all_trees = make_plots) t1k = ch_frb_rfi.transform_chain(params) p1k = rf_pipelines.pipeline(t1k) params.detrend_last = False _t1k = ch_frb_rfi.transform_chain(params) _p1k = rf_pipelines.pipeline(_t1k) t16k = [ rf_pipelines.wi_sub_pipeline(_p1k, nfreq_out=1024, nds_out=1) ] t16k += ch_frb_rfi.chains.detrender_chain(params, ix=0, jx=1) p16k = rf_pipelines.pipeline(t16k) for (pobj, suffix) in [ (p1k,'1k'), (p16k,'16k') ]: suffix2 = '' if make_plots else '-noplot' filename = '../../json_files/rfi_%s/17-12-01-two-pass%s.json' % (suffix, suffix2) rf_pipelines.utils.json_write(filename, pobj, clobber=clobber)
print >>sys.stderr, "Fatal: source file '%s' must end in .json" % srcfile if not os.path.exists(srcfile): print >>sys.stderr, "Fatal: source file '%s' does not exist" % srcfile sys.exit(1) if os.path.exists(dstdir): print >>sys.stderr, "Fatal: destination directory '%s' already exists, this is treated as an error to avoid accidentally overwriting data" % dstdir sys.exit(1) #################################################################################################### import rf_pipelines j = rf_pipelines.json_read(srcfile) stream = rf_pipelines.pipeline_object.from_json(j) dstfile = os.path.join(dstdir, '00000000.h5') writer = rf_pipelines.chime_file_writer(dstfile) writer = rf_pipelines.wi_sub_pipeline(writer, Df=16, Dt=1) pipeline = rf_pipelines.pipeline([stream, writer]) print 'Creating directory', dstdir os.makedirs(dstdir) print 'Running pipeline' pipeline.run(outdir=None)
def make_random_pipeline(): n = rand.randint(1, 5) return rf_pipelines.pipeline([ make_random_transform() for i in xrange(n) ])
# ripple effect. We compensate for this by using a _negative_ correction, i.e. deripple_fudge_factor = -0.2. deripple_fudge_factor = -0.2 nparts = 7 missing_1k_flag = False for part in xrange(nparts): # Make 16k stream. acqdir = '/data/17-10-01-16k-data/17-04-25-utkarsh-26m-part%d' % part stream_16k = rf_pipelines.chime_stream_from_acqdir(acqdir) # Append derippler. derippler = rf_pipelines.chime_16k_derippler( fudge_factor=deripple_fudge_factor) stream_16k = rf_pipelines.pipeline([stream_16k, derippler]) # Write json file for the 16k acq json_filename_16k = '../../json_files/acqs/17-04-25-utkarsh-26m-16k/part%d.json' % part rf_pipelines.utils.json_write(json_filename_16k, stream_16k, clobber=clobber) # Does the 1k acq exist? If not, print instructions for generating it. acqdir_1k = '/data/17-10-01-16k-to-1k/17-04-25-utkarsh-26m-part%d' % part acqfile_1k = os.path.join(acqdir_1k, '00000000.h5') if not os.path.exists(acqfile_1k): print "Note: 1K-channelized acq '%s' does not exist, you need to run this command:" % acqfile_1k print " ../../scripts/downsample-16k-acq.py %s /data/17-10-01-16k-to-1k/17-04-25-utkarsh-26m-part4" % json_filename_16k missing_1k_flag = True
#!/usr/bin/env python # # Creates 'rfp_bonsai.json' and 'rfp_detrender.json'. # # These are rf_pipelines json files (not frb_olympics json files) # which represent serialized rf_pipelines.pipeline_objects. import rf_pipelines d = rf_pipelines.bonsai_dedisperser( config_filename='bonsai_ntree4096_nups2.hdf5', fill_rfi_mask=False, use_analytic_normalization=True, track_global_max=True) rf_pipelines.json_write('rfp_bonsai.json', d, clobber=True) d1 = rf_pipelines.polynomial_detrender(nt_chunk=1024, axis='time', polydeg=4) d2 = rf_pipelines.spline_detrender(nt_chunk=1024, axis='freq', nbins=6) p = rf_pipelines.pipeline([d1, d2]) rf_pipelines.json_write('rfp_detrender.json', p, clobber=True)
class masklist_capturing_transform(rf_pipelines.wi_transform): def __init__(self): rf_pipelines.wi_transform.__init__(self, 'masklist_capturing_transform') def _process_chunk(self, intensity, weights, pos): if hasattr(self, 'masklist'): return self.masklist = [] for ifreq in xrange(self.nfreq): if weights[ifreq, 0] == 0.0: self.masklist.append(ifreq) s = rf_pipelines.gaussian_noise_stream(nfreq=1024, nt_tot=1024, freq_lo_MHz=400.0, freq_hi_MHz=800.0, dt_sample=1.0e-3) t1 = rf_pipelines.badchannel_mask(mask_filename) t2 = masklist_capturing_transform() p = rf_pipelines.pipeline([s, t1, t2]) p.run(outdir=None, verbosity=1) print 'The following channels were masked:', t2.masklist
#!/usr/bin/env python import rf_pipelines as rfp p1 = rfp.polynomial_detrender(nt_chunk=1024, axis='freq', polydeg=2) p2 = rfp.polynomial_detrender(nt_chunk=1024, axis='time', polydeg=2) p3 = rfp.mask_counter(nt_chunk=1024, where="after_rfi") p = rfp.pipeline([p1, p2, p3]) q = rfp.wi_sub_pipeline(p, nfreq_out=1024, nds_out=1) rfp.utils.json_write('rfi_placeholder.json', q, clobber=True)
bonsai_use_analytic_normalization = False, bonsai_hdf5_output_filename = None, bonsai_nt_per_hdf5_file = None, bonsai_fill_rfi_mask = True, var_est = False, mask_filler = False, mask_filler_w_cutoff = 0.5, bonsai_plot_threshold1 = 7, bonsai_plot_threshold2 = 10, bonsai_dynamic_plotter = False, bonsai_plot_all_trees = make_plots, detrend_last = not detrend_16k, mask_counter = True) t1k = ch_frb_rfi.transform_chain(params) p1k = rf_pipelines.pipeline(t1k) t16k = [ rf_pipelines.wi_sub_pipeline(p1k, nfreq_out=1024, nds_out=1) ] if detrend_16k: params.detrend_last = True params.mask_counter = False t16k += ch_frb_rfi.chains.detrender_chain(params, ix=1, jx=0) params.append_plotter_transform(t16k, 'dc_out_last') if write_json: assert isinstance(output_path, str) and output_path.endswith('.json') p16k = rf_pipelines.pipeline(t16k) rf_pipelines.utils.json_write(output_path, p16k, clobber=True) #rf_pipelines.utils.json_write('design-rfi-config_acq.json', s, clobber=True)
# example_gaussian_stream_with_pulse.json \ # stream # example_bonsai_transform.json # see make-jsonized-bonsai-transform.py import rf_pipelines s = rf_pipelines.gaussian_noise_stream( nfreq = 16384, nt_tot = 512*1024, freq_lo_MHz = 400.0, freq_hi_MHz = 800.0, dt_sample = 0.98304e-3, # matches value in bonsai dedisperser (from CHIME) sample_rms = 1.0, nt_chunk = 1024 ) t = rf_pipelines.frb_injector_transform( snr = 100.0, undispersed_arrival_time = 200.0, dm = 30.0, variance = 1.0**2, # should be square of 'sample_rms' above intrinsic_width = 0.005, ) # The output of this script is logically a "stream" consisting of Gaussian noise plus a # single pulse, but is implemented as a two-stage mini-pipeline: a stream object which # produces the Gaussian noise, plus a transform to add the pulse. p = rf_pipelines.pipeline([s,t]) rf_pipelines.utils.json_write('example_gaussian_stream_with_pulse.json', p, clobber=True)
variance_estimator_v1_chunk = v1_chunk, variance_estimator_v2_chunk = v2_chunk, mask_filler_w_cutoff = 0.5, bonsai_plot_threshold1 = 7, bonsai_plot_threshold2 = 10, bonsai_dynamic_plotter = False, bonsai_event_outfile = './events_example2.dat', L1Grouper_thr = 10, L1Grouper_beam = 0, L1Grouper_addr = None) # Using the specified parameters make a chain of transforms for estimating the variance. t = ch_frb_rfi.transform_chain(p) # Combine stream and transforms into a pipeline. pipeline = rf_pipelines.pipeline([s]+t) # The purpose of the first pipeline run is to create the h5 file containing variance # estimates (p.var_filename = './var_example2.h5'). We do this pipeline run using the # wrapper function run_in_scratch_dir(), which does not index the run with the web viewer. ch_frb_rfi.run_in_scratch_dir('example2', pipeline) # In the v16 API, need to "unbind" the pipeline after running, before its constituent # pipeline_objects can be reused in another pipeline run. pipeline.unbind() # Remove the variance_estimator, append the mask_filler and plotter transforms. p.var_est = False p.mask_filler = True p.make_plots = True