def test_runs(self): stream = ch_vdif_assembler.make_simulated_stream(gbps=0.1, nsec=2) assembler = ch_vdif_assembler.assembler() class ReferenceCorrelator(L0.ReferenceSqAccumMixin, L0.CallBackCorrelator): pass p1 = ReferenceCorrelator(nframe_integrate=512) p2 = L0.CallBackCorrelator(nframe_integrate=512) comparison = IntegratorComparison(p1, p2) assembler.register_processor(p1) assembler.register_processor(p2) assembler.run(stream)
def _configure_source(self): parameters = self._conf['source'] ds_type = parameters['type'] if ds_type == 'disk': data_dir = parameters['data_dir'] if data_dir == "TESTDATA": # Special value. data_dir = testdata.TEST_DATA_DIR ds = datasource.DiskSource( data_dir, ntime_chunk=self._ntime_chunk, ) elif ds_type == 'vdif': import ch_vdif_assembler vdif_source_pars = pars['vdif_source'] vdif_source_type = vdif_source_pars.pop('type') if vdif_source_type == 'network': stream = ch_vdif_assembler.make_network_stream( **vdif_source_pars) elif vdif_source_type == 'simulate': stream = ch_vdif_assembler.make_simulated_stream( **vdif_source_pars) elif vdif_source_type == 'moose-acq': stream = ch_vdif_assembler.moose_acquisition( **vdif_source_pars) elif vdif_source_type == 'file-list': stream = ch_vdif_assembler.make_file_stream( **vdif_source_pars) else: msg = "Invalid vdif source type: %s." raise ValueError(msg % vdif_source_type) assembler = ch_vdif_assembler.assembler() p = L0.CallBackCorrelator( nframe_integrate=parameters['nframe_integrate'], ) ds = datasource.vdifSource(p, ntime_chunk=ntime_chunk) assembler.register_processor(p) thread = utils.ExceptThread( target=assembler.run, args=(stream,), ) self._daemon_threads.append(thread) else: msg = "Data source type %s nor supported." raise ValueError(msg % ds_type) self._datasource = ds
def test_runs(self): stream = ch_vdif_assembler.make_simulated_stream(gbps=6., nsec=2) assembler = ch_vdif_assembler.assembler() p = DummyCorrelator(nframe_integrate=512) ntime_chunk = 563 ds = datasource.vdifSource(p, ntime_chunk=ntime_chunk) freq = ds.freq assembler.register_processor(p) thread = utils.ExceptThread( target=assembler.run, args=(stream, ), ) thread.start() n_data = 0 while True: thread.check() try: data, mask = ds.yield_chunk(timeout=0.1) except StopIteration: break except datasource.NoData: continue self.assertEqual(data.shape, (1024, ntime_chunk)) self.assertTrue(np.dtype(data.dtype) is np.dtype(np.float32)) time = ds.last_time # See L0.DummyDataMixing for data expectation. expectation = freq[:, None] * time * 3 self.assertTrue( np.allclose(data[mask == 1], expectation[mask == 1])) time_mask_inds = (time * 100) % 20 < 1 self.assertTrue(np.all(np.sum(mask, 0)[time_mask_inds] == 0)) freq_mask_inds = (freq / 1e6) % 40 < 1 self.assertTrue(np.all(np.sum(mask, 1)[freq_mask_inds] == 0)) n_data += data.shape[1] thread.check_join() ds.finalize() self.assertGreater(n_data, 1000)
def test_runs(self): stream = ch_vdif_assembler.make_simulated_stream(gbps=6., nsec=2) assembler = ch_vdif_assembler.assembler() p = DummyCorrelator(nframe_integrate=512) ntime_chunk = 563 ds = datasource.vdifSource(p, ntime_chunk=ntime_chunk) freq = ds.freq assembler.register_processor(p) thread = utils.ExceptThread( target=assembler.run, args=(stream,), ) thread.start() n_data = 0 while True: thread.check() try: data, mask = ds.yield_chunk(timeout=0.1) except StopIteration: break except datasource.NoData: continue self.assertEqual(data.shape, (1024, ntime_chunk)) self.assertTrue(np.dtype(data.dtype) is np.dtype(np.float32)) time = ds.last_time # See L0.DummyDataMixing for data expectation. expectation = freq[:,None] * time * 3 self.assertTrue(np.allclose(data[mask==1], expectation[mask==1])) time_mask_inds = (time * 100) % 20 < 1 self.assertTrue(np.all(np.sum(mask, 0)[time_mask_inds] == 0)) freq_mask_inds = (freq / 1e6) % 40 < 1 self.assertTrue(np.all(np.sum(mask, 1)[freq_mask_inds] == 0)) n_data += data.shape[1] thread.check_join() ds.finalize() self.assertGreater(n_data, 1000)
print >> sys.stderr, ' toy-python-assembler.py 41537' print >> sys.stderr, ' index-vdif-waterfalls.py waterfall_cpp_41537' print >> sys.stderr, ' index-vdif-waterfalls.py waterfall_python_41537' print >> sys.stderr, 'The index-vdif-waterfalls script makes an HTML summary page with clickable thumbnails' sys.exit(2) acq_name = sys.argv[1] import ch_vdif_assembler # Construct the stream object # ch_moose_acquisition() is a helper function which finds files on moose and returns a stream object to read them stream = ch_vdif_assembler.moose_acquisition(acq_name) # Construct an assembler object (this spawns some worker threads) assembler = ch_vdif_assembler.assembler() # Construct a C++ processor exported with cython (the only one currently exported is the waterfall_plotter) outdir = 'waterfall_cpp_' + acq_name p = ch_vdif_assembler.cpp_waterfall_plotter(outdir, is_critical=True) assembler.register_processor(p) # Construct a python processor (the reference waterfall plotter in ch_vdif_assembler.py) outdir = 'waterfall_python_' + acq_name p = ch_vdif_assembler.python_waterfall_plotter(outdir) assembler.register_processor(p) # In just a few minutes, you will have your beautiful waterfall plots, you lucky devil assembler.run(stream)
def test_runs(self): stream = ch_vdif_assembler.make_simulated_stream(nsec=5) assembler = ch_vdif_assembler.assembler() p = L0.DiskWriteCorrelator(nframe_integrate=512, outdir='tmp_corr') assembler.register_processor(p) assembler.run(stream)
def test_fast_enough(self): stream = ch_vdif_assembler.make_simulated_stream(nsec=5) assembler = ch_vdif_assembler.assembler() p = L0.BaseCorrelator(nframe_integrate=512) assembler.register_processor(p) assembler.run(stream)
for site_ix in xrange(i, j): fpga_site = self.sites[freq, pol, site_ix] t_ix = fpga_site - t0 assert 0 <= t_ix < nt if mask[freq, pol, t_ix]: self.output[freq, pol, site_ix] = data[freq, pol, t_ix] else: assert data[freq, pol, t_ix] == 0 + 0j print >> sys.stderr, "\nReading files with ch_vdif_assembler" # 2^17 was large enough to avoid drops in the example I used assembler = ch_vdif_assembler.assembler(assembler_nt=2**17) assembler.register_processor(kms_reader(sites, kms)) s = ch_vdif_assembler.make_file_stream(filename_list) assembler.run(s) (fi, pi, ti) = np.where(np.abs(kms - liam) > 0.1) n = len(fi) if n == 0: print 'Success!' sys.exit(0) # Just print first failure (fi, pi, ti) = (fi[0], pi[0], ti[0]) fpga = sites[fi, pi, ti]
for site_ix in xrange(i,j): fpga_site = self.sites[freq,pol,site_ix] t_ix = fpga_site - t0 assert 0 <= t_ix < nt if mask[freq,pol,t_ix]: self.output[freq,pol,site_ix] = data[freq,pol,t_ix] else: assert data[freq,pol,t_ix] == 0+0j print >>sys.stderr, "\nReading files with ch_vdif_assembler" # 2^17 was large enough to avoid drops in the example I used assembler = ch_vdif_assembler.assembler(assembler_nt = 2**17) assembler.register_processor(kms_reader(sites,kms)) s = ch_vdif_assembler.make_file_stream(filename_list) assembler.run(s) (fi,pi,ti) = np.where(np.abs(kms-liam) > 0.1) n = len(fi) if n == 0: print 'Success!' sys.exit(0) # Just print first failure (fi,pi,ti) = (fi[0],pi[0],ti[0]) fpga = sites[fi,pi,ti]
print >>sys.stderr, ' index-vdif-waterfalls.py waterfall_cpp_41537' print >>sys.stderr, ' index-vdif-waterfalls.py waterfall_python_41537' print >>sys.stderr, 'The index-vdif-waterfalls script makes an HTML summary page with clickable thumbnails' sys.exit(2) acq_name = sys.argv[1] import ch_vdif_assembler # Construct the stream object # ch_moose_acquisition() is a helper function which finds files on moose and returns a stream object to read them stream = ch_vdif_assembler.moose_acquisition(acq_name) # Construct an assembler object (this spawns some worker threads) assembler = ch_vdif_assembler.assembler() # Construct a C++ processor exported with cython (the only one currently exported is the waterfall_plotter) outdir = 'waterfall_cpp_' + acq_name p = ch_vdif_assembler.cpp_waterfall_plotter(outdir, is_critical=True) assembler.register_processor(p) # Construct a python processor (the reference waterfall plotter in ch_vdif_assembler.py) outdir = 'waterfall_python_' + acq_name p = ch_vdif_assembler.python_waterfall_plotter(outdir) assembler.register_processor(p) # In just a few minutes, you will have your beautiful waterfall plots, you lucky devil assembler.run(stream)