def test_runs(self):
stream = ch_vdif_assembler.make_simulated_stream(gbps=0.1, nsec=2)
assembler = ch_vdif_assembler.assembler()
class ReferenceCorrelator(L0.ReferenceSqAccumMixin,
L0.CallBackCorrelator):
pass
p1 = ReferenceCorrelator(nframe_integrate=512)
p2 = L0.CallBackCorrelator(nframe_integrate=512)
comparison = IntegratorComparison(p1, p2)
assembler.register_processor(p1)
assembler.register_processor(p2)
assembler.run(stream)
def test_runs(self):
stream = ch_vdif_assembler.make_simulated_stream(gbps=0.1, nsec=2)
assembler = ch_vdif_assembler.assembler()
class ReferenceCorrelator(L0.ReferenceSqAccumMixin,
L0.CallBackCorrelator):
pass
p1 = ReferenceCorrelator(nframe_integrate=512)
p2 = L0.CallBackCorrelator(nframe_integrate=512)
comparison = IntegratorComparison(p1, p2)
assembler.register_processor(p1)
assembler.register_processor(p2)
assembler.run(stream)
def _configure_source(self):
parameters = self._conf['source']
ds_type = parameters['type']
if ds_type == 'disk':
data_dir = parameters['data_dir']
if data_dir == "TESTDATA": # Special value.
data_dir = testdata.TEST_DATA_DIR
ds = datasource.DiskSource(
data_dir,
ntime_chunk=self._ntime_chunk,
)
elif ds_type == 'vdif':
import ch_vdif_assembler
vdif_source_pars = pars['vdif_source']
vdif_source_type = vdif_source_pars.pop('type')
if vdif_source_type == 'network':
stream = ch_vdif_assembler.make_network_stream(
**vdif_source_pars)
elif vdif_source_type == 'simulate':
stream = ch_vdif_assembler.make_simulated_stream(
**vdif_source_pars)
elif vdif_source_type == 'moose-acq':
stream = ch_vdif_assembler.moose_acquisition(
**vdif_source_pars)
elif vdif_source_type == 'file-list':
stream = ch_vdif_assembler.make_file_stream(
**vdif_source_pars)
else:
msg = "Invalid vdif source type: %s."
raise ValueError(msg % vdif_source_type)
assembler = ch_vdif_assembler.assembler()
p = L0.CallBackCorrelator(
nframe_integrate=parameters['nframe_integrate'],
)
ds = datasource.vdifSource(p, ntime_chunk=ntime_chunk)
assembler.register_processor(p)
thread = utils.ExceptThread(
target=assembler.run,
args=(stream,),
)
self._daemon_threads.append(thread)
else:
msg = "Data source type %s nor supported."
raise ValueError(msg % ds_type)
self._datasource = ds
def test_runs(self):
stream = ch_vdif_assembler.make_simulated_stream(gbps=6., nsec=2)
assembler = ch_vdif_assembler.assembler()
p = DummyCorrelator(nframe_integrate=512)
ntime_chunk = 563
ds = datasource.vdifSource(p, ntime_chunk=ntime_chunk)
freq = ds.freq
assembler.register_processor(p)
thread = utils.ExceptThread(
target=assembler.run,
args=(stream, ),
)
thread.start()
n_data = 0
while True:
thread.check()
try:
data, mask = ds.yield_chunk(timeout=0.1)
except StopIteration:
break
except datasource.NoData:
continue
self.assertEqual(data.shape, (1024, ntime_chunk))
self.assertTrue(np.dtype(data.dtype) is np.dtype(np.float32))
time = ds.last_time
# See L0.DummyDataMixing for data expectation.
expectation = freq[:, None] * time * 3
self.assertTrue(
np.allclose(data[mask == 1], expectation[mask == 1]))
time_mask_inds = (time * 100) % 20 < 1
self.assertTrue(np.all(np.sum(mask, 0)[time_mask_inds] == 0))
freq_mask_inds = (freq / 1e6) % 40 < 1
self.assertTrue(np.all(np.sum(mask, 1)[freq_mask_inds] == 0))
n_data += data.shape[1]
thread.check_join()
ds.finalize()
self.assertGreater(n_data, 1000)
def test_runs(self):
stream = ch_vdif_assembler.make_simulated_stream(gbps=6., nsec=2)
assembler = ch_vdif_assembler.assembler()
p = DummyCorrelator(nframe_integrate=512)
ntime_chunk = 563
ds = datasource.vdifSource(p, ntime_chunk=ntime_chunk)
freq = ds.freq
assembler.register_processor(p)
thread = utils.ExceptThread(
target=assembler.run,
args=(stream,),
)
thread.start()
n_data = 0
while True:
thread.check()
try:
data, mask = ds.yield_chunk(timeout=0.1)
except StopIteration:
break
except datasource.NoData:
continue
self.assertEqual(data.shape, (1024, ntime_chunk))
self.assertTrue(np.dtype(data.dtype) is np.dtype(np.float32))
time = ds.last_time
# See L0.DummyDataMixing for data expectation.
expectation = freq[:,None] * time * 3
self.assertTrue(np.allclose(data[mask==1], expectation[mask==1]))
time_mask_inds = (time * 100) % 20 < 1
self.assertTrue(np.all(np.sum(mask, 0)[time_mask_inds] == 0))
freq_mask_inds = (freq / 1e6) % 40 < 1
self.assertTrue(np.all(np.sum(mask, 1)[freq_mask_inds] == 0))
n_data += data.shape[1]
thread.check_join()
ds.finalize()
self.assertGreater(n_data, 1000)
print >> sys.stderr, ' toy-python-assembler.py 41537'
print >> sys.stderr, ' index-vdif-waterfalls.py waterfall_cpp_41537'
print >> sys.stderr, ' index-vdif-waterfalls.py waterfall_python_41537'
print >> sys.stderr, 'The index-vdif-waterfalls script makes an HTML summary page with clickable thumbnails'
sys.exit(2)
acq_name = sys.argv[1]
import ch_vdif_assembler
# Construct the stream object
# ch_moose_acquisition() is a helper function which finds files on moose and returns a stream object to read them
stream = ch_vdif_assembler.moose_acquisition(acq_name)
# Construct an assembler object (this spawns some worker threads)
assembler = ch_vdif_assembler.assembler()
# Construct a C++ processor exported with cython (the only one currently exported is the waterfall_plotter)
outdir = 'waterfall_cpp_' + acq_name
p = ch_vdif_assembler.cpp_waterfall_plotter(outdir, is_critical=True)
assembler.register_processor(p)
# Construct a python processor (the reference waterfall plotter in ch_vdif_assembler.py)
outdir = 'waterfall_python_' + acq_name
p = ch_vdif_assembler.python_waterfall_plotter(outdir)
assembler.register_processor(p)
# In just a few minutes, you will have your beautiful waterfall plots, you lucky devil
assembler.run(stream)
def test_runs(self):
stream = ch_vdif_assembler.make_simulated_stream(nsec=5)
assembler = ch_vdif_assembler.assembler()
p = L0.DiskWriteCorrelator(nframe_integrate=512, outdir='tmp_corr')
assembler.register_processor(p)
assembler.run(stream)
def test_fast_enough(self):
stream = ch_vdif_assembler.make_simulated_stream(nsec=5)
assembler = ch_vdif_assembler.assembler()
p = L0.BaseCorrelator(nframe_integrate=512)
assembler.register_processor(p)
assembler.run(stream)
for site_ix in xrange(i, j):
fpga_site = self.sites[freq, pol, site_ix]
t_ix = fpga_site - t0
assert 0 <= t_ix < nt
if mask[freq, pol, t_ix]:
self.output[freq, pol, site_ix] = data[freq, pol, t_ix]
else:
assert data[freq, pol, t_ix] == 0 + 0j
print >> sys.stderr, "\nReading files with ch_vdif_assembler"
# 2^17 was large enough to avoid drops in the example I used
assembler = ch_vdif_assembler.assembler(assembler_nt=2**17)
assembler.register_processor(kms_reader(sites, kms))
s = ch_vdif_assembler.make_file_stream(filename_list)
assembler.run(s)
(fi, pi, ti) = np.where(np.abs(kms - liam) > 0.1)
n = len(fi)
if n == 0:
print 'Success!'
sys.exit(0)
# Just print first failure
(fi, pi, ti) = (fi[0], pi[0], ti[0])
fpga = sites[fi, pi, ti]
for site_ix in xrange(i,j):
fpga_site = self.sites[freq,pol,site_ix]
t_ix = fpga_site - t0
assert 0 <= t_ix < nt
if mask[freq,pol,t_ix]:
self.output[freq,pol,site_ix] = data[freq,pol,t_ix]
else:
assert data[freq,pol,t_ix] == 0+0j
print >>sys.stderr, "\nReading files with ch_vdif_assembler"
# 2^17 was large enough to avoid drops in the example I used
assembler = ch_vdif_assembler.assembler(assembler_nt = 2**17)
assembler.register_processor(kms_reader(sites,kms))
s = ch_vdif_assembler.make_file_stream(filename_list)
assembler.run(s)
(fi,pi,ti) = np.where(np.abs(kms-liam) > 0.1)
n = len(fi)
if n == 0:
print 'Success!'
sys.exit(0)
# Just print first failure
(fi,pi,ti) = (fi[0],pi[0],ti[0])
fpga = sites[fi,pi,ti]
print >>sys.stderr, ' index-vdif-waterfalls.py waterfall_cpp_41537'
print >>sys.stderr, ' index-vdif-waterfalls.py waterfall_python_41537'
print >>sys.stderr, 'The index-vdif-waterfalls script makes an HTML summary page with clickable thumbnails'
sys.exit(2)
acq_name = sys.argv[1]
import ch_vdif_assembler
# Construct the stream object
# ch_moose_acquisition() is a helper function which finds files on moose and returns a stream object to read them
stream = ch_vdif_assembler.moose_acquisition(acq_name)
# Construct an assembler object (this spawns some worker threads)
assembler = ch_vdif_assembler.assembler()
# Construct a C++ processor exported with cython (the only one currently exported is the waterfall_plotter)
outdir = 'waterfall_cpp_' + acq_name
p = ch_vdif_assembler.cpp_waterfall_plotter(outdir, is_critical=True)
assembler.register_processor(p)
# Construct a python processor (the reference waterfall plotter in ch_vdif_assembler.py)
outdir = 'waterfall_python_' + acq_name
p = ch_vdif_assembler.python_waterfall_plotter(outdir)
assembler.register_processor(p)
# In just a few minutes, you will have your beautiful waterfall plots, you lucky devil
assembler.run(stream)
def test_runs(self):
stream = ch_vdif_assembler.make_simulated_stream(nsec=5)
assembler = ch_vdif_assembler.assembler()
p = L0.DiskWriteCorrelator(nframe_integrate=512, outdir='tmp_corr')
assembler.register_processor(p)
assembler.run(stream)
def test_fast_enough(self):
stream = ch_vdif_assembler.make_simulated_stream(nsec=5)
assembler = ch_vdif_assembler.assembler()
p = L0.BaseCorrelator(nframe_integrate=512)
assembler.register_processor(p)
assembler.run(stream)
