def write_data(tmpdir_factory):
tmpdir_l = tmpdir_factory.mktemp("freqsplit_write_lower")
tmpdir_h = tmpdir_factory.mktemp("freqsplit_write_higher")
fakevis_buffer = runner.FakeVisBuffer(
num_frames=params['total_frames'],
mode=params['mode'],
freq_ids=params['freq_ids'],
wait=False
)
write_buffer_l = runner.VisWriterBuffer(
str(tmpdir_l), 'raw')
write_buffer_h = runner.VisWriterBuffer(
str(tmpdir_h), 'raw')
test = runner.KotekanStageTester(
'freqSplit', {},
fakevis_buffer,
[write_buffer_l, write_buffer_h],
params
)
test.run()
return [write_buffer_l.load(), write_buffer_h.load()]
def pulsar_data(tmpdir_factory):
tmpdir = tmpdir_factory.mktemp("pulsar")
dump_buffer = runner.DumpVisBuffer(str(tmpdir))
dump_buffer_gated = runner.VisWriterBuffer(str(tmpdir), "raw")
# Insert an extra buffer for gated stream
dump_buffer.buffer_block.update(dump_buffer_gated.buffer_block)
dump_buffer.stage_block.update(dump_buffer_gated.stage_block)
acc_par = pulsar_params.copy()
acc_par.update({
"gating": {
"psr0": {
"mode": "pulsar",
"buf": dump_buffer_gated.name
}
},
"updatable_config": {
"psr0": "/updatable_config/psr0_config"
},
})
updatable_params = pulsar_params.copy()
updatable_params.update({
"updatable_config": {
"psr0_config": {
"kotekan_update_endpoint": "json",
"enabled": True,
"pulsar_name": "fakepsr",
"segment": 100.0,
"coeff": [pulsar_params["coeff"]],
"dm": pulsar_params["dm"],
"t_ref": [pulsar_params["t_ref"]],
"phase_ref": [pulsar_params["phase_ref"]],
"rot_freq": pulsar_params["rot_freq"],
"pulse_width": 1e-3,
}
}
})
test = runner.KotekanStageTester(
"visAccumulate",
acc_par,
runner.FakeGPUBuffer(**pulsar_params),
dump_buffer,
updatable_params,
)
test.run()
yield dump_buffer_gated.load()
def subset_data(tmpdir_factory):
tmpdir = tmpdir_factory.mktemp("subset")
fakevis_buffer = runner.FakeVisBuffer(
freq_ids=subset_params["freq_ids"],
num_frames=subset_params["total_frames"])
write_buffer = runner.VisWriterBuffer(str(tmpdir), "raw")
test = runner.KotekanStageTester("prodSubset", vis_params, fakevis_buffer,
write_buffer, subset_params)
test.run()
return write_buffer.load()
def pulsar_data(tmpdir_factory):
tmpdir = tmpdir_factory.mktemp("pulsar")
dump_buffer = runner.DumpVisBuffer(str(tmpdir))
dump_buffer_gated = runner.VisWriterBuffer(str(tmpdir), "raw")
# Insert an extra buffer for gated stream
dump_buffer.buffer_block.update(dump_buffer_gated.buffer_block)
dump_buffer.stage_block.update(dump_buffer_gated.stage_block)
acc_par = pulsar_params.copy()
acc_par.update({
'gating': {
'psr0': {
'mode': 'pulsar',
'buf': dump_buffer_gated.name
},
},
'updatable_config': {
'psr0': "/updatable_config/psr0_config"
},
})
updatable_params = pulsar_params.copy()
updatable_params.update({
'updatable_config': {
'psr0_config': {
'kotekan_update_endpoint': 'json',
'enabled': True,
'pulsar_name': 'fakepsr',
'segment': 100.,
'coeff': [pulsar_params['coeff']],
'dm': pulsar_params['dm'],
't_ref': [pulsar_params['t_ref']],
'phase_ref': [pulsar_params['phase_ref']],
'rot_freq': pulsar_params['rot_freq'],
'pulse_width': 1e-3,
}
}
})
test = runner.KotekanStageTester('visAccumulate', acc_par,
runner.FakeGPUBuffer(**pulsar_params),
dump_buffer, updatable_params)
test.run()
yield dump_buffer_gated.load()
def write_data(tmpdir_factory):
tmpdir = tmpdir_factory.mktemp("freqsub_write")
fakevis_buffer = runner.FakeVisBuffer(num_frames=params['total_frames'],
mode=params['mode'],
freq_ids=params['freq_ids'],
wait=False)
write_buffer = runner.VisWriterBuffer(str(tmpdir), 'raw')
test = runner.KotekanStageTester('freqSubset', {}, fakevis_buffer,
write_buffer, params)
test.run()
return write_buffer.load()
def write_data(tmpdir_factory):
tmpdir = tmpdir_factory.mktemp("freqsub_write")
fakevis_buffer = runner.FakeVisBuffer(
num_frames=params["total_frames"],
mode=params["mode"],
freq_ids=params["freq_ids"],
wait=False,
)
write_buffer = runner.VisWriterBuffer(str(tmpdir), "raw")
test = runner.KotekanStageTester("VisFreqSubset", {}, fakevis_buffer,
write_buffer, params)
test.run()
return write_buffer.load()
def subset_data(request, tmpdir_factory):
inputs = request.param
tmpdir = tmpdir_factory.mktemp("subset")
fakevis_buffer = runner.FakeVisBuffer(
freq_ids=subset_params["freq_ids"],
num_frames=subset_params["total_frames"])
write_buffer = runner.VisWriterBuffer(str(tmpdir), "raw")
task_params = {"inputs": inputs}
task_params.update(subset_params)
test = runner.KotekanStageTester("InputSubset", vis_params, fakevis_buffer,
write_buffer, task_params)
test.run()
return inputs, write_buffer.load()
def subset_data(tmpdir_factory):
broker_path = shutil.which("comet")
if not broker_path:
pytest.skip(
"Make sure PYTHONPATH is set to where the comet dataset broker is installed."
)
if not has_redis():
pytest.skip("Redis is not available and so comet will fail")
# run the dataset broker
broker = Popen([broker_path, "--recover", "False"])
time.sleep(1.5)
try:
tmpdir = tmpdir_factory.mktemp("freqsub_broker_fakevis")
dump_buffer_gen = runner.DumpVisBuffer(str(tmpdir))
test = runner.KotekanStageTester("FakeVis", params_fakevis, None,
dump_buffer_gen, params)
test.run()
data_gen = dump_buffer_gen.load()
### VisFreqSubset ###
tmpdir = tmpdir_factory.mktemp("freqsub_broker")
ds_id = data_gen[0].metadata.dataset_id
fakevis_buffer_subset = runner.FakeVisBuffer(
num_frames=params["total_frames"],
mode=params["mode"],
freq_ids=params["freq_ids"],
use_dataset_manager=True,
wait=False,
dataset_id="{:016x}{:016x}".format(ds_id[1], ds_id[0]),
)
dump_buffer_subset = runner.DumpVisBuffer(str(tmpdir))
test = runner.KotekanStageTester("VisFreqSubset", {},
fakevis_buffer_subset,
dump_buffer_subset, params)
test.run()
data_subset = dump_buffer_subset.load()
### freqSplit ###
time.sleep(10)
tmpdir = tmpdir_factory.mktemp("freqsplit_broker")
ds_id = data_subset[0].metadata.dataset_id
fakevis_buffer_split = runner.FakeVisBuffer(
num_frames=params["total_frames"],
mode=params["mode"],
freq_ids=params["subset_list"],
use_dataset_manager=True,
wait=False,
dataset_id="{:016x}{:016x}".format(ds_id[1], ds_id[0]),
)
dump_buffer_split_lower = runner.DumpVisBuffer(str(tmpdir))
dump_buffer_split_higher = runner.DumpVisBuffer(str(tmpdir))
test = runner.KotekanStageTester(
"freqSplit",
{},
fakevis_buffer_split,
(dump_buffer_split_lower, dump_buffer_split_higher),
params,
)
test.run()
data_split_lower = dump_buffer_split_lower.load()
data_split_higher = dump_buffer_split_higher.load()
### 2 VisWriter processes ###
tmpdir = tmpdir_factory.mktemp("freqsub_write_lower")
params_fakevis_write_lower = params_fakevis.copy()
ds_id = data_split_lower[0].metadata.dataset_id
params_fakevis_write_lower["dataset_id"] = "{:016x}{:016x}".format(
ds_id[1], ds_id[0])
# the writer is not given the subset list, it get's it through the broker
write_buffer_lower = runner.VisWriterBuffer(
str(tmpdir),
"raw",
None,
extra_config={"use_dataset_manager": True})
test = runner.KotekanStageTester("FakeVis", params_fakevis_write_lower,
None, write_buffer_lower, params)
test.run()
tmpdir = tmpdir_factory.mktemp("freqsub_write_higher")
params_fakevis_write_higher = params_fakevis.copy()
ds_id = data_split_higher[0].metadata.dataset_id
params_fakevis_write_higher["dataset_id"] = "{:016x}{:016x}".format(
ds_id[1], ds_id[0])
# the writer is not given the subset list, it get's it through the broker
write_buffer_higher = runner.VisWriterBuffer(
str(tmpdir),
"raw",
None,
extra_config={"use_dataset_manager": True})
test = runner.KotekanStageTester("FakeVis",
params_fakevis_write_higher, None,
write_buffer_higher, params)
test.run()
yield [
data_gen,
data_subset,
data_split_lower,
data_split_higher,
write_buffer_lower.load(),
write_buffer_higher.load(),
]
finally:
pid = broker.pid
os.kill(pid, signal.SIGINT)
broker.terminate()