コード例 #1
0
ファイル: test_freqsplit.py プロジェクト: leungcalvin/kotekan
def vis_data(tmpdir_factory):

    tmpdir_l = tmpdir_factory.mktemp("freqsplit_lower")
    tmpdir_h = tmpdir_factory.mktemp("freqsplit_higher")

    fakevis_buffer = runner.FakeVisBuffer(
        num_frames=params['total_frames'],
        mode=params['mode'],
        freq_ids=params['freq_ids'],
        wait=False
    )

    dump_buffer_l = runner.DumpVisBuffer(
            str(tmpdir_l))

    dump_buffer_h = runner.DumpVisBuffer(
            str(tmpdir_h))

    test = runner.KotekanStageTester(
        'freqSplit', {},
        fakevis_buffer,
        [dump_buffer_l, dump_buffer_h],
        params
    )

    test.run()

    yield [dump_buffer_l.load(), dump_buffer_h.load()]
コード例 #2
0
ファイル: test_receiveflags.py プロジェクト: mfkiwl/kotekan
def run_flagging(tmpdir_factory, cmds):
    """ Receive Flags """

    tmpdir = tmpdir_factory.mktemp("receiveflags")

    fakevis_buffer = runner.FakeVisBuffer(
        num_frames=global_params["total_frames"],
        mode=global_params["fakevis_mode"],
        cadence=global_params["cadence"],
        wait=global_params["wait"],
    )

    out_dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester(
        "ReceiveFlags",
        params,
        buffers_in=fakevis_buffer,
        buffers_out=out_dump_buffer,
        global_config=global_params,
        rest_commands=cmds,
        expect_failure=True,
    )

    test.run()

    return out_dump_buffer.load()
コード例 #3
0
ファイル: test_applygains.py プロジェクト: mfkiwl/kotekan
def apply_data(request, tmp_path_factory, gain_path, old_gains, new_gains,
               cal_broker):

    output_dir = str(tmp_path_factory.mktemp("output"))
    global_params["gains_dir"] = str(gain_path)
    global_params[
        "read_from_file"] = True if request.param == "file" else False

    # REST commands
    cmds = [[
        "post",
        "gains",
        {
            "update_id": new_update_id,
            "start_time": new_timestamp,
            "transition_interval": transition_interval,
            "new_state": new_state,
        },
    ]]

    fakevis_buffer = runner.FakeVisBuffer(
        freq_ids=global_params["freq_ids"],
        num_frames=global_params["total_frames"],
        wait=global_params["wait"],
    )

    out_dump_buffer = runner.DumpVisBuffer(output_dir)

    # Configuration for the direct dump of FakeVis's output
    fakevis_dump_conf = {
        "file_name": "fakevis_dump",
        "file_ext": "dump",
        "base_dir": output_dir,
    }

    host, port = cal_broker.server_address
    global_params.update({"broker_host": host, "broker_port": port})

    test = runner.KotekanStageTester(
        "applyGains",
        global_params,
        buffers_in=fakevis_buffer,
        buffers_out=out_dump_buffer,
        global_config=global_params,
        rest_commands=cmds,
        parallel_stage_type="rawFileWrite",
        parallel_stage_config=fakevis_dump_conf,
    )

    test.run()

    in_dump = visbuffer.VisBuffer.load_files(
        f"{output_dir}/*fakevis_dump*.dump")

    return in_dump, out_dump_buffer.load()
コード例 #4
0
def test_send_receive(tmpdir_factory):

    # Run kotekan bufferRecv
    tmpdir = tmpdir_factory.mktemp("writer")
    write_buffer = runner.DumpVisBuffer(str(tmpdir))

    # the plan is: wait 5s and then kill it
    rest_commands = [("wait", 5, None), ("get", "kill", None)]

    receiver = runner.KotekanStageTester(
        "bufferRecv",
        {},
        None,
        write_buffer,
        params_kotekan,
        rest_commands=rest_commands,
    )

    # TODO: network buffer processes should use in_buf and out_buf to please the test framework
    receiver._stages["bufferRecv_test"]["buf"] = receiver._stages[
        "bufferRecv_test"]["out_buf"]

    # Run kotekan bufferRecv in another thread
    with concurrent.futures.ThreadPoolExecutor() as executor:
        future_receiver = executor.submit(receiver.run)

        # Wait for it to start so the sender doesn't drop frames
        time.sleep(1)

        fakevis_buffer = runner.FakeVisBuffer(
            num_frames=params_kotekan["total_frames"],
            mode=params_kotekan["mode"],
            freq_ids=params_kotekan["freq_ids"],
            sleep_before=2,
            wait=False,
        )
        sender = runner.KotekanStageTester("bufferSend", {}, fakevis_buffer,
                                           None, params_kotekan)

        # TODO: network buffer processes should use in_buf and out_buf to please the test framework
        sender._stages["bufferSend_test"]["buf"] = sender._stages[
            "bufferSend_test"]["in_buf"]

        # run kotekan bufferSend
        sender.run()

        # wait for kotekan bufferRecv to finish
        future_receiver.result(timeout=7)

    assert sender.return_code == 0
    assert receiver.return_code == 0
    vis_data = write_buffer.load()

    assert len(vis_data) == params_kotekan["total_frames"]
コード例 #5
0
def data(tmpdir_factory):

    # keep all the data this test produces in a tmp directory
    tmpdir = tmpdir_factory.mktemp("name_of_the_test_case")

    # you can use FakeVisBuffer to produce fake data
    fakevis_buffer = runner.FakeVisBuffer(num_frames=params["total_frames"], mode="gaussian")

    # DumpVisBuffer can be used to dump data for testing
    dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    # KotekanStageTester is used to run kotekan with your config
    test = runner.KotekanStageTester("stageUnderTest", {}, fakevis_buffer, dump_buffer, params)
コード例 #6
0
def pulsar_data(tmpdir_factory):

    tmpdir = tmpdir_factory.mktemp("accumulate")

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester('visAccumulate', accumulate_params,
                                     runner.FakeGPUBuffer(**pulsar_params),
                                     dump_buffer, accumulate_params)

    test.run()

    yield dump_buffer.load()
コード例 #7
0
def pulsar_data(tmpdir_factory):

    tmpdir = tmpdir_factory.mktemp("pulsar")

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))
    dump_buffer_gated = runner.VisWriterBuffer(str(tmpdir), "raw")
    # Insert an extra buffer for gated stream
    dump_buffer.buffer_block.update(dump_buffer_gated.buffer_block)
    dump_buffer.stage_block.update(dump_buffer_gated.stage_block)

    acc_par = pulsar_params.copy()
    acc_par.update({
        "gating": {
            "psr0": {
                "mode": "pulsar",
                "buf": dump_buffer_gated.name
            }
        },
        "updatable_config": {
            "psr0": "/updatable_config/psr0_config"
        },
    })
    updatable_params = pulsar_params.copy()
    updatable_params.update({
        "updatable_config": {
            "psr0_config": {
                "kotekan_update_endpoint": "json",
                "enabled": True,
                "pulsar_name": "fakepsr",
                "segment": 100.0,
                "coeff": [pulsar_params["coeff"]],
                "dm": pulsar_params["dm"],
                "t_ref": [pulsar_params["t_ref"]],
                "phase_ref": [pulsar_params["phase_ref"]],
                "rot_freq": pulsar_params["rot_freq"],
                "pulse_width": 1e-3,
            }
        }
    })

    test = runner.KotekanStageTester(
        "visAccumulate",
        acc_par,
        runner.FakeGPUBuffer(**pulsar_params),
        dump_buffer,
        updatable_params,
    )

    test.run()

    yield dump_buffer_gated.load()
コード例 #8
0
def replace_data(tmpdir_factory):

    tmpdir = tmpdir_factory.mktemp("replace")

    fakevis_buffer = runner.FakeVisBuffer(
        freq=replace_params["freq"], num_frames=replace_params["total_frames"])

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester("ReplaceVis", {}, fakevis_buffer,
                                     dump_buffer, replace_params)

    test.run()

    yield dump_buffer.load()
コード例 #9
0
ファイル: test_removeev.py プロジェクト: leungcalvin/kotekan
def remove_data(tmpdir_factory):

    tmpdir = tmpdir_factory.mktemp("remove")

    fakevis_buffer = runner.FakeVisBuffer(
        freq=remove_params['freq'], num_frames=remove_params['total_frames'])

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester('removeEv', {}, fakevis_buffer,
                                     dump_buffer, remove_params)

    test.run()

    yield dump_buffer.load()
コード例 #10
0
ファイル: test_transform.py プロジェクト: leungcalvin/kotekan
def lostsamples_data(tmpdir_factory):

    tmpdir = tmpdir_factory.mktemp("lostsamples")

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester('visTransform', {'num_ev': 4}, [
        runner.FakeGPUBuffer(mode='lostsamples',
                             freq=params['freq'],
                             num_frames=params['total_frames'])
        for g in range(params['num_gpu_buffers'])
    ], dump_buffer, params)

    test.run()

    yield dump_buffer.load()
コード例 #11
0
def gaussian_data(tmpdir_factory):

    tmpdir = tmpdir_factory.mktemp("gaussian")

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester(
        'visAccumulate', {'num_ev': 4},
        runner.FakeGPUBuffer(mode='gaussian',
                             freq=gaussian_params['freq'],
                             num_frames=gaussian_params['total_frames']),
        dump_buffer, gaussian_params)

    test.run()

    yield dump_buffer.load()
コード例 #12
0
def lostsamples_data(tmpdir_factory):

    tmpdir = tmpdir_factory.mktemp("lostsamples")

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester(
        'visAccumulate', {'num_ev': 4},
        runner.FakeGPUBuffer(mode='lostsamples',
                             freq=accumulate_params['freq'],
                             num_frames=accumulate_params['total_frames']),
        dump_buffer, accumulate_params)

    test.run()

    yield dump_buffer.load()
コード例 #13
0
def pulsar_data(tmpdir_factory):

    tmpdir = tmpdir_factory.mktemp("pulsar")

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))
    dump_buffer_gated = runner.VisWriterBuffer(str(tmpdir), "raw")
    # Insert an extra buffer for gated stream
    dump_buffer.buffer_block.update(dump_buffer_gated.buffer_block)
    dump_buffer.stage_block.update(dump_buffer_gated.stage_block)

    acc_par = pulsar_params.copy()
    acc_par.update({
        'gating': {
            'psr0': {
                'mode': 'pulsar',
                'buf': dump_buffer_gated.name
            },
        },
        'updatable_config': {
            'psr0': "/updatable_config/psr0_config"
        },
    })
    updatable_params = pulsar_params.copy()
    updatable_params.update({
        'updatable_config': {
            'psr0_config': {
                'kotekan_update_endpoint': 'json',
                'enabled': True,
                'pulsar_name': 'fakepsr',
                'segment': 100.,
                'coeff': [pulsar_params['coeff']],
                'dm': pulsar_params['dm'],
                't_ref': [pulsar_params['t_ref']],
                'phase_ref': [pulsar_params['phase_ref']],
                'rot_freq': pulsar_params['rot_freq'],
                'pulse_width': 1e-3,
            }
        }
    })

    test = runner.KotekanStageTester('visAccumulate', acc_par,
                                     runner.FakeGPUBuffer(**pulsar_params),
                                     dump_buffer, updatable_params)

    test.run()

    yield dump_buffer_gated.load()
コード例 #14
0
def vis_data(tmpdir_factory):

    tmpdir = tmpdir_factory.mktemp("freqsub")

    fakevis_buffer = runner.FakeVisBuffer(num_frames=params['total_frames'],
                                          mode=params['mode'],
                                          freq_ids=params['freq_ids'],
                                          wait=False)

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester('freqSubset', {}, fakevis_buffer,
                                     dump_buffer, params)

    test.run()

    yield dump_buffer.load()
コード例 #15
0
def merge_data(tmpdir_factory):

    tmpdir = tmpdir_factory.mktemp("merge")

    fakevis_buffers = [
        runner.FakeVisBuffer(freq_ids=[f],
                             num_frames=merge_params['total_frames'])
        for f in merge_params['freq']
    ]

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester('bufferMerge', {}, fakevis_buffers,
                                     dump_buffer, merge_params)

    test.run()

    yield dump_buffer.load()
コード例 #16
0
def run_eigenvis(tdir_factory, params=None):

    if not params:
        params = default_params

    tmpdir = tdir_factory.mktemp("eigenvis")

    fakevis_buffer = runner.FakeVisBuffer(freq_ids=params["freq"],
                                          num_frames=params["total_frames"],
                                          mode=params["mode"])

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester("eigenVis", {}, fakevis_buffer,
                                     dump_buffer, params)

    test.run()
    return dump_buffer.load()
コード例 #17
0
def vis_data(tmpdir_factory):

    tmpdir = tmpdir_factory.mktemp("vis_data")

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester(
        'timeDownsample', downsamp_params,
        runner.FakeVisBuffer(
            num_frames=downsamp_params['total_frames'],
            mode=downsamp_params['fakevis_mode'],
            cadence=downsamp_params['cadence']
        ),
        dump_buffer,
        downsamp_params
    )

    test.run()

    yield dump_buffer.load()
コード例 #18
0
def complete_data(tmpdir_factory):

    tmpdir = tmpdir_factory.mktemp("no_drop")

    fakevis_buffer = runner.FakeVisBuffer(
        freq_ids=global_params["freq_ids"],
        num_frames=global_params["total_frames"])

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))
    global_params.update(dump_buffer.buffer_block)
    global_params.update(dump_buffer.stage_block)
    global_params["compress"]["out_buf"] = dump_buffer.name

    valve_params = {"out_buf": "compress_buffer"}

    test = runner.KotekanStageTester("Valve", valve_params, fakevis_buffer,
                                     None, global_params)

    test.run()

    yield dump_buffer.load()
コード例 #19
0
def complete_data(tmpdir_factory):

    tmpdir = tmpdir_factory.mktemp("no_drop")

    fakevis_buffer = runner.FakeVisBuffer(
        freq_ids=global_params['freq_ids'],
        num_frames=global_params['total_frames'])

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))
    global_params.update(dump_buffer.buffer_block)
    global_params.update(dump_buffer.stage_block)
    global_params['compress']['out_buf'] = dump_buffer.name

    valve_params = {'out_buf': 'compress_buffer'}

    test = runner.KotekanStageTester('Valve', valve_params, fakevis_buffer,
                                     None, global_params)

    test.run()

    yield dump_buffer.load()
コード例 #20
0
def apply_data(cmds, tmpdir_factory):

    apply_dir = tmpdir_factory.mktemp("apply")

    fakevis_buffer = runner.FakeVisBuffer(
        freq_ids=global_params['freq_ids'],
        num_frames=global_params['total_frames'],
        wait=global_params['wait'])

    out_dump_buffer = runner.DumpVisBuffer(str(apply_dir))

    test = runner.KotekanStageTester('applyGains',
                                     global_params,
                                     buffers_in=fakevis_buffer,
                                     buffers_out=out_dump_buffer,
                                     global_config=global_params,
                                     rest_commands=cmds)

    test.run()

    return out_dump_buffer.load()
コード例 #21
0
def gaussian_data(tmpdir_factory):

    tmpdir = tmpdir_factory.mktemp("gaussian")

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester(
        "visAccumulate",
        {},
        runner.FakeGPUBuffer(
            pattern="gaussian",
            freq=gaussian_params["freq"],
            num_frames=gaussian_params["total_frames"],
        ),
        dump_buffer,
        gaussian_params,
    )

    test.run()

    yield dump_buffer.load()
コード例 #22
0
def lostweights_data(tmpdir_factory, request):

    tmpdir = tmpdir_factory.mktemp("lostweights")

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester(
        "visAccumulate",
        {},
        runner.FakeGPUBuffer(
            pattern="lostweights",
            freq=accumulate_params["freq"],
            num_frames=accumulate_params["total_frames"],
            b=request.param,
        ),
        dump_buffer,
        accumulate_params,
    )

    test.run()

    yield (request.param, dump_buffer.load())
コード例 #23
0
def run_flagging(tmpdir_factory, cmds):
    """ Receive Flags """

    tmpdir = tmpdir_factory.mktemp("receiveflags")

    fakevis_buffer = runner.FakeVisBuffer(num_frames=params['total_frames'],
                                          mode=params['fakevis_mode'],
                                          cadence=params['cadence'],
                                          wait=params['wait'])

    out_dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester('receiveFlags',
                                     params,
                                     buffers_in=fakevis_buffer,
                                     buffers_out=out_dump_buffer,
                                     global_config=params,
                                     rest_commands=cmds)

    test.run()

    return out_dump_buffer.load()
コード例 #24
0
ファイル: test_badinputflag.py プロジェクト: mfkiwl/kotekan
def output_frames(tmpdir_factory, input_frames):

    tmpdir = tmpdir_factory.mktemp("flagbad")

    # ReadVisBuffer receives a list of frames and writes them down to disk.
    read_buffer = runner.ReadVisBuffer(str(tmpdir), input_frames)
    read_buffer.write()

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester(
        "BadInputFlag",
        {},
        read_buffer,
        dump_buffer,
        root_params,
        expect_failure=True,
    )

    test.run()

    return dump_buffer.load()
コード例 #25
0
def diagonal_data(tmpdir_factory):

    tmpdir = tmpdir_factory.mktemp("diagonal")

    fakevis_buffer = runner.FakeVisBuffer(
        freq_ids=diag_global_params["freq_ids"],
        num_frames=diag_global_params["total_frames"],
    )

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester(
        "baselineCompression",
        diag_stage_params,
        fakevis_buffer,
        dump_buffer,
        diag_global_params,
    )

    test.run()

    yield dump_buffer.load()
コード例 #26
0
ファイル: test_transform.py プロジェクト: mfkiwl/kotekan
def lostsamples_data(tmpdir_factory):

    tmpdir = tmpdir_factory.mktemp("lostsamples")

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester(
        "visTransform",
        {"num_ev": 4},
        [
            runner.FakeGPUBuffer(
                pattern="lostsamples",
                freq=params["freq"],
                num_frames=params["total_frames"],
            ) for g in range(params["num_gpu_buffers"])
        ],
        dump_buffer,
        params,
    )

    test.run()

    yield dump_buffer.load()
コード例 #27
0
def mergewait_data(tmpdir_factory):

    tmpdir = tmpdir_factory.mktemp("mergewait")

    fakevis_fast = runner.FakeVisBuffer(freq_ids=[0],
                                        cadence=0.3,
                                        wait=True,
                                        num_frames=5)

    fakevis_slow = runner.FakeVisBuffer(freq_ids=[1],
                                        cadence=5.0,
                                        wait=True,
                                        num_frames=10)

    dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester('bufferMerge', {},
                                     [fakevis_fast, fakevis_slow], dump_buffer,
                                     merge_params)

    test.run()

    yield dump_buffer.load()
コード例 #28
0
ファイル: test_truncate.py プロジェクト: mfkiwl/kotekan
def vis_data_zero_weights(tmpdir_factory):
    """ Truncated visibilities """

    tmpdir = tmpdir_factory.mktemp("vis_data_t")

    fakevis_buffer = runner.FakeVisBuffer(
        num_frames=trunc_params["total_frames"],
        mode=trunc_params["fakevis_mode"],
        cadence=trunc_params["cadence"],
        zero_weight=True,
    )

    in_dump_config = trunc_params.copy()
    in_dump_config["base_dir"] = str(tmpdir)
    in_dump_config["file_name"] = "fakevis"
    in_dump_config["file_ext"] = "dump"

    out_dump_buffer = runner.DumpVisBuffer(str(tmpdir))

    test = runner.KotekanStageTester(
        "VisTruncate",
        trunc_params,
        buffers_in=fakevis_buffer,
        buffers_out=out_dump_buffer,
        global_config=trunc_params,
        parallel_stage_type="rawFileWrite",
        parallel_stage_config=in_dump_config,
        noise="random",
    )

    test.run()

    yield (
        out_dump_buffer.load(),
        visbuffer.VisBuffer.load_files("%s/*fakevis*.dump" % str(dir)),
    )
コード例 #29
0
def run_test(
    write_dir,
    rest_commands=None,
    params=params,
    noise=False,
    name="simple",
    expect_failure=False,
):
    params["write_dir"] = write_dir

    fakevis_buffer = runner.FakeVisBuffer(
        stage_name=STAGE_NAME,
        num_frames=params["total_frames"],
        mode=params["mode"],
        freq_ids=params["freq_ids"],
        wait=True,
    )

    dump_buffer = runner.DumpVisBuffer(write_dir)

    if noise:
        fakevis_dump_conf = params.copy()
        fakevis_dump_conf["file_name"] = "fakevis_dump"
        fakevis_dump_conf["file_ext"] = "dump"
        fakevis_dump_conf["base_dir"] = write_dir

        test = runner.KotekanStageTester(
            "visTestPattern",
            params,
            buffers_in=fakevis_buffer,
            buffers_out=dump_buffer,
            global_config=params,
            parallel_stage_type="rawFileWrite",
            parallel_stage_config=fakevis_dump_conf,
            noise=True,
            rest_commands=rest_commands,
            expect_failure=expect_failure,
        )
    else:
        test = runner.KotekanStageTester(
            "visTestPattern",
            params,
            fakevis_buffer,
            dump_buffer,
            params,
            rest_commands=rest_commands,
            expect_failure=expect_failure,
        )

    test.run()

    out_data = []
    out_file = params["write_dir"] + "/" + name + ".csv"
    try:
        with open(out_file) as csvfile:
            reader = csv.DictReader(csvfile)
            for row in reader:
                out_data.append(row)
    except:
        out_data = None

    if noise:
        in_data = visbuffer.VisBuffer.load_files("%s/*fakevis_dump*.dump" %
                                                 write_dir)
    else:
        in_data = None

    return out_data, in_data, dump_buffer.load()
コード例 #30
0
def subset_data(tmpdir_factory):
    broker_path = shutil.which("comet")
    if not broker_path:
        pytest.skip(
            "Make sure PYTHONPATH is set to where the comet dataset broker is installed."
        )
    if not has_redis():
        pytest.skip("Redis is not available and so comet will fail")

    # run the dataset broker
    broker = Popen([broker_path, "--recover", "False"])
    time.sleep(1.5)

    try:
        tmpdir = tmpdir_factory.mktemp("freqsub_broker_fakevis")

        dump_buffer_gen = runner.DumpVisBuffer(str(tmpdir))

        test = runner.KotekanStageTester("FakeVis", params_fakevis, None,
                                         dump_buffer_gen, params)

        test.run()

        data_gen = dump_buffer_gen.load()

        ### VisFreqSubset ###
        tmpdir = tmpdir_factory.mktemp("freqsub_broker")

        ds_id = data_gen[0].metadata.dataset_id
        fakevis_buffer_subset = runner.FakeVisBuffer(
            num_frames=params["total_frames"],
            mode=params["mode"],
            freq_ids=params["freq_ids"],
            use_dataset_manager=True,
            wait=False,
            dataset_id="{:016x}{:016x}".format(ds_id[1], ds_id[0]),
        )

        dump_buffer_subset = runner.DumpVisBuffer(str(tmpdir))

        test = runner.KotekanStageTester("VisFreqSubset", {},
                                         fakevis_buffer_subset,
                                         dump_buffer_subset, params)

        test.run()

        data_subset = dump_buffer_subset.load()

        ### freqSplit ###
        time.sleep(10)
        tmpdir = tmpdir_factory.mktemp("freqsplit_broker")

        ds_id = data_subset[0].metadata.dataset_id
        fakevis_buffer_split = runner.FakeVisBuffer(
            num_frames=params["total_frames"],
            mode=params["mode"],
            freq_ids=params["subset_list"],
            use_dataset_manager=True,
            wait=False,
            dataset_id="{:016x}{:016x}".format(ds_id[1], ds_id[0]),
        )

        dump_buffer_split_lower = runner.DumpVisBuffer(str(tmpdir))
        dump_buffer_split_higher = runner.DumpVisBuffer(str(tmpdir))

        test = runner.KotekanStageTester(
            "freqSplit",
            {},
            fakevis_buffer_split,
            (dump_buffer_split_lower, dump_buffer_split_higher),
            params,
        )

        test.run()

        data_split_lower = dump_buffer_split_lower.load()
        data_split_higher = dump_buffer_split_higher.load()

        ### 2 VisWriter processes ###

        tmpdir = tmpdir_factory.mktemp("freqsub_write_lower")

        params_fakevis_write_lower = params_fakevis.copy()

        ds_id = data_split_lower[0].metadata.dataset_id

        params_fakevis_write_lower["dataset_id"] = "{:016x}{:016x}".format(
            ds_id[1], ds_id[0])

        # the writer is not given the subset list, it get's it through the broker
        write_buffer_lower = runner.VisWriterBuffer(
            str(tmpdir),
            "raw",
            None,
            extra_config={"use_dataset_manager": True})

        test = runner.KotekanStageTester("FakeVis", params_fakevis_write_lower,
                                         None, write_buffer_lower, params)

        test.run()

        tmpdir = tmpdir_factory.mktemp("freqsub_write_higher")

        params_fakevis_write_higher = params_fakevis.copy()
        ds_id = data_split_higher[0].metadata.dataset_id

        params_fakevis_write_higher["dataset_id"] = "{:016x}{:016x}".format(
            ds_id[1], ds_id[0])

        # the writer is not given the subset list, it get's it through the broker
        write_buffer_higher = runner.VisWriterBuffer(
            str(tmpdir),
            "raw",
            None,
            extra_config={"use_dataset_manager": True})

        test = runner.KotekanStageTester("FakeVis",
                                         params_fakevis_write_higher, None,
                                         write_buffer_higher, params)

        test.run()

        yield [
            data_gen,
            data_subset,
            data_split_lower,
            data_split_higher,
            write_buffer_lower.load(),
            write_buffer_higher.load(),
        ]
    finally:
        pid = broker.pid
        os.kill(pid, signal.SIGINT)
        broker.terminate()