def vis_data(tmpdir_factory): tmpdir_l = tmpdir_factory.mktemp("freqsplit_lower") tmpdir_h = tmpdir_factory.mktemp("freqsplit_higher") fakevis_buffer = runner.FakeVisBuffer( num_frames=params['total_frames'], mode=params['mode'], freq_ids=params['freq_ids'], wait=False ) dump_buffer_l = runner.DumpVisBuffer( str(tmpdir_l)) dump_buffer_h = runner.DumpVisBuffer( str(tmpdir_h)) test = runner.KotekanStageTester( 'freqSplit', {}, fakevis_buffer, [dump_buffer_l, dump_buffer_h], params ) test.run() yield [dump_buffer_l.load(), dump_buffer_h.load()]
def run_flagging(tmpdir_factory, cmds): """ Receive Flags """ tmpdir = tmpdir_factory.mktemp("receiveflags") fakevis_buffer = runner.FakeVisBuffer( num_frames=global_params["total_frames"], mode=global_params["fakevis_mode"], cadence=global_params["cadence"], wait=global_params["wait"], ) out_dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester( "ReceiveFlags", params, buffers_in=fakevis_buffer, buffers_out=out_dump_buffer, global_config=global_params, rest_commands=cmds, expect_failure=True, ) test.run() return out_dump_buffer.load()
def apply_data(request, tmp_path_factory, gain_path, old_gains, new_gains, cal_broker): output_dir = str(tmp_path_factory.mktemp("output")) global_params["gains_dir"] = str(gain_path) global_params[ "read_from_file"] = True if request.param == "file" else False # REST commands cmds = [[ "post", "gains", { "update_id": new_update_id, "start_time": new_timestamp, "transition_interval": transition_interval, "new_state": new_state, }, ]] fakevis_buffer = runner.FakeVisBuffer( freq_ids=global_params["freq_ids"], num_frames=global_params["total_frames"], wait=global_params["wait"], ) out_dump_buffer = runner.DumpVisBuffer(output_dir) # Configuration for the direct dump of FakeVis's output fakevis_dump_conf = { "file_name": "fakevis_dump", "file_ext": "dump", "base_dir": output_dir, } host, port = cal_broker.server_address global_params.update({"broker_host": host, "broker_port": port}) test = runner.KotekanStageTester( "applyGains", global_params, buffers_in=fakevis_buffer, buffers_out=out_dump_buffer, global_config=global_params, rest_commands=cmds, parallel_stage_type="rawFileWrite", parallel_stage_config=fakevis_dump_conf, ) test.run() in_dump = visbuffer.VisBuffer.load_files( f"{output_dir}/*fakevis_dump*.dump") return in_dump, out_dump_buffer.load()
def test_send_receive(tmpdir_factory): # Run kotekan bufferRecv tmpdir = tmpdir_factory.mktemp("writer") write_buffer = runner.DumpVisBuffer(str(tmpdir)) # the plan is: wait 5s and then kill it rest_commands = [("wait", 5, None), ("get", "kill", None)] receiver = runner.KotekanStageTester( "bufferRecv", {}, None, write_buffer, params_kotekan, rest_commands=rest_commands, ) # TODO: network buffer processes should use in_buf and out_buf to please the test framework receiver._stages["bufferRecv_test"]["buf"] = receiver._stages[ "bufferRecv_test"]["out_buf"] # Run kotekan bufferRecv in another thread with concurrent.futures.ThreadPoolExecutor() as executor: future_receiver = executor.submit(receiver.run) # Wait for it to start so the sender doesn't drop frames time.sleep(1) fakevis_buffer = runner.FakeVisBuffer( num_frames=params_kotekan["total_frames"], mode=params_kotekan["mode"], freq_ids=params_kotekan["freq_ids"], sleep_before=2, wait=False, ) sender = runner.KotekanStageTester("bufferSend", {}, fakevis_buffer, None, params_kotekan) # TODO: network buffer processes should use in_buf and out_buf to please the test framework sender._stages["bufferSend_test"]["buf"] = sender._stages[ "bufferSend_test"]["in_buf"] # run kotekan bufferSend sender.run() # wait for kotekan bufferRecv to finish future_receiver.result(timeout=7) assert sender.return_code == 0 assert receiver.return_code == 0 vis_data = write_buffer.load() assert len(vis_data) == params_kotekan["total_frames"]
def data(tmpdir_factory): # keep all the data this test produces in a tmp directory tmpdir = tmpdir_factory.mktemp("name_of_the_test_case") # you can use FakeVisBuffer to produce fake data fakevis_buffer = runner.FakeVisBuffer(num_frames=params["total_frames"], mode="gaussian") # DumpVisBuffer can be used to dump data for testing dump_buffer = runner.DumpVisBuffer(str(tmpdir)) # KotekanStageTester is used to run kotekan with your config test = runner.KotekanStageTester("stageUnderTest", {}, fakevis_buffer, dump_buffer, params)
def pulsar_data(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("accumulate") dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester('visAccumulate', accumulate_params, runner.FakeGPUBuffer(**pulsar_params), dump_buffer, accumulate_params) test.run() yield dump_buffer.load()
def pulsar_data(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("pulsar") dump_buffer = runner.DumpVisBuffer(str(tmpdir)) dump_buffer_gated = runner.VisWriterBuffer(str(tmpdir), "raw") # Insert an extra buffer for gated stream dump_buffer.buffer_block.update(dump_buffer_gated.buffer_block) dump_buffer.stage_block.update(dump_buffer_gated.stage_block) acc_par = pulsar_params.copy() acc_par.update({ "gating": { "psr0": { "mode": "pulsar", "buf": dump_buffer_gated.name } }, "updatable_config": { "psr0": "/updatable_config/psr0_config" }, }) updatable_params = pulsar_params.copy() updatable_params.update({ "updatable_config": { "psr0_config": { "kotekan_update_endpoint": "json", "enabled": True, "pulsar_name": "fakepsr", "segment": 100.0, "coeff": [pulsar_params["coeff"]], "dm": pulsar_params["dm"], "t_ref": [pulsar_params["t_ref"]], "phase_ref": [pulsar_params["phase_ref"]], "rot_freq": pulsar_params["rot_freq"], "pulse_width": 1e-3, } } }) test = runner.KotekanStageTester( "visAccumulate", acc_par, runner.FakeGPUBuffer(**pulsar_params), dump_buffer, updatable_params, ) test.run() yield dump_buffer_gated.load()
def replace_data(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("replace") fakevis_buffer = runner.FakeVisBuffer( freq=replace_params["freq"], num_frames=replace_params["total_frames"]) dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester("ReplaceVis", {}, fakevis_buffer, dump_buffer, replace_params) test.run() yield dump_buffer.load()
def remove_data(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("remove") fakevis_buffer = runner.FakeVisBuffer( freq=remove_params['freq'], num_frames=remove_params['total_frames']) dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester('removeEv', {}, fakevis_buffer, dump_buffer, remove_params) test.run() yield dump_buffer.load()
def lostsamples_data(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("lostsamples") dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester('visTransform', {'num_ev': 4}, [ runner.FakeGPUBuffer(mode='lostsamples', freq=params['freq'], num_frames=params['total_frames']) for g in range(params['num_gpu_buffers']) ], dump_buffer, params) test.run() yield dump_buffer.load()
def gaussian_data(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("gaussian") dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester( 'visAccumulate', {'num_ev': 4}, runner.FakeGPUBuffer(mode='gaussian', freq=gaussian_params['freq'], num_frames=gaussian_params['total_frames']), dump_buffer, gaussian_params) test.run() yield dump_buffer.load()
def lostsamples_data(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("lostsamples") dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester( 'visAccumulate', {'num_ev': 4}, runner.FakeGPUBuffer(mode='lostsamples', freq=accumulate_params['freq'], num_frames=accumulate_params['total_frames']), dump_buffer, accumulate_params) test.run() yield dump_buffer.load()
def pulsar_data(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("pulsar") dump_buffer = runner.DumpVisBuffer(str(tmpdir)) dump_buffer_gated = runner.VisWriterBuffer(str(tmpdir), "raw") # Insert an extra buffer for gated stream dump_buffer.buffer_block.update(dump_buffer_gated.buffer_block) dump_buffer.stage_block.update(dump_buffer_gated.stage_block) acc_par = pulsar_params.copy() acc_par.update({ 'gating': { 'psr0': { 'mode': 'pulsar', 'buf': dump_buffer_gated.name }, }, 'updatable_config': { 'psr0': "/updatable_config/psr0_config" }, }) updatable_params = pulsar_params.copy() updatable_params.update({ 'updatable_config': { 'psr0_config': { 'kotekan_update_endpoint': 'json', 'enabled': True, 'pulsar_name': 'fakepsr', 'segment': 100., 'coeff': [pulsar_params['coeff']], 'dm': pulsar_params['dm'], 't_ref': [pulsar_params['t_ref']], 'phase_ref': [pulsar_params['phase_ref']], 'rot_freq': pulsar_params['rot_freq'], 'pulse_width': 1e-3, } } }) test = runner.KotekanStageTester('visAccumulate', acc_par, runner.FakeGPUBuffer(**pulsar_params), dump_buffer, updatable_params) test.run() yield dump_buffer_gated.load()
def vis_data(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("freqsub") fakevis_buffer = runner.FakeVisBuffer(num_frames=params['total_frames'], mode=params['mode'], freq_ids=params['freq_ids'], wait=False) dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester('freqSubset', {}, fakevis_buffer, dump_buffer, params) test.run() yield dump_buffer.load()
def merge_data(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("merge") fakevis_buffers = [ runner.FakeVisBuffer(freq_ids=[f], num_frames=merge_params['total_frames']) for f in merge_params['freq'] ] dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester('bufferMerge', {}, fakevis_buffers, dump_buffer, merge_params) test.run() yield dump_buffer.load()
def run_eigenvis(tdir_factory, params=None): if not params: params = default_params tmpdir = tdir_factory.mktemp("eigenvis") fakevis_buffer = runner.FakeVisBuffer(freq_ids=params["freq"], num_frames=params["total_frames"], mode=params["mode"]) dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester("eigenVis", {}, fakevis_buffer, dump_buffer, params) test.run() return dump_buffer.load()
def vis_data(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("vis_data") dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester( 'timeDownsample', downsamp_params, runner.FakeVisBuffer( num_frames=downsamp_params['total_frames'], mode=downsamp_params['fakevis_mode'], cadence=downsamp_params['cadence'] ), dump_buffer, downsamp_params ) test.run() yield dump_buffer.load()
def complete_data(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("no_drop") fakevis_buffer = runner.FakeVisBuffer( freq_ids=global_params["freq_ids"], num_frames=global_params["total_frames"]) dump_buffer = runner.DumpVisBuffer(str(tmpdir)) global_params.update(dump_buffer.buffer_block) global_params.update(dump_buffer.stage_block) global_params["compress"]["out_buf"] = dump_buffer.name valve_params = {"out_buf": "compress_buffer"} test = runner.KotekanStageTester("Valve", valve_params, fakevis_buffer, None, global_params) test.run() yield dump_buffer.load()
def complete_data(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("no_drop") fakevis_buffer = runner.FakeVisBuffer( freq_ids=global_params['freq_ids'], num_frames=global_params['total_frames']) dump_buffer = runner.DumpVisBuffer(str(tmpdir)) global_params.update(dump_buffer.buffer_block) global_params.update(dump_buffer.stage_block) global_params['compress']['out_buf'] = dump_buffer.name valve_params = {'out_buf': 'compress_buffer'} test = runner.KotekanStageTester('Valve', valve_params, fakevis_buffer, None, global_params) test.run() yield dump_buffer.load()
def apply_data(cmds, tmpdir_factory): apply_dir = tmpdir_factory.mktemp("apply") fakevis_buffer = runner.FakeVisBuffer( freq_ids=global_params['freq_ids'], num_frames=global_params['total_frames'], wait=global_params['wait']) out_dump_buffer = runner.DumpVisBuffer(str(apply_dir)) test = runner.KotekanStageTester('applyGains', global_params, buffers_in=fakevis_buffer, buffers_out=out_dump_buffer, global_config=global_params, rest_commands=cmds) test.run() return out_dump_buffer.load()
def gaussian_data(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("gaussian") dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester( "visAccumulate", {}, runner.FakeGPUBuffer( pattern="gaussian", freq=gaussian_params["freq"], num_frames=gaussian_params["total_frames"], ), dump_buffer, gaussian_params, ) test.run() yield dump_buffer.load()
def lostweights_data(tmpdir_factory, request): tmpdir = tmpdir_factory.mktemp("lostweights") dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester( "visAccumulate", {}, runner.FakeGPUBuffer( pattern="lostweights", freq=accumulate_params["freq"], num_frames=accumulate_params["total_frames"], b=request.param, ), dump_buffer, accumulate_params, ) test.run() yield (request.param, dump_buffer.load())
def run_flagging(tmpdir_factory, cmds): """ Receive Flags """ tmpdir = tmpdir_factory.mktemp("receiveflags") fakevis_buffer = runner.FakeVisBuffer(num_frames=params['total_frames'], mode=params['fakevis_mode'], cadence=params['cadence'], wait=params['wait']) out_dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester('receiveFlags', params, buffers_in=fakevis_buffer, buffers_out=out_dump_buffer, global_config=params, rest_commands=cmds) test.run() return out_dump_buffer.load()
def output_frames(tmpdir_factory, input_frames): tmpdir = tmpdir_factory.mktemp("flagbad") # ReadVisBuffer receives a list of frames and writes them down to disk. read_buffer = runner.ReadVisBuffer(str(tmpdir), input_frames) read_buffer.write() dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester( "BadInputFlag", {}, read_buffer, dump_buffer, root_params, expect_failure=True, ) test.run() return dump_buffer.load()
def diagonal_data(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("diagonal") fakevis_buffer = runner.FakeVisBuffer( freq_ids=diag_global_params["freq_ids"], num_frames=diag_global_params["total_frames"], ) dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester( "baselineCompression", diag_stage_params, fakevis_buffer, dump_buffer, diag_global_params, ) test.run() yield dump_buffer.load()
def lostsamples_data(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("lostsamples") dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester( "visTransform", {"num_ev": 4}, [ runner.FakeGPUBuffer( pattern="lostsamples", freq=params["freq"], num_frames=params["total_frames"], ) for g in range(params["num_gpu_buffers"]) ], dump_buffer, params, ) test.run() yield dump_buffer.load()
def mergewait_data(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("mergewait") fakevis_fast = runner.FakeVisBuffer(freq_ids=[0], cadence=0.3, wait=True, num_frames=5) fakevis_slow = runner.FakeVisBuffer(freq_ids=[1], cadence=5.0, wait=True, num_frames=10) dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester('bufferMerge', {}, [fakevis_fast, fakevis_slow], dump_buffer, merge_params) test.run() yield dump_buffer.load()
def vis_data_zero_weights(tmpdir_factory): """ Truncated visibilities """ tmpdir = tmpdir_factory.mktemp("vis_data_t") fakevis_buffer = runner.FakeVisBuffer( num_frames=trunc_params["total_frames"], mode=trunc_params["fakevis_mode"], cadence=trunc_params["cadence"], zero_weight=True, ) in_dump_config = trunc_params.copy() in_dump_config["base_dir"] = str(tmpdir) in_dump_config["file_name"] = "fakevis" in_dump_config["file_ext"] = "dump" out_dump_buffer = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester( "VisTruncate", trunc_params, buffers_in=fakevis_buffer, buffers_out=out_dump_buffer, global_config=trunc_params, parallel_stage_type="rawFileWrite", parallel_stage_config=in_dump_config, noise="random", ) test.run() yield ( out_dump_buffer.load(), visbuffer.VisBuffer.load_files("%s/*fakevis*.dump" % str(dir)), )
def run_test( write_dir, rest_commands=None, params=params, noise=False, name="simple", expect_failure=False, ): params["write_dir"] = write_dir fakevis_buffer = runner.FakeVisBuffer( stage_name=STAGE_NAME, num_frames=params["total_frames"], mode=params["mode"], freq_ids=params["freq_ids"], wait=True, ) dump_buffer = runner.DumpVisBuffer(write_dir) if noise: fakevis_dump_conf = params.copy() fakevis_dump_conf["file_name"] = "fakevis_dump" fakevis_dump_conf["file_ext"] = "dump" fakevis_dump_conf["base_dir"] = write_dir test = runner.KotekanStageTester( "visTestPattern", params, buffers_in=fakevis_buffer, buffers_out=dump_buffer, global_config=params, parallel_stage_type="rawFileWrite", parallel_stage_config=fakevis_dump_conf, noise=True, rest_commands=rest_commands, expect_failure=expect_failure, ) else: test = runner.KotekanStageTester( "visTestPattern", params, fakevis_buffer, dump_buffer, params, rest_commands=rest_commands, expect_failure=expect_failure, ) test.run() out_data = [] out_file = params["write_dir"] + "/" + name + ".csv" try: with open(out_file) as csvfile: reader = csv.DictReader(csvfile) for row in reader: out_data.append(row) except: out_data = None if noise: in_data = visbuffer.VisBuffer.load_files("%s/*fakevis_dump*.dump" % write_dir) else: in_data = None return out_data, in_data, dump_buffer.load()
def subset_data(tmpdir_factory): broker_path = shutil.which("comet") if not broker_path: pytest.skip( "Make sure PYTHONPATH is set to where the comet dataset broker is installed." ) if not has_redis(): pytest.skip("Redis is not available and so comet will fail") # run the dataset broker broker = Popen([broker_path, "--recover", "False"]) time.sleep(1.5) try: tmpdir = tmpdir_factory.mktemp("freqsub_broker_fakevis") dump_buffer_gen = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester("FakeVis", params_fakevis, None, dump_buffer_gen, params) test.run() data_gen = dump_buffer_gen.load() ### VisFreqSubset ### tmpdir = tmpdir_factory.mktemp("freqsub_broker") ds_id = data_gen[0].metadata.dataset_id fakevis_buffer_subset = runner.FakeVisBuffer( num_frames=params["total_frames"], mode=params["mode"], freq_ids=params["freq_ids"], use_dataset_manager=True, wait=False, dataset_id="{:016x}{:016x}".format(ds_id[1], ds_id[0]), ) dump_buffer_subset = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester("VisFreqSubset", {}, fakevis_buffer_subset, dump_buffer_subset, params) test.run() data_subset = dump_buffer_subset.load() ### freqSplit ### time.sleep(10) tmpdir = tmpdir_factory.mktemp("freqsplit_broker") ds_id = data_subset[0].metadata.dataset_id fakevis_buffer_split = runner.FakeVisBuffer( num_frames=params["total_frames"], mode=params["mode"], freq_ids=params["subset_list"], use_dataset_manager=True, wait=False, dataset_id="{:016x}{:016x}".format(ds_id[1], ds_id[0]), ) dump_buffer_split_lower = runner.DumpVisBuffer(str(tmpdir)) dump_buffer_split_higher = runner.DumpVisBuffer(str(tmpdir)) test = runner.KotekanStageTester( "freqSplit", {}, fakevis_buffer_split, (dump_buffer_split_lower, dump_buffer_split_higher), params, ) test.run() data_split_lower = dump_buffer_split_lower.load() data_split_higher = dump_buffer_split_higher.load() ### 2 VisWriter processes ### tmpdir = tmpdir_factory.mktemp("freqsub_write_lower") params_fakevis_write_lower = params_fakevis.copy() ds_id = data_split_lower[0].metadata.dataset_id params_fakevis_write_lower["dataset_id"] = "{:016x}{:016x}".format( ds_id[1], ds_id[0]) # the writer is not given the subset list, it get's it through the broker write_buffer_lower = runner.VisWriterBuffer( str(tmpdir), "raw", None, extra_config={"use_dataset_manager": True}) test = runner.KotekanStageTester("FakeVis", params_fakevis_write_lower, None, write_buffer_lower, params) test.run() tmpdir = tmpdir_factory.mktemp("freqsub_write_higher") params_fakevis_write_higher = params_fakevis.copy() ds_id = data_split_higher[0].metadata.dataset_id params_fakevis_write_higher["dataset_id"] = "{:016x}{:016x}".format( ds_id[1], ds_id[0]) # the writer is not given the subset list, it get's it through the broker write_buffer_higher = runner.VisWriterBuffer( str(tmpdir), "raw", None, extra_config={"use_dataset_manager": True}) test = runner.KotekanStageTester("FakeVis", params_fakevis_write_higher, None, write_buffer_higher, params) test.run() yield [ data_gen, data_subset, data_split_lower, data_split_higher, write_buffer_lower.load(), write_buffer_higher.load(), ] finally: pid = broker.pid os.kill(pid, signal.SIGINT) broker.terminate()