def transpose_stack(tmpdir_factory):
# Write fake stacked data in raw format
tmpdir = str(tmpdir_factory.mktemp("writer"))
fakevis_buffer = runner.FakeVisBuffer(
freq_ids=stack_params["freq"],
num_frames=stack_params["file_length"],
cadence=stack_params["cadence"],
mode="chime",
)
# Add stacking stage
stack_buf_name = "fake_stacked"
stack_buf = {
stack_buf_name: {
"kotekan_buffer": "vis",
"metadata_pool": "vis_pool",
"num_frames": "buffer_depth",
}
}
fakevis_buffer.buffer_block.update(stack_buf)
fakevis_buffer.stage_block.update({
"fakevis_stack": {
"kotekan_stage": "baselineCompression",
"in_buf": fakevis_buffer.name,
"out_buf": stack_buf_name,
"stack_type": "chime_in_cyl",
}
})
fakevis_buffer.name = stack_buf_name
params = stack_params.copy()
params["root_path"] = tmpdir
writer = runner.KotekanStageTester(
"VisWriter",
{
"node_mode": False,
"write_ev": True,
"file_type": "raw"
},
fakevis_buffer,
None,
params,
)
writer.run()
# get raw infile
files = sorted(glob.glob(tmpdir + "/20??????T??????Z_*_corr/*.meta"))
assert len(files) == 1
infile = os.path.splitext(files[0])[0]
# Tranpose and write data
raw_buf = runner.ReadRawBuffer(infile, stack_params["chunk_size"])
outfile = tmpdir + "/transposed"
transposer = runner.KotekanStageTester(
"VisTranspose",
{
"outfile": outfile,
"infile": infile,
"chunk_size": writer_params["chunk_size"],
"comet_timeout": 120.0,
},
raw_buf,
None,
params,
)
transposer.run()
fh = h5py.File(outfile + ".h5", "r")
yield (infile, fh)
fh.close()
def transpose(tmpdir_factory):
writer_params['file_length'] = writer_params['total_frames']
# Write fake data in raw format
tmpdir = str(tmpdir_factory.mktemp("writer"))
fakevis_buffer = runner.FakeVisBuffer(
freq_ids=writer_params['freq'],
num_frames=writer_params['total_frames'],
cadence=writer_params['cadence'],
mode=writer_params['mode'],
test_pattern_value=writer_params['test_pattern_value'])
# Remove the last frequency to test handling of empty frames
fsel_buf_name = "fake_freqsel"
fsel_buf = {
fsel_buf_name: {
'kotekan_buffer': 'vis',
'metadata_pool': 'vis_pool',
'num_frames': 'buffer_depth',
}
}
fakevis_buffer.buffer_block.update(fsel_buf)
fakevis_buffer.stage_block.update({
"fakevis_fsel": {
"kotekan_stage": "visDrop",
"in_buf": fakevis_buffer.name,
"out_buf": fsel_buf_name,
"freq": [writer_params['freq'][-1]],
"log_level": "debug"
}
})
fakevis_buffer.name = fsel_buf_name
# Write fake data in hdf5 format
tmpdir_h5 = str(tmpdir_factory.mktemp("dump_h5"))
dumph5_conf = writer_params.copy()
dumph5_conf['root_path'] = str(tmpdir_h5)
dumph5_conf['file_name'] = 'dumph5'
dumph5_conf['node_mode'] = False
params = writer_params.copy()
params['root_path'] = tmpdir
writer = runner.KotekanStageTester('visWriter', {
'node_mode': False,
'write_ev': True,
'file_type': 'raw'
},
fakevis_buffer,
None,
params,
parallel_stage_type='visWriter',
parallel_stage_config=dumph5_conf,
noise="random")
writer.run()
# get raw infile
files = sorted(glob.glob(tmpdir + '/20??????T??????Z_*_corr/*.meta'))
assert len(files) == 1
infile = os.path.splitext(files[0])[0]
# get hdf5 infile
files = sorted(glob.glob(tmpdir_h5 + '/20??????T??????Z_*_corr/*.h5'))
assert len(files) == 1
infile_h5 = os.path.splitext(files[0])[0]
# Tranpose and write data
raw_buf = runner.ReadRawBuffer(infile, writer_params['chunk_size'])
outfile = tmpdir + "/transposed"
transposer = runner.KotekanStageTester(
'visTranspose', {
'outfile': outfile,
'infile': infile,
'chunk_size': writer_params['chunk_size']
}, raw_buf, None, params)
transposer.run()
fh = h5py.File(infile_h5 + '.h5', 'r')
fh_t = h5py.File(outfile + '.h5', 'r')
yield (fh_t, fh)
fh.close()
fh_t.close()
def transpose(tmpdir_factory, cal_broker):
writer_params["file_length"] = writer_params["total_frames"]
# Write fake data in raw format
tmpdir = str(tmpdir_factory.mktemp("writer"))
fakevis_buffer = runner.FakeVisBuffer(
freq_ids=writer_params["freq"],
num_frames=writer_params["total_frames"],
cadence=writer_params["cadence"],
mode=writer_params["mode"],
test_pattern_value=writer_params["test_pattern_value"],
wait=True, # make sure cal_broker has time to update
)
# Add an applyGain stage to alter the dataset ID
gain_buf_name = "gains_applied"
fakevis_buffer.buffer_block.update({
gain_buf_name: {
"kotekan_buffer": "vis",
"metadata_pool": "vis_pool",
"num_frames": "buffer_depth",
},
})
host, port = cal_broker.server_address
fakevis_buffer.stage_block.update({
"apply_gains": {
"kotekan_stage": "applyGains",
"in_buf": fakevis_buffer.name,
"out_buf": gain_buf_name,
"log_level": "debug",
"broker_host": host,
"broker_port": port,
},
})
# Remove samples from two frequency to test handling of empty frames
fsel_frac_name = "frac_freqsel"
fsel_empty_name = "empty_freqsel"
fsel_buf = {
fsel_frac_name: {
"kotekan_buffer": "vis",
"metadata_pool": "vis_pool",
"num_frames": "buffer_depth",
},
fsel_empty_name: {
"kotekan_buffer": "vis",
"metadata_pool": "vis_pool",
"num_frames": "buffer_depth",
},
}
fakevis_buffer.buffer_block.update(fsel_buf)
fakevis_buffer.stage_block.update({
"frac_fsel": {
"kotekan_stage": "visDrop",
"in_buf": gain_buf_name,
"out_buf": fsel_frac_name,
"freq": [writer_params["freq"][frac_freq]],
"frac_lost": frac_lost,
"frac_rfi": frac_rfi,
"log_level": "debug",
},
"empty_fsel": {
"kotekan_stage": "visDrop",
"in_buf": fsel_frac_name,
"out_buf": fsel_empty_name,
"freq": [writer_params["freq"][empty_freq]],
"log_level": "debug",
},
})
# update the output buffer name
fakevis_buffer.name = fsel_empty_name
# REST commands for gains update
cmds = [
["wait", 2.0, {}],
[
"post",
"gains",
{
"update_id": new_update_id,
"start_time": new_timestamp,
"transition_interval": transition_interval,
"new_state": new_state,
},
],
]
# Write out the test data in raw and HDF5 simultaneously
tmpdir_h5 = str(tmpdir_factory.mktemp("dump_h5"))
dumph5_conf = writer_params.copy()
dumph5_conf["root_path"] = str(tmpdir_h5)
dumph5_conf["file_name"] = "dumph5"
dumph5_conf["node_mode"] = False
params = writer_params.copy()
params["root_path"] = tmpdir
writer = runner.KotekanStageTester(
"VisWriter",
{
"node_mode": False,
"write_ev": True,
"file_type": "raw"
},
fakevis_buffer,
None,
params,
parallel_stage_type="VisWriter",
parallel_stage_config=dumph5_conf,
noise="random",
rest_commands=cmds,
)
writer.run()
# get raw infile
files = sorted(glob.glob(tmpdir + "/20??????T??????Z_*_corr/*.meta"))
assert len(files) == 1
infile = os.path.splitext(files[0])[0]
# get hdf5 infile
files = sorted(glob.glob(tmpdir_h5 + "/20??????T??????Z_*_corr/*.h5"))
assert len(files) == 1
infile_h5 = os.path.splitext(files[0])[0]
# Tranpose and write data
raw_buf = runner.ReadRawBuffer(infile, writer_params["chunk_size"])
outfile = tmpdir + "/transposed"
transposer = runner.KotekanStageTester(
"VisTranspose",
{
"outfile": outfile,
"chunk_size": writer_params["chunk_size"],
"comet_timeout": 120.0,
},
raw_buf,
None,
params,
)
transposer.run()
fh = h5py.File(infile_h5 + ".h5", "r")
fh_t = h5py.File(outfile + ".h5", "r")
yield (fh_t, fh)
fh.close()
fh_t.close()
def transpose_stack(tmpdir_factory):
# Write fake stacked data in raw format
tmpdir = str(tmpdir_factory.mktemp("writer"))
fakevis_buffer = runner.FakeVisBuffer(
freq_ids=stack_params['freq'],
num_frames=stack_params['file_length'],
cadence=stack_params['cadence'],
mode="chime",
)
# Add stacking stage
stack_buf_name = "fake_stacked"
stack_buf = {
stack_buf_name: {
'kotekan_buffer': 'vis',
'metadata_pool': 'vis_pool',
'num_frames': 'buffer_depth',
}
}
fakevis_buffer.buffer_block.update(stack_buf)
fakevis_buffer.stage_block.update({
"fakevis_stack": {
"kotekan_stage": "baselineCompression",
"in_buf": fakevis_buffer.name,
"out_buf": stack_buf_name,
"stack_type": "chime_in_cyl",
}
})
fakevis_buffer.name = stack_buf_name
params = stack_params.copy()
params['root_path'] = tmpdir
writer = runner.KotekanStageTester(
'visWriter',
{
'node_mode': False,
'write_ev': True,
'file_type': 'raw'
},
fakevis_buffer,
None,
params,
)
writer.run()
# get raw infile
files = sorted(glob.glob(tmpdir + '/20??????T??????Z_*_corr/*.meta'))
assert len(files) == 1
infile = os.path.splitext(files[0])[0]
# Tranpose and write data
raw_buf = runner.ReadRawBuffer(infile, stack_params['chunk_size'])
outfile = tmpdir + "/transposed"
transposer = runner.KotekanStageTester(
'visTranspose', {
'outfile': outfile,
'infile': infile,
'chunk_size': writer_params['chunk_size']
}, raw_buf, None, params)
transposer.run()
fh = h5py.File(outfile + '.h5', 'r')
yield (infile, fh)
fh.close()