def transpose_stack(tmpdir_factory): # Write fake stacked data in raw format tmpdir = str(tmpdir_factory.mktemp("writer")) fakevis_buffer = runner.FakeVisBuffer( freq_ids=stack_params["freq"], num_frames=stack_params["file_length"], cadence=stack_params["cadence"], mode="chime", ) # Add stacking stage stack_buf_name = "fake_stacked" stack_buf = { stack_buf_name: { "kotekan_buffer": "vis", "metadata_pool": "vis_pool", "num_frames": "buffer_depth", } } fakevis_buffer.buffer_block.update(stack_buf) fakevis_buffer.stage_block.update({ "fakevis_stack": { "kotekan_stage": "baselineCompression", "in_buf": fakevis_buffer.name, "out_buf": stack_buf_name, "stack_type": "chime_in_cyl", } }) fakevis_buffer.name = stack_buf_name params = stack_params.copy() params["root_path"] = tmpdir writer = runner.KotekanStageTester( "VisWriter", { "node_mode": False, "write_ev": True, "file_type": "raw" }, fakevis_buffer, None, params, ) writer.run() # get raw infile files = sorted(glob.glob(tmpdir + "/20??????T??????Z_*_corr/*.meta")) assert len(files) == 1 infile = os.path.splitext(files[0])[0] # Tranpose and write data raw_buf = runner.ReadRawBuffer(infile, stack_params["chunk_size"]) outfile = tmpdir + "/transposed" transposer = runner.KotekanStageTester( "VisTranspose", { "outfile": outfile, "infile": infile, "chunk_size": writer_params["chunk_size"], "comet_timeout": 120.0, }, raw_buf, None, params, ) transposer.run() fh = h5py.File(outfile + ".h5", "r") yield (infile, fh) fh.close()
def transpose(tmpdir_factory): writer_params['file_length'] = writer_params['total_frames'] # Write fake data in raw format tmpdir = str(tmpdir_factory.mktemp("writer")) fakevis_buffer = runner.FakeVisBuffer( freq_ids=writer_params['freq'], num_frames=writer_params['total_frames'], cadence=writer_params['cadence'], mode=writer_params['mode'], test_pattern_value=writer_params['test_pattern_value']) # Remove the last frequency to test handling of empty frames fsel_buf_name = "fake_freqsel" fsel_buf = { fsel_buf_name: { 'kotekan_buffer': 'vis', 'metadata_pool': 'vis_pool', 'num_frames': 'buffer_depth', } } fakevis_buffer.buffer_block.update(fsel_buf) fakevis_buffer.stage_block.update({ "fakevis_fsel": { "kotekan_stage": "visDrop", "in_buf": fakevis_buffer.name, "out_buf": fsel_buf_name, "freq": [writer_params['freq'][-1]], "log_level": "debug" } }) fakevis_buffer.name = fsel_buf_name # Write fake data in hdf5 format tmpdir_h5 = str(tmpdir_factory.mktemp("dump_h5")) dumph5_conf = writer_params.copy() dumph5_conf['root_path'] = str(tmpdir_h5) dumph5_conf['file_name'] = 'dumph5' dumph5_conf['node_mode'] = False params = writer_params.copy() params['root_path'] = tmpdir writer = runner.KotekanStageTester('visWriter', { 'node_mode': False, 'write_ev': True, 'file_type': 'raw' }, fakevis_buffer, None, params, parallel_stage_type='visWriter', parallel_stage_config=dumph5_conf, noise="random") writer.run() # get raw infile files = sorted(glob.glob(tmpdir + '/20??????T??????Z_*_corr/*.meta')) assert len(files) == 1 infile = os.path.splitext(files[0])[0] # get hdf5 infile files = sorted(glob.glob(tmpdir_h5 + '/20??????T??????Z_*_corr/*.h5')) assert len(files) == 1 infile_h5 = os.path.splitext(files[0])[0] # Tranpose and write data raw_buf = runner.ReadRawBuffer(infile, writer_params['chunk_size']) outfile = tmpdir + "/transposed" transposer = runner.KotekanStageTester( 'visTranspose', { 'outfile': outfile, 'infile': infile, 'chunk_size': writer_params['chunk_size'] }, raw_buf, None, params) transposer.run() fh = h5py.File(infile_h5 + '.h5', 'r') fh_t = h5py.File(outfile + '.h5', 'r') yield (fh_t, fh) fh.close() fh_t.close()
def transpose(tmpdir_factory, cal_broker): writer_params["file_length"] = writer_params["total_frames"] # Write fake data in raw format tmpdir = str(tmpdir_factory.mktemp("writer")) fakevis_buffer = runner.FakeVisBuffer( freq_ids=writer_params["freq"], num_frames=writer_params["total_frames"], cadence=writer_params["cadence"], mode=writer_params["mode"], test_pattern_value=writer_params["test_pattern_value"], wait=True, # make sure cal_broker has time to update ) # Add an applyGain stage to alter the dataset ID gain_buf_name = "gains_applied" fakevis_buffer.buffer_block.update({ gain_buf_name: { "kotekan_buffer": "vis", "metadata_pool": "vis_pool", "num_frames": "buffer_depth", }, }) host, port = cal_broker.server_address fakevis_buffer.stage_block.update({ "apply_gains": { "kotekan_stage": "applyGains", "in_buf": fakevis_buffer.name, "out_buf": gain_buf_name, "log_level": "debug", "broker_host": host, "broker_port": port, }, }) # Remove samples from two frequency to test handling of empty frames fsel_frac_name = "frac_freqsel" fsel_empty_name = "empty_freqsel" fsel_buf = { fsel_frac_name: { "kotekan_buffer": "vis", "metadata_pool": "vis_pool", "num_frames": "buffer_depth", }, fsel_empty_name: { "kotekan_buffer": "vis", "metadata_pool": "vis_pool", "num_frames": "buffer_depth", }, } fakevis_buffer.buffer_block.update(fsel_buf) fakevis_buffer.stage_block.update({ "frac_fsel": { "kotekan_stage": "visDrop", "in_buf": gain_buf_name, "out_buf": fsel_frac_name, "freq": [writer_params["freq"][frac_freq]], "frac_lost": frac_lost, "frac_rfi": frac_rfi, "log_level": "debug", }, "empty_fsel": { "kotekan_stage": "visDrop", "in_buf": fsel_frac_name, "out_buf": fsel_empty_name, "freq": [writer_params["freq"][empty_freq]], "log_level": "debug", }, }) # update the output buffer name fakevis_buffer.name = fsel_empty_name # REST commands for gains update cmds = [ ["wait", 2.0, {}], [ "post", "gains", { "update_id": new_update_id, "start_time": new_timestamp, "transition_interval": transition_interval, "new_state": new_state, }, ], ] # Write out the test data in raw and HDF5 simultaneously tmpdir_h5 = str(tmpdir_factory.mktemp("dump_h5")) dumph5_conf = writer_params.copy() dumph5_conf["root_path"] = str(tmpdir_h5) dumph5_conf["file_name"] = "dumph5" dumph5_conf["node_mode"] = False params = writer_params.copy() params["root_path"] = tmpdir writer = runner.KotekanStageTester( "VisWriter", { "node_mode": False, "write_ev": True, "file_type": "raw" }, fakevis_buffer, None, params, parallel_stage_type="VisWriter", parallel_stage_config=dumph5_conf, noise="random", rest_commands=cmds, ) writer.run() # get raw infile files = sorted(glob.glob(tmpdir + "/20??????T??????Z_*_corr/*.meta")) assert len(files) == 1 infile = os.path.splitext(files[0])[0] # get hdf5 infile files = sorted(glob.glob(tmpdir_h5 + "/20??????T??????Z_*_corr/*.h5")) assert len(files) == 1 infile_h5 = os.path.splitext(files[0])[0] # Tranpose and write data raw_buf = runner.ReadRawBuffer(infile, writer_params["chunk_size"]) outfile = tmpdir + "/transposed" transposer = runner.KotekanStageTester( "VisTranspose", { "outfile": outfile, "chunk_size": writer_params["chunk_size"], "comet_timeout": 120.0, }, raw_buf, None, params, ) transposer.run() fh = h5py.File(infile_h5 + ".h5", "r") fh_t = h5py.File(outfile + ".h5", "r") yield (fh_t, fh) fh.close() fh_t.close()
def transpose_stack(tmpdir_factory): # Write fake stacked data in raw format tmpdir = str(tmpdir_factory.mktemp("writer")) fakevis_buffer = runner.FakeVisBuffer( freq_ids=stack_params['freq'], num_frames=stack_params['file_length'], cadence=stack_params['cadence'], mode="chime", ) # Add stacking stage stack_buf_name = "fake_stacked" stack_buf = { stack_buf_name: { 'kotekan_buffer': 'vis', 'metadata_pool': 'vis_pool', 'num_frames': 'buffer_depth', } } fakevis_buffer.buffer_block.update(stack_buf) fakevis_buffer.stage_block.update({ "fakevis_stack": { "kotekan_stage": "baselineCompression", "in_buf": fakevis_buffer.name, "out_buf": stack_buf_name, "stack_type": "chime_in_cyl", } }) fakevis_buffer.name = stack_buf_name params = stack_params.copy() params['root_path'] = tmpdir writer = runner.KotekanStageTester( 'visWriter', { 'node_mode': False, 'write_ev': True, 'file_type': 'raw' }, fakevis_buffer, None, params, ) writer.run() # get raw infile files = sorted(glob.glob(tmpdir + '/20??????T??????Z_*_corr/*.meta')) assert len(files) == 1 infile = os.path.splitext(files[0])[0] # Tranpose and write data raw_buf = runner.ReadRawBuffer(infile, stack_params['chunk_size']) outfile = tmpdir + "/transposed" transposer = runner.KotekanStageTester( 'visTranspose', { 'outfile': outfile, 'infile': infile, 'chunk_size': writer_params['chunk_size'] }, raw_buf, None, params) transposer.run() fh = h5py.File(outfile + '.h5', 'r') yield (infile, fh) fh.close()