# create short data in r/i to test using that to write
arr_data = numpy.ones((vector_length,num_subchannels),
dtype=[('r', numpy.int16), ('i', numpy.int16)])
for i in range(len(arr_data)):
arr_data[i]['r'] = 2*i
arr_data[i]['i'] = 3*i
# start 2014-03-09 12:30:30 plus one sample
start_global_index = (1394368230 * sample_rate) + 1
# set up top level directory
os.system("rm -rf /tmp/hdf5 ; mkdir /tmp/hdf5");
print("Writing complex short to multiple files and subdirectores in /tmp/hdf5 channel junk0");
os.system("rm -rf /tmp/hdf5/junk0 ; mkdir /tmp/hdf5/junk0");
# init
data_object = digital_rf_hdf5.write_hdf5_channel("/tmp/hdf5/junk0", dtype_str, samples_per_file,
files_per_directory, start_global_index,
sample_rate, uuid, compression_level, checksum,
is_complex, num_subchannels, marching_periods);
# write
for i in range(7): # will write 700 samples - so creates two subdirectories
result = data_object.rf_write(arr_data);
# close
data_object.close();
print("done test");
files_per_directory = 10
# start 2014-03-09 12:30:30 plus one sample
start_global_index = (1394368230 * sample_rate) + 1
# test get unix time
dt, picoseconds = digital_rf_hdf5.get_unix_time(long(start_global_index), sample_rate)
print('For start_global_index=%i and sample_rate=%f, dt is %s and picoseconds is %i' % (start_global_index, sample_rate,
dt, picoseconds))
# set up top level directory
os.system("rm -rf /tmp/hdf5 ; mkdir /tmp/hdf5");
os.system("rm -rf /tmp/hdf52 ; mkdir /tmp/hdf52");
print("Test 0 - simple single write to multiple files, no compress, no checksum - channel 0");
os.system("rm -rf /tmp/hdf5/junk0 ; mkdir /tmp/hdf5/junk0");
data_object = digital_rf_hdf5.write_hdf5_channel("/tmp/hdf5/junk0", 'i4', 40, files_per_directory, start_global_index,
sample_rate, "FAKE_UUID_0", 0, False, True, num_subchannels=num_subchannels);
data = numpy.array(base_data, numpy.int32)
result = data_object.rf_write(data);
data_object.close();
print("done test 0.1");
print("Test 0.1 - simple single write to multiple files using r/i struct layout, no compress, no checksum - channel 0.1");
os.system("rm -rf /tmp/hdf5/junk0.1 ; mkdir /tmp/hdf5/junk0.1");
data_object = digital_rf_hdf5.write_hdf5_channel("/tmp/hdf5/junk0.1", 'i2', 40, files_per_directory, start_global_index,
sample_rate, "FAKE_UUID_0.1", 0, False, True, num_subchannels=num_subchannels);
result = data_object.rf_write(arr_data);
data_object.close();
print("done test 0.1");
print("Test 0.11 - simple single write with one subchannel to multiple files using r/i struct layout, no compress, no checksum - channel 0.1");
os.system("rm -rf /tmp/hdf5/junk0.11 ; mkdir /tmp/hdf5/junk0.11");
# data to write
data_int16 = numpy.zeros((WRITE_BLOCK_SIZE, 2), dtype='i2')
# make random
for i in range(WRITE_BLOCK_SIZE):
j = i * 2
k = i * 2 + 1
data_int16[i][0] = (j % 32768) * (j + 8192) * (j % 13)
data_int16[i][1] = (k % 32768) * (k + 8192) * (k % 13)
print('creating top level dir /tmp/benchmark')
os.system("rm -rf /tmp/benchmark ; mkdir /tmp/benchmark")
print(
"Test 0 - simple single write to multiple files, files_per_directory=0, no compress, no checksum - channel 0"
)
os.system("rm -rf /tmp/benchmark/junk0 ; mkdir /tmp/benchmark/junk0")
print("Start writing")
channelObj = digital_rf_hdf5.write_hdf5_channel('/tmp/benchmark/junk0', 'i2',
FILE_SAMPLES, FILES_PER_DIR,
start_global_index,
SAMPLE_RATE, 'Fake_uuid', 0,
False)
t = time.time()
for i in range(N_WRITES):
channelObj.rf_write(data_int16)
channelObj.close()
seconds = time.time() - t
speedMB = (N_WRITES * 4 * WRITE_BLOCK_SIZE) / (1.0E6 * seconds)
print('Total time %i seconds, speed %1.2f MB/s' % (int(seconds), speedMB))
files_per_directory = 10
# start 2014-03-09 12:30:30 plus one sample
start_global_index = (1394368230 * sample_rate) + 1
# test get unix time
dt, picoseconds = digital_rf_hdf5.get_unix_time(long(start_global_index), sample_rate)
print('For start_global_index=%i and sample_rate=%f, dt is %s and picoseconds is %i' % (start_global_index, sample_rate,
dt, picoseconds))
# set up top level directory
os.system("rm -rf /tmp/hdf5 ; mkdir /tmp/hdf5");
os.system("rm -rf /tmp/hdf52 ; mkdir /tmp/hdf52");
print("Test 0 - simple single write to multiple files, no compress, no checksum - channel 0");
os.system("rm -rf /tmp/hdf5/junk0 ; mkdir /tmp/hdf5/junk0");
data_object = digital_rf_hdf5.write_hdf5_channel("/tmp/hdf5/junk0", 'i4', 40, files_per_directory, start_global_index,
sample_rate, "FAKE_UUID_0", 0, False, True);
data = numpy.array(base_data, numpy.int32)
result = data_object.rf_write(data);
data_object.close();
print("done test 0");
print("Test 1 - use complex 1 byte ints, no compress, no checksum - channel 1");
os.system("rm -rf /tmp/hdf5/junk1 ; mkdir /tmp/hdf5/junk1");
data_object = digital_rf_hdf5.write_hdf5_channel("/tmp/hdf5/junk1", 'i1', 40, files_per_directory, start_global_index,
sample_rate, "FAKE_UUID_1", 0, False, True);
data = numpy.array(base_data, numpy.int8)
data_object.rf_write(data)
data_object.close()
print(result)
print("done test 1")
files_per_directory = 10
# start 2014-03-09 12:30:30 plus one sample
start_global_index = (1394368230 * sample_rate) + 1
# test get unix time
dt, picoseconds = digital_rf_hdf5.get_unix_time(long(start_global_index), sample_rate)
print('For start_global_index=%i and sample_rate=%f, dt is %s and picoseconds is %i' % (start_global_index, sample_rate,
dt, picoseconds))
# set up top level directory
os.system("rm -rf /tmp/hdf5 ; mkdir /tmp/hdf5");
os.system("rm -rf /tmp/hdf52 ; mkdir /tmp/hdf52");
print("Test 0 - simple single write to multiple files, no compress, no checksum - channel 0");
os.system("rm -rf /tmp/hdf5/junk0 ; mkdir /tmp/hdf5/junk0");
data_object = digital_rf_hdf5.write_hdf5_channel("/tmp/hdf5/junk0", 'i4', 40, files_per_directory, start_global_index,
sample_rate, "FAKE_UUID_0", 0, False, True, num_subchannels=num_subchannels);
data = numpy.array(base_data, numpy.int32)
result = data_object.rf_write(data);
data_object.close();
print("done test 0.1");
print("Test 0.1 - simple single write to multiple files using r/i struct layout, no compress, no checksum - channel 0.1");
os.system("rm -rf /tmp/hdf5/junk0.1 ; mkdir /tmp/hdf5/junk0.1");
data_object = digital_rf_hdf5.write_hdf5_channel("/tmp/hdf5/junk0.1", 'i2', 40, files_per_directory, start_global_index,
sample_rate, "FAKE_UUID_0.1", 0, False, True, num_subchannels=num_subchannels);
result = data_object.rf_write(arr_data);
data_object.close();
print("done test 0.1");
print("Test 0.11 - simple single write with one subchannel to multiple files using r/i struct layout, no compress, no checksum - channel 0.1");
os.system("rm -rf /tmp/hdf5/junk0.11 ; mkdir /tmp/hdf5/junk0.11");
else:
base_data.append([4,6]*num_subchannels)
# constants
sample_rate = 1.0E2
files_per_directory = 10
# start 2014-03-09 12:30:30 plus one sample
start_global_index = (1394368230 * sample_rate) + 1
# set up top level directory
os.system("rm -rf /tmp/hdf5 ; mkdir /tmp/hdf5");
print("Test 0 - simple single write to multiple files, no compress, no checksum - channel 0");
os.system("rm -rf /tmp/hdf5/junk0 ; mkdir /tmp/hdf5/junk0");
data_object = digital_rf_hdf5.write_hdf5_channel("/tmp/hdf5/junk0", 'i4', 40, files_per_directory, start_global_index,
sample_rate, "FAKE_UUID_0", 0, False, True, num_subchannels=num_subchannels);
data = numpy.array(base_data, numpy.int32)
for i in range(10):
result = data_object.rf_write(data);
data_object.close();
print("done write");
# set up fake realtime data by copying files
os.system('rm -rf /tmp/hdf52')
os.system('mkdir /tmp/hdf52')
os.system('mkdir /tmp/hdf52/junk1')
os.system('cp -r /tmp/hdf5/junk0/2014-03-09T12-30-30 /tmp/hdf52/junk1/')
os.system('mkdir /tmp/hdf52/junk1/2014-03-09T12-30-34')
files = glob.glob('/tmp/hdf5/junk0/2014-03-09T12-30-34/*')
files.sort()
for thisFile in files[:5]:
# data to write
data_int16 = numpy.zeros((WRITE_BLOCK_SIZE, 2), dtype='i2')
# make random
for i in range(WRITE_BLOCK_SIZE):
j = i*2
k = i*2 + 1
data_int16[i][0]=(j%32768)*(j+8192)*(j%13);
data_int16[i][1]=(k%32768)*(k+8192)*(k%13);
print('creating top level dir /tmp/benchmark')
os.system("rm -rf /tmp/benchmark ; mkdir /tmp/benchmark")
print("Test 0 - simple single write to multiple files, no compress, no checksum - channel 0")
os.system("rm -rf /tmp/benchmark/junk0 ; mkdir /tmp/benchmark/junk0")
print("Start writing")
channelObj = digital_rf_hdf5.write_hdf5_channel('/tmp/benchmark/junk0', 'i2', FILE_SAMPLES, FILES_PER_DIR, start_global_index, SAMPLE_RATE, 'Fake_uuid', 0, False)
t = time.time()
for i in range(N_WRITES):
channelObj.rf_write(data_int16)
channelObj.close()
seconds = time.time() - t
speedMB = (N_WRITES*4*WRITE_BLOCK_SIZE)/(1.0E6*seconds)
print('Total time %i seconds, speed %1.2f MB/s' % (int(seconds), speedMB))
print("Test 1 - simple single write to multiple files, no compress, with checksum - channel 1")
os.system("rm -rf /tmp/benchmark/junk1 ; mkdir /tmp/benchmark/junk1")
print("Start writing")
channelObj = digital_rf_hdf5.write_hdf5_channel('/tmp/benchmark/junk1', 'i2', FILE_SAMPLES, FILES_PER_DIR, start_global_index, SAMPLE_RATE, 'Fake_uuid', 0, True)
t = time.time()
for i in range(N_WRITES):
sample_rate)
print(
'For start_global_index=%i and sample_rate=%f, dt is %s and picoseconds is %i'
% (start_global_index, sample_rate, dt, picoseconds))
# set up top level directory
os.system("rm -rf /tmp/hdf5 ; mkdir /tmp/hdf5")
os.system("rm -rf /tmp/hdf52 ; mkdir /tmp/hdf52")
print(
"Test 0 - simple single write to multiple files, no compress, no checksum - channel 0"
)
os.system("rm -rf /tmp/hdf5/junk0 ; mkdir /tmp/hdf5/junk0")
data_object = digital_rf_hdf5.write_hdf5_channel("/tmp/hdf5/junk0", 'i4', 40,
files_per_directory,
start_global_index,
sample_rate, "FAKE_UUID_0", 0,
False, True)
data = numpy.array(base_data, numpy.int32)
result = data_object.rf_write(data)
data_object.close()
print("done test 0")
print("Test 1 - use complex 1 byte ints, no compress, no checksum - channel 1")
os.system("rm -rf /tmp/hdf5/junk1 ; mkdir /tmp/hdf5/junk1")
data_object = digital_rf_hdf5.write_hdf5_channel("/tmp/hdf5/junk1", 'i1', 40,
files_per_directory,
start_global_index,
sample_rate, "FAKE_UUID_1", 0,
False, True)
data = numpy.array(base_data, numpy.int8)
# start 2014-03-09 12:30:30 plus one sample
start_global_index = (1394368230 * sample_rate) + 1
# set up top level directory
os.system("rm -rf /tmp/hdf5 ; mkdir /tmp/hdf5")
print(
"Test 0 - simple single write to multiple files, no compress, no checksum - channel 0"
)
os.system("rm -rf /tmp/hdf5/junk0 ; mkdir /tmp/hdf5/junk0")
data_object = digital_rf_hdf5.write_hdf5_channel(
"/tmp/hdf5/junk0",
'i4',
40,
files_per_directory,
start_global_index,
sample_rate,
"FAKE_UUID_0",
0,
False,
True,
num_subchannels=num_subchannels)
data = numpy.array(base_data, numpy.int32)
for i in range(10):
result = data_object.rf_write(data)
data_object.close()
print("done write")
# set up fake realtime data by copying files
os.system('rm -rf /tmp/hdf52')
os.system('mkdir /tmp/hdf52')
os.system('mkdir /tmp/hdf52/junk1')