# create short data in r/i to test using that to write arr_data = numpy.ones((vector_length,num_subchannels), dtype=[('r', numpy.int16), ('i', numpy.int16)]) for i in range(len(arr_data)): arr_data[i]['r'] = 2*i arr_data[i]['i'] = 3*i # start 2014-03-09 12:30:30 plus one sample start_global_index = (1394368230 * sample_rate) + 1 # set up top level directory os.system("rm -rf /tmp/hdf5 ; mkdir /tmp/hdf5"); print("Writing complex short to multiple files and subdirectores in /tmp/hdf5 channel junk0"); os.system("rm -rf /tmp/hdf5/junk0 ; mkdir /tmp/hdf5/junk0"); # init data_object = digital_rf_hdf5.write_hdf5_channel("/tmp/hdf5/junk0", dtype_str, samples_per_file, files_per_directory, start_global_index, sample_rate, uuid, compression_level, checksum, is_complex, num_subchannels, marching_periods); # write for i in range(7): # will write 700 samples - so creates two subdirectories result = data_object.rf_write(arr_data); # close data_object.close(); print("done test");
files_per_directory = 10 # start 2014-03-09 12:30:30 plus one sample start_global_index = (1394368230 * sample_rate) + 1 # test get unix time dt, picoseconds = digital_rf_hdf5.get_unix_time(long(start_global_index), sample_rate) print('For start_global_index=%i and sample_rate=%f, dt is %s and picoseconds is %i' % (start_global_index, sample_rate, dt, picoseconds)) # set up top level directory os.system("rm -rf /tmp/hdf5 ; mkdir /tmp/hdf5"); os.system("rm -rf /tmp/hdf52 ; mkdir /tmp/hdf52"); print("Test 0 - simple single write to multiple files, no compress, no checksum - channel 0"); os.system("rm -rf /tmp/hdf5/junk0 ; mkdir /tmp/hdf5/junk0"); data_object = digital_rf_hdf5.write_hdf5_channel("/tmp/hdf5/junk0", 'i4', 40, files_per_directory, start_global_index, sample_rate, "FAKE_UUID_0", 0, False, True, num_subchannels=num_subchannels); data = numpy.array(base_data, numpy.int32) result = data_object.rf_write(data); data_object.close(); print("done test 0.1"); print("Test 0.1 - simple single write to multiple files using r/i struct layout, no compress, no checksum - channel 0.1"); os.system("rm -rf /tmp/hdf5/junk0.1 ; mkdir /tmp/hdf5/junk0.1"); data_object = digital_rf_hdf5.write_hdf5_channel("/tmp/hdf5/junk0.1", 'i2', 40, files_per_directory, start_global_index, sample_rate, "FAKE_UUID_0.1", 0, False, True, num_subchannels=num_subchannels); result = data_object.rf_write(arr_data); data_object.close(); print("done test 0.1"); print("Test 0.11 - simple single write with one subchannel to multiple files using r/i struct layout, no compress, no checksum - channel 0.1"); os.system("rm -rf /tmp/hdf5/junk0.11 ; mkdir /tmp/hdf5/junk0.11");
# data to write data_int16 = numpy.zeros((WRITE_BLOCK_SIZE, 2), dtype='i2') # make random for i in range(WRITE_BLOCK_SIZE): j = i * 2 k = i * 2 + 1 data_int16[i][0] = (j % 32768) * (j + 8192) * (j % 13) data_int16[i][1] = (k % 32768) * (k + 8192) * (k % 13) print('creating top level dir /tmp/benchmark') os.system("rm -rf /tmp/benchmark ; mkdir /tmp/benchmark") print( "Test 0 - simple single write to multiple files, files_per_directory=0, no compress, no checksum - channel 0" ) os.system("rm -rf /tmp/benchmark/junk0 ; mkdir /tmp/benchmark/junk0") print("Start writing") channelObj = digital_rf_hdf5.write_hdf5_channel('/tmp/benchmark/junk0', 'i2', FILE_SAMPLES, FILES_PER_DIR, start_global_index, SAMPLE_RATE, 'Fake_uuid', 0, False) t = time.time() for i in range(N_WRITES): channelObj.rf_write(data_int16) channelObj.close() seconds = time.time() - t speedMB = (N_WRITES * 4 * WRITE_BLOCK_SIZE) / (1.0E6 * seconds) print('Total time %i seconds, speed %1.2f MB/s' % (int(seconds), speedMB))
files_per_directory = 10 # start 2014-03-09 12:30:30 plus one sample start_global_index = (1394368230 * sample_rate) + 1 # test get unix time dt, picoseconds = digital_rf_hdf5.get_unix_time(long(start_global_index), sample_rate) print('For start_global_index=%i and sample_rate=%f, dt is %s and picoseconds is %i' % (start_global_index, sample_rate, dt, picoseconds)) # set up top level directory os.system("rm -rf /tmp/hdf5 ; mkdir /tmp/hdf5"); os.system("rm -rf /tmp/hdf52 ; mkdir /tmp/hdf52"); print("Test 0 - simple single write to multiple files, no compress, no checksum - channel 0"); os.system("rm -rf /tmp/hdf5/junk0 ; mkdir /tmp/hdf5/junk0"); data_object = digital_rf_hdf5.write_hdf5_channel("/tmp/hdf5/junk0", 'i4', 40, files_per_directory, start_global_index, sample_rate, "FAKE_UUID_0", 0, False, True); data = numpy.array(base_data, numpy.int32) result = data_object.rf_write(data); data_object.close(); print("done test 0"); print("Test 1 - use complex 1 byte ints, no compress, no checksum - channel 1"); os.system("rm -rf /tmp/hdf5/junk1 ; mkdir /tmp/hdf5/junk1"); data_object = digital_rf_hdf5.write_hdf5_channel("/tmp/hdf5/junk1", 'i1', 40, files_per_directory, start_global_index, sample_rate, "FAKE_UUID_1", 0, False, True); data = numpy.array(base_data, numpy.int8) data_object.rf_write(data) data_object.close() print(result) print("done test 1")
else: base_data.append([4,6]*num_subchannels) # constants sample_rate = 1.0E2 files_per_directory = 10 # start 2014-03-09 12:30:30 plus one sample start_global_index = (1394368230 * sample_rate) + 1 # set up top level directory os.system("rm -rf /tmp/hdf5 ; mkdir /tmp/hdf5"); print("Test 0 - simple single write to multiple files, no compress, no checksum - channel 0"); os.system("rm -rf /tmp/hdf5/junk0 ; mkdir /tmp/hdf5/junk0"); data_object = digital_rf_hdf5.write_hdf5_channel("/tmp/hdf5/junk0", 'i4', 40, files_per_directory, start_global_index, sample_rate, "FAKE_UUID_0", 0, False, True, num_subchannels=num_subchannels); data = numpy.array(base_data, numpy.int32) for i in range(10): result = data_object.rf_write(data); data_object.close(); print("done write"); # set up fake realtime data by copying files os.system('rm -rf /tmp/hdf52') os.system('mkdir /tmp/hdf52') os.system('mkdir /tmp/hdf52/junk1') os.system('cp -r /tmp/hdf5/junk0/2014-03-09T12-30-30 /tmp/hdf52/junk1/') os.system('mkdir /tmp/hdf52/junk1/2014-03-09T12-30-34') files = glob.glob('/tmp/hdf5/junk0/2014-03-09T12-30-34/*') files.sort() for thisFile in files[:5]:
# data to write data_int16 = numpy.zeros((WRITE_BLOCK_SIZE, 2), dtype='i2') # make random for i in range(WRITE_BLOCK_SIZE): j = i*2 k = i*2 + 1 data_int16[i][0]=(j%32768)*(j+8192)*(j%13); data_int16[i][1]=(k%32768)*(k+8192)*(k%13); print('creating top level dir /tmp/benchmark') os.system("rm -rf /tmp/benchmark ; mkdir /tmp/benchmark") print("Test 0 - simple single write to multiple files, no compress, no checksum - channel 0") os.system("rm -rf /tmp/benchmark/junk0 ; mkdir /tmp/benchmark/junk0") print("Start writing") channelObj = digital_rf_hdf5.write_hdf5_channel('/tmp/benchmark/junk0', 'i2', FILE_SAMPLES, FILES_PER_DIR, start_global_index, SAMPLE_RATE, 'Fake_uuid', 0, False) t = time.time() for i in range(N_WRITES): channelObj.rf_write(data_int16) channelObj.close() seconds = time.time() - t speedMB = (N_WRITES*4*WRITE_BLOCK_SIZE)/(1.0E6*seconds) print('Total time %i seconds, speed %1.2f MB/s' % (int(seconds), speedMB)) print("Test 1 - simple single write to multiple files, no compress, with checksum - channel 1") os.system("rm -rf /tmp/benchmark/junk1 ; mkdir /tmp/benchmark/junk1") print("Start writing") channelObj = digital_rf_hdf5.write_hdf5_channel('/tmp/benchmark/junk1', 'i2', FILE_SAMPLES, FILES_PER_DIR, start_global_index, SAMPLE_RATE, 'Fake_uuid', 0, True) t = time.time() for i in range(N_WRITES):
sample_rate) print( 'For start_global_index=%i and sample_rate=%f, dt is %s and picoseconds is %i' % (start_global_index, sample_rate, dt, picoseconds)) # set up top level directory os.system("rm -rf /tmp/hdf5 ; mkdir /tmp/hdf5") os.system("rm -rf /tmp/hdf52 ; mkdir /tmp/hdf52") print( "Test 0 - simple single write to multiple files, no compress, no checksum - channel 0" ) os.system("rm -rf /tmp/hdf5/junk0 ; mkdir /tmp/hdf5/junk0") data_object = digital_rf_hdf5.write_hdf5_channel("/tmp/hdf5/junk0", 'i4', 40, files_per_directory, start_global_index, sample_rate, "FAKE_UUID_0", 0, False, True) data = numpy.array(base_data, numpy.int32) result = data_object.rf_write(data) data_object.close() print("done test 0") print("Test 1 - use complex 1 byte ints, no compress, no checksum - channel 1") os.system("rm -rf /tmp/hdf5/junk1 ; mkdir /tmp/hdf5/junk1") data_object = digital_rf_hdf5.write_hdf5_channel("/tmp/hdf5/junk1", 'i1', 40, files_per_directory, start_global_index, sample_rate, "FAKE_UUID_1", 0, False, True) data = numpy.array(base_data, numpy.int8)
# start 2014-03-09 12:30:30 plus one sample start_global_index = (1394368230 * sample_rate) + 1 # set up top level directory os.system("rm -rf /tmp/hdf5 ; mkdir /tmp/hdf5") print( "Test 0 - simple single write to multiple files, no compress, no checksum - channel 0" ) os.system("rm -rf /tmp/hdf5/junk0 ; mkdir /tmp/hdf5/junk0") data_object = digital_rf_hdf5.write_hdf5_channel( "/tmp/hdf5/junk0", 'i4', 40, files_per_directory, start_global_index, sample_rate, "FAKE_UUID_0", 0, False, True, num_subchannels=num_subchannels) data = numpy.array(base_data, numpy.int32) for i in range(10): result = data_object.rf_write(data) data_object.close() print("done write") # set up fake realtime data by copying files os.system('rm -rf /tmp/hdf52') os.system('mkdir /tmp/hdf52') os.system('mkdir /tmp/hdf52/junk1')