def test01_MultidimLeaves(self):
"""Creating new nodes with multidimensional time elements."""
# Table creation.
class MyTimeRow(tb.IsDescription):
intcol = tb.IntCol(shape=(2, 1))
t32col = tb.Time32Col(shape=(2, 1))
t64col = tb.Time64Col(shape=(2, 1))
self.h5file.create_table('/', 'table', MyTimeRow)
# VLArray creation.
self.h5file.create_vlarray('/', 'vlarray4',
tb.Time32Atom(shape=(2, 1)))
self.h5file.create_vlarray('/', 'vlarray8',
tb.Time64Atom(shape=(2, 1)))
# EArray creation.
self.h5file.create_earray('/',
'earray4',
tb.Time32Atom(),
shape=(0, 2, 1))
self.h5file.create_earray('/',
'earray8',
tb.Time64Atom(),
shape=(0, 2, 1))
def test03b_Compare64EArray(self):
"Comparing several written and read 64-bit time values in an EArray."
# Create test EArray with data.
h5file = tables.openFile(self.h5fname,
'w',
title="Test for comparing Time64 E arrays")
ea = h5file.createEArray('/',
'test',
tables.Time64Atom(),
shape=(0, 2))
# Size of the test.
nrows = ea.nrowsinbuf + 34 # Add some more rows than buffer.
# Only for home checks; the value above should check better
# the I/O with multiple buffers.
##nrows = 10
for i in xrange(nrows):
j = i * 2
ea.append(((j + 0.012, j + 1 + 0.012), ))
h5file.close()
# Check the written data.
h5file = tables.openFile(self.h5fname)
arr = h5file.root.test.read()
h5file.close()
orig_val = numpy.arange(0, nrows * 2, dtype=numpy.int32) + 0.012
orig_val.shape = (nrows, 2)
if common.verbose:
print "Original values:", orig_val
print "Retrieved values:", arr
self.assertTrue(allequal(arr, orig_val),
"Stored and retrieved values do not match.")
def test03b_Compare64EArray(self):
"""Comparing several written and read 64-bit time values in an
EArray."""
# Create test EArray with data.
ea = self.h5file.create_earray('/',
'test',
tb.Time64Atom(),
shape=(0, 2))
# Size of the test.
nrows = ea.nrowsinbuf + 34 # Add some more rows than buffer.
# Only for home checks; the value above should check better
# the I/O with multiple buffers.
# nrows = 10
for i in range(nrows):
j = i * 2
ea.append(((j + 0.012, j + 1 + 0.012), ))
self._reopen()
# Check the written data.
arr = self.h5file.root.test.read()
self.h5file.close()
orig_val = np.arange(0, nrows * 2, dtype=np.int32) + 0.012
orig_val.shape = (nrows, 2)
if common.verbose:
print("Original values:", orig_val)
print("Retrieved values:", arr)
self.assertTrue(common.allequal(arr, orig_val),
"Stored and retrieved values do not match.")
def test00_UnidimLeaves(self):
"Creating new nodes with unidimensional time elements."
# Table creation.
class MyTimeRow(tables.IsDescription):
intcol = tables.IntCol()
t32col = tables.Time32Col()
t64col = tables.Time64Col()
self.h5file.create_table('/', 'table', MyTimeRow)
# VLArray creation.
self.h5file.create_vlarray('/', 'vlarray4', tables.Time32Atom())
self.h5file.create_vlarray('/', 'vlarray8', tables.Time64Atom())
# EArray creation.
self.h5file.create_earray('/', 'earray4',
tables.Time32Atom(), shape=(0,))
self.h5file.create_earray('/', 'earray8',
tables.Time64Atom(), shape=(0,))
def test03_Compare64EArray(self):
"""Comparing written 64-bit time data with read data in an EArray."""
wtime = 1234567890.123456
# Create test EArray with data.
ea = self.h5file.create_earray(
'/', 'test', tables.Time64Atom(), shape=(0,))
ea.append((wtime,))
self._reopen()
# Check the written data.
rtime = self.h5file.root.test[0]
self.h5file.close()
self.assertTrue(allequal(rtime, wtime),
"Stored and retrieved values do not match.")
def test03_Compare64EArray(self):
"Comparing written 64-bit time data with read data in an EArray."
wtime = 1234567890.123456
# Create test EArray with data.
h5file = tables.openFile(self.h5fname,
'w',
title="Test for comparing Time64 EArrays")
ea = h5file.createEArray('/', 'test', tables.Time64Atom(), shape=(0, ))
ea.append((wtime, ))
h5file.close()
# Check the written data.
h5file = tables.openFile(self.h5fname)
rtime = h5file.root.test[0]
h5file.close()
self.assertTrue(allequal(rtime, wtime),
"Stored and retrieved values do not match.")
class OpenTestCase(common.TempFileMixin, common.PyTablesTestCase):
"""Tests opening a file with Time nodes."""
# The description used in the test Table.
class MyTimeRow(tb.IsDescription):
t32col = tb.Time32Col(shape=(2, 1))
t64col = tb.Time64Col(shape=(2, 1))
# The atoms used in the test VLArrays.
myTime32Atom = tb.Time32Atom(shape=(2, 1))
myTime64Atom = tb.Time64Atom(shape=(2, 1))
def setUp(self):
super().setUp()
# Create test Table.
self.h5file.create_table('/', 'table', self.MyTimeRow)
# Create test VLArrays.
self.h5file.create_vlarray('/', 'vlarray4', self.myTime32Atom)
self.h5file.create_vlarray('/', 'vlarray8', self.myTime64Atom)
self._reopen()
def test00_OpenFile(self):
"""Opening a file with Time nodes."""
# Test the Table node.
tbl = self.h5file.root.table
self.assertEqual(tbl.coldtypes['t32col'],
self.MyTimeRow.columns['t32col'].dtype,
"Column dtypes do not match.")
self.assertEqual(tbl.coldtypes['t64col'],
self.MyTimeRow.columns['t64col'].dtype,
"Column dtypes do not match.")
# Test the VLArray nodes.
vla4 = self.h5file.root.vlarray4
self.assertEqual(vla4.atom.dtype, self.myTime32Atom.dtype,
"Atom types do not match.")
self.assertEqual(vla4.atom.shape, self.myTime32Atom.shape,
"Atom shapes do not match.")
vla8 = self.h5file.root.vlarray8
self.assertEqual(vla8.atom.dtype, self.myTime64Atom.dtype,
"Atom types do not match.")
self.assertEqual(vla8.atom.shape, self.myTime64Atom.shape,
"Atom shapes do not match.")
def test01_OpenFileStype(self):
"""Opening a file with Time nodes, comparing Atom.stype."""
# Test the Table node.
tbl = self.h5file.root.table
self.assertEqual(tbl.coltypes['t32col'],
self.MyTimeRow.columns['t32col'].type,
"Column types do not match.")
self.assertEqual(tbl.coltypes['t64col'],
self.MyTimeRow.columns['t64col'].type,
"Column types do not match.")
# Test the VLArray nodes.
vla4 = self.h5file.root.vlarray4
self.assertEqual(vla4.atom.type, self.myTime32Atom.type,
"Atom types do not match.")
vla8 = self.h5file.root.vlarray8
self.assertEqual(vla8.atom.type, self.myTime64Atom.type,
"Atom types do not match.")
class CompareTestCase(common.TempFileMixin, common.PyTablesTestCase):
"""Tests whether stored and retrieved time data is kept the same."""
# The description used in the test Table.
class MyTimeRow(tb.IsDescription):
t32col = tb.Time32Col(pos=0)
t64col = tb.Time64Col(shape=(2, ), pos=1)
# The atoms used in the test VLArrays.
myTime32Atom = tb.Time32Atom(shape=(2, ))
myTime64Atom = tb.Time64Atom(shape=(2, ))
def test00_Compare32VLArray(self):
"""Comparing written 32-bit time data with read data in a VLArray."""
wtime = np.array((1_234_567_890, ) * 2, np.int32)
# Create test VLArray with data.
vla = self.h5file.create_vlarray('/', 'test', self.myTime32Atom)
vla.append(wtime)
self._reopen()
# Check the written data.
rtime = self.h5file.root.test.read()[0][0]
self.h5file.close()
self.assertTrue(common.allequal(rtime, wtime),
"Stored and retrieved values do not match.")
def test01_Compare64VLArray(self):
"""Comparing written 64-bit time data with read data in a VLArray."""
wtime = np.array((1_234_567_890.123456, ) * 2, np.float64)
# Create test VLArray with data.
vla = self.h5file.create_vlarray('/', 'test', self.myTime64Atom)
vla.append(wtime)
self._reopen()
# Check the written data.
rtime = self.h5file.root.test.read()[0][0]
self.h5file.close()
self.assertTrue(common.allequal(rtime, wtime),
"Stored and retrieved values do not match.")
def test01b_Compare64VLArray(self):
"""Comparing several written and read 64-bit time values in a
VLArray."""
# Create test VLArray with data.
vla = self.h5file.create_vlarray('/', 'test', self.myTime64Atom)
# Size of the test.
nrows = vla.nrowsinbuf + 34 # Add some more rows than buffer.
# Only for home checks; the value above should check better
# the I/O with multiple buffers.
# nrows = 10
for i in range(nrows):
j = i * 2
vla.append((j + 0.012, j + 1 + 0.012))
self._reopen()
# Check the written data.
arr = self.h5file.root.test.read()
self.h5file.close()
arr = np.array(arr)
orig_val = np.arange(0, nrows * 2, dtype=np.int32) + 0.012
orig_val.shape = (nrows, 1, 2)
if common.verbose:
print("Original values:", orig_val)
print("Retrieved values:", arr)
self.assertTrue(common.allequal(arr, orig_val),
"Stored and retrieved values do not match.")
def test02_CompareTable(self):
"""Comparing written time data with read data in a Table."""
wtime = 1_234_567_890.123456
# Create test Table with data.
tbl = self.h5file.create_table('/', 'test', self.MyTimeRow)
row = tbl.row
row['t32col'] = int(wtime)
row['t64col'] = (wtime, wtime)
row.append()
self._reopen()
# Check the written data.
recarr = self.h5file.root.test.read(0)
self.h5file.close()
self.assertEqual(recarr['t32col'][0], int(wtime),
"Stored and retrieved values do not match.")
comp = (recarr['t64col'][0] == np.array((wtime, wtime)))
self.assertTrue(np.alltrue(comp),
"Stored and retrieved values do not match.")
def test02b_CompareTable(self):
"""Comparing several written and read time values in a Table."""
# Create test Table with data.
tbl = self.h5file.create_table('/', 'test', self.MyTimeRow)
# Size of the test.
nrows = tbl.nrowsinbuf + 34 # Add some more rows than buffer.
# Only for home checks; the value above should check better
# the I/O with multiple buffers.
# nrows = 10
row = tbl.row
for i in range(nrows):
row['t32col'] = i
j = i * 2
row['t64col'] = (j + 0.012, j + 1 + 0.012)
row.append()
self._reopen()
# Check the written data.
recarr = self.h5file.root.test.read()
self.h5file.close()
# Time32 column.
orig_val = np.arange(nrows, dtype=np.int32)
if common.verbose:
print("Original values:", orig_val)
print("Retrieved values:", recarr['t32col'][:])
self.assertTrue(np.alltrue(recarr['t32col'][:] == orig_val),
"Stored and retrieved values do not match.")
# Time64 column.
orig_val = np.arange(0, nrows * 2, dtype=np.int32) + 0.012
orig_val.shape = (nrows, 2)
if common.verbose:
print("Original values:", orig_val)
print("Retrieved values:", recarr['t64col'][:])
self.assertTrue(
common.allequal(recarr['t64col'][:], orig_val, np.float64),
"Stored and retrieved values do not match.")
def test03_Compare64EArray(self):
"""Comparing written 64-bit time data with read data in an EArray."""
wtime = 1_234_567_890.123456
# Create test EArray with data.
ea = self.h5file.create_earray('/',
'test',
tb.Time64Atom(),
shape=(0, ))
ea.append((wtime, ))
self._reopen()
# Check the written data.
rtime = self.h5file.root.test[0]
self.h5file.close()
self.assertTrue(common.allequal(rtime, wtime),
"Stored and retrieved values do not match.")
def test03b_Compare64EArray(self):
"""Comparing several written and read 64-bit time values in an
EArray."""
# Create test EArray with data.
ea = self.h5file.create_earray('/',
'test',
tb.Time64Atom(),
shape=(0, 2))
# Size of the test.
nrows = ea.nrowsinbuf + 34 # Add some more rows than buffer.
# Only for home checks; the value above should check better
# the I/O with multiple buffers.
# nrows = 10
for i in range(nrows):
j = i * 2
ea.append(((j + 0.012, j + 1 + 0.012), ))
self._reopen()
# Check the written data.
arr = self.h5file.root.test.read()
self.h5file.close()
orig_val = np.arange(0, nrows * 2, dtype=np.int32) + 0.012
orig_val.shape = (nrows, 2)
if common.verbose:
print("Original values:", orig_val)
print("Retrieved values:", arr)
self.assertTrue(common.allequal(arr, orig_val),
"Stored and retrieved values do not match.")
class CompareTestCase(common.PyTablesTestCase):
"Tests whether stored and retrieved time data is kept the same."
# The description used in the test Table.
class MyTimeRow(tables.IsDescription):
t32col = tables.Time32Col(pos=0)
t64col = tables.Time64Col(shape=(2, ), pos=1)
# The atoms used in the test VLArrays.
myTime32Atom = tables.Time32Atom(shape=(2, ))
myTime64Atom = tables.Time64Atom(shape=(2, ))
def setUp(self):
"""setUp() -> None
This method sets the following instance attributes:
* 'h5fname', the name of the temporary HDF5 file
"""
self.h5fname = tempfile.mktemp(suffix='.h5')
def tearDown(self):
"""tearDown() -> None
Removes 'h5fname'.
"""
os.remove(self.h5fname)
def test00_Compare32VLArray(self):
"Comparing written 32-bit time data with read data in a VLArray."
wtime = numpy.array((1234567890, ) * 2, numpy.int32)
# Create test VLArray with data.
h5file = tables.openFile(self.h5fname,
'w',
title="Test for comparing Time32 VL arrays")
vla = h5file.createVLArray('/', 'test', self.myTime32Atom)
vla.append(wtime)
h5file.close()
# Check the written data.
h5file = tables.openFile(self.h5fname)
rtime = h5file.root.test.read()[0][0]
h5file.close()
self.assertTrue(allequal(rtime, wtime),
"Stored and retrieved values do not match.")
def test01_Compare64VLArray(self):
"Comparing written 64-bit time data with read data in a VLArray."
wtime = numpy.array((1234567890.123456, ) * 2, numpy.float64)
# Create test VLArray with data.
h5file = tables.openFile(self.h5fname,
'w',
title="Test for comparing Time64 VL arrays")
vla = h5file.createVLArray('/', 'test', self.myTime64Atom)
vla.append(wtime)
h5file.close()
# Check the written data.
h5file = tables.openFile(self.h5fname)
rtime = h5file.root.test.read()[0][0]
h5file.close()
self.assertTrue(allequal(rtime, wtime),
"Stored and retrieved values do not match.")
def test01b_Compare64VLArray(self):
"Comparing several written and read 64-bit time values in a VLArray."
# Create test VLArray with data.
h5file = tables.openFile(self.h5fname,
'w',
title="Test for comparing Time64 VL arrays")
vla = h5file.createVLArray('/', 'test', self.myTime64Atom)
# Size of the test.
nrows = vla.nrowsinbuf + 34 # Add some more rows than buffer.
# Only for home checks; the value above should check better
# the I/O with multiple buffers.
#nrows = 10
for i in xrange(nrows):
j = i * 2
vla.append((j + 0.012, j + 1 + 0.012))
h5file.close()
# Check the written data.
h5file = tables.openFile(self.h5fname)
arr = h5file.root.test.read()
h5file.close()
arr = numpy.array(arr)
orig_val = numpy.arange(0, nrows * 2, dtype=numpy.int32) + 0.012
orig_val.shape = (nrows, 1, 2)
if common.verbose:
print "Original values:", orig_val
print "Retrieved values:", arr
self.assertTrue(allequal(arr, orig_val),
"Stored and retrieved values do not match.")
def test02_CompareTable(self):
"Comparing written time data with read data in a Table."
wtime = 1234567890.123456
# Create test Table with data.
h5file = tables.openFile(self.h5fname,
'w',
title="Test for comparing Time tables")
tbl = h5file.createTable('/', 'test', self.MyTimeRow)
row = tbl.row
row['t32col'] = int(wtime)
row['t64col'] = (wtime, wtime)
row.append()
h5file.close()
# Check the written data.
h5file = tables.openFile(self.h5fname)
recarr = h5file.root.test.read(0)
h5file.close()
self.assertEqual(recarr['t32col'][0], int(wtime),
"Stored and retrieved values do not match.")
comp = (recarr['t64col'][0] == numpy.array((wtime, wtime)))
self.assertTrue(numpy.alltrue(comp),
"Stored and retrieved values do not match.")
def test02b_CompareTable(self):
"Comparing several written and read time values in a Table."
# Create test Table with data.
h5file = tables.openFile(self.h5fname,
'w',
title="Test for comparing Time tables")
tbl = h5file.createTable('/', 'test', self.MyTimeRow)
# Size of the test.
nrows = tbl.nrowsinbuf + 34 # Add some more rows than buffer.
# Only for home checks; the value above should check better
# the I/O with multiple buffers.
##nrows = 10
row = tbl.row
for i in xrange(nrows):
row['t32col'] = i
j = i * 2
row['t64col'] = (j + 0.012, j + 1 + 0.012)
row.append()
h5file.close()
# Check the written data.
h5file = tables.openFile(self.h5fname)
recarr = h5file.root.test.read()
h5file.close()
# Time32 column.
orig_val = numpy.arange(nrows, dtype=numpy.int32)
if common.verbose:
print "Original values:", orig_val
print "Retrieved values:", recarr['t32col'][:]
self.assertTrue(numpy.alltrue(recarr['t32col'][:] == orig_val),
"Stored and retrieved values do not match.")
# Time64 column.
orig_val = numpy.arange(0, nrows * 2, dtype=numpy.int32) + 0.012
orig_val.shape = (nrows, 2)
if common.verbose:
print "Original values:", orig_val
print "Retrieved values:", recarr['t64col'][:]
self.assertTrue(allequal(recarr['t64col'][:], orig_val, numpy.float64),
"Stored and retrieved values do not match.")
def test03_Compare64EArray(self):
"Comparing written 64-bit time data with read data in an EArray."
wtime = 1234567890.123456
# Create test EArray with data.
h5file = tables.openFile(self.h5fname,
'w',
title="Test for comparing Time64 EArrays")
ea = h5file.createEArray('/', 'test', tables.Time64Atom(), shape=(0, ))
ea.append((wtime, ))
h5file.close()
# Check the written data.
h5file = tables.openFile(self.h5fname)
rtime = h5file.root.test[0]
h5file.close()
self.assertTrue(allequal(rtime, wtime),
"Stored and retrieved values do not match.")
def test03b_Compare64EArray(self):
"Comparing several written and read 64-bit time values in an EArray."
# Create test EArray with data.
h5file = tables.openFile(self.h5fname,
'w',
title="Test for comparing Time64 E arrays")
ea = h5file.createEArray('/',
'test',
tables.Time64Atom(),
shape=(0, 2))
# Size of the test.
nrows = ea.nrowsinbuf + 34 # Add some more rows than buffer.
# Only for home checks; the value above should check better
# the I/O with multiple buffers.
##nrows = 10
for i in xrange(nrows):
j = i * 2
ea.append(((j + 0.012, j + 1 + 0.012), ))
h5file.close()
# Check the written data.
h5file = tables.openFile(self.h5fname)
arr = h5file.root.test.read()
h5file.close()
orig_val = numpy.arange(0, nrows * 2, dtype=numpy.int32) + 0.012
orig_val.shape = (nrows, 2)
if common.verbose:
print "Original values:", orig_val
print "Retrieved values:", arr
self.assertTrue(allequal(arr, orig_val),
"Stored and retrieved values do not match.")
class OpenTestCase(common.PyTablesTestCase):
"Tests opening a file with Time nodes."
# The description used in the test Table.
class MyTimeRow(tables.IsDescription):
t32col = tables.Time32Col(shape=(2, 1))
t64col = tables.Time64Col(shape=(2, 1))
# The atoms used in the test VLArrays.
myTime32Atom = tables.Time32Atom(shape=(2, 1))
myTime64Atom = tables.Time64Atom(shape=(2, 1))
def setUp(self):
"""setUp() -> None
This method sets the following instance attributes:
* 'h5fname', the name of the temporary HDF5 file with '/table',
'/vlarray4' and '/vlarray8' nodes.
"""
self.h5fname = tempfile.mktemp(suffix='.h5')
h5file = tables.openFile(self.h5fname,
'w',
title="Test for creating time leaves")
# Create test Table.
h5file.createTable('/', 'table', self.MyTimeRow)
# Create test VLArrays.
h5file.createVLArray('/', 'vlarray4', self.myTime32Atom)
h5file.createVLArray('/', 'vlarray8', self.myTime64Atom)
h5file.close()
def tearDown(self):
"""tearDown() -> None
Removes 'h5fname'.
"""
os.remove(self.h5fname)
def test00_OpenFile(self):
"Opening a file with Time nodes."
h5file = tables.openFile(self.h5fname)
# Test the Table node.
tbl = h5file.root.table
self.assertEqual(tbl.coldtypes['t32col'],
self.MyTimeRow.columns['t32col'].dtype,
"Column dtypes do not match.")
self.assertEqual(tbl.coldtypes['t64col'],
self.MyTimeRow.columns['t64col'].dtype,
"Column dtypes do not match.")
# Test the VLArray nodes.
vla4 = h5file.root.vlarray4
self.assertEqual(vla4.atom.dtype, self.myTime32Atom.dtype,
"Atom types do not match.")
self.assertEqual(vla4.atom.shape, self.myTime32Atom.shape,
"Atom shapes do not match.")
vla8 = h5file.root.vlarray8
self.assertEqual(vla8.atom.dtype, self.myTime64Atom.dtype,
"Atom types do not match.")
self.assertEqual(vla8.atom.shape, self.myTime64Atom.shape,
"Atom shapes do not match.")
h5file.close()
def test01_OpenFileStype(self):
"Opening a file with Time nodes, comparing Atom.stype."
h5file = tables.openFile(self.h5fname)
# Test the Table node.
tbl = h5file.root.table
self.assertEqual(tbl.coltypes['t32col'],
self.MyTimeRow.columns['t32col'].type,
"Column types do not match.")
self.assertEqual(tbl.coltypes['t64col'],
self.MyTimeRow.columns['t64col'].type,
"Column types do not match.")
# Test the VLArray nodes.
vla4 = h5file.root.vlarray4
self.assertEqual(vla4.atom.type, self.myTime32Atom.type,
"Atom types do not match.")
vla8 = h5file.root.vlarray8
self.assertEqual(vla8.atom.type, self.myTime64Atom.type,
"Atom types do not match.")
h5file.close()
def tdms_to_hdf5(tdms_file,
h5_file,
load_data=True,
chan_map='',
memmap=True,
compression_level=0):
"""
Converts TDMS data output from the LabView DAQ software used in the Viventi Lab to
record from multiplexing neural implants. This method will most likely not interpret
other TDMS files: see npTDMS for general file handling.
Parameters
----------
tdms_file : path (string)
h5_file : path (string)
chan_map : path (string)
Optional table specifying a channel permutation. The first p rows
of the outgoing H5 file will be the contents of these channels in
sequence. The next (N-p) rows will be any channels not specified,
in the order they are found.
memmap : bool
compression_level : int
Optionally compress the outgoing H5 rows with zlib compression.
This can reduce the time cost caused by disk access.
"""
map_dir = tempfile.gettempdir() if memmap else None
with tables.open_file(h5_file, mode='w') as h5_file:
tdms_file = nptdms.TdmsFile(tdms_file, memmap_dir=map_dir)
# assume for now there is only a single group -- see more files later
# Catch an API change (older version first)
try:
t_group = tdms_file.groups()[0]
group = tdms_file.object(t_group)
chans = tdms_file.group_channels(t_group)
except AttributeError:
group = tdms_file.groups()[0]
# Headstage channels and BNC channels are presently lumped into the "data" HDF5 array.. it might make sense
# to separate them
chans = group.channels()
n_col = len(chans)
n_row = len(chans[0])
# The H5 file will be constructed as follows:
# * create a Group for the info section
# * create a CArray with zlib(3) compression for the data channels
# * create separate Arrays for special values
# (SampRate[SamplingRate], numRow[nrRows], numCol[nrColumns],
# OSR[OverSampling], numChan[nrColumns+nrBNCs])
special_conversion = dict(SamplingRate='sampRate',
nrRows='numRow',
nrColumns='numCol',
OverSampling='OSR')
h5_info = h5_file.create_group(h5_file.root, 'info')
for (key, val) in group.properties.items():
if isinstance(val, str):
# pytables doesn't support strings as arrays
arr = h5_file.create_vlarray(h5_info,
key,
atom=tables.ObjectAtom())
arr.append(val)
elif isinstance(val, np.datetime64):
h5_file.create_array(h5_info,
key,
obj=val.astype('f8'),
atom=tables.Time64Atom())
else:
h5_file.create_array(h5_info, key, obj=val)
if key in special_conversion:
print('caught', key)
# Put this array at the top level with new name
h5_file.create_array('/', special_conversion[key], obj=val)
# do extra extra conversions
try:
num_chan = group.properties['nrColumns'] + group.properties[
'nrBNCs']
h5_file.create_array(h5_file.root, 'numChan', num_chan)
except KeyError:
pass
try:
mux_ratio = group.properties['OverSampling'] * group.properties[
'nrRows']
Fs = float(group.properties['SamplingRate']) / mux_ratio
h5_file.create_array(h5_file.root, 'Fs', Fs)
except KeyError:
print('Could not determine sampling rate')
h5_file.flush()
if not load_data:
return h5_file
# now get down to the data
atom = tables.Float64Atom()
if compression_level > 0:
filters = tables.Filters(complevel=compression_level,
complib='zlib')
else:
filters = None
d_array = h5_file.create_earray(h5_file.root,
'data',
atom=atom,
shape=(0, n_row),
filters=filters,
expectedrows=n_col)
# create a reverse lookup to index channels by number
col_mapping = dict([(ch.properties['NI_ArrayColumn'], ch)
for ch in chans])
# If a channel permutation is requested, lay down channels
# in that order. Otherwise go in sequential order.
if chan_map:
chan_map = np.loadtxt(chan_map).astype('i')
if chan_map.ndim > 1:
print(chan_map.shape)
# the actual channel permutation is in the 1st column
# the array matrix coordinates are in the next columns
chan_ij = chan_map[:, 1:3]
chan_map = chan_map[:, 0]
else:
chan_ij = None
# do any channels not specified at the end
if len(chan_map) < n_col:
left_out = set(range(n_col)).difference(chan_map.tolist())
left_out = sorted(left_out)
chan_map = np.r_[chan_map, left_out]
else:
chan_map = list(range(n_col))
chan_ij = None
for n in chan_map:
# get TDMS column
ch = col_mapping[n]
# make a temp array here.. if all data in memory, then this is
# slightly wasteful, but if it is mmap'd then this is more flexible
d = ch.data[:]
d_array.append(d[None, :])
print('copied channel', ch.path, d_array.shape)
if chan_ij is not None:
h5_file.create_array(h5_file.root, 'channel_ij', obj=chan_ij)
return h5_file
