def setUp(self):
"""
This method sets the following instance attributes:
* ``h5fname``: the name of the temporary HDF5 file.
* ``h5file``: the writable, temporary HDF5 file with a ``/test`` node.
* ``fnode``: the readable file node in ``/test``, with text in it.
"""
super(ReadlineTestCase, self).setUp()
linesep = self.line_separator
# Fill the node file with some text.
fnode = filenode.new_node(self.h5file, where='/', name='test')
#fnode.line_separator = linesep
fnode.write(linesep)
data = 'short line%sshort line%s%s' % ((linesep.decode('ascii'),) * 3)
data = data.encode('ascii')
fnode.write(data)
fnode.write(b'long line ' * 20 + linesep)
fnode.write(b'unterminated')
fnode.close()
# Re-open it for reading.
self.fnode = filenode.open_node(self.h5file.get_node('/test'))
def setUp(self):
"""This method sets the following instance attributes:
* ``h5fname``: the name of the temporary HDF5 file.
* ``h5file``: the writable, temporary HDF5 file with a ``/test`` node.
* ``fnode``: the readable file node in ``/test``, with text in it.
"""
super(ReadlineTestCase, self).setUp()
linesep = self.line_separator
# Fill the node file with some text.
fnode = filenode.new_node(self.h5file, where='/', name='test')
#fnode.line_separator = linesep
fnode.write(linesep)
data = 'short line%sshort line%s%s' % ((linesep.decode('ascii'),) * 3)
data = data.encode('ascii')
fnode.write(data)
fnode.write(b'long line ' * 20 + linesep)
fnode.write(b'unterminated')
fnode.close()
# Re-open it for reading.
self.fnode = filenode.open_node(self.h5file.get_node('/test'))
def test_write_json_blob(hdf_file_path, mock_key, json_data):
hdf._write_json_blob(hdf_file_path, mock_key, json_data)
with tables.open_file(str(hdf_file_path)) as file:
node = file.get_node(mock_key.path)
with filenode.open_node(node) as file_node:
data = json.load(file_node)
assert data == json_data
def filenodeTotalRows(leaf):
"""Traverse the whole filenode and count its number of rows."""
# Warning: this is SLOW for large files
counter = 0
with filenode.open_node(leaf, 'r') as f:
for l in f:
counter += 1
return counter
def load(path: Union[str,
Path], entity_key: str, filter_terms: Optional[List[str]],
column_filters: Optional[List[str]]) -> Any:
"""Loads data from an HDF file.
Parameters
----------
path
The path to the HDF file to load the data from.
entity_key
A representation of the internal HDF path where the data is located.
filter_terms
An optional list of terms used to filter the rows in the data.
The terms must be formatted in a way that is suitable for use with
the ``where`` argument of :func:`pd.read_hdf`. Only
filters applying to existing columns in the data are used.
column_filters
An optional list of columns to load from the data.
Raises
------
ValueError
If the path or entity_key are improperly formatted.
Returns
-------
The data stored at the the given key in the HDF file.
"""
path = _get_valid_hdf_path(path)
entity_key = EntityKey(entity_key)
with tables.open_file(str(path)) as file:
node = file.get_node(entity_key.path)
if isinstance(node, tables.earray.EArray):
# This should be a json encoded document rather than a pandas dataframe
with filenode.open_node(node) as file_node:
data = json.load(file_node)
else:
filter_terms = _get_valid_filter_terms(filter_terms,
node.table.colnames)
with pd.HDFStore(str(path), complevel=9, mode='r') as store:
metadata = store.get_storer(
entity_key.path
).attrs.metadata # NOTE: must use attrs. write this up
if metadata.get('is_empty', False):
data = pd.read_hdf(path, entity_key.path, where=filter_terms)
data = data.set_index(list(
data.columns)) # undoing transform performed on write
else:
data = pd.read_hdf(path,
entity_key.path,
where=filter_terms,
columns=column_filters)
return data
def filenodeTotalRows(leaf):
"""Traverse the whole filenode and count its number of rows."""
# Warning: this is SLOW for large files
counter = 0
with filenode.open_node(leaf, 'r') as f:
for l in f:
counter += 1
return counter
def retrieve_internal_file(self, h5name, where='/attached'):
"""
returns the content of a n internal file stored with store_internal_file() or written directly
"""
v = self.hf.get_node(where=where, name=h5name)
F = filenode.open_node(v, 'r')
content = F.read()
F.close()
return content
def retrieve_object(self, h5name, where='/', access='r'):
"""
retrieve a python object stored with store_internal_object()
"""
v = self.hf.get_node(where=where, name=h5name)
F = filenode.open_node(v, 'r')
js = F.read()
F.close()
obj = json.loads(js.decode())
return obj
def _file_node(self, path):
"""Unlocked version of file_node"""
if self.test is False:
self.log.warning('CoreFile.file_node: no test', self.path)
return ''
node = self._get_node(path)
self.log.debug('open file node', path)
node = filenode.open_node(node, 'r')
r = node.read()
# node.close()
return r
def __init__(self, h5_group, auto_flush=True):
assert self.is_dict_node(h5_group)
h5_group = self._get_pyt_group(h5_group)
self._h5_group = h5_group
self.auto_flush = auto_flush
# Load dict data from the file node.
dict_node = getattr(h5_group, self._pyobject_data_node)
with closing(filenode.open_node(dict_node)) as f:
self._pyobject_data = json.load(f, object_hook=self._object_hook)
def __init__(self, h5_group, auto_flush=True):
assert self.is_dict_node(h5_group)
h5_group = self._get_pyt_group(h5_group)
self._h5_group = h5_group
self.auto_flush = auto_flush
# Load dict data from the file node.
dict_node = getattr(h5_group, self._pyobject_data_node)
with closing(filenode.open_node(dict_node)) as f:
self._pyobject_data = json.loads(f.read().decode("ascii"),
object_hook=self._object_hook)
def open_internal_file(self, h5name, access='r', where='/attached'):
"""
opens a node called h5name in the file, which can be accessed as a file.
returns a file stram which can be used as a classical file.
access is either
'r' : for reading an existing node
'w' : create a node for writing into it
'a' : for appending in an existing node
file is stored in a h5 group called h5name
eg.
F = h5.open_internal_file('myfile.txt', 'w', where='/files')
# create a node called '/files/myfile.txt' (node 'myfile.txt' in the group '/files')
F.writelines(text)
F.close()
# and write some text into it
# then, latter on
F = h5.open_internal_file('myfile.txt', 'r', where='/files')
textback = F.read()
F.close()
This is used to add parameter files, audit_trail, etc... to spike/hdf5 files
it is based on the filenode module from pytables
"""
import warnings
if access == 'r':
v = self.hf.get_node(where=where, name=h5name)
F = filenode.open_node(v, 'r')
elif access == 'a':
v = self.hf.get_node(where=where, name=h5name)
F = filenode.open_node(v, 'a+')
elif access == 'w':
with warnings.catch_warnings(
): # remove warnings, as the dot in name activates them
warnings.simplefilter("ignore")
F = filenode.new_node(self.hf, where=where, name=h5name)
return F
def test00_OpenFileRead(self):
"Opening an existing file node for reading."
node = self.h5file.get_node('/test')
fnode = filenode.open_node(node)
self.assertEqual(
fnode.node, node, "filenode.open_node() opened the wrong node.")
self.assertEqual(
fnode.mode, 'r',
"File was opened with an invalid mode %s." % repr(fnode.mode))
self.assertEqual(
fnode.tell(), 0L,
"Pointer is not positioned at the beginning of the file.")
fnode.close()
def test00_OpenFileRead(self):
"""Opening an existing file node for reading."""
node = self.h5file.get_node('/test')
fnode = filenode.open_node(node)
self.assertEqual(
fnode.node, node, "filenode.open_node() opened the wrong node.")
self.assertEqual(
fnode.mode, 'r',
"File was opened with an invalid mode %s." % repr(fnode.mode))
self.assertEqual(
fnode.tell(), 0L,
"Pointer is not positioned at the beginning of the file.")
fnode.close()
def test_acquisitionProcess(self):
"""Test if acquisition process works"""
self.obj.start_acquisition()
print('acquisition started')
p = multiprocessing.Process(target=self.parallel_process,
kwargs={'spr':csutil.sharedProcessResources})
p.daemon = self.obj._daemon_acquisition_process
sleep(3)
p.start()
sleep(2)
self.assertFalse(self.obj.root['initTest'])
self.assertTrue(self.obj.root['isRunning'])
p.join()
sleep(3)
self.obj.stop_acquisition(writeLevel=5)
hdf = tables.open_file(self.obj.measure.get('measureFile'), mode='r')
elp = hdf.root.conf.attrs.elapsed
self.assertGreater(elp, 10)
node = filenode.open_node(hdf.root.conf, 'r')
node.seek(0)
tree = node.read()
node.close()
d = loads(tree)
self.assertEqual(
elp, d['instrument']['measure']['self']['elapsed']['current'])
print('ELAPSED', elp)
T = self.T
pos = self.pos
sT = hdf.root.dsrv.dev1.val.cols.v
spos = hdf.root.instrument.sample0.anerr.cols.v
# Few more points at the beginning!
self.assertEqual(len(sT), self.N)
self.assertEqual(len(spos), self.N)
eT = sT[:] - np.array(T)
eT = eT.sum()
epos = spos[:] - np.array(pos)
epos = epos.sum()
msg = 'Saved data do not correspond to predetermined values. Total err=%.2f'
self.assertEqual(eT, 0, msg=msg % eT)
self.assertEqual(epos, 0, msg=msg % epos)
hdf.close()
def setUp(self):
"""
This method sets the following instance attributes:
* ``h5fname``: the name of the temporary HDF5 file.
* ``h5file``: the writable, temporary HDF5 file with a ``/test`` node.
* ``fnode``: the readable file node in ``/test``.
"""
self.h5fname = tempfile.mktemp(suffix='.h5')
self.oldh5fname = self._testFilename(self.oldh5fname)
oldh5f = tables.open_file(self.oldh5fname)
oldh5f.copy_file(self.h5fname)
oldh5f.close()
self.h5file = tables.open_file(
self.h5fname, 'r+',
title="Test for file node old version compatibility")
self.fnode = filenode.open_node(self.h5file.root.test, 'a+')
def setUp(self):
"""
This method sets the following instance attributes:
* ``h5fname``: the name of the temporary HDF5 file.
* ``h5file``: the writable, temporary HDF5 file with a ``/test`` node.
* ``fnode``: the readable file node in ``/test``.
"""
self.h5fname = tempfile.mktemp(suffix='.h5')
self.oldh5fname = self._testFilename(self.oldh5fname)
oldh5f = tables.open_file(self.oldh5fname)
oldh5f.copy_file(self.h5fname)
oldh5f.close()
self.h5file = tables.open_file(
self.h5fname, 'r+',
title="Test for file node old version compatibility")
self.fnode = filenode.open_node(self.h5file.root.test, 'a+')
def setUp(self):
"""setUp() -> None
This method sets the following instance attributes:
* 'datafile', the opened data file
* 'h5fname', the name of the temporary HDF5 file
* 'h5file', the writable, temporary HDF5 file with a '/test' node
* 'fnode', the readable file node in '/test', with data in it
"""
self.datafname = self._testFilename(self.datafname)
self.datafile = open(self.datafname, 'rb')
super(ReadFileTestCase, self).setUp()
fnode = filenode.new_node(self.h5file, where='/', name='test')
copyFileToFile(self.datafile, fnode)
fnode.close()
self.datafile.seek(0)
self.fnode = filenode.open_node(self.h5file.get_node('/test'))
def setUp(self):
"""setUp() -> None
This method sets the following instance attributes:
* 'datafile', the opened data file
* 'h5fname', the name of the temporary HDF5 file
* 'h5file', the writable, temporary HDF5 file with a '/test' node
* 'fnode', the readable file node in '/test', with data in it
"""
self.datafname = self._testFilename(self.datafname)
self.datafile = open(self.datafname, 'rb')
super(ReadFileTestCase, self).setUp()
fnode = filenode.new_node(self.h5file, where='/', name='test')
copyFileToFile(self.datafile, fnode)
fnode.close()
self.datafile.seek(0)
self.fnode = filenode.open_node(self.h5file.get_node('/test'))
def test_0_data(self):
"""Conversion of data and configuration"""
converter = convert.Converter(iut.db3_path)
converter.get_outpath('00001S', force=True, keep_img=True)
op = converter.convert(max_num_images = 10)
self.assertTrue(op, 'Conversion Failed')
t = tables.open_file(op, mode='r')
n = filenode.open_node(t.root.conf)
tree = pickle.loads(n.read())
measure = tree['hsm']['measure']['self']
self.assertEqual(measure['nSamples']['current'], 1)
sumT = t.root.kiln.T
nrows = len(sumT)
inidim = getattr(t.root.hsm.sample0.h.attrs, 'initialDimension', None)
t0 = sumT[0][0]
T0 = sumT[0][1]
h0 = t.root.hsm.sample0.h[0][1]
log = t.root.log[:]
t.close()
self.check_standard(op)
self.check_logging(op)
self.check_import(op, hsm_names)
self.check_images(op,'Image')
self.check_curve(op)
def get_test_data(self, table, file_path, add_uid_to_incremental_ids_table):
conf = getattr(table.root, 'conf', False)
if '/userdata' in table:
active_version = str(
table.get_node_attr('/userdata', 'active_version')).strip()
if active_version:
version_node = getattr(table.root, active_version, False)
if version_node is not False:
versioned_conf = getattr(version_node, 'conf', False)
if versioned_conf is not False:
conf = version_node.conf
# Load configuration
node = filenode.open_node(conf, 'r')
node.seek(0)
tree = node.read()
node.close()
tree = pickle.loads(tree)
# ##
# Test row
# ##
test = {}
dbdir = os.path.dirname(self.dbPath)
relative_path = "." + \
file_path[len(dbdir):] if file_path.startswith(
dbdir) else file_path
relative_path = '/'.join(relative_path.split(os.sep))
test['file'] = relative_path
instrument = conf.attrs.instrument
test['instrument'] = instrument
if not tree.has_key(instrument):
print tree.keys()
self.log.debug('Instrument tree missing')
return False
for p in 'name,comment,nSamples,zerotime,elapsed,id'.split(','):
test[p] = tree[instrument]['measure']['self'][p]['current']
zerotime = test['zerotime']
test['serial'] = conf.attrs.serial
uid = getattr(conf.attrs, 'uid', False)
if not uid or len(uid) < 2:
self.log.debug('UID attribute not found')
sname = tree[instrument]['measure']['id']
test['uid'] = hashlib.md5(
'%s_%s_%i' % (test['serial'], test['zerotime'], sname)).hexdigest()
else:
test['uid'] = uid
test['flavour'] = 'Standard'
v = []
for k in 'file,serial,uid,id,zerotime,instrument,flavour,name,elapsed,nSamples,comment'.split(','):
v.append(testColConverter[k](test[k]))
# ok=digisign.verify(table)
# Performance problem: should be only verified on request.
ok = False
print 'File verify:', ok
v.append(ok)
return v, tree, instrument, test
def __init__(self, name=None, dir=None, file=None, new=False, biography=None):
"""
Args:
name (str): subject ID
dir (str): path where the .h5 file is located, if `None`, `prefs.DATADIR` is used
file (str): load a subject from a filename. if `None`, ignored.
new (bool): if True, a new file is made (a new file is made if one does not exist anyway)
biography (dict): If making a new subject file, a dictionary with biographical data can be passed
"""
self.STRUCTURE = [
('/data', '/', 'data', 'group'),
('/history', '/', 'history' 'group'),
('/history/hashes', '/history', 'hashes', self.Hash_Table),
('/history/history', '/history', 'history', self.History_Table),
('/history/weights', '/history', 'weights', self.Weight_Table),
('/history/past_protocols', '/history', 'past_protocols', 'group'),
('/info', '/', 'info', 'group')
]
self.lock = threading.Lock()
if not dir:
try:
dir = prefs.DATADIR
except AttributeError:
dir = os.path.split(file)[0]
if not name:
if not file:
Exception('Need to either have a name or a file, how else would we find the .h5 file?')
if not os.path.isfile(file):
Exception('no file was found at passed file: {}'.format(file))
self.file = file
else:
if file:
Warning('file passed, but so was name, defaulting to using name + dir')
self.name = str(name)
self.file = os.path.join(dir, name + '.h5')
if new or not os.path.isfile(self.file):
self.new_subject_file(biography)
# before we open, make sure we have the stuff we need
self.ensure_structure()
h5f = self.open_hdf()
if not name:
try:
self.name = h5f.root.info._v_attrs['name']
except KeyError:
Warning('No Name attribute saved, trying to recover from filename')
self.name = os.path.splitext(os.path.split(file)[-1])[0]
# If subject has a protocol, load it to a dict
self.current = None
self.step = None
self.protocol_name = None
if "/current" in h5f:
# We load the info from 'current' but don't keep the node open
# Stash it as a dict so better access from Python
current_node = filenode.open_node(h5f.root.current)
protocol_string = current_node.readall()
self.current = json.loads(protocol_string)
self.step = int(current_node.attrs['step'])
self.protocol_name = current_node.attrs['protocol_name']
# We will get handles to trial and continuous data when we start running
self.current_trial = None
# Is the subject currently running (ie. we expect data to be incoming)
# Used to keep the subject object alive, otherwise we close the file whenever we don't need it
self.running = False
# We use a threading queue to dump data into a kept-alive h5f file
self.data_queue = None
self.thread = None
self.did_graduate = threading.Event()
# Every time we are initialized we stash the git hash
history_row = h5f.root.history.hashes.row
history_row['time'] = self.get_timestamp()
try:
history_row['hash'] = prefs.HASH
except AttributeError:
history_row['hash'] = ''
history_row.append()
# we have to always open and close the h5f
_ = self.close_hdf(h5f)
# This script is based on a script by Ivan Vilata.
"""How to use the filenode module."""
from tables.nodes import filenode
import tables
h5file = tables.open_file('fnode.h5', 'w')
fnode = filenode.new_node(h5file, where='/', name='fnode_test')
print >> fnode, "This is a test text line."
print >> fnode, "And this is another one."
print >> fnode
fnode.write("Of course, file methods can also be used.")
fnode.close()
node = h5file.root.fnode_test
fnode = filenode.open_node(node, 'a+')
print >> fnode, "This is a new line."
fnode.attrs.content_type = 'text/plain; charset=us-ascii'
fnode.attrs.author = "Ivan Vilata i Balaguer"
fnode.attrs.creation_date = '2004-10-20T13:25:25+0200'
fnode.attrs.keywords_en = ["FileNode", "test", "metadata"]
fnode.attrs.keywords_ca = ["FileNode", "prova", "metadades"]
fnode.attrs.owner = 'ivan'
fnode.attrs.acl = {'ivan': 'rw', '@users': 'r'}
fnode.close()
h5file.close()
"""How to use the filenode module."""
from tables.nodes import filenode
import tables
h5file = tables.open_file('fnode.h5', 'w')
fnode = filenode.new_node(h5file, where='/', name='fnode_test')
print >> fnode, "This is a test text line."
print >> fnode, "And this is another one."
print >> fnode
fnode.write("Of course, file methods can also be used.")
fnode.close()
node = h5file.root.fnode_test
fnode = filenode.open_node(node, 'a+')
print >> fnode, "This is a new line."
fnode.attrs.content_type = 'text/plain; charset=us-ascii'
fnode.attrs.author = "Ivan Vilata i Balaguer"
fnode.attrs.creation_date = '2004-10-20T13:25:25+0200'
fnode.attrs.keywords_en = ["FileNode", "test", "metadata"]
fnode.attrs.keywords_ca = ["FileNode", "prova", "metadades"]
fnode.attrs.owner = 'ivan'
fnode.attrs.acl = {'ivan': 'rw', '@users': 'r'}
fnode.close()
h5file.close()
def _get_model(self):
fn = filenode.open_node(self.root.model, 'r')
extractor = pickle.load(fn)
fn.close()
return extractor