def _create_coincidences_tables(file, group, station_groups):
"""Setup coincidence tables
:return: the created coincidences group.
"""
coin_group = group + '/coincidences'
# Create coincidences table
description = storage.Coincidence
s_columns = {
's%d' % station: tables.BoolCol(pos=p)
for p, station in enumerate(station_groups, 12)
}
description.columns.update(s_columns)
coincidences = file.create_table(coin_group,
'coincidences',
description,
createparents=True)
# Create c_index
file.create_vlarray(coin_group, 'c_index', tables.UInt32Col(shape=2))
# Create and fill s_index
s_index = file.create_vlarray(coin_group, 's_index', tables.VLStringAtom())
for station_group in itervalues(station_groups):
s_index.append(station_group['group'].encode('utf-8'))
return coincidences._v_parent
def _prepare_coincidence_tables(self):
"""Create coincidence tables
These are the same as the tables created by
:class:`~sapphire.analysis.coincidences.CoincidencesESD`.
This makes it easy to link events detected by multiple stations.
"""
self.coincidence_group = self.data.create_group(self.output_path,
'coincidences',
createparents=True)
try:
self.coincidence_group._v_attrs.cluster = self.cluster
except tables.HDF5ExtError:
warnings.warn('Unable to store cluster object, to large for HDF.')
description = storage.Coincidence
s_columns = {
's%d' % station.number: tables.BoolCol(pos=p)
for p, station in enumerate(self.cluster.stations, 12)
}
description.columns.update(s_columns)
self.coincidences = self.data.create_table(self.coincidence_group,
'coincidences', description)
self.c_index = self.data.create_vlarray(self.coincidence_group,
'c_index',
tables.UInt32Col(shape=2))
self.s_index = self.data.create_vlarray(self.coincidence_group,
's_index',
tables.VLStringAtom())
def _make_str_vlarray(h5file: tables.File, name: str,
attribute: List[str]) -> None:
vlarray = h5file.create_vlarray(h5file.root,
name=name,
atom=tables.VLStringAtom())
for a in attribute:
vlarray.append(a)
def get_or_create_node(file, cluster, node):
"""Get an existing node or create a new one
:param file: the PyTables data file
:param cluster: the parent (cluster) node
:param node: the node (e.g. events, blobs)
"""
try:
node = file.get_node(cluster, node)
except tables.NoSuchNodeError:
if node == 'events':
node = file.create_table(cluster, 'events', HisparcEvent,
'HiSPARC coincidences table')
elif node == 'errors':
node = file.create_table(cluster, 'errors', HisparcError,
'HiSPARC error messages')
elif node == 'comparator':
node = file.create_table(cluster, 'comparator',
HisparcComparatorData,
'HiSPARC comparator messages')
elif node == 'blobs':
node = file.create_vlarray(cluster, 'blobs', tables.VLStringAtom(),
'HiSPARC binary data')
elif node == 'config':
node = file.create_table(cluster, 'config', HisparcConfiguration,
'HiSPARC configuration messages')
elif node == 'weather':
node = file.create_table(cluster, 'weather', HisparcWeather,
'HiSPARC weather data')
file.flush()
return node
def formatFile(self):
if '/keys' not in self.file:
self.file.createVLArray("/",
"keys",
tables.VLStringAtom(),
filters=self.filter)
if '/vals' not in self.file:
self.file.createVLArray("/",
"vals",
tables.ObjectAtom(),
filters=self.filter)
try:
self.fileKeys = self.file.getNode("/", "keys", classname="VLArray")
self.fileVals = self.file.getNode("/", "vals", classname="VLArray")
except tables.NoSuchNodeError:
raise AttributeError("Incorrect File Structure: %s" % \
self.filename)
if not isinstance(self.fileKeys.atom, tables.VLStringAtom) or \
not isinstance(self.fileVals.atom, tables.ObjectAtom):
raise AttributeError("Incorrect File Structure: %s" % \
self.filename)
self.constraints()
def _removeStaleKeyVal(self, index):
self.repack = True
keys = self.fileKeys.read()
vals = self.fileVals.read()
self.file.removeNode("/keys")
self.file.removeNode("/vals")
self.file.flush()
self.file.createVLArray("/",
"keys",
tables.VLStringAtom(),
filters=self.filter)
self.file.createVLArray("/",
"vals",
tables.ObjectAtom(),
filters=self.filter)
self.fileKeys = self.file.root.keys
self.fileVals = self.file.root.vals
for (curKey, curVal) in zip(keys[0:index], vals[0:index]):
self.fileKeys.append(curKey)
self.fileVals.append(curVal)
for (curKey, curVal) in zip(keys[index + 1:], vals[index + 1:]):
self.fileKeys.append(curKey)
self.fileVals.append(curVal)
def init_log2(config):
"""second attempt at log init function: called from Blockspy3 init, triggered by topblock arg of Blockspy3"""
global h5file, loginit, lognodes
# ts = datetime.datetime.now().strftime("%Y%m%d-%H%M%S-%f")
# experiment_name = config.keys()[0]
experiment = "%s" % (config["id"]) # , ts)
tblfile = "data/%s.h5" % (experiment)
h5file = tb.open_file(tblfile, mode = "w", title = "%s" % (experiment))
root = h5file.root
storage_version = "v2"
# create VLArray for storing the graph configuration
conf_array = h5file.create_vlarray(root, 'conf', tb.VLStringAtom(),
"Variable Length Config String")
conf_array.append(str(config))
# FIXME: log git commit
# create topblock array
# a = tb.Float64Atom()
# lognodes[parent.id] = h5file.create_earray(root, "%s_obuf" % (parent.id), a, (parent.odim, 0))
# # create arrays for each node's data
# for nodek, nodev in nodes.items():
# print("init_log: node", nodek)
# a = tb.Float64Atom()
# # node_fieldkeys =
# # for node_fieldkey in ["%s_%03d" % (nodek, i) for i in range(nodev.odim)]:
# # tdef[node_fieldkey] = tb.Float32Col()
# # tdef[nodek] = tb.Float32Col(shape=(nodev.odim, 1))
# # return tdef
# lognodes[nodev.id] = h5file.create_earray(root, "%s_obuf" % (nodev.id), a, (nodev.odim, 0))
# loginit = True # this doesn't work yet persistently, why?
loginit = True
def setUp(self):
h5file = tables.openFile("test.h5", "w")
h5file.createVLArray("/",
"keys",
tables.VLStringAtom(),
filters=tables.Filters(complevel=1))
keys = h5file.root.keys
h5file.createVLArray("/",
"vals",
tables.ObjectAtom(),
filters=tables.Filters(complevel=1))
vals = h5file.root.vals
keys.append('1')
vals.append(MyObj('1'))
keys.append('2')
vals.append(MyObj('2'))
keys.append('list')
vals.append([1, 2, 3])
h5file.close()
self.shelf = PytableShelf('test.h5')
def _create_vlarray(self, group: tables.Group, name: str,
data: VLArray) -> None:
assert len(data), "VLArray must have at least one element"
was_dict = isinstance(data, dict)
if was_dict:
data = [data[i] for i in sorted(data.keys())]
types = set(type(v) for v in data)
assert len(
types) == 1, f"More than one type found in VLArray {name}: {types}"
if str in types:
atom = tables.VLUnicodeAtom()
elif bytes in types:
atom = tables.VLStringAtom()
else:
data = [
v if isinstance(v, np.ndarray) else np.asarray(v) for v in data
]
dtypes = set(v.dtype for v in data)
assert len(
dtypes
) == 1, f"More than one dtype found in VLArray {name}: {dtypes}"
atom = tables.Atom.from_dtype(dtypes.pop())
_d: tables.VLArray = self.h5f.create_vlarray(group,
name,
atom,
filters=self.filters)
for v in data:
_d.append(v)
_d.set_attr("was_dict", was_dict)
self.h5f.flush()
def _checkCreateDataTable(self, f):
if (not 'ImageData' in dir(self.h5f.root)) and (not 'PZFImageData'
in dir(self.h5f.root)):
if isinstance(f, str):
#is a PZF file
f = PZFFormat.loads(f)[0]
f.reshape((1, ) + f.shape[:2])
framesize = f.shape[1:3]
self.dshape[:2] = framesize
if not self.usePZFFormat:
filt = tables.Filters(self.complevel,
self.complib,
shuffle=True)
self.imageData = self.h5f.create_earray(
self.h5f.root,
'ImageData',
tables.UInt16Atom(), (0, ) + tuple(framesize),
filters=filt,
chunkshape=(1, ) + tuple(framesize))
else:
self.compImageData = self.h5f.create_vlarray(
self.h5f.root, 'PZFImageData', tables.VLStringAtom())
self.compImageData.attrs.framesize = framesize
def store_coincidences(self, station_numbers=None):
"""Store the previously found coincidences.
After having searched for coincidences, you can store the more
user-friendly results in the ``coincidences`` group using this
method. It also created a ``c_index`` and ``s_index`` table to
find the source events.
:param station_numbers: optional list of station_numbers.
If given these will be used to attach correct numbers to the
station column names in the coincidences table. Otherwise
they will simply be numbered by id. This list must be the
same length as the station_groups.
"""
n_coincidences = len(self._src_c_index)
if station_numbers is not None:
if len(station_numbers) != len(self.station_groups):
raise RuntimeError(
"Number of station numbers must equal number of groups.")
self.station_numbers = station_numbers
s_columns = {
's%d' % number: tables.BoolCol(pos=p)
for p, number in enumerate(station_numbers, 12)
}
else:
self.station_numbers = None
s_columns = {
's%d' % n: tables.BoolCol(pos=(n + 12))
for n, _ in enumerate(self.station_groups)
}
description = storage.Coincidence
description.columns.update(s_columns)
self.coincidences = self.data.create_table(self.coincidence_group,
'coincidences',
description,
expectedrows=n_coincidences)
self.c_index = []
for coincidence in pbar(self._src_c_index, show=self.progress):
self._store_coincidence(coincidence)
c_index = self.data.create_vlarray(self.coincidence_group,
'c_index',
tables.UInt32Col(shape=2),
expectedrows=n_coincidences)
for observables_idx in pbar(self.c_index, show=self.progress):
c_index.append(observables_idx)
c_index.flush()
s_index = self.data.create_vlarray(self.coincidence_group,
's_index',
tables.VLStringAtom(),
expectedrows=len(
self.station_groups))
for station_group in self.station_groups:
s_index.append(station_group.encode('utf-8'))
s_index.flush()
def create_file_names_array(h5file):
# vlarray of strings only supports a single column, so the file_key is implicitly the array index
array = h5file.create_vlarray("/", "file_names", tables.VLStringAtom(),
"File names with index corresponding to Files table key")
array.append("*") # index/key 0 is reserved for this
array.flush()
return array
def _create_group_and_tables(self, group):
"""Create group and tables in HDF5 file.
:param group: name of the group in which to create the tables.
"""
self.group = self.data.create_group('/', group)
self.data.create_table(self.group, 'events', HisparcEvent)
self.data.create_vlarray(self.group, 'blobs', tables.VLStringAtom())
def log_pd_store_config_final(conf):
# store the final config for this run via the node attribute hack
# df_conf = pd.DataFrame([[0]])
# log_store['conf_final'] = df_conf
# log_store.get_storer('conf_final').attrs.conf = "" # (conf)
h5file = log_store._handle
conf_array = h5file.create_vlarray(log_store.root, 'conf_final',
tb.VLStringAtom(),
"Variable Length Config String")
conf_array.append(str(conf))
def setup(self, datafile):
logging.debug("Setting up HDF5 file %s" % self._filename)
session = os.path.splitext(os.path.basename(datafile.filename))[0]
logging.debug("Found session for datafile: %s" % session)
self._group = self._file.createGroup("/", session, \
"Data for session %s" % session)
self._events = self._file.createTable(self._group, "events", \
Event, "Events")
self._codec = self._file.createTable(self._group, "codec", \
CodecEntry, "Codec")
self._values = self._file.createVLArray(self._group, "values", \
tables.VLStringAtom(), "Values", expectedsizeinMB=0.0001)
def store_coincidences(self, cluster=None):
"""Store the previously found coincidences.
After having searched for coincidences, you can store the more
user-friendly results in the ``coincidences`` group using this
method. It also created a ``c_index`` and ``s_index`` table to
find the source events.
"""
n_coincidences = len(self._src_c_index)
if cluster:
self.cluster = cluster
self.coincidence_group._v_attrs.cluster = cluster
s_columns = {
's%d' % station.number: tables.BoolCol(pos=p)
for p, station in enumerate(cluster.stations, 12)
}
else:
self.cluster = None
s_columns = {
's%d' % n: tables.BoolCol(pos=(n + 12))
for n, _ in enumerate(self.station_groups)
}
description = storage.Coincidence
description.columns.update(s_columns)
self.coincidences = self.data.create_table(self.coincidence_group,
'coincidences',
description,
expectedrows=n_coincidences)
self.c_index = []
for coincidence in pbar(self._src_c_index, show=self.progress):
self._store_coincidence(coincidence)
c_index = self.data.create_vlarray(self.coincidence_group,
'c_index',
tables.UInt32Col(shape=2),
expectedrows=n_coincidences)
for observables_idx in pbar(self.c_index, show=self.progress):
c_index.append(observables_idx)
c_index.flush()
s_index = self.data.create_vlarray(self.coincidence_group,
's_index',
tables.VLStringAtom(),
expectedrows=len(
self.station_groups))
for station_group in self.station_groups:
s_index.append(station_group)
s_index.flush()
def append(self, key, item):
if key not in self.data_:
filters = tb.Filters(complevel=5, complib='blosc')
if isinstance(item, np.ndarray):
atom = tb.Atom.from_type(item.dtype.name, item.shape[1:])
else:
atom = tb.VLStringAtom()
self.data_[key] = self.h5f_.create_vlarray(self.h5f_.root, key,
atom, filters=filters)
print('Creating VLArray, and appending to key {}'.format(key))
print(self.data_[key])
self.data_[key].append(self.pack(item))
def write_dict(filename, data, name, title):
logging.debug("Writing dictionary %s to hdf5 file %s" % (name, filename))
with utils.H5Maker(filename, 'a') as f:
# f = tables.openFile(filename, 'a')
if '/%s' % name in f:
logging.debug("Removing existing /%s from %s" % (name, filename))
f.removeNode('/%s' % name, recursive=True)
g = f.createGroup('/', name, title)
keys = f.createVLArray(g,
'keys',
tables.VLStringAtom(),
"keys",
expectedsizeinMB=0.0001)
values = f.createVLArray(g,
'values',
tables.VLStringAtom(),
"values",
expectedsizeinMB=0.0001)
for (k, v) in data.iteritems():
keys.append(k)
values.append(str(v))
f.flush()
def testWrongTypes2(self):
h5file = tables.openFile("wrongTypes2.h5", mode="w")
h5file.createVLArray("/",
"keys",
tables.VLStringAtom(),
filters=tables.Filters(complevel=1))
keys = h5file.root.keys
keys.append("key1")
keys.append("key2")
h5file.createVLArray("/",
"vals",
tables.VLStringAtom(),
filters=tables.Filters(complevel=1))
vals = h5file.root.vals
vals.append("5555")
vals.append("aaaaaa")
h5file.close()
self.expectException("wrongTypes2.h5")
def _create_table_list(self, name, example):
"""
Create a new table within the HDF file, where the tables shape and its
datatype are determined by *example*.
The modified version for creating table with appendList
"""
type_map = {
np.dtype(np.float64): tables.Float64Atom(),
np.dtype(np.float32): tables.Float32Atom(),
np.dtype(np.int): tables.Int64Atom(),
np.dtype(np.int8): tables.Int8Atom(),
np.dtype(np.uint8): tables.UInt8Atom(),
np.dtype(np.int16): tables.Int16Atom(),
np.dtype(np.uint16): tables.UInt16Atom(),
np.dtype(np.int32): tables.Int32Atom(),
np.dtype(np.uint32): tables.UInt32Atom(),
np.dtype(np.bool): tables.BoolAtom(),
}
try:
if type(example) == np.ndarray:
h5type = type_map[example.dtype]
elif type(example) == list and type(example[0]) == str:
h5type = tables.VLStringAtom()
except KeyError:
raise TypeError("Don't know how to handle dtype '%s'" %
example.dtype)
if type(example) == np.ndarray:
h5dim = (0, ) + example.shape[1:]
h5 = self.h5
filters = tables.Filters(complevel=self.compression_level,
complib='zlib',
shuffle=True)
self.tables[name] = h5.create_earray(h5.root,
name,
h5type,
h5dim,
filters=filters)
elif type(example) == list and type(example[0]) == str:
h5 = self.h5
filters = tables.Filters(complevel=self.compression_level,
complib='zlib',
shuffle=True)
self.tables[name] = h5.create_vlarray(h5.root,
name,
h5type,
filters=filters)
self.types[name] = type(example)
def _create_table(self, name, example):
"""
Create a new table within the HDF file, where the tables shape and its
datatype are determined by *example*.
"""
type_map = {
np.dtype(np.float64): tables.Float64Atom(),
np.dtype(np.float32): tables.Float32Atom(),
np.dtype(np.int): tables.Int64Atom(),
np.dtype(np.int8): tables.Int8Atom(),
np.dtype(np.uint8): tables.UInt8Atom(),
np.dtype(np.int16): tables.Int16Atom(),
np.dtype(np.uint16): tables.UInt16Atom(),
np.dtype(np.int32): tables.Int32Atom(),
np.dtype(np.uint32): tables.UInt32Atom(),
np.dtype(np.bool): tables.BoolAtom(),
}
try:
if type(example) == np.ndarray:
h5type = type_map[example.dtype]
elif type(example) == str:
h5type = tables.VLStringAtom()
except KeyError:
raise TypeError(
"Could not create table %s because of unknown dtype '%s'" %
(name, example.dtype)) #+ ", of name: " % example.shape)
if type(example) == np.ndarray:
h5dim = (0, ) + example.shape
h5 = self.h5
filters = tables.Filters(complevel=self.compression_level,
complib='zlib',
shuffle=True)
self.tables[name] = h5.create_earray(h5.root,
name,
h5type,
h5dim,
filters=filters)
elif type(example) == str:
h5 = self.h5
filters = tables.Filters(complevel=self.compression_level,
complib='zlib',
shuffle=True)
self.tables[name] = h5.create_vlarray(h5.root,
name,
h5type,
filters=filters)
self.types[name] = type(example)
def _ensure_heirarchy(self):
r = self.root
db = self.db
filters = tb.Filters(complib=b'zlib', complevel=1)
if 'replays' not in r:
db.create_vlarray(r, 'replays', atom=tb.VLStringAtom())
if 'metadata' not in r:
db.create_table(r,
'metadata',
filters=filters,
description=METADATA_DESC)
if 'actions' not in r:
db.create_earray(r,
'actions',
atom=tb.Int64Atom(),
shape=(0, 2, NSTEPS),
filters=filters)
def _create_table(self, name, example, parent=None):
"""
Create a new table within the HDF file, where the tables shape and its
datatype are determined by *example*.
"""
h5 = self.h5
filters = tables.Filters(complevel=self.compression_level,
complib='zlib',
shuffle=True)
if parent is None:
parent = h5.root
if type(example) == str:
h5type = tables.VLStringAtom()
h5.createVLArray(parent, name, h5type, filters=filters)
return
if type(example) == dict:
self.h5.createGroup(parent, name)
return
#If we get here then we're dealing with numpy arrays
example = np.asarray(example)
#MODIFICATION: appended name everywhere and introduced string
type_map = {
np.dtype(np.float64).name: tables.Float64Atom(),
np.dtype(np.float32).name: tables.Float32Atom(),
np.dtype(np.int).name: tables.Int64Atom(),
np.dtype(np.int8).name: tables.Int8Atom(),
np.dtype(np.uint8).name: tables.UInt8Atom(),
np.dtype(np.int16).name: tables.Int16Atom(),
np.dtype(np.uint16).name: tables.UInt16Atom(),
np.dtype(np.int32).name: tables.Int32Atom(),
np.dtype(np.uint32).name: tables.UInt32Atom(),
np.dtype(np.bool).name: tables.BoolAtom(),
# Maximal string length of 128 per string - change if needed
'string32': tables.StringAtom(128)
}
try:
h5type = type_map[example.dtype.name]
h5dim = (0, ) + example.shape
h5.createEArray(parent, name, h5type, h5dim, filters=filters)
except KeyError:
raise TypeError("Don't know how to handle dtype '%s'" %
example.dtype)
def __init__(self,
file_name=None,
chromosomes=None,
mode='a',
tmpdir=None,
_table_name_chromosomes='chromosomes'):
"""
Build :class:`~Genome` from a list of chromosomes or load
previously saved object.
:param file_name: Path of the file to load or to save to.
:param chromosomes: List of chromosomes to load into this
object.
"""
FileGroup.__init__(self,
_table_name_chromosomes,
file_name,
mode=mode,
tmpdir=tmpdir)
# check if this is an existing regions file
try:
self._sequences = self._group.sequences
except t.NoSuchNodeError:
self._sequences = self.file.create_vlarray(self._group,
'sequences',
t.VLStringAtom())
try:
self._chromosome_table = self._group.chromosomes
except t.NoSuchNodeError:
try:
self._chromosome_table = self.file.create_table(
self._group, 'chromosomes', Genome.ChromosomeDefinition)
except t.FileModeError:
self._chromosome_table = None
pass
if chromosomes is not None:
if isinstance(chromosomes, Chromosome):
chromosomes = [chromosomes]
for chromosome in chromosomes:
self.add_chromosome(chromosome)
def _appendToTable(self, tablename, data):
with tablesLock:
try:
table = getattr(self._h5file.root, tablename)
table.append(data)
except AttributeError:
# we don't have a table with that name - create one
if isinstance(data, six.string_types):
table = self._h5file.create_vlarray(
self._h5file.root, tablename, tables.VLStringAtom())
table.append(data)
else:
self._h5file.create_table(self._h5file.root,
tablename,
data,
filters=tables.Filters(
complevel=5, shuffle=True),
expectedrows=500000)
if (tablename == 'PZFImageData'):
from PYME.IO import PZFFormat
#special case for pzf data - also build an index table
frameNum = PZFFormat.load_header(data)['FrameNum']
#record a mapping from frame number to the row we added
idx_entry = np.array(
[frameNum, table.nrows - 1],
dtype='i4').view(dtype=[('FrameNum',
'i4'), ('Position', 'i4')])
try:
index = getattr(self._h5file.root, 'PZFImageIndex')
index.append(idx_entry)
except AttributeError:
self._h5file.create_table(self._h5file.root,
'PZFImageIndex',
idx_entry,
filters=tables.Filters(
complevel=5, shuffle=True),
expectedrows=50000)
self._pzf_index = None
def _create_coincidences_tables(file, station_groups):
"""Setup coincidence tables"""
group = '/coincidences'
# Create coincidences table
description = storage.Coincidence
s_columns = {'s%d' % station: tables.BoolCol(pos=p)
for p, station in enumerate(station_groups.iterkeys(), 12)}
description.columns.update(s_columns)
coincidences = file.create_table(group, 'coincidences', description,
createparents=True)
# Create c_index
file.create_vlarray(group, 'c_index', tables.UInt32Col(shape=2))
# Create and fill s_index
s_index = file.create_vlarray(group, 's_index', tables.VLStringAtom())
for station_group in station_groups.itervalues():
s_index.append(station_group['group'])
return coincidences
def log_pd_store_config_initial(conf):
"""log_pd_store_config_initial
store the initial config in a separate table '/conf'
Arguments:
- conf: configuration dictionary
"""
# # table attribute hack
# df_conf = pd.DataFrame([[0]])
# log_store['conf'] = df_conf
# print "logging.log_pd_store_config_initial conf = %s" % conf
# log_store.get_storer('conf').attrs.conf = conf
# get h5 handle from pandas
h5file = log_store._handle
# create VLArray for storing the graph configuration
conf_array = h5file.create_vlarray(log_store.root, 'conf',
tb.VLStringAtom(),
"Variable Length Config String")
# store it
conf_array.append(str(conf))
def _setup_file(self):
validate_path(self._events_path)
if self._events_path not in self.file:
g, n = parse_path(self._events_path)
if g not in self.file:
r, gn = parse_path(g)
self.file.createGroup(r, gn)
self.file.createTable(g, n, HDF5Event)
validate_path(self._values_path)
if self._values_path not in self.file:
g, n = parse_path(self._values_path)
if g not in self.file:
r, gn = parse_path(g)
self.file.createGroup(r, gn)
self.file.createVLArray(g,
n,
tables.VLStringAtom(),
expectedsizeinMB=self._vsize)
# force recaching of nodes
self._events_node = None
self._values_node = None
self._cache_nodes()
def _add_vlarray(
file: tables.File,
where: tables.group,
name: str,
to_store: List[Union[str, HomList]],
) -> None:
"""
Adds a ragged array to a tables file. Each row in the array is
populated by an element of `to_store`, which can contain either strings or
lists of any of the types supported by PyTables. This includes
floats, integers and other scalar data types.
Parameters
----------
file : tables.File
where : tables.Group
name : str
to_store : list of either str or lists of scalars
"""
if to_store:
if isinstance(to_store[0], str):
to_store = [s.encode("utf-8") for s in to_store]
atom = tables.VLStringAtom()
else:
to_store = [np.array(ll) for ll in to_store]
atom = tables.Atom.from_dtype(to_store[0].dtype)
else:
atom = tables.StringAtom(itemsize=1)
vla = file.create_vlarray(
where,
name,
atom=atom,
filters=compression_filter,
expectedrows=len(to_store),
)
for s in to_store:
vla.append(s)
def log_tb_init(config):
"""log_tb: log to tables
log init function: global, called from topblock init
"""
# global handles
global h5file, loginit, lognodes
# experiment signature
# experiment = "%s" % (config["id"])
# filename
tblfile = '%s_log_tb.h5' % (config['params']['datafile_expr'], )
# tblfile = "data/%s.h5" % (experiment)
# h5 file
h5file = tb.open_file(tblfile, mode="w", title="%s" % (experiment))
root = h5file.root
storage_version = "v2"
# create VLArray for storing the graph configuration
conf_array = h5file.create_vlarray(root, 'conf', tb.VLStringAtom(),
"Variable Length Config String")
conf_array.append(str(config))
# FIXME: log git commit
# create topblock array
# a = tb.Float64Atom()
# lognodes[parent.id] = h5file.create_earray(root, "%s_obuf" % (parent.id), a, (parent.odim, 0))
# # create arrays for each node's data
# for nodek, nodev in nodes.items():
# print("init_log: node", nodek)
# a = tb.Float64Atom()
# # node_fieldkeys =
# # for node_fieldkey in ["%s_%03d" % (nodek, i) for i in range(nodev.odim)]:
# # tdef[node_fieldkey] = tb.Float32Col()
# # tdef[nodek] = tb.Float32Col(shape=(nodev.odim, 1))
# # return tdef
# lognodes[nodev.id] = h5file.create_earray(root, "%s_obuf" % (nodev.id), a, (nodev.odim, 0))
# loginit = True # this doesn't work yet persistently, why?
loginit = True
