def __init__(self, path: str,
event_table_desc: Type[tables.IsDescription]):
"""
Writer for a collection of Photoelectrons to HDF5 file using pytables
Stores the photoelectron info as variable length arrays
Parameters
----------
path : str
Path to store the file (overwrites if already exists)
event_table_desc : Type[tables.IsDescription]
Uninstanced `tables.IsDescription` class describing the columns of
the event metadata table
"""
self._file = tables.File(path, mode='w', filters=FILTERS)
group = self._file.create_group(self._file.root, "data", "Event Data")
self._event_metadata_table = self._file.create_table(
group, "event_metadata", event_table_desc, "Event Metadata")
self._pixel_column = self._file.create_vlarray(
group, "photoelectron_arrival_pixel", tables.UInt16Atom(shape=()),
"Pixel hit by the photoelectron")
self._time_column = self._file.create_vlarray(
group, "photoelectron_arrival_time", tables.Float64Atom(shape=()),
"Arrival time of the photoelectrons")
self._charge_column = self._file.create_vlarray(
group, "photoelectron_measured_charge",
tables.Float64Atom(shape=()),
"Charge reported by photosensor for each photoelectron")
def compute_particles(hrfile,
centers,
nr_particles=10,
avg_dist=10.0,
avg_radial_vel=1.0,
avg_rand_vel=0.1,
min_mass=0.01,
max_mass=10.0):
'''compute nr_particles around the given center of gravity,
with average distane avg_dist and average velocity avg_vel'''
particles = h5file.createGroup(h5file.root, 'particles')
position = h5file.create_earray(particles, 'position',
tables.Float64Atom(), (0, 3),
'particle positions')
velocity = h5file.create_earray(particles, 'velocity',
tables.Float64Atom(), (0, 3),
'particle velocities')
mass = h5file.create_earray(particles, 'mass', tables.Float64Atom(), (0, ),
'particle masses')
center = np.sum(centers, axis=0)
pos = avg_dist * (center + np.random.randn(nr_particles, 3))
position.append(pos)
vel = (avg_rand_vel * np.random.randn(nr_particles, 3) + avg_radial_vel *
(pos - center))
velocity.append(vel)
mass.append(np.random.uniform(min_mass, max_mass, nr_particles))
h5file.flush()
def init_db(self, db, size):
""" Takes a Pytables Group object (group) and the total number of samples expected and
expands or creates the necessary groups.
"""
D = self.dist0.D
objroot = db.root.object
db.createEArray(objroot.objfxn, 'mu0', t.Float64Atom(shape=(D,)), (0,), expectedrows=size)
db.createEArray(objroot.objfxn, 'mu1', t.Float64Atom(shape=(D,)), (0,), expectedrows=size)
db.createEArray(objroot.objfxn, 'sigma0', t.Float64Atom(shape=(D,D)), (0,), expectedrows=size)
db.createEArray(objroot.objfxn, 'sigma1', t.Float64Atom(shape=(D,D)), (0,), expectedrows=size)
db.createEArray(objroot.objfxn, 'c', t.Float64Atom(), (0,), expectedrows=size)
def __init__(self, n_dict, spk, dt, debug=False, LPU_id=None):
self.num_neurons = len(n_dict['id'])
self.LPU_id = None
super(Olfactory_receptor, self).__init__(n_dict, spk, dt, debug,
LPU_id)
self.debug = debug
self.dt = dt
#self.steps = max(int(round(dt / 1e-5)),1)
#self.ddt = dt / self.steps
self.I_drive = garray.to_gpu(
np.asarray(np.zeros((self.num_neurons, 1)), dtype=np.double))
binding_rate_data = np.random.rand(self.num_neurons, 1)
self.binding_rate = garray.to_gpu(
np.asarray(binding_rate_data, dtype=np.double))
init_state = np.zeros((self.num_neurons, 13))
init_state[:, 4] = 1e-5
init_state[:, 7] = 1
self.state = garray.to_gpu(np.asarray(init_state, dtype=np.double))
#self.V = V
self.spk = spk
self.spk_flag = 0
self.V = garray.to_gpu(np.asarray(n_dict['V'], dtype=np.double))
#self.spk = garray.to_gpu(np.asarray(np.zeros((self.num_neurons,1)), dtype=np.int32))
self.V_prev = garray.to_gpu(
np.asarray(n_dict['V_prev'], dtype=np.double))
self.X_1 = garray.to_gpu(np.asarray(n_dict['X_1'], dtype=np.double))
self.X_2 = garray.to_gpu(np.asarray(n_dict['X_2'], dtype=np.double))
self.X_3 = garray.to_gpu(np.asarray(n_dict['X_3'], dtype=np.double))
#self.I_ext = garray.to_gpu(np.asarray([1]*self.num_neurons, dtype=np.double))
#cuda.memcpy_htod(int(self.V), np.asarray(n_dict['V'], dtype=np.double))
#cuda.memcpy_htod(int(self.spk), np.asarray(np.zeros((self.num_neurons,1)), dtype=np.double))
self.update_olfactory_transduction = self.get_olfactory_transduction_kernel(
)
self.update_hhn = self.get_multi_step_hhn_kernel()
if self.debug:
if self.LPU_id is None:
self.LPU_id = "anon"
self.I_file = tables.openFile(self.LPU_id + "_I.h5", mode="w")
self.I_file.createEArray("/","array", \
tables.Float64Atom(), (0,self.num_neurons))
self.V_file = tables.openFile(self.LPU_id + "_V.h5", mode="w")
self.V_file.createEArray("/","array", \
tables.Float64Atom(), (0,self.num_neurons))
def initUV(L,M,N):
f = tb.open_file('./matrix-pt.h5', 'w')
filters = tb.Filters(complevel=5, complib='blosc')
ad = f.create_carray(f.root, 'a', tb.Float64Atom(), (L,M), filters=filters)
a = np.random.normal(0, .1, (M))*0.1
for i in range(L):
ad[i,:] = a
bd = f.create_carray(f.root, 'b', tb.Float64Atom(), (N, M), filters=filters)
for i in range(N):
bd[i,:] = a
return ad , bd
def write_new_file(f_name, Phi, m, n):
file_name = f_name + '.h5'
#f = tb.open_file('Phi.h5', 'w')
f = tb.open_file(file_name, 'w')
filters = tb.Filters(complevel=5, complib='blosc')
#because by default numpy (in other scripts) operates using float64, this is necessary
out = f.create_carray(f.root,
'data',
tb.Float64Atom(),
shape=(m, n),
filters=filters)
print "h5 file created, now putting Phi from memory to file..."
step = 1000 #this is the number of rows we calculate each loop (example was using bl)
#this may not the most efficient value
#look into buffersize usage in PyTables and adopt the buffersite of the
#carray accordingly to improve specifically fetching performance
#b = b.tocsc() #we slice b on columns, csc improves performance #not necessary in this case
#this can also be changed to slice on rows instead of columns (which is what will be done for Phi)
for i in range(0, n, step):
try:
out[:, i:min(i + step, n)] = Phi[:, i:min(
i + step, n
)] # initially, example was using this => (a.dot(b[:,i:min(i+bl, l)])).toarray()
except Exception as e:
print e
break
print i
print "Phi saving done, closing file..."
f.close()
def contact_maps_from_traj(pdb_file,
traj_file,
contact_cutoff=8.0,
savefile=None):
"""
Get contact map from trajectory.
"""
mda_traj = mda.Universe(pdb_file, traj_file)
traj_length = len(mda_traj.trajectory)
ca = mda_traj.select_atoms('name CA')
if savefile:
savefile = os.path.abspath(savefile)
outfile = tables.open_file(savefile, 'w')
atom = tables.Float64Atom()
cm_table = outfile.create_earray(outfile.root,
'contact_maps',
atom,
shape=(traj_length, 0))
contact_matrices = []
for frame in mda_traj.trajectory:
cm_matrix = (distances.self_distance_array(ca.positions) <
contact_cutoff) * 1.0
contact_matrices.append(cm_matrix)
if savefile:
cm_table.append(contact_matrices)
outfile.close()
return contact_matrices
def save_data(main_path, exp_path, data):
"""
save_data(main_path, exp_path, data)
Parameters
----------
main_path : Str
exp_path : Str
data : ndarray
Returns
-------
"""
try:
# Saving Parameters
atom = tables.Float64Atom() # declare data type
fsave = tables.open_file(os.path.join(main_path, exp_path + '.h5'),
mode='w') # open tables object
ds = fsave.create_earray(
fsave.root,
'data',
atom, # create data store
[0, data.shape[1], data.shape[2]])
ds.append(data) # append data
fsave.close() # close tables object
return 1
except:
print('File could not be saved')
return 0
def write_jointState(self, topic_group, data):
fields = ['position', 'velocity', 'effort', 'time']
self.pytable_writer_helper(topic_group, fields, tables.Float64Atom(),
data)
self.pytable_writer_helper(topic_group, ['name'],
tables.StringAtom(itemsize=20), data)
def write_zlib(data):
zlib_filter = tables.Filters(complib='zlib', complevel=9)
zlib_out = tables.openFile('zlib.h5', mode='w')
atom = tables.Float64Atom()
ca = zlib_out.createCArray(zlib_out.root, 'ca', atom, data.shape, filters=zlib_filter)
ca[:] = data
zlib_out.close()
def _draw_frame(self, framedata):
self.framedata = framedata
try:
try:
myGUI.ppathh
except:
myGUI.set_path()
spectrum = myGUI.acquireData() - self.background
self.wv_ax = myGUI.wv
self.plot_spectrum(spectrum)
if myGUI.measure_chkBox.isChecked():
self.dt = myGUI.measure_SpinBox.value()
if time.time() > self.timestamp + self.dt:
self.filename = str(myGUI.ppathh + "sp_array_" +
str(self.save_file_num) + '.h5')
if os.path.exists(self.filename):
f = tables.open_file(self.filename, mode='a')
f.root.data.append(spectrum)
f.close()
else:
data_to_save = np.row_stack((self.wv_ax, spectrum))
f = tables.open_file(self.filename, mode='w')
data_format = tables.Float64Atom()
array_c = f.create_earray(f.root, 'data', data_format,
(0, ))
array_c.append(self.wv_ax)
array_c.append(spectrum)
f.close()
self.timestamp = time.time()
except:
pass
def _initialize(self):
if not self._disabled:
self._logger.debug('Initializing.')
self._logger.debug('Disabled = %s' % self._disabled)
if self._out_file: # for resets
self._out_file.close()
info = self.traverse_back_and_find("mne_info")
col_size = info["nchan"]
self._out_file = tables.open_file(self.output_path, mode="w")
atom = tables.Float64Atom()
self.output_array = self._out_file.create_earray(
self._out_file.root, "data", atom, (col_size, 0))
self.timestamps_array = self._out_file.create_earray(
self._out_file.root, "timestamps", atom, (1, 0))
self.ch_names = self._out_file.create_array(
self._out_file.root,
"ch_names",
np.array(info["ch_names"]),
"Channel names in data",
)
self._out_file.root.data.attrs.sfreq = info["sfreq"]
try:
fwd = self.traverse_back_and_find("_fwd")
self._out_file.create_array(
self._out_file.root,
"src_xyz",
fwd['source_rr'],
"Source space coordinates",
)
except Exception as e:
self._logger.exception(e)
self._logger.warning('Forward model not found.'
' Skip adding source coordinates.')
def generate_dataset():
FOLDER = "car_images/"
FILENAME = "car_dataset_%dx%d.h5" % (IMG_WIDTH, IMG_HEIGHT)
TARGET_SIZE = 50000
os.chdir(ROOT)
fd = tables.open_file(FILENAME, mode='w')
atom = tables.Float64Atom()
filters = tables.Filters(complevel=5, complib='blosc')
dataset = fd.create_earray(fd.root,
'data',
atom, (0, 3, IMG_WIDTH, IMG_HEIGHT),
filters=filters,
expectedrows=TARGET_SIZE)
os.chdir(ROOT + "/" + FOLDER)
count = 0
for f in glob.glob("*.jpg"):
img = Image.open(f)
count += 1
print("%d : %s" % (count, f))
img = img.resize((IMG_WIDTH, IMG_HEIGHT))
arr = np.asarray(img)
arr = np.reshape(arr, (1, arr.shape[2], arr.shape[1], arr.shape[0]))
dataset.append(arr)
fd.close()
def create_data_file(self):
total_lines = self.filesize
repeats = 1000
batch_size = int(total_lines / repeats)
self.filesize = batch_size * repeats
filename = self.filename
#initial creation of the data file (if statement is added for multiprocessing). To be safe don't have a preexisting file with the same filename in the database folder
if not os.path.exists(filename):
ROW_SIZE = self.architecture[0] + self.architecture[-1]
f = tables.open_file(filename, mode='w')
atom = tables.Float64Atom()
array_c = f.create_earray(f.root, 'data', atom, (0, ROW_SIZE))
f.close()
f = tables.open_file(filename,
mode='a') #file opened for concatenation
for i in range(repeats):
print(int(i / repeats * 100),
"% complete (data file generation) ",
end="\r")
x_input = self.generate_input(batch_size=batch_size)
y_output = self.predict(x_input)
data_rows = np.concatenate((x_input, y_output), axis=1)
f.root.data.append(data_rows)
f.close()
else:
print(
'Data file exists. Waiting for 200 seconds before proceeding.')
time.sleep(200)
def unify_channel(out_fname, in_fnames, process_signs):
"""
do unification for one channel
"""
# find total number of spikes
num_pos, num_neg, nsamp = count_spikes(in_fnames)
print('Positive: {}, Negative: {}'.format(num_pos, num_neg))
print('opening ' + out_fname)
outfile = tables.open_file(out_fname, 'w')
print(num_pos, num_neg, nsamp)
start = {}
stop = {}
operate_signs = []
for sign, nspk in zip(('pos', 'neg'), (num_pos, num_neg)):
if sign not in process_signs:
print('Skipping {} as requested'.format(sign))
continue
if nspk:
operate_signs.append(sign)
start[sign] = 0
else:
print('{} has no {} spikes'.format(out_fname, sign))
continue
outfile.create_group('/', sign)
outfile.create_array('/' + sign,
'times',
atom=tables.Float64Atom(),
shape=(nspk, ))
outfile.create_array('/' + sign,
'spikes',
atom=tables.Float32Atom(),
shape=(nspk, nsamp))
for h5file in in_fnames:
fid = tables.open_file(h5file, 'r')
for sign in operate_signs:
# times
data = fid.get_node('/' + sign + '/times')
stop[sign] = start[sign] + data.shape[0]
out = outfile.get_node('/' + sign + '/times')
print("Copying {} spikes to {}-{}".format(data.shape[0],
start[sign], stop[sign]))
out[start[sign]:stop[sign]] = data[:]
data = fid.get_node('/' + sign + '/spikes')
out = outfile.get_node('/' + sign + '/spikes')
out[start[sign]:stop[sign]] = data[:]
start[sign] = stop[sign]
fid.close()
outfile.close()
def check_and_write(array, filename, NUM_ENTRIES):
if not os.path.isfile(filename):
f = tables.open_file(filename, mode='w')
atom = tables.Float64Atom()
array_c = f.create_earray(f.root, 'data', atom, (0, 8))
print (array)
array_c.append(array)
f.close()
print ("new file")
return True
f = tables.open_file(filename, mode='r')
i = 0
check = False
for item in f.root.data[0]:
c = np.array((f.root.data[i:i+NUM_ENTRIES,0:]))
idx = np.where(abs((c[:,np.newaxis,:] - array)).sum(axis=2) == 0)
i = i + NUM_ENTRIES
if len(idx[0]) == NUM_ENTRIES:
check = True
break
f.close()
if check:
print ("Duplicate Solution")
return False
else:
print ("Unique Solution")
f = tables.open_file(filename, mode='a')
f.root.data.append(array)
f.close()
return True
def check_and_write(array, filename, NUM_ENTRIES, count2):
if not os.path.isfile(filename):
f = tables.open_file(filename, mode='w')
atom = tables.Float64Atom()
array_c = f.create_earray(f.root, 'data', atom, (0, 8))
print(array)
array_c.append(array)
f.close()
print("new file")
return True
f = tables.open_file(filename, mode='r')
i = 0
check = 0
count = 0
for arr in f.root.data:
if (count % 7 == 0):
c = np.array((f.root.data[count:count + NUM_ENTRIES, 0:]))
idx = np.where(abs((c[:, np.newaxis, :] - array)).sum(axis=2) == 0)
if len(idx[0]) == NUM_ENTRIES:
check += 1
count += 1
f.close()
if check > 0:
print("Duplicate Solution")
print(check)
return False, check
else:
print("Unique Solution")
f = tables.open_file(filename, mode='a')
f.root.data.append(array)
f.close()
return True, 0
def __init__(self,
hdf5Node,
compLib='zlib',
compLevel=1,
expectedRows=300,
**kwargs):
'''
Args:
hdf5Node (str): Path of the node to store data in HDF5 file.
compLib (str): Compression library, should be one of the following:
zlib, lzo, bzip2, blosc, blosc:blosclz, blosc:lz4,
blosc:lz4hc, blosc:snappy, blosc:zlib, blosc:zstd
compLevel: Level of compression can vary from 0 (no compression)
to 9 (maximum compression)
'''
if not isinstance(hdf5Node, str):
raise TypeError('`hdf5Node` should be a string')
if contains(hdf5Node):
raise NameError('HDF5 node %s already exists' % hdf5Node)
self._hdf5Node = hdf5Node
self._hdf5Filters = tb.Filters(complib=compLib, complevel=compLevel)
self._lock = threading.Lock()
self._partial = None
createEArray(hdf5Node,
tb.Float64Atom(), (0, 2),
'',
self._hdf5Filters,
expectedrows=expectedRows)
super().__init__(**kwargs)
def write_bz2(data):
bz2_filter = tables.Filters(complib='bzip2', complevel=9)
bz2_out = tables.openFile('bzip2.h5', mode='w')
atom = tables.Float64Atom()
ca = bz2_out.createCArray(bz2_out.root, 'ca', atom, data.shape, filters=bz2_filter)
ca[:] = data
bz2_out.close()
def assign_attribute(obj_attr, attr_name, path, node):
""" subfunction to serialize a given attribute """
if isinstance(obj_attr, pq.Quantity) or isinstance(obj_attr, np.ndarray):
if not lazy:
# we need to simplify custom quantities
if isinstance(obj_attr, pq.Quantity):
for un in obj_attr.dimensionality.keys():
if not un.name in pq.units.__dict__ or \
not isinstance(pq.units.__dict__[un.name], pq.Quantity):
obj_attr = obj_attr.simplified
break
# we try to create new array first, so not to loose the
# data in case of any failure
if obj_attr.size == 0:
atom = tb.Float64Atom(shape=(1,))
new_arr = self._data.createEArray(path, attr_name + "__temp", atom, shape=(0,), expectedrows=1)
else:
new_arr = self._data.createArray(path, attr_name + "__temp", obj_attr)
if hasattr(obj_attr, "dimensionality"):
for un in obj_attr.dimensionality.items():
new_arr._f_setAttr("unit__" + un[0].name, un[1])
try:
self._data.removeNode(path, attr_name)
except:
pass # there is no array yet or object is new
self._data.renameNode(path, attr_name, name=attr_name + "__temp")
elif obj_attr is not None:
node._f_setAttr(attr_name, obj_attr)
def write_lzo(data):
lzo_filter = tables.Filters(complib='lzo', complevel=9)
lzo_out = tables.openFile('lzo.h5', mode='w')
atom = tables.Float64Atom()
ca = lzo_out.createCArray(lzo_out.root, 'ca', atom, data.shape, filters=lzo_filter)
ca[:] = data
lzo_out.close()
def compute_vector_field(h5file,
centers,
xs,
ys,
zs,
max_field=50.0,
group='vector'):
'''compute a vector field that decreases quadratically with the
distance to the center'''
vector = h5file.createGroup(h5file.root, group)
dims = ['x', 'y', 'z']
for i, dim in enumerate(dims):
comp = h5file.create_earray(vector, dim, tables.Float64Atom(),
(len(xs), len(ys), 0),
'field {0}-component'.format(dim))
for z in zs:
coords = np.meshgrid(xs, ys, np.array(z))
field_slice = np.zeros((len(xs), len(ys), 1))
for center in centers:
dist = np.sqrt((coords[0] - center[0])**2 +
(coords[1] - center[1])**2 +
(coords[2] - center[2])**2 + 1.0)
field_slice += max_field * (coords[i] - center[i]) / dist**2
comp.append(field_slice)
h5file.flush()
def _make_float_vlarray(h5file: tables.File, name: str,
attribute: np.ndarray) -> None:
vlarray = h5file.create_vlarray(h5file.root,
name=name,
atom=tables.Float64Atom(shape=()))
for a in attribute:
vlarray.append(a)
def resize(h5file, start, stop):
ensure_tables()
# TODO is there any smarter and more efficient way to this?
data = h5file.getNode('/', "Data")
try:
gcolumns = h5file.createGroup('/', "Data_", "Data")
except tables.exceptions.NodeError:
h5file.removeNode('/', "Data_", 1)
gcolumns = h5file.createGroup('/', "Data_", "Data")
start = 0 if start is None else start
stop = gcolumns.X.nrows if stop is None else stop
atom = tables.Float32Atom() if config.floatX == 'float32' else tables.Float64Atom()
filters = DenseDesignMatrixPyTables.filters
x = h5file.createCArray(gcolumns, 'X', atom = atom, shape = ((stop - start, data.X.shape[1])),
title = "Data values", filters = filters)
y = h5file.createCArray(gcolumns, 'y', atom = atom, shape = ((stop - start, 10)),
title = "Data targets", filters = filters)
x[:] = data.X[start:stop]
y[:] = data.y[start:stop]
h5file.removeNode('/', "Data", 1)
h5file.renameNode('/', "Data", "Data_")
h5file.flush()
return h5file, gcolumns
def init_hdf5(path, shapes):
"""
Initialize hdf5 file to be used ba dataset
"""
x_shape, y_shape = shapes
# make pytables
ensure_tables()
h5file = tables.openFile(path, mode="w", title="SVHN Dataset")
gcolumns = h5file.createGroup(h5file.root, "Data", "Data")
atom = tables.Float32Atom(
) if config.floatX == 'float32' else tables.Float64Atom()
filters = DenseDesignMatrixPyTables.filters
h5file.createCArray(gcolumns,
'X',
atom=atom,
shape=x_shape,
title="Data values",
filters=filters)
h5file.createCArray(gcolumns,
'y',
atom=atom,
shape=y_shape,
title="Data targets",
filters=filters)
return h5file, gcolumns
def __init__(self, store_shape, file):
try:
import tables
except:
raise Exception('Use of DiskBackedStorage required PyTables to be installed (try pip install tables).')
self.__store_shape__ = store_shape
self.__array__ = np.empty(store_shape)
self.__max_rows__ = store_shape[0]
self.__current_index__ = 0
if type(file) in (str, unicode):
self.__f__ = tables.open_file(file, 'a')
else:
self.__f__ = file
# check whether samples already exist in the file
try:
self.__f__.get_node('/samples')
raise Exception('File already contains samples!')
except:
pass
self.__dest_table__ = self.__f__.create_earray('/', 'samples', tables.Float64Atom(), (0, store_shape[1]))
def save_hdf5(self, filepath):
import tables
import datetime
h5f = tables.open_file(filepath, 'w')
filters = tables.Filters(complevel=5, complib='zlib', shuffle=True)
h5f.set_node_attr(h5f.root, 'version', self._version)
h5f.set_node_attr(h5f.root, 'created_at', self.created_at)
h5f.set_node_attr(
h5f.root, 'saved_at',
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC"))
h5f.set_node_attr(h5f.root, 'label', self.label)
h5f.set_node_attr(h5f.root, 'units', self.units)
metadata_node = h5f.create_group(h5f.root, 'metadata')
self.metadata.save_pytables(metadata_node)
params = h5f.create_carray(h5f.root,
'values',
tables.Float64Atom(),
self.values.shape,
filters=filters)
params[:] = self.values
h5f.close()
def init_hdf5(self, path, shapes):
"""
.. todo::
WRITEME properly
Initialize hdf5 file to be used ba dataset
"""
x_shape, y_shape = shapes
# make pytables
ensure_tables()
h5file = tables.openFile(path, mode="w", title="SVHN Dataset")
gcolumns = h5file.createGroup(h5file.root, "Data", "Data")
atom = (tables.Float32Atom()
if config.floatX == 'float32' else tables.Float64Atom())
h5file.createCArray(gcolumns,
'X',
atom=atom,
shape=x_shape,
title="Data values",
filters=self.filters)
h5file.createCArray(gcolumns,
'y',
atom=atom,
shape=y_shape,
title="Data targets",
filters=self.filters)
return h5file, gcolumns
def write_hist(group_name: 'string with folder name to save histograms',
table_name: 'histogram name',
entries: 'np.array with bin content',
bins: 'list of np.array of bins',
out_of_range: 'np.array lenght=2 with events out of range',
errors: 'np.array with bins uncertainties',
labels: 'list with labels of the histogram',
scales: 'list with the scales of the histogram'):
try:
hist_group = getattr(file.root, group_name)
except tb.NoSuchNodeError:
hist_group = file.create_group(file.root, group_name)
if table_name in hist_group:
raise ValueError(f"Histogram {table_name} already exists")
vlarray = file.create_vlarray(hist_group,
table_name + '_bins',
atom=tb.Float64Atom(shape=()),
filters=tbl.filters(compression))
for ibin in bins:
vlarray.append(ibin)
add_carray(hist_group, table_name, entries)
add_carray(hist_group, table_name + '_outRange', out_of_range)
add_carray(hist_group, table_name + '_errors', errors)
file.create_array(hist_group, table_name + '_labels', labels)
file.create_array(hist_group, table_name + '_scales', scales)
def store_datadist(data_source, model, batch_size=10, fname='datamatrix.h5'):
"""
Store the log-probability matrix for a given method.
"""
# Turn on evaluation mode which disables dropout.
model.eval()
if args.model == 'QRNN': model.reset()
hidden = model.init_hidden(batch_size)
# Initialize a data matrix structure which can be stored directly to the disk.
f = tables.open_file(filename, mode='w')
atom = tables.Float64Atom()
array_c = f.create_earray(f.root, 'data', atom, (0, 10000))
# Add a row sequentially to the matrix for each different context.
for i in range(0, data_source.size(0) - 1, args.bptt):
data, targets = get_batch(data_source, i, args, evaluation=True)
output, weight, bias, hidden = model(data, hidden)
pred_targets = torch.mm(output, weight.t()) + bias
hidden = repackage_hidden(hidden)
datadist = nn.LogSoftmax()(pred_targets)
array_c.append(datadist.detach().cpu().numpy())
# Close file.
f.close()
