def _get_pytable_desc(self, col_list, col_names): d = {} for c, cn in zip(col_list, col_names): col_type = tables.Col.from_dtype(c.dtype) d[cn] = col_type return tables.Description(d)
def __descr_from_dtype__(dtype_, shape_={}): """ Get a description instance and byteorder from a (nested) NumPy dtype. Parameters ---------- dtype_: dtype dtype of a ndarray shape_: dict a dictionary of column's shapes returns ------- r: tables.Description Description of a new table """ fields = {} fbyteorder = "|" for (name, (dtype, pos)) in list(dtype_.fields.items()): kind = dtype.base.kind shape = shape_.get(name, ()) byteorder = dtype.base.byteorder if byteorder in "><=": if fbyteorder not in ["|", byteorder]: raise NotImplementedError( "record arrays with mixed byteorders " "are not supported yet, sorry") fbyteorder = byteorder # Non-nested column if kind in "biufSc": # col = tables.Col.from_dtype(dtype, pos=pos) if len(shape) > 1: col = _typeDict[dtype.base.type](shape=shape) else: col = _typeDict[dtype.type]() # Nested column elif kind == "V" and dtype.shape in [(), (1, )]: col, _ = __descr_from_dtype__(dtype) col._v_pos = pos else: raise NotImplementedError( "record arrays with columns with type description ``%s`` " "are not supported yet, sorry" % dtype) fields[name] = col return tables.Description(fields)
def descr_from_dtype(self, dtype_, shape_={}): """ Get a description instance and byteorder from a (nested) NumPy dtype. INPUTS: dtype_ dtype dtype of a ndarray shape_ dict a dictionary of column's shapes OUTPUTS: Description of a new table """ fields = {} fbyteorder = '|' for (name, (dtype, pos)) in dtype_.fields.items(): kind = dtype.base.kind shape = shape_.get(name, ()) byteorder = dtype.base.byteorder if byteorder in '><=': if fbyteorder not in ['|', byteorder]: raise NotImplementedError( "record arrays with mixed byteorders " "are not supported yet, sorry") fbyteorder = byteorder # Non-nested column if kind in 'biufSc': #col = tables.Col.from_dtype(dtype, pos=pos) if len(shape) > 1: col = typeDict[dtype.type](shape=shape) else: col = typeDict[dtype.type]() # Nested column elif kind == 'V' and dtype.shape in [(), (1, )]: col, _ = self.descr_from_dtype(dtype) col._v_pos = pos else: raise NotImplementedError( "record arrays with columns with type description ``%s`` " "are not supported yet, sorry" % dtype) fields[name] = col return tables.Description(fields)
def dtype_to_pytables(dtype): """ Convert NumPy dtype to PyTable descriptor Examples -------- >>> from tables import Int32Col, StringCol, Time64Col >>> dt = np.dtype([('name', 'S7'), ('amount', 'i4'), ('time', 'M8[us]')]) >>> dtype_to_pytables(dt) # doctest: +SKIP {'amount': Int32Col(shape=(), dflt=0, pos=1), 'name': StringCol(itemsize=7, shape=(), dflt='', pos=0), 'time': Time64Col(shape=(), dflt=0.0, pos=2)} """ d = {} for pos, name in enumerate(dtype.names): dt, _ = dtype.fields[name] if issubclass(dt.type, np.datetime64): tdtype = tb.Description({name: tb.Time64Col(pos=pos)}), else: tdtype = tb.descr_from_dtype(np.dtype([(name, dt)])) el = tdtype[0] # removed dependency on toolz -DJC getattr(el, name)._v_pos = pos d.update(el._v_colobjects) return d
def dtype_to_table(dtype): """ Convert a NumPy dtype to a PyTables Table description Essentially just :ref:`tables.descr_from_dtype` but it works on :ref:`np.datetime64` Args: dtype (np.dtype): NumPy data type Returns: dict: PyTables description """ desc = {} for idx, name in enumerate(dtype.names): _dt, _ = dtype.fields[name] if issubclass(_dt.type, np.datetime64): tb_dtype = tb.Description({name: tb.Time64Col(pos=idx)}) else: tb_dtype, byteorder = tb.descr_from_dtype(np.dtype([(name, _dt)])) _tb_dtype = tb_dtype._v_colobjects _tb_dtype[name]._v_pos = idx desc.update(_tb_dtype) return desc
def createFile(filename, totalrows, recsize, verbose): # Open a 'n'ew file dd = db.DB() if recsize == "big": isrec = tb.Description(Big) elif recsize == "medium": isrec = Medium() else: isrec = tb.Description(Small) # dd.set_re_len(struct.calcsize(isrec._v_fmt)) # fixed length records dd.open(filename, db.DB_RECNO, db.DB_CREATE | db.DB_TRUNCATE) rowswritten = 0 # Get the record object associated with the new table if recsize == "big": isrec = Big() arr = np.array(np.arange(32), type=np.float64) arr2 = np.array(np.arange(32), type=np.float64) elif recsize == "medium": isrec = Medium() arr = np.array(np.arange(2), type=np.float64) else: isrec = Small() # print d # Fill the table if recsize == "big" or recsize == "medium": d = { "name": " ", "float1": 1.0, "float2": 2.0, "ADCcount": 12, "grid_i": 1, "grid_j": 1, "pressure": 1.9, "energy": 1.8, } for i in range(totalrows): #d['name'] = 'Particle: %6d' % (i) #d['TDCcount'] = i % 256 d['ADCcount'] = (i * 256) % (1 << 16) if recsize == "big": #d.float1 = np.array([i]*32, np.float64) #d.float2 = np.array([i**2]*32, np.float64) arr[0] = 1.1 d['float1'] = arr arr2[0] = 2.2 d['float2'] = arr2 pass else: d['float1'] = float(i) d['float2'] = float(i) d['grid_i'] = i d['grid_j'] = 10 - i d['pressure'] = float(i * i) d['energy'] = d['pressure'] dd.append(cPickle.dumps(d)) # dd.append(struct.pack(isrec._v_fmt, # d['name'], d['float1'], d['float2'], # d['ADCcount'], # d['grid_i'], d['grid_j'], # d['pressure'], d['energy'])) else: d = {"var1": " ", "var2": 1, "var3": 12.1e10} for i in range(totalrows): d['var1'] = str(i) d['var2'] = i d['var3'] = 12.1e10 dd.append(cPickle.dumps(d)) #dd.append( # struct.pack(isrec._v_fmt, d['var1'], d['var2'], d['var3'])) rowswritten += totalrows # Close the file dd.close() return (rowswritten, struct.calcsize(isrec._v_fmt))
def __init__(self, h5file, reconstructor, cam_id2camns=None, min_observations_to_save=0, textlog_save_lines=None, dynamic_model_name=None, dynamic_model=None, fake_timestamp=None, debug=False): self.cam_id2camns = cam_id2camns self.min_observations_to_save = min_observations_to_save self.debug = debug self.kalman_saver_info_instance = flydra_kalman_utils.KalmanSaveInfo( name=dynamic_model_name) kalman_estimates_description = ( self.kalman_saver_info_instance.get_description()) filters = tables.Filters(1, complib='zlib') # compress self.h5file = h5file reconstructor.save_to_h5file(self.h5file) self.h5_xhat = self.h5file.create_table( self.h5file.root, 'kalman_estimates', kalman_estimates_description, "Kalman a posteriori estimates of tracked object", filters=filters) self.h5_xhat.attrs.dynamic_model_name = dynamic_model_name self.h5_xhat.attrs.dynamic_model = dynamic_model self.h5_obs = self.h5file.create_table( self.h5file.root, 'ML_estimates', FilteredObservations, "observations of tracked object", filters=filters) self.h5_2d_obs_next_idx = 0 # Note that ML_estimates_2d_idxs_type() should # match dtype with tro.observations_2d. self.h5_2d_obs = self.h5file.create_vlarray( self.h5file.root, 'ML_estimates_2d_idxs', ML_estimates_2d_idxs_type(), "camns and idxs") self.obj_id = 0 self.h5textlog = self.h5file.create_table(self.h5file.root, 'textlog', TextLogDescription, 'text log') if 1: textlog_row = self.h5textlog.row cam_id = 'mainbrain' if fake_timestamp is None: timestamp = time.time() else: timestamp = fake_timestamp list_of_textlog_data = [(timestamp, cam_id, timestamp, text) for text in textlog_save_lines] for textlog_data in list_of_textlog_data: (mainbrain_timestamp, cam_id, host_timestamp, message) = textlog_data textlog_row['mainbrain_timestamp'] = mainbrain_timestamp textlog_row['cam_id'] = cam_id textlog_row['host_timestamp'] = host_timestamp textlog_row['message'] = message textlog_row.append() self.h5textlog.flush() self.h5_xhat_names = PT.Description( kalman_estimates_description().columns)._v_names self.h5_obs_names = PT.Description( FilteredObservations().columns)._v_names self.all_kalman_calibration_data = []
from distutils.version import LooseVersion import tables as PT import tables import flydra_core.data_descriptions from flydra_core.reconstruct import Reconstructor import time import flydra_analysis.version Info2D = flydra_core.data_descriptions.Info2D Info2DCol_description = tables.Description(Info2D().columns)._v_nested_descr CamSyncInfo = flydra_core.data_descriptions.CamSyncInfo TextLogDescription = flydra_core.data_descriptions.TextLogDescription def startup_message(h5textlog, fps): textlog_row = h5textlog.row cam_id = 'mainbrain' timestamp = time.time() # This line is important (including the formatting). It is # read by flydra_analysis.a2.check_atmel_clock. list_of_textlog_data = [ (timestamp, cam_id, timestamp, ('MainBrain running at %s fps, ' '(flydra_version %s, ' 'time_tzname0 %s)' % ( str(fps), flydra_analysis.version.__version__, time.tzname[0], ))),
import sys, os import tables import numpy as np # Use simplejson or Python 2.6 json, prefer simplejson. try: import simplejson as json except ImportError: import json import data_format AnalogInputWordstreamDescription = data_format.AnalogInputWordstreamDescription AnalogInputWordstream_dtype = tables.Description( AnalogInputWordstreamDescription().columns)._v_nestedDescr TimeDataDescription = data_format.TimeDataDescription TimeData_dtype = tables.Description( TimeDataDescription().columns)._v_nestedDescr def doit(filename): base, ext = os.path.splitext(filename) output_fname = base + '.h5' print 'converting %s to %s' % (filename, output_fname) contents = open(filename).read() input = json.loads(contents) h5 = tables.openFile(output_fname, mode='w') stream_ain_table = h5.createTable(h5.root, 'ain_wordstream', AnalogInputWordstreamDescription, "AIN data")
def convert_to_flydrah5(bag_file, topic_name="pointcloud", out_h5=None, reconstructor=None): import rosbag if out_h5 is None: out_h5 = bag_file + ".h5" bag = rosbag.Bag(bag_file, "r") h5file = tables.open_file(out_h5, mode="w", title="Flydra data file (from ROS bag)") ct = h5file.create_table # shorthand root = h5file.root # shorthand # save data as both "observations" (ML estimates) and "kalman estimates" (MAP estimates) FilteredObservations = flydra_kalman_utils.FilteredObservations h5data3d_ML_estimates = ct(root, "ML_estimates", FilteredObservations, "3d data (input to Kalman filter)") h5_obs_names = tables.Description(FilteredObservations().columns)._v_names # we're not actually doing any kalman filtering, so just get a model with position kalman_saver_info_instance = flydra_kalman_utils.KalmanSaveInfo( name="mamarama, units: mm") KalmanEstimatesDescription = kalman_saver_info_instance.get_description() h5data3d_kalman_estimates = ct(root, "kalman_estimates", KalmanEstimatesDescription, "3d data") h5_xhat_names = tables.Description( KalmanEstimatesDescription().columns)._v_names obj_id = 0 for topic, cloud, t in bag.read_messages(topics=[topic_name]): pts = [] for p in read_points(cloud): pts.append((p[0], p[1], p[2], 0, 0, 0)) # velocity = 0 obj_id += 1 pts = np.array(pts) # save observations shape1d = (len(pts), ) shape2d_6 = (len(pts), 6) this_idxs = np.zeros(shape1d, dtype=np.uint64) # this_idxs = numpy.array( this_idxs, dtype=numpy.uint64 ) # becomes obs_2d_idx (index into 'ML_estimates_2d_idxs') observations_frames = np.arange(len(pts), dtype=np.uint64) obj_id_array = np.empty(observations_frames.shape, dtype=np.uint32) obj_id_array.fill(obj_id) observations_data = np.array(pts[:, :3], dtype=np.float32) observations_Lcoords = np.zeros(shape2d_6, dtype=np.float32) list_of_obs = [ observations_data[:, i] for i in range(observations_data.shape[1]) ] list_of_lines = [ observations_Lcoords[:, i] for i in range(observations_Lcoords.shape[1]) ] array_list = ([obj_id_array, observations_frames] + list_of_obs + [this_idxs] + list_of_lines) obs_recarray = np.rec.fromarrays(array_list, names=h5_obs_names) h5data3d_ML_estimates.append(obs_recarray) h5data3d_ML_estimates.flush() # save xhat info (kalman estimates) shape3d_6x6 = (len(pts), 6, 6) frames = np.arange(len(pts), dtype=np.uint64) timestamps = np.zeros(shape1d, dtype=np.float64) xhat_data = np.array(pts, dtype=np.float32) P_data_full = np.zeros(shape3d_6x6, dtype=np.float32) obj_id_array = np.empty(frames.shape, dtype=np.uint32) obj_id_array.fill(obj_id) list_of_xhats = [xhat_data[:, i] for i in range(xhat_data.shape[1])] ksii = kalman_saver_info_instance list_of_Ps = ksii.covar_mats_to_covar_entries(P_data_full) xhats_recarray = np.rec.fromarrays( [obj_id_array, frames, timestamps] + list_of_xhats + list_of_Ps, names=h5_xhat_names, ) h5data3d_kalman_estimates.append(xhats_recarray) h5data3d_kalman_estimates.flush() if reconstructor is not None: R = flydra_core.reconstruct.Reconstructor(reconstructor) R.save_to_h5file(h5file) h5file.close() bag.close()