def __init__(self, name, **keys):
DataNode.__init__(self, 'zerod', name, **keys)
cnx = self.db_connection
self._channels_name = QueueSetting('%s_channels' % self.db_name,
connection=cnx)
self._channels = {}
for channel_name in self._channels_name:
self._channels[channel_name] = QueueSetting(
'%s_%s' % (self.db_name, channel_name), connection=cnx)
class Dataset0D(DataNode):
class DataChannel(object):
def __init__(self,channel_db_name) :
cnx = node.get_default_connection()
self._queue = QueueSetting(channel_db_name,
connection=cnx).get_proxy()
def get(self,from_index,to_index = None):
if to_index is None:
return self._queue[from_index]
else:
return self._queue[from_index:to_index]
def __init__(self,name,**keys):
DataNode.__init__(self,'zerod',name,**keys)
cnx = self.db_connection
self._channels_name = QueueSetting('%s_channels' % self.db_name(),connection=cnx)
self._channels = {}
for channel_name in self._channels_name:
self._channels[channel_name] = QueueSetting('%s_%s' % (self.db_name(),channel_name),
connection=cnx)
def channel_name(self) :
return self._channels.get()
def store(self,signal,event_dict) :
if signal == "new_data":
channel_data = event_dict.get("channel_data")
if channel_data is None:
#warning
return
for channel_name,data in channel_data.iteritems():
queue = self._channels.get(channel_name)
if queue is None:
self._channels_name.append(channel_name)
queue = QueueSetting('%s_%s' % (self.db_name(),channel_name),
connection=self.db_connection)
queue.extend(data)
self._channels[channel_name] = queue
else:
queue.extend(data)
#@brief get data channel object
def get_channel(self,channel_name = None) :
if channel_name is None:
channel_name = self._channels[0]
channel_db_name = '%s_%s' % (self.db_name(),channel_name)
return Dataset0D.DataChannel(channel_db_name)
def set_ttl(self):
DataNode.set_ttl(self)
for channel in self._channels.itervalues():
channel.ttl(DataNode.default_time_to_live)
class DataNodeContainer(DataNode):
def __init__(self, node_type, name, parent=None, connection=None, create=False):
DataNode.__init__(self, node_type, name,
parent=parent, connection=connection, create=create)
children_queue_name = '%s_children_list' % self.db_name
self._children = QueueSetting(
children_queue_name, connection=connection)
def add_children(self, *child):
if len(child) > 1:
self._children.extend([c.db_name for c in child])
else:
self._children.append(child[0].db_name)
def children(self, from_id=0, to_id=-1):
"""Iter over children.
@return an iterator
@param from_id start child index
@param to_id last child index
"""
for child_name in self._children.get(from_id, to_id):
new_child = get_node(child_name)
if new_child is not None:
yield new_child
else:
self._children.remove(child_name) # clean
@property
def last_child(self):
return get_node(self._children.get(-1))
def __init__(self, node_type, name, parent=None, connection=None, create=False):
DataNode.__init__(self, node_type, name,
parent=parent, connection=connection, create=create)
children_queue_name = '%s_children_list' % self.db_name
self._children = QueueSetting(
children_queue_name, connection=connection)
def store(self,signal,event_dict) :
if signal == "new_data":
channel_data = event_dict.get("channel_data")
if channel_data is None:
#warning
return
for channel_name,data in channel_data.iteritems():
queue = self._channels.get(channel_name)
if queue is None:
self._channels_name.append(channel_name)
queue = QueueSetting('%s_%s' % (self.db_name(),channel_name),
connection=self.db_connection)
queue.extend(data)
self._channels[channel_name] = queue
else:
queue.extend(data)
def __init__(self,
node_type,
name,
parent=None,
connection=None,
create=False):
DataNode.__init__(self,
node_type,
name,
parent=parent,
connection=connection,
create=create)
children_queue_name = '%s_children_list' % self.db_name
self._children = QueueSetting(children_queue_name,
connection=connection)
def __init__(self,name,**keys):
DataNode.__init__(self,'zerod',name,**keys)
cnx = self.db_connection
self._channels_name = QueueSetting('%s_channels' % self.db_name(),connection=cnx)
self._channels = {}
for channel_name in self._channels_name:
self._channels[channel_name] = QueueSetting('%s_%s' % (self.db_name(),channel_name),
connection=cnx)
def __init__(self, name, **keys):
shape = keys.pop('shape', None)
dtype = keys.pop('dtype', None)
DataNode.__init__(self, 'channel', name, **keys)
if keys.get('create', False):
if shape is not None:
self.info["shape"] = shape
if dtype is not None:
self.info["dtype"] = dtype
cnx = self.db_connection
self._queue = QueueSetting("%s_data" % self.db_name,
connection=cnx,
read_type_conversion=functools.partial(
data_from_bytes,
shape=shape,
dtype=dtype),
write_type_conversion=data_to_bytes)
class DataChannel(object):
def __init__(self, channel_db_name, cnx):
self._queue = QueueSetting(channel_db_name, connection=cnx)
def get(self, from_index, to_index=None):
if to_index is None:
return self._queue[from_index]
else:
return self._queue[from_index:to_index]
def __len__(self):
return self._queue.__len__()
class DataChannel(object):
def __init__(self, channel_db_name, cnx):
self._queue = QueueSetting(channel_db_name,
connection=cnx)
def get(self, from_index, to_index=None):
if to_index is None:
return self._queue[from_index]
else:
return self._queue[from_index:to_index]
def __len__(self):
return self._queue.__len__()
class ChannelDataNode(DataNode):
def __init__(self, name, **keys):
shape = keys.pop('shape', None)
dtype = keys.pop('dtype', None)
DataNode.__init__(self, 'channel', name, **keys)
if keys.get('create', False):
if shape is not None:
self.info["shape"] = shape
if dtype is not None:
self.info["dtype"] = dtype
cnx = self.db_connection
self._queue = QueueSetting("%s_data" % self.db_name,
connection=cnx,
read_type_conversion=functools.partial(
data_from_bytes,
shape=shape,
dtype=dtype),
write_type_conversion=data_to_bytes)
def store(self, signal, event_dict, cnx=None):
if signal == "new_data":
data = event_dict.get("data")
channel = event_dict.get("channel")
if len(channel.shape) == data.ndim:
self._queue.append(data, cnx=cnx)
else:
self._queue.extend(data, cnx=cnx)
def get(self, from_index, to_index=None, cnx=None):
if to_index is None:
return self._queue.get(from_index, from_index, cnx=cnx)
else:
return self._queue.get(from_index, to_index, cnx=cnx)
def __len__(self, cnx=None):
return self._queue.__len__(cnx=cnx)
@property
def shape(self):
return self.info.get("shape")
@property
def dtype(self):
return self.info.get("dtype")
def _get_db_names(self):
db_names = DataNode._get_db_names(self)
db_names.append(self.db_name + "_data")
return db_names
def __init__(self, name, **keys):
shape = keys.pop('shape', None)
dtype = keys.pop('dtype', None)
DataNode.__init__(self, 'channel', name, **keys)
if keys.get('create', False):
if shape is not None:
self.info["shape"] = shape
if dtype is not None:
self.info["dtype"] = dtype
cnx = self.db_connection
self._queue = QueueSetting("%s_data" % self.db_name, connection=cnx,
read_type_conversion=functools.partial(data_from_bytes, shape=shape, dtype=dtype),
write_type_conversion=data_to_bytes)
class ChannelDataNode(DataNode):
def __init__(self, name, **keys):
shape = keys.pop('shape', None)
dtype = keys.pop('dtype', None)
DataNode.__init__(self, 'channel', name, **keys)
if keys.get('create', False):
if shape is not None:
self.info["shape"] = shape
if dtype is not None:
self.info["dtype"] = dtype
cnx = self.db_connection
self._queue = QueueSetting("%s_data" % self.db_name, connection=cnx,
read_type_conversion=functools.partial(data_from_bytes, shape=shape, dtype=dtype),
write_type_conversion=data_to_bytes)
def store(self, signal, event_dict, cnx=None):
if signal == "new_data":
data = event_dict.get("data")
channel = event_dict.get("channel")
if len(channel.shape) == data.ndim:
self._queue.append(data, cnx=cnx)
else:
self._queue.extend(data, cnx=cnx)
def get(self, from_index, to_index=None, cnx=None):
if to_index is None:
return self._queue.get(from_index, from_index, cnx=cnx)
else:
return self._queue.get(from_index, to_index, cnx=cnx)
def __len__(self, cnx=None):
return self._queue.__len__(cnx=cnx)
@property
def shape(self):
return self.info.get("shape")
@property
def dtype(self):
return self.info.get("dtype")
def _get_db_names(self):
db_names = DataNode._get_db_names(self)
db_names.append(self.db_name+"_data")
return db_names
class DataNodeContainer(DataNode):
def __init__(self,
node_type,
name,
parent=None,
connection=None,
create=False):
DataNode.__init__(self,
node_type,
name,
parent=parent,
connection=connection,
create=create)
children_queue_name = '%s_children_list' % self.db_name
self._children = QueueSetting(children_queue_name,
connection=connection)
def add_children(self, *child):
if len(child) > 1:
self._children.extend([c.db_name for c in child])
else:
self._children.append(child[0].db_name)
def children(self, from_id=0, to_id=-1):
"""Iter over children.
@return an iterator
@param from_id start child index
@param to_id last child index
"""
for child_name in self._children.get(from_id, to_id):
new_child = get_node(child_name)
if new_child is not None:
yield new_child
else:
self._children.remove(child_name) # clean
@property
def last_child(self):
return get_node(self._children.get(-1))
def __init__(self,node_type,name,parent = None, connection = None, create=False):
if connection is None:
connection = client.get_cache(db=1)
db_name = '%s:%s' % (parent.db_name(),name) if parent else name
self._data = Struct(db_name,
connection=connection)
children_queue_name = '%s_children_list' % db_name
self._children = QueueSetting(children_queue_name,
connection=connection)
info_hash_name = '%s_info' % db_name
self._info = HashObjSetting(info_hash_name,
connection=connection)
self.db_connection = connection
if create:
self._data.name = name
self._data.db_name = db_name
self._data.node_type = node_type
if parent:
self._data.parent = parent.db_name()
parent.add_children(self)
def store(self, signal, event_dict):
if signal == "new_data":
channel_data = event_dict.get("channel_data")
if channel_data is None:
# warning
return
for channel_name, data in channel_data.iteritems():
if data.size == 0:
continue
queue = self._channels.get(channel_name)
if queue is None:
self._channels_name.append(channel_name)
queue = QueueSetting('%s_%s' %
(self.db_name, channel_name),
connection=self.db_connection)
self._channels[channel_name] = queue
try:
iter(data)
except:
queue.append(data)
else:
queue.extend(data)
class DataNode(object):
default_time_to_live = 24*3600 # 1 day
@staticmethod
def exists(name,parent = None, connection = None):
if connection is None:
connection = client.get_cache(db=1)
db_name = '%s:%s' % (parent.db_name(),name) if parent else name
return db_name if connection.exists(db_name) else None
def __init__(self,node_type,name,parent = None, connection = None, create=False):
if connection is None:
connection = client.get_cache(db=1)
db_name = '%s:%s' % (parent.db_name(),name) if parent else name
self._data = Struct(db_name,
connection=connection)
children_queue_name = '%s_children_list' % db_name
self._children = QueueSetting(children_queue_name,
connection=connection)
info_hash_name = '%s_info' % db_name
self._info = HashObjSetting(info_hash_name,
connection=connection)
self.db_connection = connection
if create:
self._data.name = name
self._data.db_name = db_name
self._data.node_type = node_type
if parent:
self._data.parent = parent.db_name()
parent.add_children(self)
def db_name(self):
return self._data.db_name
def name(self):
return self._data.name
def type(self):
return self._data.node_type
def iterator(self):
return DataNodeIterator(self)
def add_children(self,*child):
if len(child) > 1:
children_no = self._children.extend([c.db_name() for c in child])
else:
children_no = self._children.append(child[0].db_name())
def connect(self, signal, callback):
dispatcher.connect(callback, signal, self)
def parent(self):
parent_name = self._data.parent
if parent_name:
parent = get_node(parent_name)
if parent is None: # clean
del self._data.parent
return parent
#@brief iter over children
#@return an iterator
#@param from_id start child index
#@param to_id last child index
def children(self,from_id = 0,to_id = -1):
for child_name in self._children.get(from_id,to_id):
new_child = get_node(child_name)
if new_child is not None:
yield new_child
else:
self._children.remove(child_name) # clean
def last_child(self):
return get_node(self._children.get(-1))
def set_info(self,key,values):
self._info[keys] = values
if self._ttl > 0:
self._info.ttl(self._ttl)
def info_iteritems(self):
return self._info.iteritems()
def info_get(self,name):
return self._info.get(name)
def data_update(self,keys):
self._data.update(keys)
def set_ttl(self):
redis_conn = client.get_cache(db=1)
redis_conn.expire(self.db_name(), DataNode.default_time_to_live)
self._children.ttl(DataNode.default_time_to_live)
self._info.ttl(DataNode.default_time_to_live)
parent = self.parent()
if parent:
parent.set_ttl()
def store(self, signal, event_dict):
pass
def __init__(self, channel_db_name, cnx):
self._queue = QueueSetting(channel_db_name, connection=cnx)
class Dataset0D(DataNode):
class DataChannel(object):
def __init__(self, channel_db_name, cnx):
self._queue = QueueSetting(channel_db_name, connection=cnx)
def get(self, from_index, to_index=None):
if to_index is None:
return self._queue[from_index]
else:
return self._queue[from_index:to_index]
def __len__(self):
return self._queue.__len__()
def __init__(self, name, **keys):
DataNode.__init__(self, 'zerod', name, **keys)
cnx = self.db_connection
self._channels_name = QueueSetting('%s_channels' % self.db_name,
connection=cnx)
self._channels = {}
for channel_name in self._channels_name:
self._channels[channel_name] = QueueSetting(
'%s_%s' % (self.db_name, channel_name), connection=cnx)
def channels_name(self):
return list(self._channels_name)
def store(self, signal, event_dict):
if signal == "new_data":
channel_data = event_dict.get("channel_data")
if channel_data is None:
# warning
return
for channel_name, data in channel_data.iteritems():
if data.size == 0:
continue
queue = self._channels.get(channel_name)
if queue is None:
self._channels_name.append(channel_name)
queue = QueueSetting('%s_%s' %
(self.db_name, channel_name),
connection=self.db_connection)
self._channels[channel_name] = queue
try:
iter(data)
except:
queue.append(data)
else:
queue.extend(data)
#@brief get data channel object
def get_channel(self, channel_name=None, check_exists=True, cnx=None):
if channel_name is None:
channel_name = self._channels_name[0]
elif check_exists and channel_name not in self._channels_name:
raise ValueError("Unknown channel %s" % channel_name)
channel_db_name = '%s_%s' % (self.db_name, channel_name)
return Dataset0D.DataChannel(
channel_db_name, self.db_connection if cnx is None else cnx)
def get_all_channels(self):
"""
return all channels for this node
the return is a dict {channel_name:DataChannel}
"""
return dict(((chan_name, self.get_channel(chan_name))
for chan_name in self._channels_name))
def _get_db_names(self):
db_names = DataNode._get_db_names(self)
db_names.append(self._channels_name._name)
db_names.extend(
(channel._name for channel in self._channels.itervalues()))
return db_names
class Dataset0D(DataNode):
class DataChannel(object):
def __init__(self, channel_db_name, cnx):
self._queue = QueueSetting(channel_db_name,
connection=cnx)
def get(self, from_index, to_index=None):
if to_index is None:
return self._queue[from_index]
else:
return self._queue[from_index:to_index]
def __len__(self):
return self._queue.__len__()
def __init__(self, name, **keys):
DataNode.__init__(self, 'zerod', name, **keys)
cnx = self.db_connection
self._channels_name = QueueSetting(
'%s_channels' % self.db_name, connection=cnx)
self._channels = {}
for channel_name in self._channels_name:
self._channels[channel_name] = QueueSetting('%s_%s' % (self.db_name, channel_name),
connection=cnx)
def channels_name(self):
return list(self._channels_name)
def store(self, signal, event_dict):
if signal == "new_data":
channel_data = event_dict.get("channel_data")
if channel_data is None:
# warning
return
for channel_name, data in channel_data.iteritems():
if data.size == 0:
continue
queue = self._channels.get(channel_name)
if queue is None:
self._channels_name.append(channel_name)
queue = QueueSetting('%s_%s' % (self.db_name, channel_name),
connection=self.db_connection)
self._channels[channel_name] = queue
try:
iter(data)
except:
queue.append(data)
else:
queue.extend(data)
#@brief get data channel object
def get_channel(self, channel_name=None, check_exists=True, cnx=None):
if channel_name is None:
channel_name = self._channels_name[0]
elif check_exists and channel_name not in self._channels_name:
raise ValueError("Unknown channel %s" % channel_name)
channel_db_name = '%s_%s' % (self.db_name, channel_name)
return Dataset0D.DataChannel(channel_db_name, self.db_connection if cnx is None else cnx)
def get_all_channels(self):
"""
return all channels for this node
the return is a dict {channel_name:DataChannel}
"""
return dict(((chan_name, self.get_channel(chan_name))
for chan_name in self._channels_name))
def _get_db_names(self):
db_names = DataNode._get_db_names(self)
db_names.append(self._channels_name._name)
db_names.extend(
(channel._name for channel in self._channels.itervalues()))
return db_names
def __init__(self, channel_db_name, cnx):
self._queue = QueueSetting(channel_db_name,
connection=cnx)