class DataNodeContainer(DataNode): def __init__(self, node_type, name, parent=None, connection=None, create=False): DataNode.__init__(self, node_type, name, parent=parent, connection=connection, create=create) children_queue_name = '%s_children_list' % self.db_name self._children = QueueSetting( children_queue_name, connection=connection) def add_children(self, *child): if len(child) > 1: self._children.extend([c.db_name for c in child]) else: self._children.append(child[0].db_name) def children(self, from_id=0, to_id=-1): """Iter over children. @return an iterator @param from_id start child index @param to_id last child index """ for child_name in self._children.get(from_id, to_id): new_child = get_node(child_name) if new_child is not None: yield new_child else: self._children.remove(child_name) # clean @property def last_child(self): return get_node(self._children.get(-1))
class ChannelDataNode(DataNode): def __init__(self, name, **keys): shape = keys.pop('shape', None) dtype = keys.pop('dtype', None) DataNode.__init__(self, 'channel', name, **keys) if keys.get('create', False): if shape is not None: self.info["shape"] = shape if dtype is not None: self.info["dtype"] = dtype cnx = self.db_connection self._queue = QueueSetting("%s_data" % self.db_name, connection=cnx, read_type_conversion=functools.partial( data_from_bytes, shape=shape, dtype=dtype), write_type_conversion=data_to_bytes) def store(self, signal, event_dict, cnx=None): if signal == "new_data": data = event_dict.get("data") channel = event_dict.get("channel") if len(channel.shape) == data.ndim: self._queue.append(data, cnx=cnx) else: self._queue.extend(data, cnx=cnx) def get(self, from_index, to_index=None, cnx=None): if to_index is None: return self._queue.get(from_index, from_index, cnx=cnx) else: return self._queue.get(from_index, to_index, cnx=cnx) def __len__(self, cnx=None): return self._queue.__len__(cnx=cnx) @property def shape(self): return self.info.get("shape") @property def dtype(self): return self.info.get("dtype") def _get_db_names(self): db_names = DataNode._get_db_names(self) db_names.append(self.db_name + "_data") return db_names
class ChannelDataNode(DataNode): def __init__(self, name, **keys): shape = keys.pop('shape', None) dtype = keys.pop('dtype', None) DataNode.__init__(self, 'channel', name, **keys) if keys.get('create', False): if shape is not None: self.info["shape"] = shape if dtype is not None: self.info["dtype"] = dtype cnx = self.db_connection self._queue = QueueSetting("%s_data" % self.db_name, connection=cnx, read_type_conversion=functools.partial(data_from_bytes, shape=shape, dtype=dtype), write_type_conversion=data_to_bytes) def store(self, signal, event_dict, cnx=None): if signal == "new_data": data = event_dict.get("data") channel = event_dict.get("channel") if len(channel.shape) == data.ndim: self._queue.append(data, cnx=cnx) else: self._queue.extend(data, cnx=cnx) def get(self, from_index, to_index=None, cnx=None): if to_index is None: return self._queue.get(from_index, from_index, cnx=cnx) else: return self._queue.get(from_index, to_index, cnx=cnx) def __len__(self, cnx=None): return self._queue.__len__(cnx=cnx) @property def shape(self): return self.info.get("shape") @property def dtype(self): return self.info.get("dtype") def _get_db_names(self): db_names = DataNode._get_db_names(self) db_names.append(self.db_name+"_data") return db_names
def store(self,signal,event_dict) : if signal == "new_data": channel_data = event_dict.get("channel_data") if channel_data is None: #warning return for channel_name,data in channel_data.iteritems(): queue = self._channels.get(channel_name) if queue is None: self._channels_name.append(channel_name) queue = QueueSetting('%s_%s' % (self.db_name(),channel_name), connection=self.db_connection) queue.extend(data) self._channels[channel_name] = queue else: queue.extend(data)
class DataNodeContainer(DataNode): def __init__(self, node_type, name, parent=None, connection=None, create=False): DataNode.__init__(self, node_type, name, parent=parent, connection=connection, create=create) children_queue_name = '%s_children_list' % self.db_name self._children = QueueSetting(children_queue_name, connection=connection) def add_children(self, *child): if len(child) > 1: self._children.extend([c.db_name for c in child]) else: self._children.append(child[0].db_name) def children(self, from_id=0, to_id=-1): """Iter over children. @return an iterator @param from_id start child index @param to_id last child index """ for child_name in self._children.get(from_id, to_id): new_child = get_node(child_name) if new_child is not None: yield new_child else: self._children.remove(child_name) # clean @property def last_child(self): return get_node(self._children.get(-1))
def store(self, signal, event_dict): if signal == "new_data": channel_data = event_dict.get("channel_data") if channel_data is None: # warning return for channel_name, data in channel_data.iteritems(): if data.size == 0: continue queue = self._channels.get(channel_name) if queue is None: self._channels_name.append(channel_name) queue = QueueSetting('%s_%s' % (self.db_name, channel_name), connection=self.db_connection) self._channels[channel_name] = queue try: iter(data) except: queue.append(data) else: queue.extend(data)
class DataNode(object): default_time_to_live = 24*3600 # 1 day @staticmethod def exists(name,parent = None, connection = None): if connection is None: connection = client.get_cache(db=1) db_name = '%s:%s' % (parent.db_name(),name) if parent else name return db_name if connection.exists(db_name) else None def __init__(self,node_type,name,parent = None, connection = None, create=False): if connection is None: connection = client.get_cache(db=1) db_name = '%s:%s' % (parent.db_name(),name) if parent else name self._data = Struct(db_name, connection=connection) children_queue_name = '%s_children_list' % db_name self._children = QueueSetting(children_queue_name, connection=connection) info_hash_name = '%s_info' % db_name self._info = HashObjSetting(info_hash_name, connection=connection) self.db_connection = connection if create: self._data.name = name self._data.db_name = db_name self._data.node_type = node_type if parent: self._data.parent = parent.db_name() parent.add_children(self) def db_name(self): return self._data.db_name def name(self): return self._data.name def type(self): return self._data.node_type def iterator(self): return DataNodeIterator(self) def add_children(self,*child): if len(child) > 1: children_no = self._children.extend([c.db_name() for c in child]) else: children_no = self._children.append(child[0].db_name()) def connect(self, signal, callback): dispatcher.connect(callback, signal, self) def parent(self): parent_name = self._data.parent if parent_name: parent = get_node(parent_name) if parent is None: # clean del self._data.parent return parent #@brief iter over children #@return an iterator #@param from_id start child index #@param to_id last child index def children(self,from_id = 0,to_id = -1): for child_name in self._children.get(from_id,to_id): new_child = get_node(child_name) if new_child is not None: yield new_child else: self._children.remove(child_name) # clean def last_child(self): return get_node(self._children.get(-1)) def set_info(self,key,values): self._info[keys] = values if self._ttl > 0: self._info.ttl(self._ttl) def info_iteritems(self): return self._info.iteritems() def info_get(self,name): return self._info.get(name) def data_update(self,keys): self._data.update(keys) def set_ttl(self): redis_conn = client.get_cache(db=1) redis_conn.expire(self.db_name(), DataNode.default_time_to_live) self._children.ttl(DataNode.default_time_to_live) self._info.ttl(DataNode.default_time_to_live) parent = self.parent() if parent: parent.set_ttl() def store(self, signal, event_dict): pass