示例#1
0
 def __init__(self, name, **keys):
     DataNode.__init__(self, 'zerod', name, **keys)
     cnx = self.db_connection
     self._channels_name = QueueSetting('%s_channels' % self.db_name,
                                        connection=cnx)
     self._channels = {}
     for channel_name in self._channels_name:
         self._channels[channel_name] = QueueSetting(
             '%s_%s' % (self.db_name, channel_name), connection=cnx)
示例#2
0
文件: zerod.py 项目: mguijarr/bliss
class Dataset0D(DataNode):
    class DataChannel(object):
        def __init__(self,channel_db_name) :
            cnx = node.get_default_connection()
            self._queue = QueueSetting(channel_db_name,
                                       connection=cnx).get_proxy()
        def get(self,from_index,to_index = None):
            if to_index is None:
                return self._queue[from_index]
            else:
                return self._queue[from_index:to_index]

    def __init__(self,name,**keys):
        DataNode.__init__(self,'zerod',name,**keys)
        cnx = self.db_connection
        self._channels_name = QueueSetting('%s_channels' % self.db_name(),connection=cnx)
        self._channels = {}
        for channel_name in self._channels_name:
            self._channels[channel_name] = QueueSetting('%s_%s' % (self.db_name(),channel_name),
                                                        connection=cnx)
    def channel_name(self) :
        return self._channels.get()

    def store(self,signal,event_dict) :
        if signal == "new_data":
            channel_data = event_dict.get("channel_data")
            if channel_data is None:
                #warning
                return
            for channel_name,data in channel_data.iteritems():
                queue = self._channels.get(channel_name)
                if queue is None:
                    self._channels_name.append(channel_name)
                    queue = QueueSetting('%s_%s' % (self.db_name(),channel_name),
                                         connection=self.db_connection)
                    queue.extend(data)
                    self._channels[channel_name] = queue
                else:
                    queue.extend(data)

    #@brief get data channel object
    def get_channel(self,channel_name = None) :
        if channel_name is None:
            channel_name = self._channels[0]
        channel_db_name = '%s_%s' % (self.db_name(),channel_name)
        return Dataset0D.DataChannel(channel_db_name)

    def set_ttl(self):
        DataNode.set_ttl(self)
        for channel in self._channels.itervalues():
            channel.ttl(DataNode.default_time_to_live)
示例#3
0
class DataNodeContainer(DataNode):
    def __init__(self, node_type, name, parent=None, connection=None, create=False):
        DataNode.__init__(self, node_type, name,
                          parent=parent, connection=connection, create=create)

        children_queue_name = '%s_children_list' % self.db_name
        self._children = QueueSetting(
            children_queue_name, connection=connection)

    def add_children(self, *child):
        if len(child) > 1:
            self._children.extend([c.db_name for c in child])
        else:
            self._children.append(child[0].db_name)

    def children(self, from_id=0, to_id=-1):
        """Iter over children.

        @return an iterator
        @param from_id start child index
        @param to_id last child index
        """
        for child_name in self._children.get(from_id, to_id):
            new_child = get_node(child_name)
            if new_child is not None:
                yield new_child
            else:
                self._children.remove(child_name)  # clean

    @property
    def last_child(self):
        return get_node(self._children.get(-1))
示例#4
0
    def __init__(self, node_type, name, parent=None, connection=None, create=False):
        DataNode.__init__(self, node_type, name,
                          parent=parent, connection=connection, create=create)

        children_queue_name = '%s_children_list' % self.db_name
        self._children = QueueSetting(
            children_queue_name, connection=connection)
示例#5
0
文件: zerod.py 项目: mguijarr/bliss
 def store(self,signal,event_dict) :
     if signal == "new_data":
         channel_data = event_dict.get("channel_data")
         if channel_data is None:
             #warning
             return
         for channel_name,data in channel_data.iteritems():
             queue = self._channels.get(channel_name)
             if queue is None:
                 self._channels_name.append(channel_name)
                 queue = QueueSetting('%s_%s' % (self.db_name(),channel_name),
                                      connection=self.db_connection)
                 queue.extend(data)
                 self._channels[channel_name] = queue
             else:
                 queue.extend(data)
示例#6
0
    def __init__(self,
                 node_type,
                 name,
                 parent=None,
                 connection=None,
                 create=False):
        DataNode.__init__(self,
                          node_type,
                          name,
                          parent=parent,
                          connection=connection,
                          create=create)

        children_queue_name = '%s_children_list' % self.db_name
        self._children = QueueSetting(children_queue_name,
                                      connection=connection)
示例#7
0
文件: zerod.py 项目: mguijarr/bliss
 def __init__(self,name,**keys):
     DataNode.__init__(self,'zerod',name,**keys)
     cnx = self.db_connection
     self._channels_name = QueueSetting('%s_channels' % self.db_name(),connection=cnx)
     self._channels = {}
     for channel_name in self._channels_name:
         self._channels[channel_name] = QueueSetting('%s_%s' % (self.db_name(),channel_name),
                                                     connection=cnx)
示例#8
0
    def __init__(self, name, **keys):
        shape = keys.pop('shape', None)
        dtype = keys.pop('dtype', None)

        DataNode.__init__(self, 'channel', name, **keys)

        if keys.get('create', False):
            if shape is not None:
                self.info["shape"] = shape
            if dtype is not None:
                self.info["dtype"] = dtype

        cnx = self.db_connection
        self._queue = QueueSetting("%s_data" % self.db_name,
                                   connection=cnx,
                                   read_type_conversion=functools.partial(
                                       data_from_bytes,
                                       shape=shape,
                                       dtype=dtype),
                                   write_type_conversion=data_to_bytes)
示例#9
0
    class DataChannel(object):
        def __init__(self, channel_db_name, cnx):
            self._queue = QueueSetting(channel_db_name, connection=cnx)

        def get(self, from_index, to_index=None):
            if to_index is None:
                return self._queue[from_index]
            else:
                return self._queue[from_index:to_index]

        def __len__(self):
            return self._queue.__len__()
示例#10
0
    class DataChannel(object):
        def __init__(self, channel_db_name, cnx):
            self._queue = QueueSetting(channel_db_name,
                                       connection=cnx)

        def get(self, from_index, to_index=None):
            if to_index is None:
                return self._queue[from_index]
            else:
                return self._queue[from_index:to_index]

        def __len__(self):
            return self._queue.__len__()
示例#11
0
class ChannelDataNode(DataNode):
    def __init__(self, name, **keys):
        shape = keys.pop('shape', None)
        dtype = keys.pop('dtype', None)

        DataNode.__init__(self, 'channel', name, **keys)

        if keys.get('create', False):
            if shape is not None:
                self.info["shape"] = shape
            if dtype is not None:
                self.info["dtype"] = dtype

        cnx = self.db_connection
        self._queue = QueueSetting("%s_data" % self.db_name,
                                   connection=cnx,
                                   read_type_conversion=functools.partial(
                                       data_from_bytes,
                                       shape=shape,
                                       dtype=dtype),
                                   write_type_conversion=data_to_bytes)

    def store(self, signal, event_dict, cnx=None):
        if signal == "new_data":
            data = event_dict.get("data")
            channel = event_dict.get("channel")
            if len(channel.shape) == data.ndim:
                self._queue.append(data, cnx=cnx)
            else:
                self._queue.extend(data, cnx=cnx)

    def get(self, from_index, to_index=None, cnx=None):
        if to_index is None:
            return self._queue.get(from_index, from_index, cnx=cnx)
        else:
            return self._queue.get(from_index, to_index, cnx=cnx)

    def __len__(self, cnx=None):
        return self._queue.__len__(cnx=cnx)

    @property
    def shape(self):
        return self.info.get("shape")

    @property
    def dtype(self):
        return self.info.get("dtype")

    def _get_db_names(self):
        db_names = DataNode._get_db_names(self)
        db_names.append(self.db_name + "_data")
        return db_names
示例#12
0
    def __init__(self, name, **keys):
        shape = keys.pop('shape', None)
        dtype = keys.pop('dtype', None)

        DataNode.__init__(self, 'channel', name, **keys)
    
        if keys.get('create', False):
            if shape is not None:
                self.info["shape"] = shape
            if dtype is not None:
                self.info["dtype"] = dtype

        cnx = self.db_connection
        self._queue = QueueSetting("%s_data" % self.db_name, connection=cnx,
                                   read_type_conversion=functools.partial(data_from_bytes, shape=shape, dtype=dtype),
                                   write_type_conversion=data_to_bytes)
示例#13
0
class ChannelDataNode(DataNode):
    def __init__(self, name, **keys):
        shape = keys.pop('shape', None)
        dtype = keys.pop('dtype', None)

        DataNode.__init__(self, 'channel', name, **keys)
    
        if keys.get('create', False):
            if shape is not None:
                self.info["shape"] = shape
            if dtype is not None:
                self.info["dtype"] = dtype

        cnx = self.db_connection
        self._queue = QueueSetting("%s_data" % self.db_name, connection=cnx,
                                   read_type_conversion=functools.partial(data_from_bytes, shape=shape, dtype=dtype),
                                   write_type_conversion=data_to_bytes)

    def store(self, signal, event_dict, cnx=None):
        if signal == "new_data":
            data = event_dict.get("data")
            channel = event_dict.get("channel")
            if len(channel.shape) == data.ndim:
                self._queue.append(data, cnx=cnx)
            else:
                self._queue.extend(data, cnx=cnx)

    def get(self, from_index, to_index=None, cnx=None):
        if to_index is None:
            return self._queue.get(from_index, from_index, cnx=cnx)
        else:
            return self._queue.get(from_index, to_index, cnx=cnx)

    def __len__(self, cnx=None):
        return self._queue.__len__(cnx=cnx)

    @property
    def shape(self):
        return self.info.get("shape")

    @property
    def dtype(self):
        return self.info.get("dtype")

    def _get_db_names(self):
        db_names = DataNode._get_db_names(self)
        db_names.append(self.db_name+"_data")
        return db_names
示例#14
0
class DataNodeContainer(DataNode):
    def __init__(self,
                 node_type,
                 name,
                 parent=None,
                 connection=None,
                 create=False):
        DataNode.__init__(self,
                          node_type,
                          name,
                          parent=parent,
                          connection=connection,
                          create=create)

        children_queue_name = '%s_children_list' % self.db_name
        self._children = QueueSetting(children_queue_name,
                                      connection=connection)

    def add_children(self, *child):
        if len(child) > 1:
            self._children.extend([c.db_name for c in child])
        else:
            self._children.append(child[0].db_name)

    def children(self, from_id=0, to_id=-1):
        """Iter over children.

        @return an iterator
        @param from_id start child index
        @param to_id last child index
        """
        for child_name in self._children.get(from_id, to_id):
            new_child = get_node(child_name)
            if new_child is not None:
                yield new_child
            else:
                self._children.remove(child_name)  # clean

    @property
    def last_child(self):
        return get_node(self._children.get(-1))
示例#15
0
 def __init__(self,node_type,name,parent = None, connection = None, create=False):
     if connection is None:
         connection = client.get_cache(db=1)
     db_name = '%s:%s' % (parent.db_name(),name) if parent else name
     self._data = Struct(db_name,
                         connection=connection)
     children_queue_name = '%s_children_list' % db_name
     self._children = QueueSetting(children_queue_name,
                                   connection=connection)
     info_hash_name = '%s_info' % db_name
     self._info = HashObjSetting(info_hash_name,
                                 connection=connection)
     self.db_connection = connection
     
     if create:
         self._data.name = name
         self._data.db_name = db_name
         self._data.node_type = node_type
         if parent: 
             self._data.parent = parent.db_name()
             parent.add_children(self)
示例#16
0
 def store(self, signal, event_dict):
     if signal == "new_data":
         channel_data = event_dict.get("channel_data")
         if channel_data is None:
             # warning
             return
         for channel_name, data in channel_data.iteritems():
             if data.size == 0:
                 continue
             queue = self._channels.get(channel_name)
             if queue is None:
                 self._channels_name.append(channel_name)
                 queue = QueueSetting('%s_%s' %
                                      (self.db_name, channel_name),
                                      connection=self.db_connection)
                 self._channels[channel_name] = queue
             try:
                 iter(data)
             except:
                 queue.append(data)
             else:
                 queue.extend(data)
示例#17
0
class DataNode(object):
    default_time_to_live = 24*3600 # 1 day
    
    @staticmethod
    def exists(name,parent = None, connection = None):
        if connection is None:
            connection = client.get_cache(db=1)
        db_name = '%s:%s' % (parent.db_name(),name) if parent else name
        return db_name if connection.exists(db_name) else None

    def __init__(self,node_type,name,parent = None, connection = None, create=False):
        if connection is None:
            connection = client.get_cache(db=1)
        db_name = '%s:%s' % (parent.db_name(),name) if parent else name
        self._data = Struct(db_name,
                            connection=connection)
        children_queue_name = '%s_children_list' % db_name
        self._children = QueueSetting(children_queue_name,
                                      connection=connection)
        info_hash_name = '%s_info' % db_name
        self._info = HashObjSetting(info_hash_name,
                                    connection=connection)
        self.db_connection = connection
        
        if create:
            self._data.name = name
            self._data.db_name = db_name
            self._data.node_type = node_type
            if parent: 
                self._data.parent = parent.db_name()
                parent.add_children(self)

    def db_name(self):
        return self._data.db_name

    def name(self):
        return self._data.name

    def type(self):
        return self._data.node_type

    def iterator(self):
        return DataNodeIterator(self)

    def add_children(self,*child):
        if len(child) > 1:
            children_no = self._children.extend([c.db_name() for c in child])
        else:
            children_no = self._children.append(child[0].db_name())

    def connect(self, signal, callback):
        dispatcher.connect(callback, signal, self)

    def parent(self):
        parent_name = self._data.parent
        if parent_name:
            parent = get_node(parent_name)
            if parent is None:  # clean
                del self._data.parent
            return parent

    #@brief iter over children
    #@return an iterator
    #@param from_id start child index
    #@param to_id last child index
    def children(self,from_id = 0,to_id = -1):
        for child_name in self._children.get(from_id,to_id):
            new_child = get_node(child_name)
            if new_child is not None:
                yield new_child
            else:
                self._children.remove(child_name) # clean

    def last_child(self):
        return get_node(self._children.get(-1))

    def set_info(self,key,values):
        self._info[keys] = values
        if self._ttl > 0:
            self._info.ttl(self._ttl)

    def info_iteritems(self):
        return self._info.iteritems()

    def info_get(self,name):
        return self._info.get(name)

    def data_update(self,keys):
        self._data.update(keys)

    def set_ttl(self):
        redis_conn = client.get_cache(db=1)
	redis_conn.expire(self.db_name(), DataNode.default_time_to_live)
	self._children.ttl(DataNode.default_time_to_live)
	self._info.ttl(DataNode.default_time_to_live)
        parent = self.parent()
	if parent:
	   parent.set_ttl()

    def store(self, signal, event_dict):
        pass
示例#18
0
 def __init__(self, channel_db_name, cnx):
     self._queue = QueueSetting(channel_db_name, connection=cnx)
示例#19
0
class Dataset0D(DataNode):
    class DataChannel(object):
        def __init__(self, channel_db_name, cnx):
            self._queue = QueueSetting(channel_db_name, connection=cnx)

        def get(self, from_index, to_index=None):
            if to_index is None:
                return self._queue[from_index]
            else:
                return self._queue[from_index:to_index]

        def __len__(self):
            return self._queue.__len__()

    def __init__(self, name, **keys):
        DataNode.__init__(self, 'zerod', name, **keys)
        cnx = self.db_connection
        self._channels_name = QueueSetting('%s_channels' % self.db_name,
                                           connection=cnx)
        self._channels = {}
        for channel_name in self._channels_name:
            self._channels[channel_name] = QueueSetting(
                '%s_%s' % (self.db_name, channel_name), connection=cnx)

    def channels_name(self):
        return list(self._channels_name)

    def store(self, signal, event_dict):
        if signal == "new_data":
            channel_data = event_dict.get("channel_data")
            if channel_data is None:
                # warning
                return
            for channel_name, data in channel_data.iteritems():
                if data.size == 0:
                    continue
                queue = self._channels.get(channel_name)
                if queue is None:
                    self._channels_name.append(channel_name)
                    queue = QueueSetting('%s_%s' %
                                         (self.db_name, channel_name),
                                         connection=self.db_connection)
                    self._channels[channel_name] = queue
                try:
                    iter(data)
                except:
                    queue.append(data)
                else:
                    queue.extend(data)

    #@brief get data channel object
    def get_channel(self, channel_name=None, check_exists=True, cnx=None):
        if channel_name is None:
            channel_name = self._channels_name[0]
        elif check_exists and channel_name not in self._channels_name:
            raise ValueError("Unknown channel %s" % channel_name)

        channel_db_name = '%s_%s' % (self.db_name, channel_name)
        return Dataset0D.DataChannel(
            channel_db_name, self.db_connection if cnx is None else cnx)

    def get_all_channels(self):
        """
        return all channels for this node
        the return is a dict {channel_name:DataChannel}
        """
        return dict(((chan_name, self.get_channel(chan_name))
                     for chan_name in self._channels_name))

    def _get_db_names(self):
        db_names = DataNode._get_db_names(self)
        db_names.append(self._channels_name._name)
        db_names.extend(
            (channel._name for channel in self._channels.itervalues()))
        return db_names
示例#20
0
class Dataset0D(DataNode):
    class DataChannel(object):
        def __init__(self, channel_db_name, cnx):
            self._queue = QueueSetting(channel_db_name,
                                       connection=cnx)

        def get(self, from_index, to_index=None):
            if to_index is None:
                return self._queue[from_index]
            else:
                return self._queue[from_index:to_index]

        def __len__(self):
            return self._queue.__len__()

    def __init__(self, name, **keys):
        DataNode.__init__(self, 'zerod', name, **keys)
        cnx = self.db_connection
        self._channels_name = QueueSetting(
            '%s_channels' % self.db_name, connection=cnx)
        self._channels = {}
        for channel_name in self._channels_name:
            self._channels[channel_name] = QueueSetting('%s_%s' % (self.db_name, channel_name),
                                                        connection=cnx)

    def channels_name(self):
        return list(self._channels_name)

    def store(self, signal, event_dict):
        if signal == "new_data":
            channel_data = event_dict.get("channel_data")
            if channel_data is None:
                # warning
                return
            for channel_name, data in channel_data.iteritems():
                if data.size == 0:
                    continue
                queue = self._channels.get(channel_name)
                if queue is None:
                    self._channels_name.append(channel_name)
                    queue = QueueSetting('%s_%s' % (self.db_name, channel_name),
                                         connection=self.db_connection)
                    self._channels[channel_name] = queue
                try:
                    iter(data)
                except:
                    queue.append(data)
                else:
                    queue.extend(data)

    #@brief get data channel object
    def get_channel(self, channel_name=None, check_exists=True, cnx=None):
        if channel_name is None:
            channel_name = self._channels_name[0]
        elif check_exists and channel_name not in self._channels_name:
            raise ValueError("Unknown channel %s" % channel_name)

        channel_db_name = '%s_%s' % (self.db_name, channel_name)
        return Dataset0D.DataChannel(channel_db_name, self.db_connection if cnx is None else cnx)

    def get_all_channels(self):
        """
        return all channels for this node
        the return is a dict {channel_name:DataChannel}
        """
        return dict(((chan_name, self.get_channel(chan_name))
                     for chan_name in self._channels_name))

    def _get_db_names(self):
        db_names = DataNode._get_db_names(self)
        db_names.append(self._channels_name._name)
        db_names.extend(
            (channel._name for channel in self._channels.itervalues()))
        return db_names
示例#21
0
 def __init__(self, channel_db_name, cnx):
     self._queue = QueueSetting(channel_db_name,
                                connection=cnx)