Example #1
0
    def __init__(self, app_name, self_name, cluster_list, max_partition,
                 partition_update_cb, zk_server):

        # Initialize local variables
        self._zk_server = zk_server
        self._cluster_list = set(cluster_list)
        self._max_partition = max_partition
        self._update_cb = partition_update_cb
        self._curr_part_ownership_list = []
        self._target_part_ownership_list = []
        self._con_hash = ConsistentHash(cluster_list)
        self._name = self_name

        # some sanity check
        if not (self._name in cluster_list):
            raise ValueError('cluster list is missing local server name')

        # initialize logging and other stuff
        logging.basicConfig()
        self._conn_state = None
        self._sandesh_connection_info_update(status='INIT', message='')

        # connect to zookeeper
        self._zk = KazooClient(zk_server)
        while True:
            try:
                self._zk.start()
                break
            except gevent.event.Timeout as e:
                # Update connection info
                self._sandesh_connection_info_update(status='DOWN',
                                                     message=str(e))
                gevent.sleep(1)
            # Zookeeper is also throwing exception due to delay in master election
            except Exception as e:
                # Update connection info
                self._sandesh_connection_info_update(status='DOWN',
                                                     message=str(e))
                gevent.sleep(1)
        # Update connection info
        self._sandesh_connection_info_update(status='UP', message='')
        # Done connecting to ZooKeeper

        # create a lock array to contain locks for each partition
        self._part_locks = []
        for part in range(0, self._max_partition):
            lockpath = "/lockpath/" + app_name + "/" + str(part)
            l = self._zk.Lock(lockpath, self._name)
            self._part_locks.append(l)

        # initialize partition # to lock acquire greenlet dictionary
        self._part_lock_task_dict = {}

        # update target partition ownership list
        for part in range(0, self._max_partition):
            if (self._con_hash.get_node(str(part)) == self._name):
                self._target_part_ownership_list.append(part)

        # update current ownership list
        self._acquire_partition_ownership()
def process(udp_clients):
    client_ring = ConsistentHash(udp_clients)
    hash_codes = set()
    # PUT all users.
    for u in USERS:
        data_bytes, key = serialize_PUT(u)
        response = client_ring.get_node(key).send(data_bytes)
        print(response)
        hash_codes.add(str(response.decode()))

    print(
        f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
    )

    # GET all users.
    for hc in hash_codes:
        print(hc)
        data_bytes, key = serialize_GET(hc)
        response = client_ring.get_node(key).send(data_bytes)
        print(response)

    # DELETE
    for hc in hash_codes:
        print(hc)
        data_bytes, key = serialize_DELETE(hc)
        response = client_ring.get_node(key).send(data_bytes)
        print(response)
Example #3
0
 def test_something(self):
     shards = [u'redis1', u'redis2', u'redis3', u'redis4']
     users = [
         u'-84942321036308', u'-76029520310209', u'-68343931116147',
         u'-54921760962352'
     ]
     sharder = ConsistentHash(shards)
     expected = ['redis4', 'redis4', 'redis2', 'redis3']
     actual = []
     for user in users:
         shard = sharder.get_shard(user)
         actual.append(shard)
     self.assertEqual(expected, actual)
Example #4
0
 def _consistent_hash(self, members):
     if self._con_hash is None:
         self._con_hash = ConsistentHash(members)
         self._supress_log('members:', self._con_hash.nodes)
     cur, updtd = set(self._con_hash.nodes), set(members)
     if cur != updtd:
         newm = updtd - cur
         rmvd = cur - updtd
         if newm:
             self._supress_log('new workers:', newm)
             self._con_hash.add_nodes(list(newm))
         if rmvd:
             self._supress_log('workers left:', rmvd)
             self._con_hash.del_nodes(list(rmvd))
     return self._con_hash
 def _consistent_hash(self, members):
     if self._con_hash is None:
         self._con_hash = ConsistentHash(members)
         self._logger.error('members: %s' % (str(self._con_hash.nodes)))
     cur, updtd = set(self._con_hash.nodes), set(members)
     if cur != updtd:
         newm = updtd - cur
         rmvd = cur - updtd
         if newm:
             self._logger.error('new members: %s' % (str(newm)))
             self._con_hash.add_nodes(list(newm))
         if rmvd:
             self._logger.error('members left: %s' % (str(rmvd)))
             self._con_hash.del_nodes(list(rmvd))
     return self._con_hash
 def __init__(self, *args, **kwargs):
     """ 
     A memcache subclass. It currently allows you to add or delete a new 
     host at run time. It also checks if a memcache server is down and 
     automatically readjusts if a memcached server is not reachable.
     """
     super(ConsistentMemcachedClient, self).__init__(*args, **kwargs)
     self.hash_manager = ConsistentHash(self.servers)
 def _remove_dead_server(self, server):
     """
     Reconfigure hashing by removing the server that is not responding
     """
     try:
         self.hash_manager.nodes.remove(server)
         self.hash_manager = ConsistentHash(self.hash_manager.nodes)
     except ValueError:
         raise ValueError('no data store is functioning, cannot process request')
class MemcacheClient(memcache.Client):
    """ A memcache subclass. It currently allows you to add a new host at run
time.

only 1/N th cache is wiped out, where N is the number of cache servers.
"""
    def __init__(self, replicas=1,*args, **kwargs):
        self.consistent_hash=ConsistentHash(replicas=replicas)
        super(MemcacheClient, self).__init__(*args, **kwargs)
        

    def _get_server(self, key):
        """ Current implementation of Memcac he client"""
        for i in range(memcache.Client._SERVER_RETRIES):
          server = self.buckets[self.get_server_index(key)]
          if server.connect():
            return server, key
          return None, None
        
    def set_servers(self, servers):
        ret=super(MemcacheClient, self).set_servers(servers)
        self.consistent_hash.setup_servers(servers=self.servers)
        return ret
   
    def get_server_index(self,key):
        '''Returns the number of the machine which key gets sent to.'''
        return self.consistent_hash.get_machine(key)
        

    def add_server(self, server):
        """ Adds a host at runtime to client"""
        # Create a new host entry
        server = memcache._Host(
            server, self.debug, dead_retry=self.dead_retry,
            socket_timeout=self.socket_timeout,
            flush_on_reconnect=self.flush_on_reconnect
        )
        # Add this to our server choices
        self.servers.append(server)
        # Update our buckets
        self.buckets.append(server)
        self.consistent_hash.add_machine(server,len(self.servers)-1)
Example #9
0
 def redistribute_data(self):
     cache = copy.deepcopy(self.storage.map)
     for key, value in cache.iteritems():
         # set key and value in the corresponding node
         self.set(key, value)
         # remove data from this node if necessary
         node_id = ConsistentHash(len(self.config.node_ips),
                                  NUM_HASH_REPLICAS).get_machine(key)
         if node_id != self.config.node_id and \
             (node_id + 1) % len(self.config.node_ips) != self.config.node_id:
             self.storage.remove(key)
class MemcacheClient(memcache.Client):
    """ A memcache subclass. It currently allows you to add a new host at run
time.

only 1/N th cache is wiped out, where N is the number of cache servers.
"""
    def __init__(self, replicas=1, *args, **kwargs):
        self.consistent_hash = ConsistentHash(replicas=replicas)
        super(MemcacheClient, self).__init__(*args, **kwargs)

    def _get_server(self, key):
        """ Current implementation of Memcac he client"""
        for i in range(memcache.Client._SERVER_RETRIES):
            server = self.buckets[self.get_server_index(key)]
            if server.connect():
                return server, key
            return None, None

    def set_servers(self, servers):
        ret = super(MemcacheClient, self).set_servers(servers)
        self.consistent_hash.setup_servers(servers=self.servers)
        return ret

    def get_server_index(self, key):
        '''Returns the number of the machine which key gets sent to.'''
        return self.consistent_hash.get_machine(key)

    def add_server(self, server):
        """ Adds a host at runtime to client"""
        # Create a new host entry
        server = memcache._Host(server,
                                self.debug,
                                dead_retry=self.dead_retry,
                                socket_timeout=self.socket_timeout,
                                flush_on_reconnect=self.flush_on_reconnect)
        # Add this to our server choices
        self.servers.append(server)
        # Update our buckets
        self.buckets.append(server)
        self.consistent_hash.add_machine(server, len(self.servers) - 1)
Example #11
0
    def __init__(self, app_name, self_name, cluster_list, max_partition,
                 partition_update_cb, zk_server):

        # Initialize local variables
        self._zk_server = zk_server
        self._cluster_list = set(cluster_list)
        self._max_partition = max_partition
        self._update_cb = partition_update_cb
        self._curr_part_ownership_list = []
        self._target_part_ownership_list = []
        self._con_hash = ConsistentHash(cluster_list)
        self._name = self_name

        # some sanity check
        if not (self._name in cluster_list):
            raise ValueError('cluster list is missing local server name')

        # connect to zookeeper
        self._zk = KazooClient(zk_server)
        self._zk.start()

        # create a lock array to contain locks for each partition
        self._part_locks = []
        for part in range(0, self._max_partition):
            lockpath = "/lockpath/" + app_name + "/" + str(part)
            l = self._zk.Lock(lockpath, self._name)
            self._part_locks.append(l)

        # initialize partition # to lock acquire greenlet dictionary
        self._part_lock_task_dict = {}

        # update target partition ownership list
        for part in range(0, self._max_partition):
            if (self._con_hash.get_node(str(part)) == self._name):
                self._target_part_ownership_list.append(part)

        # update current ownership list
        self._acquire_partition_ownership()
Example #12
0
 def set(self, key, value):
     ''' Add the pair key-value to the cache cluster. If the data is not supposed
     to be stored in this node according to the hash function, the node calls the
     right node in the cluster.
     '''
     node_id = ConsistentHash(len(self.config.node_ips),
                              NUM_HASH_REPLICAS).get_machine(key)
     if node_id == self.config.node_id:
         self.storage.add(key, value)
         next_node = (node_id + 1) % len(self.config.node_ips)
         # store replica in another node
         NodeClient.set_replica(self.config.node_ips[next_node], key, value)
     else:
         # set in node
         NodeClient.set(self.config.node_ips[node_id], key, value)
 def _consistent_hash(self, members):
     if self._con_hash is None:
         self._con_hash = ConsistentHash(members)
         self._logger.error('members: %s' % (str(self._con_hash.nodes)))
     cur, updtd = set(self._con_hash.nodes), set(members)
     if cur != updtd:
         newm = updtd - cur
         rmvd = cur - updtd
         if newm:
             self._logger.error('new members: %s' % (str(newm)))
             self._con_hash.add_nodes(list(newm))
         if rmvd:
             self._logger.error('members left: %s' % (str(rmvd)))
             self._con_hash.del_nodes(list(rmvd))
     return self._con_hash
 def _consistent_hash(self, members):
     if self._con_hash is None:
         self._con_hash = ConsistentHash(members)
         self._supress_log('members:', self._con_hash.nodes)
     cur, updtd = set(self._con_hash.nodes), set(members)
     if cur != updtd:
         newm = updtd - cur
         rmvd = cur - updtd
         if newm:
             self._supress_log('new workers:', newm)
             self._con_hash.add_nodes(list(newm))
         if rmvd:
             self._supress_log('workers left:', rmvd)
             self._con_hash.del_nodes(list(rmvd))
     return self._con_hash
Example #15
0
    def get(self, key):
        ''' Get the value associated to key. If the key is not found in this node, the
        node calls the right node in the cluster
        '''
        # Due to data replication, key may be in this node even
        # if the corresponding hash points to a different node
        value = self.storage.get(key)
        if value is not None:
            return value

        # if value is not found, search in corresponding node
        node_id = ConsistentHash(len(self.config.node_ips),
                                 NUM_HASH_REPLICAS).get_machine(key)
        if node_id == self.config.node_id:
            # The key being searched does not exist in the cluster
            return None
        # get value from machine
        return NodeClient.get(self.config.node_ips[node_id], key)['value']
def process(udp_clients):

    udp_client = udp_clients
    hash_codes = set()
    client_ring = NodeRing(udp_client)

    # these couple lines are for the rendezvous hash
    # weights = [5, 5, 5, 5]
    # seeds = ["node1", "serve2", "salt1", "clasuter"]
    # instance_rendezvous = RendezvousHash(weights, seeds)

    # uncomment/comment this to apply consistent hash
    instance_consistent_hash = ConsistentHash(NODES)

    client_ring.apply_new_hash(instance_consistent_hash)

    # PUT all users.
    for u in USERS:
        data_bytes, key = serialize_PUT(u)
        response = put(key, data_bytes, client_ring)
        hash_codes.add(response)
    print(hash_codes)

    print(
        f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
    )

    for hc in hash_codes:
        # print(hc)
        data_bytes, key = serialize_GET(hc)
        response = get(key, data_bytes, client_ring)
        # print(response)

    for hc in hash_codes:
        data_bytes, key = serialize_DELETE(hc)
        print("keys from DELETE OPERATAIONS:", data_bytes, key)
        response = delete(key, data_bytes, client_ring)
        print(response)
class ConsistentMemcachedClient(Client):
    """
    Consistent Memcached Client attempts to create a scalable Memcached
    cluster that uses Consistent Hashing (using the ketama algorithm).
    In any distributed caching setup, adding or deleting servers disrupts
    the entire hashing and results in significant redistribution of keys.
    A consistent hashing function will significantly decrease the chances
    of a key being hashed to a different slot.
    
    A good explanation for the algorithm is found here:
    http://michaelnielsen.org/blog/consistent-hashing/
    """

    # The timeout period before marking a server as a dead
    _RETRY_GAP = 0.1

    def __init__(self, *args, **kwargs):
        """ 
        A memcache subclass. It currently allows you to add or delete a new 
        host at run time. It also checks if a memcache server is down and 
        automatically readjusts if a memcached server is not reachable.
        """
        super(ConsistentMemcachedClient, self).__init__(*args, **kwargs)
        self.hash_manager = ConsistentHash(self.servers)
        
    def _reconfigure_hashing(self):
        """
        If a server can be reached add it to the list of available servers.
        If a server cannot be reached, delete it from the list of available
        servers.
        """
        for server in self.servers:
            if self._is_server_alive(server, sleep=False):
                self._add_alive_server(server)
        for server in self.hash_manager.nodes:
            if not self._is_server_alive(server, sleep=False):
                self._remove_dead_server(server)            
    
    def _add_alive_server(self, server):
        """
        Add a server to the hash manager
        """
        if server not in self.hash_manager.nodes:
            self.hash_manager.add_nodes([server])
    
    def _is_server_alive(self, server, sleep=True):
        """
        Check is server is alive Client._SERVER_RETRIES times
        """
        for i in range(Client._SERVER_RETRIES):
            try:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                if not sock.connect_ex(server.address):
                    return True
                if sleep:
                    time.sleep(Client._RETRY_GAP)
            finally:
                sock.close()
        return False
        
    def _remove_dead_server(self, server):
        """
        Reconfigure hashing by removing the server that is not responding
        """
        try:
            self.hash_manager.nodes.remove(server)
            self.hash_manager = ConsistentHash(self.hash_manager.nodes)
        except ValueError:
            raise ValueError('no data store is functioning, cannot process request')
        
    def _get_server(self, key):
        """ 
        Returns the most likely server to hold the key
        """
        self._reconfigure_hashing()
        
        server = self.hash_manager.get_node(key)
        
        if not self.buckets:
            return None, None
        
        for i in range(Client._SERVER_RETRIES):
            try:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                if server and server.connect() and not sock.connect_ex(server.address):
                    return server, key
                time.sleep(Client._RETRY_GAP)
            finally:
                sock.close()
        return None, None
class ConsistentScheduler(object):
    """
        LibPartitionHelper abstract out workers and work_items, and their
        mapping to partitions. So application can only deal with the work
        items it owns, without bothering about partition mapping.

        This class also provides syncronization premitives to ensure apps
        to clean up b4 giving up their partitions
    """

    _MAX_WAIT_4_ALLOCATION = 6 + randint(0, 9)

    def __init__(
        self,
        service_name=None,
        zookeeper="127.0.0.1:2181",
        delete_hndlr=None,
        add_hndlr=None,
        bucketsize=47,
        item2part_func=None,
        partitioner=None,
        logger=None,
    ):
        if logger:
            self._logger = logger
        else:
            self._logger = logging.getLogger(__name__)
        self._service_name = service_name or os.path.basename(sys.argv[0])
        self._item2part_func = item2part_func or self._device2partition
        self._zookeeper_srvr = zookeeper
        self._bucketsize = bucketsize
        self._delete_hndlr = delete_hndlr
        self._add_hndlr = add_hndlr
        self._partitioner = partitioner or self._partitioner_func
        self._partitions = {}
        self._con_hash = None
        self._last_log = ""
        self._last_log_cnt = 0
        self._partition_set = map(str, range(self._bucketsize))
        self._zk_path = "/".join(["/contrail_cs", self._service_name])
        self._zk = KazooClient(self._zookeeper_srvr)
        self._zk.add_listener(self._zk_lstnr)
        self._conn_state = None
        while True:
            try:
                self._zk.start()
                break
            except gevent.event.Timeout as e:
                # Update connection info
                self._sandesh_connection_info_update(status="DOWN", message=str(e))
                gevent.sleep(1)
            # Zookeeper is also throwing exception due to delay in master election
            except Exception as e:
                # Update connection info
                self._sandesh_connection_info_update(status="DOWN", message=str(e))
                gevent.sleep(1)
        self._pc = self._zk.SetPartitioner(
            path=self._zk_path, set=self._partition_set, partition_func=self._partitioner
        )
        self._wait_allocation = 0
        gevent.sleep(0)

    def _sandesh_connection_info_update(self, status, message):
        from pysandesh.connection_info import ConnectionState
        from pysandesh.gen_py.process_info.ttypes import ConnectionStatus, ConnectionType

        new_conn_state = getattr(ConnectionStatus, status)
        ConnectionState.update(
            conn_type=ConnectionType.ZOOKEEPER,
            name="Zookeeper",
            status=new_conn_state,
            message=message,
            server_addrs=self._zookeeper_srvr.split(","),
        )

        if (self._conn_state and self._conn_state != ConnectionStatus.DOWN) and new_conn_state == ConnectionStatus.DOWN:
            msg = "Connection to Zookeeper down: %s" % (message)
            self._supress_log(msg)
        if self._conn_state and self._conn_state != new_conn_state and new_conn_state == ConnectionStatus.UP:
            msg = "Connection to Zookeeper ESTABLISHED"
            self._supress_log(msg)

        self._conn_state = new_conn_state

    # end _sandesh_connection_info_update

    def _zk_lstnr(self, state):
        if state == KazooState.CONNECTED:
            # Update connection info
            self._sandesh_connection_info_update(status="UP", message="")
        elif state == KazooState.LOST:
            # Lost the session with ZooKeeper Server
            # Best of option we have is to exit the process and restart all
            # over again
            self._sandesh_connection_info_update(status="DOWN", message="Connection to Zookeeper lost")
            os._exit(2)
        elif state == KazooState.SUSPENDED:
            # Update connection info
            self._sandesh_connection_info_update(status="INIT", message="Connection to zookeeper lost. Retrying")

    def schedule(self, items, lock_timeout=30):
        gevent.sleep(0)
        ret = False
        if self._pc.failed:
            raise Exception("Lost or unable to acquire partition")
        elif self._pc.release:
            self._supress_log("Releasing...")
            self._release()
        elif self._pc.allocating:
            self._supress_log("Waiting for allocation...")
            self._pc.wait_for_acquire(lock_timeout)
            if self._wait_allocation < self._MAX_WAIT_4_ALLOCATION:
                self._wait_allocation += 1
            else:
                raise StopIteration("Giving up after %d tries!" % (self._wait_allocation))
        elif self._pc.acquired:
            self._supress_log("got work: ", list(self._pc))
            ret = True
            self._wait_allocation = 0
            self._populate_work_items(items)
            self._supress_log(
                "work items: ", self._items2name(self.work_items()), "from the list", self._items2name(items)
            )
        return ret

    def work_items(self):
        return sum(self._partitions.values(), [])

    def finish(self):
        self._inform_delete(self._partitions.keys())
        self._pc.finish()

    def _items2name(self, items):
        return map(lambda x: x.name, items)

    def _supress_log(self, *s):
        slog = " ".join(map(str, s))
        dl = ""
        if slog != self._last_log_cnt:
            if self._last_log_cnt:
                dl += " " * 4
                dl += "." * 8
                dl += "[last print repeats %d times]" % self._last_log_cnt
                self._last_log_cnt = 0
            dl += slog
            self._last_log = slog
            self._logger.debug(dl)
        else:
            self._last_log_cnt += 1

    def _consistent_hash(self, members):
        if self._con_hash is None:
            self._con_hash = ConsistentHash(members)
            self._supress_log("members:", self._con_hash.nodes)
        cur, updtd = set(self._con_hash.nodes), set(members)
        if cur != updtd:
            newm = updtd - cur
            rmvd = cur - updtd
            if newm:
                self._supress_log("new workers:", newm)
                self._con_hash.add_nodes(list(newm))
            if rmvd:
                self._supress_log("workers left:", rmvd)
                self._con_hash.del_nodes(list(rmvd))
        return self._con_hash

    def _consistent_hash_get_node(self, members, partition):
        return self._consistent_hash(members).get_node(partition)

    def _partitioner_func(self, identifier, members, _partitions):
        return [p for p in _partitions if self._consistent_hash_get_node(members, p) == identifier]

    def _release(self):
        old = set(self._pc)
        new = set(self._partitioner(self._pc._identifier, list(self._pc._party), self._partition_set))
        rmvd = old - new
        added = new - old
        if rmvd:
            self._inform_delete(list(rmvd))
        if added:
            self._inform_will_add(list(added))
        self._pc.release_set()

    def _list_items_in(self, partitions):
        return sum([self._partitions[k] for k in partitions if k in self._partitions], [])

    def _inform_will_add(self, partitions):
        if callable(self._add_hndlr):
            self._add_hndlr(self._list_items_in(partitions))

    def _inform_delete(self, partitions):
        if callable(self._delete_hndlr):
            self._delete_hndlr(self._list_items_in(partitions))

    def _populate_work_items(self, items):
        self._refresh_work_items()
        for i in items:
            part = str(self._item2part_func(i.name))
            if part in list(self._pc):
                if part not in self._partitions:
                    self._partitions[part] = []
                if i.name not in map(lambda x: x.name, self._partitions[part]):
                    self._partitions[part].append(i)
        self._logger.debug(
            "@populate_work_items(%s): done!"
            % " ".join(map(lambda v: str(v[0]) + ":" + ",".join(map(lambda x: x.name, v[1])), self._partitions.items()))
        )
        gevent.sleep(0)

    def _device2partition(self, key):
        return struct.unpack("Q", hashlib.md5(key).digest()[-8:])[0] % self._bucketsize

    def _refresh_work_items(self):
        for k in self._partitions:
            self._partitions[k] = []
 def __init__(self, replicas=1,*args, **kwargs):
     self.consistent_hash=ConsistentHash(replicas=replicas)
     super(MemcacheClient, self).__init__(*args, **kwargs)
class ConsistentScheduler(object):
    '''
        LibPartitionHelper abstract out workers and work_items, and their
        mapping to partitions. So application can only deal with the work
        items it owns, without bothering about partition mapping.

        This class also provides syncronization premitives to ensure apps
        to clean up b4 giving up their partitions
    '''
    _MAX_WAIT_4_ALLOCATION = 6 + randint(0, 9)

    def __init__(self,
                 service_name=None,
                 zookeeper='127.0.0.1:2181',
                 delete_hndlr=None,
                 add_hndlr=None,
                 bucketsize=47,
                 item2part_func=None,
                 partitioner=None,
                 logger=None):
        if logger:
            self._logger = logger
        else:
            self._logger = logging.getLogger(__name__)
        self._service_name = service_name or os.path.basename(sys.argv[0])
        self._item2part_func = item2part_func or self._device2partition
        self._zookeeper_srvr = zookeeper
        self._bucketsize = bucketsize
        self._delete_hndlr = delete_hndlr
        self._add_hndlr = add_hndlr
        self._partitioner = partitioner or self._partitioner_func
        self._partitions = {}
        self._con_hash = None
        self._last_log = ''
        self._last_log_cnt = 0
        self._partition_set = map(str, range(self._bucketsize))
        self._zk_path = '/'.join(['/contrail_cs', self._service_name])
        self._zk = KazooClient(self._zookeeper_srvr)
        self._zk.add_listener(self._zk_lstnr)
        self._zk.start()
        self._pc = self._zk.SetPartitioner(path=self._zk_path,
                                           set=self._partition_set,
                                           partition_func=self._partitioner)
        self._wait_allocation = 0
        gevent.sleep(0)

    def _zk_lstnr(self, state):
        self._supress_log('zk state change to %s' % str(state))

    def schedule(self, items, lock_timeout=30):
        gevent.sleep(0)
        ret = False
        if self._pc.failed:
            raise Exception("Lost or unable to acquire partition")
        elif self._pc.release:
            self._supress_log('Releasing...')
            self._release()
        elif self._pc.allocating:
            self._supress_log('Waiting for allocation...')
            self._pc.wait_for_acquire(lock_timeout)
            if self._wait_allocation < self._MAX_WAIT_4_ALLOCATION:
                self._wait_allocation += 1
            else:
                raise StopIteration('Giving up after %d tries!' %
                                    (self._wait_allocation))
        elif self._pc.acquired:
            self._supress_log('got work: ', list(self._pc))
            ret = True
            self._wait_allocation = 0
            self._populate_work_items(items)
            self._supress_log('work items: ',
                              self._items2name(self.work_items()),
                              'from the list', self._items2name(items))
        return ret

    def work_items(self):
        return sum(self._partitions.values(), [])

    def finish(self):
        self._inform_delete(self._partitions.keys())
        self._pc.finish()

    def _items2name(self, items):
        return map(lambda x: x.name, items)

    def _supress_log(self, *s):
        slog = ' '.join(map(str, s))
        dl = ''
        if slog != self._last_log_cnt:
            if self._last_log_cnt:
                dl += ' ' * 4
                dl += '.' * 8
                dl += '[last print repeats %d times]' % self._last_log_cnt
                self._last_log_cnt = 0
            dl += slog
            self._last_log = slog
            self._logger.debug(dl)
        else:
            self._last_log_cnt += 1

    def _consistent_hash(self, members):
        if self._con_hash is None:
            self._con_hash = ConsistentHash(members)
            self._supress_log('members:', self._con_hash.nodes)
        cur, updtd = set(self._con_hash.nodes), set(members)
        if cur != updtd:
            newm = updtd - cur
            rmvd = cur - updtd
            if newm:
                self._supress_log('new workers:', newm)
                self._con_hash.add_nodes(list(newm))
            if rmvd:
                self._supress_log('workers left:', rmvd)
                self._con_hash.del_nodes(list(rmvd))
        return self._con_hash

    def _consistent_hash_get_node(self, members, partition):
        return self._consistent_hash(members).get_node(partition)

    def _partitioner_func(self, identifier, members, _partitions):
        return [p for p in _partitions \
                if self._consistent_hash_get_node(members, p) == identifier]

    def _release(self):
        old = set(self._pc)
        new = set(
            self._partitioner(self._pc._identifier, list(self._pc._party),
                              self._partition_set))
        rmvd = old - new
        added = new - old
        if rmvd:
            self._inform_delete(list(rmvd))
        if added:
            self._inform_will_add(list(added))
        self._pc.release_set()

    def _list_items_in(self, partitions):
        return sum([self._partitions[k] for k in partitions if k in \
                    self._partitions], [])

    def _inform_will_add(self, partitions):
        if callable(self._add_hndlr):
            self._add_hndlr(self._list_items_in(partitions))

    def _inform_delete(self, partitions):
        if callable(self._delete_hndlr):
            self._delete_hndlr(self._list_items_in(partitions))

    def _populate_work_items(self, items):
        self._refresh_work_items()
        for i in items:
            part = str(self._item2part_func(i.name))
            if part in list(self._pc):
                if part not in self._partitions:
                    self._partitions[part] = []
                if i.name not in map(lambda x: x.name, self._partitions[part]):
                    self._partitions[part].append(i)
        self._logger.debug('@populate_work_items(%s): done!' % ' '.join(
            map(
                lambda v: str(v[0]) + ':' + ','.join(
                    map(lambda x: x.name, v[1])), self._partitions.items())))
        gevent.sleep(0)

    def _device2partition(self, key):
        return struct.unpack(
            'Q',
            hashlib.md5(key).digest()[-8:])[0] % self._bucketsize

    def _refresh_work_items(self):
        for k in self._partitions:
            self._partitions[k] = []
Example #21
0
class PartitionClient(object):
    """ Client Class for the Partition Library
    Example usage:
    ---------------------
    import libpartition
    from libpartition.libpartition import PartitionClient

    def own_change_cb(l):
            print "ownership change:" + str(l)

    c = PartitionClient("test", "s1", ["s1", "s2", "s3"], 32, 
            own_change_cb, "zookeeper_s1")

    ##do some real work now"
    if (c.own_partition(1)):
        ...... do something with partition #1 .....
        .........
    ...
    c.update_cluster_list(["s1", "s2"])
    ...
    ----------------------
    You should not call any partition library routine from within the 
    callback function

    Args:
        app_name(str): Name of the app for which partition cluster is used
        self_name(str): Name of the local cluster node (can be ip address)
        cluster_list(list): List of all the nodes in the cluster including 
            local node
        max_partition(int): Partition space always go from 0..max_partition-1
        partition_update_cb: Callback function invoked when partition
            ownership list is updated.x
        zk_server(str): <zookeeper server>:<zookeeper server port>
    """
    def __init__(self,
                 app_name,
                 self_name,
                 cluster_list,
                 max_partition,
                 partition_update_cb,
                 zk_server,
                 logger=None):

        # Initialize local variables
        self._zk_server = zk_server
        self._cluster_list = set(cluster_list)
        self._max_partition = max_partition
        self._update_cb = partition_update_cb
        self._curr_part_ownership_list = []
        self._target_part_ownership_list = []
        self._con_hash = ConsistentHash(cluster_list)
        self._name = self_name

        # some sanity check
        if not (self._name in cluster_list):
            raise ValueError('cluster list is missing local server name')

        # initialize logging and other stuff
        if logger is None:
            logging.basicConfig()
            self._logger = logging
        else:
            self._logger = logger
        self._conn_state = None
        self._sandesh_connection_info_update(status='INIT', message='')

        # connect to zookeeper
        self._zk = KazooClient(zk_server)
        while True:
            try:
                self._zk.start()
                break
            except gevent.event.Timeout as e:
                # Update connection info
                self._sandesh_connection_info_update(status='DOWN',
                                                     message=str(e))
                gevent.sleep(1)
            # Zookeeper is also throwing exception due to delay in master election
            except Exception as e:
                # Update connection info
                self._sandesh_connection_info_update(status='DOWN',
                                                     message=str(e))
                gevent.sleep(1)
        # Update connection info
        self._sandesh_connection_info_update(status='UP', message='')
        # Done connecting to ZooKeeper

        # create a lock array to contain locks for each partition
        self._part_locks = []
        for part in range(0, self._max_partition):
            lockpath = "/lockpath/" + app_name + "/" + str(part)
            l = self._zk.Lock(lockpath, self._name)
            self._part_locks.append(l)

        # initialize partition # to lock acquire greenlet dictionary
        self._part_lock_task_dict = {}

        self._logger.error("initial servers:" + str(self._cluster_list))

        # update target partition ownership list
        for part in range(0, self._max_partition):
            if (self._con_hash.get_node(str(part)) == self._name):
                self._target_part_ownership_list.append(part)

        # update current ownership list
        self._acquire_partition_ownership()

    #end __init__

    def _sandesh_connection_info_update(self, status, message):
        from pysandesh.connection_info import ConnectionState
        from pysandesh.gen_py.process_info.ttypes import ConnectionStatus, \
            ConnectionType
        from pysandesh.gen_py.sandesh.ttypes import SandeshLevel

        new_conn_state = getattr(ConnectionStatus, status)
        ConnectionState.update(conn_type=ConnectionType.ZOOKEEPER,
                               name='Zookeeper',
                               status=new_conn_state,
                               message=message,
                               server_addrs=self._zk_server.split(','))

        if (self._conn_state and self._conn_state != ConnectionStatus.DOWN
                and new_conn_state == ConnectionStatus.DOWN):
            msg = 'Connection to Zookeeper down: %s' % (message)
            self._logger.error(msg)
        if (self._conn_state and self._conn_state != new_conn_state
                and new_conn_state == ConnectionStatus.UP):
            msg = 'Connection to Zookeeper ESTABLISHED'
            self._logger.error(msg)

        self._conn_state = new_conn_state

    # end _sandesh_connection_info_update

    # following routine is the greenlet task function to acquire the lock
    # for a partition
    def _acquire_lock(self, part):
        # lock for the partition
        l = self._part_locks[part]

        # go in an infinite loop waiting to acquire the lock
        try:
            while True:
                ret = l.acquire(blocking=False)
                if ret == True:
                    self._logger.error("Acquired lock for:" + str(part))
                    self._curr_part_ownership_list.append(part)
                    self._update_cb(self._curr_part_ownership_list)
                    return True
                else:
                    gevent.sleep(1)
        except CancelledError:
            self._logger.error("Lock acquire cancelled for:" + str(part))
            return False
        except Exception as ex:
            # TODO: If we have a non-KazooException, the lock object
            #       may get stuck in the "cancelled" state
            self._logger.error("Lock acquire unexpected error!: " + str(ex))
            # This exception should get propogated to main thread
            raise SystemExit
            return False

    #end _acquire_lock

    # get rid of finished spawned tasks from datastructures
    def _cleanup_greenlets(self):
        for part in self._part_lock_task_dict.keys():
            if (self._part_lock_task_dict[part].ready()):
                del self._part_lock_task_dict[part]

    #end _cleanup_greenlets

    # following routine launches tasks to acquire partition locks
    def _acquire_partition_ownership(self):
        # cleanup any finished greenlets
        self._cleanup_greenlets()

        # this variable will help us decide if we need to call callback
        updated_curr_ownership = False

        # list of partitions for which locks have to be released
        release_lock_list = []

        self._logger.error("known servers: %s" %
                           self._con_hash.get_all_nodes())

        for part in range(0, self._max_partition):
            if (part in self._target_part_ownership_list):
                if (part in self._curr_part_ownership_list):
                    # do nothing, I already have ownership of this partition
                    self._logger.error("No need to acquire ownership of:" +
                                       str(part))
                else:
                    # I need to acquire lock for this partition before I own
                    if (part in self._part_lock_task_dict.keys()):
                        try:
                            self._part_lock_task_dict[part].get(block=False)
                        except:
                            # do nothing there is already a greenlet running to
                            # acquire the lock
                            self._logger.error("Already a greenlet running to"
                                               " acquire:" + str(part))
                            continue

                        # Greenlet died without getting ownership. Cleanup
                        self._logger.error("Cleanup stale greenlet running to"
                                           " acquire:" + str(part))
                        del self._part_lock_task_dict[part]

                    self._logger.error("Starting greenlet running to"
                                       " acquire:" + str(part))
                    # launch the greenlet to acquire the loc, k
                    g = Greenlet.spawn(self._acquire_lock, part)
                    self._part_lock_task_dict[part] = g

            else:
                # give up ownership of the partition

                # cancel any lock acquisition which is ongoing
                if (part in self._part_lock_task_dict.keys()):
                    try:
                        self._part_lock_task_dict[part].get(block=False)
                    except:

                        self._logger.error(
                            "canceling lock acquisition going on \
                            for:" + str(part))
                        # Cancelling the lock should result in killing the gevent
                        self._part_locks[part].cancel()
                        self._part_lock_task_dict[part].get(block=True)

                    del self._part_lock_task_dict[part]

                if (part in self._curr_part_ownership_list):
                    release_lock_list.append(part)
                    self._curr_part_ownership_list.remove(part)
                    updated_curr_ownership = True
                    self._logger.error("giving up ownership of:" + str(part))

        if (updated_curr_ownership is True):
            # current partition membership was updated call the callback
            self._update_cb(self._curr_part_ownership_list)

        if (len(release_lock_list) != 0):
            # release locks which were acquired
            for part in release_lock_list:
                self._logger.error("release the lock which was acquired:" + \
                        str(part))
                try:
                    self._part_locks[part].release()
                    self._logger.error("fully gave up ownership of:" +
                                       str(part))
                except:
                    pass

    #end _acquire_partition_ownership

    def update_cluster_list(self, cluster_list):
        """ Updates the cluster node list
        Args:
            cluster_list(list): New list of names of the nodes in 
                the cluster
        Returns:
            None
        """
        # some sanity check
        if not (self._name in cluster_list):
            raise ValueError('cluster list is missing local server name')

        new_cluster_list = set(cluster_list)
        new_servers = list(new_cluster_list.difference(self._cluster_list))
        deleted_servers = list(
            set(self._cluster_list).difference(new_cluster_list))
        self._cluster_list = set(cluster_list)
        self._logger.error("deleted servers:" + str(deleted_servers))
        self._logger.error("new servers:" + str(new_servers))

        # update the hash structure
        if new_servers:
            self._con_hash.add_nodes(new_servers)
        if deleted_servers:
            self._con_hash.del_nodes(deleted_servers)

        # update target partition ownership list
        self._target_part_ownership_list = []
        for part in range(0, self._max_partition):
            if (self._con_hash.get_node(str(part)) == self._name):
                if not (part in self._target_part_ownership_list):
                    self._target_part_ownership_list.append(part)

        # update current ownership list
        self._acquire_partition_ownership()

    #end update_cluster_list

    def own_partition(self, part_no):
        """ Returns ownership information of a partition
        Args:
            part_no(int) : Partition no 
        Returns:
            True if partition is owned by the local node
            False if partition is not owned by the local node
        """
        return part_no in self._curr_part_ownership_list

    #end own_partition

    def close(self):
        """ Closes any connections and frees up any data structures
        Args:
        Returns:
            None
        """
        # clean up greenlets
        for part in self._part_lock_task_dict.keys():
            try:
                self._part_lock_task_dict[part].kill()
            except:
                pass

        # close zookeeper
        try:
            self._zk.stop()
        except:
            pass
        try:
            self._zk.close()
        except:
            pass
class ConsistentScheduler(object):
    '''
        LibPartitionHelper abstract out workers and work_items, and their
        mapping to partitions. So application can only deal with the work
        items it owns, without bothering about partition mapping.

        This class also provides syncronization premitives to ensure apps
        to clean up b4 giving up their partitions
    '''
    _MAX_WAIT_4_ALLOCATION = 6 + randint(0, 9)

    def __init__(self, service_name=None, zookeeper='127.0.0.1:2181',
                 delete_hndlr=None, add_hndlr=None, bucketsize=47,
                 item2part_func=None, partitioner=None, logger=None, 
                 cluster_id=''):
        if logger:
            self._logger = logger
        else:
            self._logger = logging.getLogger(__name__)
        self._service_name = service_name or os.path.basename(sys.argv[0])
        self._item2part_func = item2part_func or self._device2partition
        self._zookeeper_srvr = zookeeper
        self._zk = None
        self._bucketsize = bucketsize
        self._delete_hndlr = delete_hndlr
        self._add_hndlr = add_hndlr
        self._partitioner = partitioner or self._partitioner_func
        self._partitions = {}
        self._con_hash = None
        self._last_log = ''
        self._last_log_cnt = 0
        self._partition_set = map(str, range(self._bucketsize))

        self._cluster_id = cluster_id
        if self._cluster_id:
            self._zk_path = '/'+self._cluster_id + '/contrail_cs' + '/'+self._service_name
        else:
            self._zk_path = '/'.join(['/contrail_cs', self._service_name])
        self._conn_state = None
        self._sandesh_connection_info_update(status='INIT', message='')

        while True:
            self._logger.error("Consistent scheduler zk start")
            self._zk = KazooClient(self._zookeeper_srvr,
                handler=SequentialGeventHandler())
            self._zk.add_listener(self._zk_lstnr)
            try:
                self._zk.start()
                while self._conn_state != ConnectionStatus.UP:
                    gevent.sleep(1)
                break
            except Exception as e:
                # Update connection info
                self._sandesh_connection_info_update(status='DOWN',
                                                     message=str(e))
                self._zk.remove_listener(self._zk_lstnr)
                try:
                    self._zk.stop()
                    self._zk.close()
                except Exception as ex:
                    template = "Exception {0} in Consistent scheduler zk stop/close. Args:\n{1!r}"
                    messag = template.format(type(ex).__name__, ex.args)
                    self._logger.error("%s : traceback %s for %s" % \
                        (messag, traceback.format_exc(), self._service_name))
                finally:
                    self._zk = None
                gevent.sleep(1)
        self._pc = self._zk.SetPartitioner(path=self._zk_path,
                                           set=self._partition_set,
                                           partition_func=self._partitioner)
        self._wait_allocation = 0
        gevent.sleep(0)

    def _sandesh_connection_info_update(self, status, message):
        new_conn_state = getattr(ConnectionStatus, status)
        ConnectionState.update(conn_type = ConnectionType.ZOOKEEPER,
                name = 'Zookeeper', status = new_conn_state,
                message = message,
                server_addrs = self._zookeeper_srvr.split(','))

        if ((self._conn_state and self._conn_state != ConnectionStatus.DOWN) and
            new_conn_state == ConnectionStatus.DOWN):
            msg = 'Connection to Zookeeper down: %s' %(message)
            self._supress_log(msg)
        if (self._conn_state and self._conn_state != new_conn_state and
            new_conn_state == ConnectionStatus.UP):
            msg = 'Connection to Zookeeper ESTABLISHED'
            self._supress_log(msg)

        self._conn_state = new_conn_state
    # end _sandesh_connection_info_update

    def _zk_lstnr(self, state):
        self._logger.error("Consistent scheduler listen %s" % str(state))
        if state == KazooState.CONNECTED:
            # Update connection info
            self._sandesh_connection_info_update(status='UP', message='')
        elif state == KazooState.LOST:
            self._logger.error("Consistent scheduler connection LOST")
            # Lost the session with ZooKeeper Server
            # Best of option we have is to exit the process and restart all 
            # over again
            self._sandesh_connection_info_update(status='DOWN',
                                      message='Connection to Zookeeper lost')
            os._exit(2)
        elif state == KazooState.SUSPENDED:
            self._logger.error("Consistent scheduler connection SUSPENDED")
            # Update connection info
            self._sandesh_connection_info_update(status='INIT',
                message = 'Connection to zookeeper lost. Retrying')

    def schedule(self, items, lock_timeout=30):
        gevent.sleep(0)
        ret = False
        if self._pc.failed:
            self._logger.error('Lost or unable to acquire partition')
            os._exit(2)
        elif self._pc.release:
            self._supress_log('Releasing...')
            self._release()
        elif self._pc.allocating:
            self._supress_log('Waiting for allocation...')
            self._pc.wait_for_acquire(lock_timeout)
            if self._wait_allocation < self._MAX_WAIT_4_ALLOCATION:
                self._wait_allocation += 1
            else:
                self._logger.error('Giving up after %d tries!' %
                    (self._wait_allocation))
                os._exit(2)
        elif self._pc.acquired:
            self._supress_log('got work: ', list(self._pc))
            ret = True
            self._wait_allocation = 0
            self._populate_work_items(items)
            self._supress_log('work items: ',
                              self._items2name(self.work_items()),
                              'from the list',
                              self._items2name(items))
        return ret

    def members(self):
        return list(self._con_hash.nodes)

    def partitions(self):
        return list(self._pc)

    def work_items(self):
        return sum(self._partitions.values(), [])

    def finish(self):
        self._inform_delete(self._partitions.keys())
        self._pc.finish()
        self._zk.remove_listener(self._zk_lstnr)
        gevent.sleep(1)
        try:
            self._zk.stop()
        except:
            self._logger.error("Stopping kazooclient failed")
        else:
            self._logger.error("Stopping kazooclient successful")
        try:
            self._zk.close()
        except:
            self._logger.error("Closing kazooclient failed")
        else:
            self._logger.error("Closing kazooclient successful")

    def _items2name(self, items):
        return map(lambda x: x.name, items)

    def _supress_log(self, *s):
        slog = ' '.join(map(str, s))
        dl = ''
        if slog != self._last_log_cnt:
            if self._last_log_cnt:
                dl += ' ' * 4
                dl += '.' * 8
                dl += '[last print repeats %d times]' % self._last_log_cnt
                self._last_log_cnt = 0
            dl += slog
            self._last_log = slog
            self._logger.debug(dl)
        else:
            self._last_log_cnt += 1

    def _consistent_hash(self, members):
        if self._con_hash is None:
            self._con_hash = ConsistentHash(members)
            self._logger.error('members: %s' % (str(self._con_hash.nodes)))
        cur, updtd = set(self._con_hash.nodes), set(members)
        if cur != updtd:
            newm = updtd - cur
            rmvd = cur - updtd
            if newm:
                self._logger.error('new members: %s' % (str(newm)))
                self._con_hash.add_nodes(list(newm))
            if rmvd:
                self._logger.error('members left: %s' % (str(rmvd)))
                self._con_hash.del_nodes(list(rmvd))
        return self._con_hash

    def _consistent_hash_get_node(self, members, partition):
        return self._consistent_hash(members).get_node(partition)

    def _partitioner_func(self, identifier, members, _partitions):
        partitions = [p for p in _partitions \
            if self._consistent_hash_get_node(members, p) == identifier]
        self._logger.error('partitions: %s' % (str(partitions)))
        return partitions

    def _release(self):
        old = set(self._pc)
        new = set(self._partitioner(self._pc._identifier,
                                   list(self._pc._party),
                                   self._partition_set))
        rmvd = old - new
        added = new - old
        if rmvd:
            self._inform_delete(list(rmvd))
        if added:
            self._inform_will_add(list(added))
        self._pc.release_set()

    def _list_items_in(self, partitions):
        return sum([self._partitions[k] for k in partitions if k in \
                    self._partitions], [])

    def _inform_will_add(self, partitions):
        if callable(self._add_hndlr):
            self._add_hndlr(self._list_items_in(partitions))

    def _inform_delete(self, partitions):
        if callable(self._delete_hndlr):
            self._delete_hndlr(self._list_items_in(partitions))

    def _populate_work_items(self, items):
        self._refresh_work_items()
        for i in items:
            part = str(self._item2part_func(i.name))
            if part in list(self._pc):
                if part not in self._partitions:
                    self._partitions[part] = []
                if i.name not in map(lambda x: x.name,
                                     self._partitions[part]):
                    self._partitions[part].append(i)
        self._logger.debug('@populate_work_items(%s): done!' % ' '.join(
                map(lambda v: str(v[0]) + ':' + ','.join(map(
                        lambda x: x.name, v[1])), self._partitions.items())))
        gevent.sleep(0)

    def _device2partition(self, key):
        return struct.unpack('Q', hashlib.md5(key).digest(
                    )[-8:])[0] % self._bucketsize

    def _refresh_work_items(self):
        for k in self._partitions:
            self._partitions[k] = []
 def __init__(self):
     self.config = self._get_redis_config()
     self._connections = {}
     self._consistent_hash = ConsistentHash(self.config.hosts)
     self._establish_connections(self.config)
class PartitionClient(object):
    """ Client Class for the Partition Library
    Example usage:
    ---------------------
    import libpartition
    from libpartition.libpartition import PartitionClient

    def own_change_cb(l):
            print "ownership change:" + str(l)

    c = PartitionClient("test", "s1", ["s1", "s2", "s3"], 32, 
            own_change_cb, "zookeeper_s1")

    ##do some real work now"
    if (c.own_partition(1)):
        ...... do something with partition #1 .....
        .........
    ...
    c.update_cluster_list(["s1", "s2"])
    ...
    ----------------------
    You should not call any partition library routine from within the 
    callback function

    Args:
        app_name(str): Name of the app for which partition cluster is used
        self_name(str): Name of the local cluster node (can be ip address)
        cluster_list(list): List of all the nodes in the cluster including 
            local node
        max_partition(int): Partition space always go from 0..max_partition-1
        partition_update_cb: Callback function invoked when partition
            ownership list is updated.x
        zk_server(str): <zookeeper server>:<zookeeper server port>
    """
    def __init__(
            self, app_name, self_name, cluster_list, max_partition,
            partition_update_cb, zk_server, logger = None):
       
        # Initialize local variables
        self._zk_server = zk_server
        self._cluster_list = set(cluster_list)
        self._max_partition = max_partition
        self._update_cb = partition_update_cb
        self._curr_part_ownership_list = []
        self._target_part_ownership_list = []
        self._con_hash = ConsistentHash(cluster_list)
        self._name = self_name

        # some sanity check
        if not(self._name in cluster_list):
            raise ValueError('cluster list is missing local server name')

        # initialize logging and other stuff
        if logger is None:
            logging.basicConfig()
            self._logger = logging
        else:
            self._logger = logger
        self._conn_state = None
        self._sandesh_connection_info_update(status='INIT', message='')

        # connect to zookeeper
        self._zk = KazooClient(zk_server)
        while True:
            try:
                self._zk.start()
                break
            except gevent.event.Timeout as e:
                # Update connection info
                self._sandesh_connection_info_update(status='DOWN',
                                                     message=str(e))
                gevent.sleep(1)
            # Zookeeper is also throwing exception due to delay in master election
            except Exception as e:
                # Update connection info
                self._sandesh_connection_info_update(status='DOWN',
                                                     message=str(e))
                gevent.sleep(1)
        # Update connection info
        self._sandesh_connection_info_update(status='UP', message='')
        # Done connecting to ZooKeeper

        # create a lock array to contain locks for each partition
        self._part_locks = []
        for part in range(0, self._max_partition):
            lockpath = "/lockpath/"+ app_name + "/" + str(part)
            l = self._zk.Lock(lockpath, self._name)
            self._part_locks.append(l)

        # initialize partition # to lock acquire greenlet dictionary
        self._part_lock_task_dict = {}
       
        self._logger.error("initial servers:" + str(self._cluster_list))

        # update target partition ownership list
        for part in range(0, self._max_partition):
            if (self._con_hash.get_node(str(part)) == self._name):
                self._target_part_ownership_list.append(part)

        # update current ownership list
        self._acquire_partition_ownership()

    #end __init__

    def _sandesh_connection_info_update(self, status, message):
        from pysandesh.connection_info import ConnectionState
        from pysandesh.gen_py.process_info.ttypes import ConnectionStatus, \
            ConnectionType
        from pysandesh.gen_py.sandesh.ttypes import SandeshLevel

        new_conn_state = getattr(ConnectionStatus, status)
        ConnectionState.update(conn_type = ConnectionType.ZOOKEEPER,
                name = 'Zookeeper', status = new_conn_state,
                message = message,
                server_addrs = self._zk_server.split(','))

        if (self._conn_state and self._conn_state != ConnectionStatus.DOWN and
                new_conn_state == ConnectionStatus.DOWN):
            msg = 'Connection to Zookeeper down: %s' %(message)
            self._logger.error(msg)
        if (self._conn_state and self._conn_state != new_conn_state and
                new_conn_state == ConnectionStatus.UP):
            msg = 'Connection to Zookeeper ESTABLISHED'
            self._logger.error(msg)

        self._conn_state = new_conn_state
    # end _sandesh_connection_info_update


    # following routine is the greenlet task function to acquire the lock
    # for a partition
    def _acquire_lock(self, part):
        # lock for the partition
        l = self._part_locks[part]

        # go in an infinite loop waiting to acquire the lock
        try:
            while True:
                ret = l.acquire(blocking=False)
                if ret == True:
                    self._logger.error("Acquired lock for:" + str(part))
                    self._curr_part_ownership_list.append(part)
                    self._update_cb(self._curr_part_ownership_list)
                    return True
                else:
                    gevent.sleep(1)
        except CancelledError:
            self._logger.error("Lock acquire cancelled for:" + str(part))
            return False
        except Exception as ex:
            # TODO: If we have a non-KazooException, the lock object
            #       may get stuck in the "cancelled" state
            self._logger.error("Lock acquire unexpected error!: " + str(ex))
            # This exception should get propogated to main thread
            raise SystemExit
            return False
    #end _acquire_lock

    # get rid of finished spawned tasks from datastructures
    def _cleanup_greenlets(self):
        for part in self._part_lock_task_dict.keys():
            if (self._part_lock_task_dict[part].ready()):
                del self._part_lock_task_dict[part]
    #end _cleanup_greenlets 

    # following routine launches tasks to acquire partition locks
    def _acquire_partition_ownership(self):
        # cleanup any finished greenlets
        self._cleanup_greenlets()

        # this variable will help us decide if we need to call callback
        updated_curr_ownership = False 

        # list of partitions for which locks have to be released
        release_lock_list = []

        self._logger.info("known servers: %s" % self._con_hash.get_all_nodes())

        for part in range(0, self._max_partition):
            if (part in self._target_part_ownership_list):
                if (part in self._curr_part_ownership_list):
                    # do nothing, I already have ownership of this partition
                    self._logger.info("No need to acquire ownership of:" +
                            str(part))
                else:
                    # I need to acquire lock for this partition before I own
                    if (part in self._part_lock_task_dict.keys()):
                        try:
                            self._part_lock_task_dict[part].get(block=False)
                        except:
                            # do nothing there is already a greenlet running to
                            # acquire the lock
                            self._logger.error("Already a greenlet running to" 
                                    " acquire:" + str(part))
                            continue

                        # Greenlet died without getting ownership. Cleanup
                        self._logger.error("Cleanup stale greenlet running to" 
                                " acquire:" + str(part))
                        del self._part_lock_task_dict[part]

                    self._logger.error("Starting greenlet running to" 
                            " acquire:" + str(part))
                    # launch the greenlet to acquire the loc, k
                    g = Greenlet.spawn(self._acquire_lock, part)
                    self._part_lock_task_dict[part] = g

            else:
                # give up ownership of the partition

                # cancel any lock acquisition which is ongoing 
                if (part in self._part_lock_task_dict.keys()):
                    try:
                        self._part_lock_task_dict[part].get(block=False)
                    except:
                        
                        self._logger.error("canceling lock acquisition going on \
                            for:" + str(part))
                        # Cancelling the lock should result in killing the gevent
                        self._part_locks[part].cancel()
                        self._part_lock_task_dict[part].get(block=True)
                        
                    del self._part_lock_task_dict[part]
                        
                if (part in self._curr_part_ownership_list):
                    release_lock_list.append(part)
                    self._curr_part_ownership_list.remove(part)
                    updated_curr_ownership = True
                    self._logger.error("giving up ownership of:" + str(part))

        if (updated_curr_ownership is True):
            # current partition membership was updated call the callback 
            self._update_cb(self._curr_part_ownership_list)

        if (len(release_lock_list) != 0):
            # release locks which were acquired
            for part in release_lock_list:
                self._logger.error("release the lock which was acquired:" + \
                        str(part))
                try:
                    self._part_locks[part].release()
                    self._logger.error("fully gave up ownership of:" + str(part))
                except:
                    pass
    #end _acquire_partition_ownership

    def update_cluster_list(self, cluster_list):
        """ Updates the cluster node list
        Args:
            cluster_list(list): New list of names of the nodes in 
                the cluster
        Returns:
            None
        """
        # some sanity check
        if not(self._name in cluster_list):
            raise ValueError('cluster list is missing local server name')

        new_cluster_list = set(cluster_list)
        new_servers = list(new_cluster_list.difference(
            self._cluster_list))
        deleted_servers = list(set(self._cluster_list).difference(
            new_cluster_list)) 
        self._cluster_list = set(cluster_list)

        # update the hash structure
        if new_servers:
            self._logger.error("new servers:" + str(new_servers))
            self._con_hash.add_nodes(new_servers)
        if deleted_servers:
            self._logger.error("deleted servers:" + str(deleted_servers))
            self._con_hash.del_nodes(deleted_servers)

        # update target partition ownership list
        self._target_part_ownership_list = []
        for part in range(0, self._max_partition):
            if (self._con_hash.get_node(str(part)) == self._name):
                if not (part in self._target_part_ownership_list):
                    self._target_part_ownership_list.append(part)

        # update current ownership list
        self._acquire_partition_ownership()

    #end update_cluster_list

    def own_partition(self, part_no):
        """ Returns ownership information of a partition
        Args:
            part_no(int) : Partition no 
        Returns:
            True if partition is owned by the local node
            False if partition is not owned by the local node
        """
        return part_no in self._curr_part_ownership_list 
    #end own_partition

    def close(self):
        """ Closes any connections and frees up any data structures
        Args:
        Returns:
            None
        """
        # clean up greenlets
        for part in self._part_lock_task_dict.keys():
            try:
                self._part_lock_task_dict[part].kill()
            except:
                pass

        # close zookeeper
        try:
            self._zk.stop()
        except:
            pass
        try:
            self._zk.close()
        except:
            pass
    def __init__(
            self, app_name, self_name, cluster_list, max_partition,
            partition_update_cb, zk_server, logger = None):
       
        # Initialize local variables
        self._zk_server = zk_server
        self._cluster_list = set(cluster_list)
        self._max_partition = max_partition
        self._update_cb = partition_update_cb
        self._curr_part_ownership_list = []
        self._target_part_ownership_list = []
        self._con_hash = ConsistentHash(cluster_list)
        self._name = self_name

        # some sanity check
        if not(self._name in cluster_list):
            raise ValueError('cluster list is missing local server name')

        # initialize logging and other stuff
        if logger is None:
            logging.basicConfig()
            self._logger = logging
        else:
            self._logger = logger
        self._conn_state = None
        self._sandesh_connection_info_update(status='INIT', message='')

        # connect to zookeeper
        self._zk = KazooClient(zk_server)
        while True:
            try:
                self._zk.start()
                break
            except gevent.event.Timeout as e:
                # Update connection info
                self._sandesh_connection_info_update(status='DOWN',
                                                     message=str(e))
                gevent.sleep(1)
            # Zookeeper is also throwing exception due to delay in master election
            except Exception as e:
                # Update connection info
                self._sandesh_connection_info_update(status='DOWN',
                                                     message=str(e))
                gevent.sleep(1)
        # Update connection info
        self._sandesh_connection_info_update(status='UP', message='')
        # Done connecting to ZooKeeper

        # create a lock array to contain locks for each partition
        self._part_locks = []
        for part in range(0, self._max_partition):
            lockpath = "/lockpath/"+ app_name + "/" + str(part)
            l = self._zk.Lock(lockpath, self._name)
            self._part_locks.append(l)

        # initialize partition # to lock acquire greenlet dictionary
        self._part_lock_task_dict = {}
       
        self._logger.error("initial servers:" + str(self._cluster_list))

        # update target partition ownership list
        for part in range(0, self._max_partition):
            if (self._con_hash.get_node(str(part)) == self._name):
                self._target_part_ownership_list.append(part)

        # update current ownership list
        self._acquire_partition_ownership()
class ConsistentScheduler(object):
    """
        LibPartitionHelper abstract out workers and work_items, and their
        mapping to partitions. So application can only deal with the work
        items it owns, without bothering about partition mapping.

        This class also provides syncronization premitives to ensure apps
        to clean up b4 giving up their partitions
    """

    _MAX_WAIT_4_ALLOCATION = 6 + randint(0, 9)

    def __init__(
        self,
        service_name=None,
        zookeeper="127.0.0.1:2181",
        delete_hndlr=None,
        add_hndlr=None,
        bucketsize=47,
        item2part_func=None,
        partitioner=None,
        logger=None,
    ):
        if logger:
            self._logger = logger
        else:
            self._logger = logging.getLogger(__name__)
        self._service_name = service_name or os.path.basename(sys.argv[0])
        self._item2part_func = item2part_func or self._device2partition
        self._zookeeper_srvr = zookeeper
        self._bucketsize = bucketsize
        self._delete_hndlr = delete_hndlr
        self._add_hndlr = add_hndlr
        self._partitioner = partitioner or self._partitioner_func
        self._partitions = {}
        self._con_hash = None
        self._last_log = ""
        self._last_log_cnt = 0
        self._partition_set = map(str, range(self._bucketsize))
        self._zk_path = "/".join(["/contrail_cs", self._service_name])
        self._zk = KazooClient(self._zookeeper_srvr)
        self._zk.add_listener(self._zk_lstnr)
        self._zk.start()
        self._pc = self._zk.SetPartitioner(
            path=self._zk_path, set=self._partition_set, partition_func=self._partitioner
        )
        self._wait_allocation = 0
        gevent.sleep(0)

    def _zk_lstnr(self, state):
        self._supress_log("zk state change to %s" % str(state))

    def schedule(self, items, lock_timeout=30):
        gevent.sleep(0)
        ret = False
        if self._pc.failed:
            raise Exception("Lost or unable to acquire partition")
        elif self._pc.release:
            self._supress_log("Releasing...")
            self._release()
        elif self._pc.allocating:
            self._supress_log("Waiting for allocation...")
            self._pc.wait_for_acquire(lock_timeout)
            if self._wait_allocation < self._MAX_WAIT_4_ALLOCATION:
                self._wait_allocation += 1
            else:
                raise StopIteration("Giving up after %d tries!" % (self._wait_allocation))
        elif self._pc.acquired:
            self._supress_log("got work: ", list(self._pc))
            ret = True
            self._wait_allocation = 0
            self._populate_work_items(items)
            self._supress_log(
                "work items: ", self._items2name(self.work_items()), "from the list", self._items2name(items)
            )
        return ret

    def work_items(self):
        return sum(self._partitions.values(), [])

    def finish(self):
        self._inform_delete(self._partitions.keys())
        self._pc.finish()

    def _items2name(self, items):
        return map(lambda x: x.name, items)

    def _supress_log(self, *s):
        slog = " ".join(map(str, s))
        dl = ""
        if slog != self._last_log_cnt:
            if self._last_log_cnt:
                dl += " " * 4
                dl += "." * 8
                dl += "[last print repeats %d times]" % self._last_log_cnt
                self._last_log_cnt = 0
            dl += slog
            self._last_log = slog
            self._logger.debug(dl)
        else:
            self._last_log_cnt += 1

    def _consistent_hash(self, members):
        if self._con_hash is None:
            self._con_hash = ConsistentHash(members)
            self._supress_log("members:", self._con_hash.nodes)
        cur, updtd = set(self._con_hash.nodes), set(members)
        if cur != updtd:
            newm = updtd - cur
            rmvd = cur - updtd
            if newm:
                self._supress_log("new workers:", newm)
                self._con_hash.add_nodes(list(newm))
            if rmvd:
                self._supress_log("workers left:", rmvd)
                self._con_hash.del_nodes(list(rmvd))
        return self._con_hash

    def _consistent_hash_get_node(self, members, partition):
        return self._consistent_hash(members).get_node(partition)

    def _partitioner_func(self, identifier, members, _partitions):
        return [p for p in _partitions if self._consistent_hash_get_node(members, p) == identifier]

    def _release(self):
        old = set(self._pc)
        new = set(self._partitioner(self._pc._identifier, list(self._pc._party), self._partition_set))
        rmvd = old - new
        added = new - old
        if rmvd:
            self._inform_delete(list(rmvd))
        if added:
            self._inform_will_add(list(added))
        self._pc.release_set()

    def _list_items_in(self, partitions):
        return sum([self._partitions[k] for k in partitions if k in self._partitions], [])

    def _inform_will_add(self, partitions):
        if callable(self._add_hndlr):
            self._add_hndlr(self._list_items_in(partitions))

    def _inform_delete(self, partitions):
        if callable(self._delete_hndlr):
            self._delete_hndlr(self._list_items_in(partitions))

    def _populate_work_items(self, items):
        self._refresh_work_items()
        for i in items:
            part = str(self._item2part_func(i.name))
            if part in list(self._pc):
                if part not in self._partitions:
                    self._partitions[part] = []
                if i.name not in map(lambda x: x.name, self._partitions[part]):
                    self._partitions[part].append(i)
        self._logger.debug(
            "@populate_work_items(%s): done!"
            % " ".join(map(lambda v: str(v[0]) + ":" + ",".join(map(lambda x: x.name, v[1])), self._partitions.items()))
        )
        gevent.sleep(0)

    def _device2partition(self, key):
        return struct.unpack("Q", hashlib.md5(key).digest()[-8:])[0] % self._bucketsize

    def _refresh_work_items(self):
        for k in self._partitions:
            self._partitions[k] = []
Example #27
0
    def __init__(self,
                 app_name,
                 self_name,
                 cluster_list,
                 max_partition,
                 partition_update_cb,
                 zk_server,
                 logger=None):

        # Initialize local variables
        self._zk_server = zk_server
        self._cluster_list = set(cluster_list)
        self._max_partition = max_partition
        self._update_cb = partition_update_cb
        self._curr_part_ownership_list = []
        self._target_part_ownership_list = []
        self._con_hash = ConsistentHash(cluster_list)
        self._name = self_name

        # some sanity check
        if not (self._name in cluster_list):
            raise ValueError('cluster list is missing local server name')

        # initialize logging and other stuff
        if logger is None:
            logging.basicConfig()
            self._logger = logging
        else:
            self._logger = logger
        self._conn_state = None
        self._sandesh_connection_info_update(status='INIT', message='')

        # connect to zookeeper
        while True:
            self._logger.error("Libpartition zk start")
            self._zk = KazooClient(zk_server)
            self._zk.add_listener(self._zk_listen)
            try:
                self._zk.start()
                while self._conn_state != ConnectionStatus.UP:
                    gevent.sleep(1)
                break
            except Exception as e:
                # Update connection info
                self._sandesh_connection_info_update(status='DOWN',
                                                     message=str(e))
                self._zk.remove_listener(self._zk_listen)
                try:
                    self._zk.stop()
                    self._zk.close()
                except Exception as ex:
                    template = "Exception {0} in Libpartition zk stop/close. Args:\n{1!r}"
                    messag = template.format(type(ex).__name__, ex.args)
                    self._logger.error("%s : traceback %s for %s" % \
                        (messag, traceback.format_exc(), self._name))
                finally:
                    self._zk = None
                gevent.sleep(1)

        # create a lock array to contain locks for each partition
        self._part_locks = []
        for part in range(0, self._max_partition):
            lockpath = "/lockpath/" + app_name + "/" + str(part)
            l = self._zk.Lock(lockpath, self._name)
            self._part_locks.append(l)

        # initialize partition # to lock acquire greenlet dictionary
        self._part_lock_task_dict = {}

        self._logger.error("initial servers:" + str(self._cluster_list))

        # update target partition ownership list
        for part in range(0, self._max_partition):
            if (self._con_hash.get_node(str(part)) == self._name):
                self._target_part_ownership_list.append(part)

        # update current ownership list
        self._acquire_partition_ownership()
class ShardedRedisConnection(object):
    """Maintain a list of several Redis backends in a sharded manner.

    This class establishes several pools based off of the IP addresses
    resolved from the ``hostname`` part of the ``REDIS_URI`` environment
    variable.  Any reads, writes, or deletes will be delegated to the proper
    Redis backend by determining which shard the query should be directed
    to.

    Additionally, the ``info`` method is available to gather health
    information across all of the servers in the backend.  This data can
    be used to determine the health of the service.

    .. note::

        The hostname in the ``REDIS_URI`` will be DNS resolved and a
        connection will be opened for each address returned in the
        answer section.  You can specify a Round-Robin DNS record and
        we will open a connection to all hosts returned.

    """
    def __init__(self):
        self.config = self._get_redis_config()
        self._connections = {}
        self._consistent_hash = ConsistentHash(self.config.hosts)
        self._establish_connections(self.config)

    def _get_redis_config(self):
        """Construct a Redis config object from the URI env-var."""
        LOGGER.debug('Creating connection info for "%s"',
                     os.environ['REDIS_URI'])

        broken = urlsplit(os.environ['REDIS_URI'])

        if broken.scheme != 'redis':
            raise RuntimeError('Non "redis://" URI provided in REDIS_URI!')

        _, _, ip_addresses = socket.gethostbyname_ex(broken.hostname)

        if not ip_addresses:
            raise RuntimeError('Unable to find Redis in DNS!')

        ttl = DEFAULT_TTL
        if broken.query:
            # parse_qs returns a list of values given a key
            ttl = parse_qs(broken.query).get('ttl', [ttl])[0]

        return _redis_config(
            hosts=ip_addresses,
            port=broken.port or DEFAULT_PORT,
            db=broken.path[1:],  # Remove the leading /
            ttl=int(ttl),
        )

    def _establish_connections(self, config):
        """Create Redis connection pool objects for each server shard."""
        LOGGER.debug('Establishing Redis connection pools')
        for host in config.hosts:
            LOGGER.debug('Opening Redis connection to host "%s"', host)
            self._connections[host] = redis.StrictRedis(
                host=host,
                port=config.port,
                db=config.db,
            )

    def _get_shard_connection(self, key):
        """Get a connection for a Redis shard given a ``key``."""
        LOGGER.debug('Getting Redis host shard given key "%s"', key)
        host = self._consistent_hash.get_node(key)
        LOGGER.debug('Got Redis host shard "%s" given key "%s"', host, key)
        return self._connections[host]

    def set(self, key, value, ttl=None):
        """Set ``key`` to ``value`` in a Redis shard."""
        LOGGER.debug('Setting Redis key "%s"', key)
        ttl = ttl or self.config.ttl
        connection = self._get_shard_connection(key)
        connection.set(key, value, ex=ttl)

    def get(self, key):
        """Get a ``key`` in a Redis shard."""
        LOGGER.debug('Getting Redis value given key "%s"', key)
        connection = self._get_shard_connection(key)
        return connection.get(key)

    def delete(self, key):
        """Delete a ``key`` in a Redis shard."""
        LOGGER.debug('Deleting Redis key "%s"', key)
        connection = self._get_shard_connection(key)
        connection.delete(key)

    def sadd(self, key, *values):
        """Add the specified ``values`` to the set stored at ``key``."""
        LOGGER.debug('Getting Redis key "%s" set members', key)
        connection = self._get_shard_connection(key)
        return connection.sadd(key, *values)

    def smembers(self, key):
        """Return all members of the set stored at ``key``."""
        LOGGER.debug('Getting Redis key "%s" set members', key)
        connection = self._get_shard_connection(key)
        return connection.smembers(key)

    def sismember(self, key, member):
        """Returns if ``member`` is a member of the set stored at ``key``."""
        LOGGER.debug('Checkin for membership at Redis key "%s"', key)
        connection = self._get_shard_connection(key)
        return connection.sismember(key, member)

    def info(self):
        """Return a list of the health of each connected redis server.

        :rtype: list
        :returns: A list of the server info from all of the server shards.

        """
        LOGGER.info('Getting Redis server stats')
        stats = []
        for host, connection in self._connections.items():
            LOGGER.debug('Getting Redis health for host "%s"', host)
            stats.append(connection.info())

        return stats
class ConsistentScheduler(object):
    '''
        LibPartitionHelper abstract out workers and work_items, and their
        mapping to partitions. So application can only deal with the work
        items it owns, without bothering about partition mapping.

        This class also provides syncronization premitives to ensure apps
        to clean up b4 giving up their partitions
    '''
    _MAX_WAIT_4_ALLOCATION = 6 + randint(0, 9)

    def __init__(self,
                 service_name=None,
                 zookeeper='127.0.0.1:2181',
                 delete_hndlr=None,
                 add_hndlr=None,
                 bucketsize=47,
                 item2part_func=None,
                 partitioner=None,
                 logger=None,
                 cluster_id=''):
        if logger:
            self._logger = logger
        else:
            self._logger = logging.getLogger(__name__)
        self._service_name = service_name or os.path.basename(sys.argv[0])
        self._item2part_func = item2part_func or self._device2partition
        self._zookeeper_srvr = zookeeper
        self._zk = None
        self._bucketsize = bucketsize
        self._delete_hndlr = delete_hndlr
        self._add_hndlr = add_hndlr
        self._partitioner = partitioner or self._partitioner_func
        self._partitions = {}
        self._con_hash = None
        self._last_log = ''
        self._last_log_cnt = 0
        self._partition_set = map(str, range(self._bucketsize))

        self._cluster_id = cluster_id
        if self._cluster_id:
            self._zk_path = '/' + self._cluster_id + '/contrail_cs' + '/' + self._service_name
        else:
            self._zk_path = '/'.join(['/contrail_cs', self._service_name])
        self._conn_state = None
        self._sandesh_connection_info_update(status='INIT', message='')

        while True:
            self._logger.error("Consistent scheduler zk start")
            self._zk = KazooClient(self._zookeeper_srvr,
                                   handler=SequentialGeventHandler())
            self._zk.add_listener(self._zk_lstnr)
            try:
                self._zk.start()
                while self._conn_state != ConnectionStatus.UP:
                    gevent.sleep(1)
                break
            except Exception as e:
                # Update connection info
                self._sandesh_connection_info_update(status='DOWN',
                                                     message=str(e))
                self._zk.remove_listener(self._zk_lstnr)
                try:
                    self._zk.stop()
                    self._zk.close()
                except Exception as ex:
                    template = "Exception {0} in Consistent scheduler zk stop/close. Args:\n{1!r}"
                    messag = template.format(type(ex).__name__, ex.args)
                    self._logger.error("%s : traceback %s for %s" % \
                        (messag, traceback.format_exc(), self._service_name))
                finally:
                    self._zk = None
                gevent.sleep(1)
        self._pc = self._zk.SetPartitioner(path=self._zk_path,
                                           set=self._partition_set,
                                           partition_func=self._partitioner)
        self._wait_allocation = 0
        gevent.sleep(0)

    def _sandesh_connection_info_update(self, status, message):
        new_conn_state = getattr(ConnectionStatus, status)
        ConnectionState.update(conn_type=ConnectionType.ZOOKEEPER,
                               name='Zookeeper',
                               status=new_conn_state,
                               message=message,
                               server_addrs=self._zookeeper_srvr.split(','))

        if ((self._conn_state and self._conn_state != ConnectionStatus.DOWN)
                and new_conn_state == ConnectionStatus.DOWN):
            msg = 'Connection to Zookeeper down: %s' % (message)
            self._supress_log(msg)
        if (self._conn_state and self._conn_state != new_conn_state
                and new_conn_state == ConnectionStatus.UP):
            msg = 'Connection to Zookeeper ESTABLISHED'
            self._supress_log(msg)

        self._conn_state = new_conn_state

    # end _sandesh_connection_info_update

    def _zk_lstnr(self, state):
        self._logger.error("Consistent scheduler listen %s" % str(state))
        if state == KazooState.CONNECTED:
            # Update connection info
            self._sandesh_connection_info_update(status='UP', message='')
        elif state == KazooState.LOST:
            self._logger.error("Consistent scheduler connection LOST")
            # Lost the session with ZooKeeper Server
            # Best of option we have is to exit the process and restart all
            # over again
            self._sandesh_connection_info_update(
                status='DOWN', message='Connection to Zookeeper lost')
            os._exit(2)
        elif state == KazooState.SUSPENDED:
            self._logger.error("Consistent scheduler connection SUSPENDED")
            # Update connection info
            self._sandesh_connection_info_update(
                status='INIT',
                message='Connection to zookeeper lost. Retrying')

    def schedule(self, items, lock_timeout=30):
        gevent.sleep(0)
        ret = False
        if self._pc.failed:
            self._logger.error('Lost or unable to acquire partition')
            os._exit(2)
        elif self._pc.release:
            self._supress_log('Releasing...')
            self._release()
        elif self._pc.allocating:
            self._supress_log('Waiting for allocation...')
            self._pc.wait_for_acquire(lock_timeout)
            if self._wait_allocation < self._MAX_WAIT_4_ALLOCATION:
                self._wait_allocation += 1
            else:
                self._logger.error('Giving up after %d tries!' %
                                   (self._wait_allocation))
                os._exit(2)
        elif self._pc.acquired:
            self._supress_log('got work: ', list(self._pc))
            ret = True
            self._wait_allocation = 0
            self._populate_work_items(items)
            self._supress_log('work items: ',
                              self._items2name(self.work_items()),
                              'from the list', self._items2name(items))
        return ret

    def members(self):
        return list(self._con_hash.nodes)

    def partitions(self):
        return list(self._pc)

    def work_items(self):
        return sum(self._partitions.values(), [])

    def finish(self):
        self._inform_delete(self._partitions.keys())
        self._pc.finish()
        self._zk.remove_listener(self._zk_lstnr)
        gevent.sleep(1)
        try:
            self._zk.stop()
        except:
            self._logger.error("Stopping kazooclient failed")
        else:
            self._logger.error("Stopping kazooclient successful")
        try:
            self._zk.close()
        except:
            self._logger.error("Closing kazooclient failed")
        else:
            self._logger.error("Closing kazooclient successful")

    def _items2name(self, items):
        return map(lambda x: x.name, items)

    def _supress_log(self, *s):
        slog = ' '.join(map(str, s))
        dl = ''
        if slog != self._last_log_cnt:
            if self._last_log_cnt:
                dl += ' ' * 4
                dl += '.' * 8
                dl += '[last print repeats %d times]' % self._last_log_cnt
                self._last_log_cnt = 0
            dl += slog
            self._last_log = slog
            self._logger.debug(dl)
        else:
            self._last_log_cnt += 1

    def _consistent_hash(self, members):
        if self._con_hash is None:
            self._con_hash = ConsistentHash(members)
            self._logger.error('members: %s' % (str(self._con_hash.nodes)))
        cur, updtd = set(self._con_hash.nodes), set(members)
        if cur != updtd:
            newm = updtd - cur
            rmvd = cur - updtd
            if newm:
                self._logger.error('new members: %s' % (str(newm)))
                self._con_hash.add_nodes(list(newm))
            if rmvd:
                self._logger.error('members left: %s' % (str(rmvd)))
                self._con_hash.del_nodes(list(rmvd))
        return self._con_hash

    def _consistent_hash_get_node(self, members, partition):
        return self._consistent_hash(members).get_node(partition)

    def _partitioner_func(self, identifier, members, _partitions):
        partitions = [p for p in _partitions \
            if self._consistent_hash_get_node(members, p) == identifier]
        self._logger.error('partitions: %s' % (str(partitions)))
        return partitions

    def _release(self):
        old = set(self._pc)
        new = set(
            self._partitioner(self._pc._identifier, list(self._pc._party),
                              self._partition_set))
        rmvd = old - new
        added = new - old
        if rmvd:
            self._inform_delete(list(rmvd))
        if added:
            self._inform_will_add(list(added))
        self._pc.release_set()

    def _list_items_in(self, partitions):
        return sum([self._partitions[k] for k in partitions if k in \
                    self._partitions], [])

    def _inform_will_add(self, partitions):
        if callable(self._add_hndlr):
            self._add_hndlr(self._list_items_in(partitions))

    def _inform_delete(self, partitions):
        if callable(self._delete_hndlr):
            self._delete_hndlr(self._list_items_in(partitions))

    def _populate_work_items(self, items):
        self._refresh_work_items()
        for i in items:
            part = str(self._item2part_func(i.name))
            if part in list(self._pc):
                if part not in self._partitions:
                    self._partitions[part] = []
                if i.name not in map(lambda x: x.name, self._partitions[part]):
                    self._partitions[part].append(i)
        self._logger.debug('@populate_work_items(%s): done!' % ' '.join(
            map(
                lambda v: str(v[0]) + ':' + ','.join(
                    map(lambda x: x.name, v[1])), self._partitions.items())))
        gevent.sleep(0)

    def _device2partition(self, key):
        return struct.unpack(
            'Q',
            hashlib.md5(key).digest()[-8:])[0] % self._bucketsize

    def _refresh_work_items(self):
        for k in self._partitions:
            self._partitions[k] = []
Example #30
0
class PartitionClient(object):
    """ Client Class for the Partition Library
    Example usage:
    ---------------------
    import libpartition
    from libpartition import PartitionClient

    def own_change_cb(l):
            print "ownership change:" + str(l)

    c = PartitionClient("test", "s1", ["s1", "s2", "s3"], 32, 
            own_change_cb, "zookeeper_s1")

    ##do some real work now"
    if (c.own_partition(1)):
        ...... do something with partition #1 .....
        .........
    ...
    c.update_cluster_list(["s1", "s2"])
    ...
    ----------------------
    You should not call any partition library routine from within the 
    callback function

    Args:
        app_name(str): Name of the app for which partition cluster is used
        self_name(str): Name of the local cluster node (can be ip address)
        cluster_list(list): List of all the nodes in the cluster including 
            local node
        max_partition(int): Partition space always go from 0..max_partition-1
        partition_update_cb: Callback function invoked when partition
            ownership list is updated.x
        zk_server(str): <zookeeper server>:<zookeeper server port>
    """
    def __init__(self, app_name, self_name, cluster_list, max_partition,
                 partition_update_cb, zk_server):

        # Initialize local variables
        self._zk_server = zk_server
        self._cluster_list = set(cluster_list)
        self._max_partition = max_partition
        self._update_cb = partition_update_cb
        self._curr_part_ownership_list = []
        self._target_part_ownership_list = []
        self._con_hash = ConsistentHash(cluster_list)
        self._name = self_name

        # some sanity check
        if not (self._name in cluster_list):
            raise ValueError('cluster list is missing local server name')

        # connect to zookeeper
        self._zk = KazooClient(zk_server)
        self._zk.start()

        # create a lock array to contain locks for each partition
        self._part_locks = []
        for part in range(0, self._max_partition):
            lockpath = "/lockpath/" + app_name + "/" + str(part)
            l = self._zk.Lock(lockpath, self._name)
            self._part_locks.append(l)

        # initialize partition # to lock acquire greenlet dictionary
        self._part_lock_task_dict = {}

        # update target partition ownership list
        for part in range(0, self._max_partition):
            if (self._con_hash.get_node(str(part)) == self._name):
                self._target_part_ownership_list.append(part)

        # update current ownership list
        self._acquire_partition_ownership()

    #end __init__

    # following routine is the greenlet task function to acquire the lock
    # for a partition
    def _acquire_lock(self, part):
        # lock for the partition
        l = self._part_locks[part]

        while True:
            if (l.cancelled == True):
                # a lock acquisition is getting cancelled let's wait
                logging.info("lock acquisition is getting cancelled, \
                        lets wait")
                gevent.sleep(1)
            else:
                break

        # go in an infinite loop waiting to acquire the lock
        while True:
            ret = l.acquire(blocking=False)
            if ret == True:

                logging.info("Acquired lock for:" + str(part))
                self._curr_part_ownership_list.append(part)
                self._update_cb(self._curr_part_ownership_list)
                return ret
            else:
                gevent.sleep(1)

    #end _acquire_lock

    # get rid of finished spawned tasks from datastructures
    def _cleanup_greenlets(self):
        for part in self._part_lock_task_dict.keys():
            if (self._part_lock_task_dict[part].ready()):
                del self._part_lock_task_dict[part]

    #end _cleanup_greenlets

    # following routine launches tasks to acquire partition locks
    def _acquire_partition_ownership(self):
        # cleanup any finished greenlets
        self._cleanup_greenlets()

        # this variable will help us decide if we need to call callback
        updated_curr_ownership = False

        for part in range(0, self._max_partition):
            if (part in self._target_part_ownership_list):
                if (part in self._curr_part_ownership_list):
                    # do nothing, I already have ownership of this partition
                    logging.info("No need to acquire ownership of:" +
                                 str(part))
                else:
                    # I need to acquire lock for this partition before I own
                    if (part in self._part_lock_task_dict.keys()):
                        # do nothing there is already a greenlet running to
                        # acquire the lock
                        logging.info("Already a greenlet running to"
                                     " acquire:" + str(part))
                    else:
                        # launch the greenlet to acquire the loc, k
                        g = Greenlet.spawn(self._acquire_lock, part)
                        self._part_lock_task_dict[part] = g

            else:
                # give up ownership of the partition

                # cancel any lock acquisition which is ongoing
                if (part in self._part_lock_task_dict.keys()):
                    # kill the greenlet trying to get the lock for this
                    # partition
                    self._part_lock_task_dict[part].kill()
                    del self._part_lock_task_dict[part]

                    logging.info("canceling lock acquisition going on \
                            for:" + str(part))
                    try:
                        self._part_locks[part].cancel()
                    except:
                        pass

                if (part in self._curr_part_ownership_list):
                    # release if lock was already acquired
                    logging.info("release the lock which was acquired:" + \
                            str(part))
                    try:
                        self._part_locks[part].release()
                    except:
                        pass

                    self._curr_part_ownership_list.remove(part)
                    updated_curr_ownership = True
                    logging.info("gave up ownership of:" + str(part))

        if (updated_curr_ownership is True):
            # current partition membership was updated call the callback
            self._update_cb(self._curr_part_ownership_list)

    #end _acquire_partition_ownership

    def update_cluster_list(self, cluster_list):
        """ Updates the cluster node list
        Args:
            cluster_list(list): New list of names of the nodes in 
                the cluster
        Returns:
            None
        """
        # some sanity check
        if not (self._name in cluster_list):
            raise ValueError('cluster list is missing local server name')

        new_cluster_list = set(cluster_list)
        new_servers = list(new_cluster_list.difference(self._cluster_list))
        deleted_servers = list(
            set(self._cluster_list).difference(new_cluster_list))
        self._cluster_list = cluster_list
        logging.info("deleted servers:" + str(deleted_servers))
        logging.info("new servers:" + str(new_servers))

        # update the hash structure
        if new_servers:
            self._con_hash.add_nodes(new_servers)
        if deleted_servers:
            self._con_hash.del_nodes(deleted_servers)

        # update target partition ownership list
        self._target_part_ownership_list = []
        for part in range(0, self._max_partition):
            if (self._con_hash.get_node(str(part)) == self._name):
                if not (part in self._target_part_ownership_list):
                    self._target_part_ownership_list.append(part)

        # update current ownership list
        self._acquire_partition_ownership()

    #end update_cluster_list

    def own_partition(self, part_no):
        """ Returns ownership information of a partition
        Args:
            part_no(int) : Partition no 
        Returns:
            True if partition is owned by the local node
            False if partition is not owned by the local node
        """
        return part_no in self._curr_part_ownership_list

    #end own_partition

    def close(self):
        """ Closes any connections and frees up any data structures
        Args:
        Returns:
            None
        """
        # clean up greenlets
        for part in self._part_lock_task_dict.keys():
            try:
                self._part_lock_task_dict[part].kill()
            except:
                pass

        # close zookeeper
        try:
            self._zk.stop()
        except:
            pass
        try:
            self._zk.close()
        except:
            pass
    def __init__(
            self, app_name, self_name, cluster_list, max_partition,
            partition_update_cb, zk_server, logger = None):
       
        # Initialize local variables
        self._zk_server = zk_server
        self._cluster_list = set(cluster_list)
        self._max_partition = max_partition
        self._update_cb = partition_update_cb
        self._curr_part_ownership_list = []
        self._target_part_ownership_list = []
        self._con_hash = ConsistentHash(cluster_list)
        self._name = self_name

        # some sanity check
        if not(self._name in cluster_list):
            raise ValueError('cluster list is missing local server name')

        # initialize logging and other stuff
        if logger is None:
            logging.basicConfig()
            self._logger = logging
        else:
            self._logger = logger
        self._conn_state = None
        self._sandesh_connection_info_update(status='INIT', message='')

        # connect to zookeeper
        while True:
            self._logger.error("Libpartition zk start")
            self._zk = KazooClient(zk_server)
            self._zk.add_listener(self._zk_listen)
            try:
                self._zk.start()
                while self._conn_state != ConnectionStatus.UP:
                    gevent.sleep(1)
                break
            except Exception as e:
                # Update connection info
                self._sandesh_connection_info_update(status='DOWN',
                                                     message=str(e))
                self._zk.remove_listener(self._zk_listen)
                try:
                    self._zk.stop()
                    self._zk.close()
                except Exception as ex:
                    template = "Exception {0} in Libpartition zk stop/close. Args:\n{1!r}"
                    messag = template.format(type(ex).__name__, ex.args)
                    self._logger.error("%s : traceback %s for %s" % \
                        (messag, traceback.format_exc(), self._name))
                finally:
                    self._zk = None
                gevent.sleep(1)

        # create a lock array to contain locks for each partition
        self._part_locks = []
        for part in range(0, self._max_partition):
            lockpath = "/lockpath/"+ app_name + "/" + str(part)
            l = self._zk.Lock(lockpath, self._name)
            self._part_locks.append(l)

        # initialize partition # to lock acquire greenlet dictionary
        self._part_lock_task_dict = {}
       
        self._logger.error("initial servers:" + str(self._cluster_list))

        # update target partition ownership list
        for part in range(0, self._max_partition):
            if (self._con_hash.get_node(str(part)) == self._name):
                self._target_part_ownership_list.append(part)

        # update current ownership list
        self._acquire_partition_ownership()
 def __init__(self, replicas=1, *args, **kwargs):
     self.consistent_hash = ConsistentHash(replicas=replicas)
     super(MemcacheClient, self).__init__(*args, **kwargs)