def test_job_with_RoundRobinRouter(self): pubsub = self.get_cluster('nydus.db.routers.RoundRobinRouter') jobs = {} size = 20 # put jobs onto the queue for x in xrange(0, size): jps = pubsub.job('testjob') jid = jps.put(str(x)) if id(jps) not in jobs: jobs[id(jps)] = [] jobs[id(jps)].append(jid) # make sure that we are reusing the job items self.assertEqual(len(jobs), 5) for k, v in jobs.iteritems(): self.assertEqual(len(v), size / 5) # make sure we fishi for x in xrange(0, size): jps = pubsub.job('testjob') jid, job, cancel_count = jps.get() jps.finish(jid) self.assertEqual(len(jobs), 5) for x in range(len(pubsub)): ps = pubsub.get_conn('testjob') jps = ps.job('testjob') self.assertEqual(jps.get_ids(), [])
def _build_circle(self): """ Creates hash ring. """ total_weight = 0 for node in self._nodes: total_weight += self._weights.get(node, 1) for node in self._nodes: weight = self._weights.get(node, 1) ks = math.floor((40 * len(self._nodes) * weight) / total_weight) for i in xrange(0, int(ks)): b_key = self._md5_digest('%s-%s-salt' % (node, i)) for l in xrange(0, 4): key = ((b_key[3 + l * 4] << 24) | (b_key[2 + l * 4] << 16) | (b_key[1 + l * 4] << 8) | b_key[l * 4]) self._hashring[key] = node self._sorted_keys.append(key) self._sorted_keys.sort()
def execute(self, path, args, kwargs): connections = self.__connections_for(path, args=args, kwargs=kwargs) results = [] for conn in connections: for retry in xrange(self.max_connection_retries): func = conn for piece in path.split('.'): func = getattr(func, piece) try: results.append(func(*args, **kwargs)) except tuple(conn.retryable_exceptions) as e: if not self.router.retryable: raise e elif retry == self.max_connection_retries - 1: raise self.MaxRetriesExceededError(e) else: conn = self.__connections_for(path, retry_for=conn.num, args=args, kwargs=kwargs)[0] else: break # If we only had one db to query, we simply return that res if len(results) == 1: return results[0] else: return results
def _route(self, attr, args, kwargs, **fkwargs): now = time.time() for i in xrange(len(self.cluster)): db_num = next(self._hosts_cycler) marked_down_at = self._down_connections.get(db_num, False) if not marked_down_at or (marked_down_at + self.retry_timeout <= now): return [db_num] else: raise self.HostListExhausted()
def test(k): data = {} for i in xrange(REQUESTS): tower = k.get_node('a' + str(i)) data.setdefault(tower, 0) data[tower] += 1 print('Number of caches on each node: ') print(data) print('') print(k.get_node('Aplple')) print(k.get_node('Hello')) print(k.get_node('Data')) print(k.get_node('Computer'))
def setUp(self): self.hosts = dict((i, {}) for i in xrange(5)) self.cluster = BaseCluster(router=self.Router, hosts=self.hosts, backend=DummyConnection) self.router = self.cluster.router
def __init__(self, workers=10): self.queue = Queue() self.workers = [] self.tasks = [] for worker in xrange(workers): self.workers.append(Worker(self.queue))