def test_job_with_RoundRobinRouter(self):
pubsub = self.get_cluster('nydus.db.routers.RoundRobinRouter')
jobs = {}
size = 20
# put jobs onto the queue
for x in xrange(0, size):
jps = pubsub.job('testjob')
jid = jps.put(str(x))
if id(jps) not in jobs:
jobs[id(jps)] = []
jobs[id(jps)].append(jid)
# make sure that we are reusing the job items
self.assertEqual(len(jobs), 5)
for k, v in jobs.iteritems():
self.assertEqual(len(v), size / 5)
# make sure we fishi
for x in xrange(0, size):
jps = pubsub.job('testjob')
jid, job, cancel_count = jps.get()
jps.finish(jid)
self.assertEqual(len(jobs), 5)
for x in range(len(pubsub)):
ps = pubsub.get_conn('testjob')
jps = ps.job('testjob')
self.assertEqual(jps.get_ids(), [])
def test_job_with_RoundRobinRouter(self):
pubsub = self.get_cluster('nydus.db.routers.RoundRobinRouter')
jobs = {}
size = 20
# put jobs onto the queue
for x in xrange(0, size):
jps = pubsub.job('testjob')
jid = jps.put(str(x))
if id(jps) not in jobs:
jobs[id(jps)] = []
jobs[id(jps)].append(jid)
# make sure that we are reusing the job items
self.assertEqual(len(jobs), 5)
for k, v in jobs.iteritems():
self.assertEqual(len(v), size / 5)
# make sure we fishi
for x in xrange(0, size):
jps = pubsub.job('testjob')
jid, job, cancel_count = jps.get()
jps.finish(jid)
self.assertEqual(len(jobs), 5)
for x in range(len(pubsub)):
ps = pubsub.get_conn('testjob')
jps = ps.job('testjob')
self.assertEqual(jps.get_ids(), [])
def _build_circle(self):
"""
Creates hash ring.
"""
total_weight = 0
for node in self._nodes:
total_weight += self._weights.get(node, 1)
for node in self._nodes:
weight = self._weights.get(node, 1)
ks = math.floor((40 * len(self._nodes) * weight) / total_weight)
for i in xrange(0, int(ks)):
b_key = self._md5_digest('%s-%s-salt' % (node, i))
for l in xrange(0, 4):
key = ((b_key[3 + l * 4] << 24) |
(b_key[2 + l * 4] << 16) |
(b_key[1 + l * 4] << 8) |
b_key[l * 4])
self._hashring[key] = node
self._sorted_keys.append(key)
self._sorted_keys.sort()
def _build_circle(self):
"""
Creates hash ring.
"""
total_weight = 0
for node in self._nodes:
total_weight += self._weights.get(node, 1)
for node in self._nodes:
weight = self._weights.get(node, 1)
ks = math.floor((40 * len(self._nodes) * weight) / total_weight)
for i in xrange(0, int(ks)):
b_key = self._md5_digest('%s-%s-salt' % (node, i))
for l in xrange(0, 4):
key = ((b_key[3 + l * 4] << 24) |
(b_key[2 + l * 4] << 16) |
(b_key[1 + l * 4] << 8) |
b_key[l * 4])
self._hashring[key] = node
self._sorted_keys.append(key)
self._sorted_keys.sort()
def execute(self, path, args, kwargs):
connections = self.__connections_for(path, args=args, kwargs=kwargs)
results = []
for conn in connections:
for retry in xrange(self.max_connection_retries):
func = conn
for piece in path.split('.'):
func = getattr(func, piece)
try:
results.append(func(*args, **kwargs))
except tuple(conn.retryable_exceptions) as e:
if not self.router.retryable:
raise e
elif retry == self.max_connection_retries - 1:
raise self.MaxRetriesExceededError(e)
else:
conn = self.__connections_for(path, retry_for=conn.num, args=args, kwargs=kwargs)[0]
else:
break
# If we only had one db to query, we simply return that res
if len(results) == 1:
return results[0]
else:
return results
def _route(self, attr, args, kwargs, **fkwargs):
now = time.time()
for i in xrange(len(self.cluster)):
db_num = next(self._hosts_cycler)
marked_down_at = self._down_connections.get(db_num, False)
if not marked_down_at or (marked_down_at + self.retry_timeout <= now):
return [db_num]
else:
raise self.HostListExhausted()
def _route(self, attr, args, kwargs, **fkwargs):
now = time.time()
for i in xrange(len(self.cluster)):
db_num = next(self._hosts_cycler)
marked_down_at = self._down_connections.get(db_num, False)
if not marked_down_at or (marked_down_at + self.retry_timeout <=
now):
return [db_num]
else:
raise self.HostListExhausted()
def test(k):
data = {}
for i in xrange(REQUESTS):
tower = k.get_node('a' + str(i))
data.setdefault(tower, 0)
data[tower] += 1
print('Number of caches on each node: ')
print(data)
print('')
print(k.get_node('Aplple'))
print(k.get_node('Hello'))
print(k.get_node('Data'))
print(k.get_node('Computer'))
def test(k):
data = {}
for i in xrange(REQUESTS):
tower = k.get_node('a' + str(i))
data.setdefault(tower, 0)
data[tower] += 1
print('Number of caches on each node: ')
print(data)
print('')
print(k.get_node('Aplple'))
print(k.get_node('Hello'))
print(k.get_node('Data'))
print(k.get_node('Computer'))
def setUp(self):
self.hosts = dict((i, {}) for i in xrange(5))
self.cluster = BaseCluster(router=self.Router, hosts=self.hosts, backend=DummyConnection)
self.router = self.cluster.router
def __init__(self, workers=10):
self.queue = Queue()
self.workers = []
self.tasks = []
for worker in xrange(workers):
self.workers.append(Worker(self.queue))
def setUp(self):
self.hosts = dict((i, {}) for i in xrange(5))
self.cluster = BaseCluster(router=self.Router,
hosts=self.hosts,
backend=DummyConnection)
self.router = self.cluster.router
