def testAddNode(self):
data = range(100)
new_port = random.randint(10000, 30000)
new_node = 'localhost:%s' % new_port
new_rpc_server = ColaRPCServer(('localhost', new_port))
thd = threading.Thread(target=new_rpc_server.serve_forever)
thd.setDaemon(True)
thd.start()
new_dir = tempfile.mkdtemp()
ns = list(self.nodes)
ns.append(new_node)
new_mq = MessageQueue(new_dir, new_rpc_server, new_node, ns)
try:
self.mq0.add_node(new_node)
self.mq1.add_node(new_node)
self.mq2.add_node(new_node)
self.mq0.put(data)
self.assertEqual(data, sorted(self.mq0.get(size=100)))
finally:
try:
new_rpc_server.shutdown()
new_mq.shutdown()
finally:
shutil.rmtree(new_dir)
def testMqProxy(self):
self.mq = MessageQueue(self.dir_, None, self.addr, self.addrs,
copies=0, n_priorities=1)
self.proxy = MpMessageQueueClient(self.mq.new_connection('0'))
try:
test_obj = Url(u'http://qinxuye.me/三星')
self.proxy.put(test_obj, )
self.assertEqual(self.proxy.get(), test_obj)
test_obj = u'三星'
self.proxy.put(test_obj, )
self.assertEqual(self.proxy.get(), test_obj)
finally:
self.mq.shutdown()
def init_mq(self):
mq_store_dir = os.path.join(self.root, 'store')
mq_backup_dir = os.path.join(self.root, 'backup')
if not os.path.exists(mq_store_dir):
os.makedirs(mq_store_dir)
if not os.path.exists(mq_backup_dir):
os.makedirs(mq_backup_dir)
self.mq = MessageQueue(self.nodes,
self.local,
self.rpc_server,
copies=self.copies)
self.mq.init_store(mq_store_dir,
mq_backup_dir,
verify_exists_hook=self._init_bloom_filter())
def init_mq(self):
mq_dir = os.path.join(self.working_dir, 'mq')
copies = self.job_desc.settings.job.copies
n_priorities = self.job_desc.settings.job.priorities
kw = {
'app_name': self.job_name,
'copies': copies,
'n_priorities': n_priorities,
'deduper': self.deduper
}
self.mq = MessageQueue(mq_dir, self.rpc_server, self.ctx.worker_addr,
self.ctx.addrs[:], **kw)
# register shutdown callback
self.shutdown_callbacks.append(self.mq.shutdown)
def setUp(self):
ports = tuple([random.randint(10000, 30000) for _ in range(3)])
self.nodes = ['localhost:%s'%port for port in ports]
self.dirs = [tempfile.mkdtemp() for _ in range(len(ports))]
self.size = len(ports)
for i in range(self.size):
setattr(self, 'rpc_server%s'%i, ColaRPCServer(('localhost', ports[i])))
setattr(self, 'mq%s'%i,
MessageQueue(self.dirs[i], getattr(self, 'rpc_server%s'%i),
self.nodes[i], self.nodes[:])
)
thd = threading.Thread(target=getattr(self, 'rpc_server%s'%i).serve_forever)
thd.setDaemon(True)
thd.start()
self.client = MessageQueueClient(self.nodes)
def init_mq(self,
nodes,
local_node,
loc,
verify_exists_hook=None,
copies=1):
mq_store_dir = os.path.join(loc, 'store')
mq_backup_dir = os.path.join(loc, 'backup')
if not os.path.exists(mq_store_dir):
os.mkdir(mq_store_dir)
if not os.path.exists(mq_backup_dir):
os.mkdir(mq_backup_dir)
# MQ relative
self.mq = MessageQueue(nodes,
local_node,
self.rpc_server,
copies=copies)
self.mq.init_store(mq_store_dir,
mq_backup_dir,
verify_exists_hook=verify_exists_hook)
def setUp(self):
ports = (11111, 11211, 11311)
self.nodes = ['localhost:%s' % port for port in ports]
self.dirs = [tempfile.mkdtemp() for _ in range(2 * len(ports))]
self.size = len(ports)
for i in range(self.size):
setattr(self, 'rpc_server%s' % i,
ColaRPCServer(('localhost', ports[i])))
setattr(
self, 'mq%s' % i,
MessageQueue(self.nodes[:], self.nodes[i],
getattr(self, 'rpc_server%s' % i)))
getattr(self, 'mq%s' % i).init_store(self.dirs[2 * i],
self.dirs[2 * i + 1])
thd = threading.Thread(target=getattr(self, 'rpc_server%s' %
i).serve_forever)
thd.setDaemon(True)
thd.start()
self.client = MessageQueueClient(self.nodes)
def __init__(self, nodes, copies=1):
self.nodes = nodes
self.hash_ring = HashRing(self.nodes)
self.copies = max(min(len(self.nodes) - 1, copies), 0)
self.mq = MessageQueue(nodes, copies=copies)
