def put_starts(master=None):
if master is None:
master = ['%s:%s' % (get_ip(), getattr(user_config.master, 'port'))]
print('master:%s' % master)
jobs = client_call(master, 'runnable_jobs')
app_name = ''
for a, j in jobs.items():
if j == "douban movie":
app_name = a
break
if not app_name:
raise Exception('douban movie job has not upload')
nodes = client_call(master, 'list_workers')
addrs = []
default_addr = master.split(':')[0]
for ap, s in nodes:
a, p = ap.split(':')
if a.lower() == 'localhost':
addrs.append('%s:%s' % (default_addr, p))
else:
addrs.append(ap)
mq_client = MessageQueueClient(addrs, app_name)
print('get:%s' % mq_client.get())
urls = []
size = 0
for url in starts:
urls.append(url)
size += 1
if size >= PUTSIZE:
mq_client.put(urls)
size = 0
urls = []
if len(urls) > 0:
mq_client.put(urls)
class Test(unittest.TestCase):
def setUp(self):
ports = (11111, 11211, 11311)
self.nodes = ['localhost:%s'%port for port in ports]
self.dirs = [tempfile.mkdtemp() for _ in range(2*len(ports))]
self.size = len(ports)
for i in range(self.size):
setattr(self, 'rpc_server%s'%i, ColaRPCServer(('localhost', ports[i])))
setattr(self, 'mq%s'%i,
MessageQueue(self.nodes[:], self.nodes[i], getattr(self, 'rpc_server%s'%i))
)
getattr(self, 'mq%s'%i).init_store(self.dirs[2*i], self.dirs[2*i+1])
thd = threading.Thread(target=getattr(self, 'rpc_server%s'%i).serve_forever)
thd.setDaemon(True)
thd.start()
self.client = MessageQueueClient(self.nodes)
def tearDown(self):
try:
for i in range(self.size):
getattr(self, 'rpc_server%s'%i).shutdown()
getattr(self, 'mq%s'%i).shutdown()
finally:
for d in self.dirs:
shutil.rmtree(d)
def testMQ(self):
mq = self.mq0
data = [str(random.randint(10000, 50000)) for _ in range(20)]
mq.put(data)
gets = []
while True:
get = mq.get()
if get is None:
break
gets.append(get)
self.assertEqual(sorted(data), sorted(gets))
# test mq client
data = str(random.randint(10000, 50000))
self.client.put(data)
get = self.client.get()
self.assertEqual(data, get)
def testAddOrRemoveNode(self):
mq = self.mq0
data = [str(i) for i in range(100)]
mq.put(data)
self.mq2.shutdown()
self.assertEqual(len(self.nodes), 3)
self.mq0.remove_node(self.nodes[2])
self.assertEqual(len(self.nodes), 3)
self.mq1.remove_node(self.nodes[2])
gets = []
while True:
get = mq.get()
if get is None:
break
gets.append(get)
self.assertEqual(sorted(data), sorted(gets))
class Test(unittest.TestCase):
def setUp(self):
ports = (11111, 11211, 11311)
self.nodes = ['localhost:%s' % port for port in ports]
self.dirs = [tempfile.mkdtemp() for _ in range(2 * len(ports))]
self.size = len(ports)
for i in range(self.size):
setattr(self, 'rpc_server%s' % i,
ColaRPCServer(('localhost', ports[i])))
setattr(
self, 'mq%s' % i,
MessageQueue(self.nodes[:], self.nodes[i],
getattr(self, 'rpc_server%s' % i)))
getattr(self, 'mq%s' % i).init_store(self.dirs[2 * i],
self.dirs[2 * i + 1])
thd = threading.Thread(target=getattr(self, 'rpc_server%s' %
i).serve_forever)
thd.setDaemon(True)
thd.start()
self.client = MessageQueueClient(self.nodes)
def tearDown(self):
try:
for i in range(self.size):
getattr(self, 'rpc_server%s' % i).shutdown()
getattr(self, 'mq%s' % i).shutdown()
finally:
for d in self.dirs:
shutil.rmtree(d)
def testMQ(self):
mq = self.mq0
data = [str(random.randint(10000, 50000)) for _ in range(20)]
mq.put(data)
gets = []
while True:
get = mq.get()
if get is None:
break
gets.append(get)
self.assertEqual(sorted(data), sorted(gets))
# test mq client
data = str(random.randint(10000, 50000))
self.client.put(data)
get = self.client.get()
self.assertEqual(data, get)
def testAddOrRemoveNode(self):
mq = self.mq0
data = [str(i) for i in range(100)]
mq.put(data)
self.mq2.shutdown()
self.assertEqual(len(self.nodes), 3)
self.mq0.remove_node(self.nodes[2])
self.assertEqual(len(self.nodes), 3)
self.mq1.remove_node(self.nodes[2])
gets = []
while True:
get = mq.get()
if get is None:
break
gets.append(get)
self.assertEqual(sorted(data), sorted(gets))