class Test(unittest.TestCase):
def setUp(self):
self.dir_ = tempfile.mkdtemp()
self.addr = '127.0.0.1'
self.addrs = [self.addr, ]
def tearDown(self):
try:
self.mq.shutdown()
finally:
shutil.rmtree(self.dir_)
def testMqProxy(self):
self.mq = MessageQueue(self.dir_, None, self.addr, self.addrs,
copies=0, n_priorities=1)
self.proxy = MpMessageQueueClient(self.mq.new_connection('0'))
try:
test_obj = Url('http://qinxuye.me')
self.proxy.put(test_obj, )
self.assertEqual(self.proxy.get(), test_obj)
finally:
self.mq.shutdown()
class Test(unittest.TestCase):
def setUp(self):
self.dir_ = tempfile.mkdtemp()
self.addr = '127.0.0.1'
self.addrs = [
self.addr,
]
def tearDown(self):
try:
self.mq.shutdown()
finally:
shutil.rmtree(self.dir_)
def testMqProxy(self):
self.mq = MessageQueue(self.dir_,
None,
self.addr,
self.addrs,
copies=0,
n_priorities=1)
self.proxy = MpMessageQueueClient(self.mq.new_connection('0'))
try:
test_obj = Url('http://qinxuye.me')
self.proxy.put(test_obj, )
self.assertEqual(self.proxy.get(), test_obj)
finally:
self.mq.shutdown()
def testMqProxy(self):
self.mq = MessageQueue(self.dir_, None, self.addr, self.addrs,
copies=0, n_priorities=1)
self.proxy = MpMessageQueueClient(self.mq.new_connection('0'))
try:
test_obj = Url(u'http://qinxuye.me/三星')
self.proxy.put(test_obj, )
self.assertEqual(self.proxy.get(), test_obj)
test_obj = u'三星'
self.proxy.put(test_obj, )
self.assertEqual(self.proxy.get(), test_obj)
finally:
self.mq.shutdown()
def testMqProxy(self):
self.mq = MessageQueue(self.dir_, None, self.addr, self.addrs,
copies=0, n_priorities=1)
self.proxy = MpMessageQueueClient(self.mq.new_connection('0'))
try:
test_obj = Url('http://qinxuye.me')
self.proxy.put(test_obj, )
self.assertEqual(self.proxy.get(), test_obj)
finally:
self.mq.shutdown()
def run_containers(n_containers,
n_instances,
working_dir,
job_def_path,
job_name,
env,
mq,
counter_server,
budget_server,
speed_server,
stopped,
nonsuspend,
idle_statuses,
block=False,
is_multi_process=False,
is_local=False,
master_ip=None,
offset=0):
processes = []
acc = offset * n_instances
for container_id in range(n_containers):
n_tasks = n_instances / n_containers
if container_id < n_instances % n_containers:
n_tasks += 1
if not is_multi_process:
mq_client = mq
else:
conn = mq.new_connection(container_id)
mq_client = MpMessageQueueClient(conn)
container = Container(container_id,
working_dir,
job_def_path,
job_name,
env,
mq_client,
counter_server,
budget_server,
speed_server,
stopped,
nonsuspend,
idle_statuses,
n_tasks=n_tasks,
is_local=is_local,
master_ip=master_ip,
task_start_id=acc)
if is_multi_process:
process = multiprocessing.Process(target=container.run,
args=(True, ))
process.start()
processes.append(process)
else:
thread = threading.Thread(target=container.run, args=(True, ))
thread.start()
processes.append(thread)
acc += n_tasks
if block:
[process.join() for process in processes]
return processes
