def test_protobuf_process_remote_dispatch():
context1 = Context()
context1.start()
context2 = Context()
context2.start()
try:
ping_pong(context1, context2)
finally:
context1.stop()
context2.stop()
def test_mesos_executor_abort_on_disconnection(self):
os.environ['MESOS_SLAVE_PID'] = str(self.slave.pid)
shutdown_event, shutdown_call, shutdown_side_effect = self.mock_method(
)
executor = mock.create_autospec(Executor, spec_set=True)
executor.shutdown = mock.Mock(side_effect=shutdown_side_effect)
driver_context = Context()
driver_context.start()
self.close_contexts.append(driver_context)
driver = PesosExecutorDriver(executor, context=driver_context)
assert driver.start() == mesos_pb2.DRIVER_RUNNING
driver.executor_process.connected.wait(timeout=MAX_TIMEOUT)
assert driver.executor_process.connected.is_set()
self.slave.register_event.wait(timeout=MAX_TIMEOUT)
assert self.slave.register_event.is_set()
# kill connection
conn = driver_context._connections[self.slave.pid]
conn.close()
# abort event
shutdown_event.wait(timeout=MAX_TIMEOUT)
assert shutdown_event.is_set()
assert shutdown_call.mock_calls == [mock.call(driver)]
def test_simple_process():
parameter = []
event = threading.Event()
class DerpProcess(Process):
def __init__(self, **kw):
super(DerpProcess, self).__init__('derp', **kw)
@Process.install('ping')
def ping(self, value):
parameter.append(value)
event.set()
context = Context()
context.start()
derp = DerpProcess()
pid = context.spawn(derp)
context.dispatch(pid, 'ping', 42)
event.wait(timeout=1.0)
assert event.is_set()
assert parameter == [42]
context.stop()
def setUp(self):
self.__pong = subprocess.Popen(
'vagrant ssh -- LIBPROCESS_IP=192.168.33.2 LIBPROCESS_PORT=31337 GLOG_v=5 ./pong',
shell=True)
self.ping_pid = PID.from_string('(1)@192.168.33.2:31337')
self.context = Context(ip='192.168.33.1')
self.context.start()
def __init__(self, executor, context=None):
self.context = context or Context()
self.executor = executor
self.executor_process = None
self.executor_pid = None
self.status = mesos.DRIVER_NOT_STARTED
self.lock = threading.Condition()
def test_link_race_condition():
context1 = Context()
context1.start()
context2 = Context()
context2.start()
class Leader(Process):
def __init__(self):
super(Leader, self).__init__('leader')
self.uuid = None
@Process.install('register')
def register(self, from_pid, uuid):
log.debug('Leader::register(%s, %s)' % (from_pid, uuid))
self.send(from_pid, 'registered', uuid)
class Follower(Process):
def __init__(self, leader):
super(Follower, self).__init__('follower')
self.leader = leader
self.uuid = uuid.uuid4().bytes
self.registered = threading.Event()
def initialize(self):
super(Follower, self).initialize()
self.link(self.leader.pid)
self.send(self.leader.pid, 'register', self.uuid)
@Process.install('registered')
def registered(self, from_pid, uuid):
log.debug('Follower::registered(%s, %s)' % (from_pid, uuid))
assert uuid == self.uuid
assert from_pid == self.leader.pid
self.registered.set()
leader = Leader()
context1.spawn(leader)
follower = Follower(leader)
context2.spawn(follower)
follower.registered.wait(timeout=MAX_TIMEOUT)
assert follower.registered.is_set()
context1.stop()
context2.stop()
def test_protobuf_process_local_dispatch():
context = Context()
context.start()
try:
ping_pong(context, context)
finally:
context.stop()
def setUp(self):
self.framework_id = mesos_pb2.FrameworkID(value=self.FRAMEWORK_ID)
self.framework_info = mesos_pb2.FrameworkInfo(
user='******',
name='fake_framework_name',
)
self.context = Context()
self.context.start()
self.master = MockMaster()
self.context.spawn(self.master)
self.close_contexts = []
def test_multi_thread_multi_scatter():
with ephemeral_context() as context:
gather = GatherProcess()
context.spawn(gather)
scatters = [ScatterThread(gather.pid, 3, Context()) for k in range(5)]
for scatter in scatters:
scatter.context.start()
try:
startjoin(context, scatters)
finally:
for scatter in scatters:
scatter.context.stop()
def listen(identifier):
"""
Launch a listener and return the compactor context.
"""
context = Context()
process = WebProcess(identifier)
context.spawn(process)
log.info("Launching PID %s", process.pid)
return process, context
def test_simple_message_pass():
context = Context()
context.start()
receiver = SimpleReceiver()
sender = SimpleSender()
context.spawn(receiver)
context.spawn(sender)
sender.send_simple(receiver.pid, 'simple_framework', 'simple_pid')
receiver.receive_event.wait(timeout=1)
assert receiver.receive_event.is_set()
context.stop()
def __init__(self,
scheduler,
framework,
master_uri,
credential=None,
context=None):
self.context = context or Context()
self.scheduler = scheduler
self.scheduler_process = None
self.master_uri = master_uri
self.framework = framework
self.lock = threading.Condition()
self.status = mesos.DRIVER_NOT_STARTED
self.detector = None
self.credential = credential
def test_mesos_executor_driver_init():
context = Context()
context.start()
# spawn slave
slave = MockSlave()
context.spawn(slave)
os.environ['MESOS_SLAVE_PID'] = str(slave.pid)
executor = Executor()
driver = MesosExecutorDriver(executor, context=context)
assert driver.start() == mesos.DRIVER_RUNNING
assert driver.stop() == mesos.DRIVER_STOPPED
assert driver.join() == mesos.DRIVER_STOPPED
context.stop()
def test_mesos_executor_register():
context = Context()
context.start()
# spawn slave
slave = MockSlave()
context.spawn(slave)
os.environ['MESOS_SLAVE_PID'] = str(slave.pid)
executor = mock.MagicMock()
executor.registered = mock.MagicMock()
driver = PesosExecutorDriver(executor, context=context)
assert driver.start() == mesos.DRIVER_RUNNING
command_info = mesos.CommandInfo(value='wat')
framework_id = mesos.FrameworkID(value='fake_framework_id')
executor_id = mesos.ExecutorID(value='fake_executor_id')
executor_info = mesos.ExecutorInfo(
executor_id=executor_id,
framework_id=framework_id,
command=command_info
)
framework_info = mesos.FrameworkInfo(user='******', name='fake_framework_name')
slave.send_registered(
driver.executor_process.pid,
executor_info,
framework_id,
framework_info
)
driver.executor_process.connected.wait(timeout=1)
assert driver.executor_process.connected.is_set()
executor.registered.assert_called_with(driver, executor_info, framework_info, slave.slave_info)
assert driver.stop() == mesos.DRIVER_STOPPED
context.stop()
def test_link_exit_remote(self):
parent_context = Context()
parent_context.start()
parent = ParentProcess()
parent_context.spawn(parent)
child = ChildProcess()
self.context.spawn(child)
parent.send(child.pid, 'link_me')
child.link_event.wait(timeout=1.0)
assert child.link_event.is_set()
assert not child.exit_event.is_set()
parent_context.terminate(parent.pid)
parent_context.stop()
child.send(parent.pid, 'this_will_break')
child.exit_event.wait(timeout=1)
assert child.exit_event.is_set()
def test_mesos_executor_reregister(self):
os.environ['MESOS_SLAVE_PID'] = str(self.slave.pid)
os.environ['MESOS_CHECKPOINT'] = '1'
os.environ['MESOS_RECOVERY_TIMEOUT'] = '15mins'
reregistered_event, reregistered_call, reregistered_side_effect = self.mock_method(
)
executor = mock.create_autospec(Executor, spec_set=True)
executor.reregistered = mock.Mock(side_effect=reregistered_side_effect)
driver_context = Context()
driver_context.start()
self.close_contexts.append(driver_context)
driver = PesosExecutorDriver(executor, context=driver_context)
assert driver.start() == mesos_pb2.DRIVER_RUNNING
self.slave.register_event.wait(timeout=MAX_TIMEOUT)
assert self.slave.register_event.is_set()
driver.executor_process.connected.wait(timeout=MAX_TIMEOUT)
assert driver.executor_process.connected.is_set()
# kill connection, but expect reconnect
conn = driver_context._connections[self.slave.pid]
conn.close()
# simulate slave reconnecting
self.slave.send_reconnect(driver.executor_process.pid)
# wait for reregister event
self.slave.reregister_event.wait(timeout=MAX_TIMEOUT)
assert self.slave.reregister_event.is_set()
driver.executor_process.connected.wait(timeout=MAX_TIMEOUT)
assert driver.executor_process.connected.is_set()
# reregistered event
reregistered_event.wait(timeout=MAX_TIMEOUT)
assert reregistered_event.is_set()
assert reregistered_call.mock_calls == [
mock.call(driver, self.slave.slave_info)
]
def setUp(self):
command_info = mesos_pb2.CommandInfo(value='wat')
framework_id = mesos_pb2.FrameworkID(value=self.FRAMEWORK_ID)
executor_id = mesos_pb2.ExecutorID(value=self.EXECUTOR_ID)
self.executor_info = mesos_pb2.ExecutorInfo(
executor_id=executor_id,
framework_id=framework_id,
command=command_info,
)
self.framework_info = mesos_pb2.FrameworkInfo(
user='******',
name='fake_framework_name',
)
self.context = Context()
self.context.start()
self.slave = MockSlave(
mesos_pb2.SlaveID(value=self.SLAVE_ID),
{self.EXECUTOR_ID: self.executor_info},
{self.FRAMEWORK_ID: self.framework_info},
)
self.context.spawn(self.slave)
self.close_contexts = []