def test_on_acknowledged(mocker):
agent_addr = 'mock_addr:12345'
framework_id = str(uuid.uuid4())
executor_id = str(uuid.uuid4())
env = {
'MESOS_LOCAL': 'true',
'MESOS_AGENT_ENDPOINT': agent_addr,
'MESOS_FRAMEWORK_ID': framework_id,
'MESOS_EXECUTOR_ID': executor_id,
}
mocker.patch('os.environ', env)
exc = mocker.Mock()
driver = MesosExecutorDriver(exc)
driver._started = True
assert driver.updates == {}
assert driver.tasks == {}
tid = str(uuid.uuid4())
uid = uuid.uuid4()
driver.updates[uid] = mocker.Mock()
driver.tasks[tid] = mocker.Mock()
event = {
'type': 'ACKNOWLEDGED',
'acknowledged': {
'task_id': {
'value': tid
},
'uuid': encode_data(uid.bytes)
}
}
driver.on_event(event)
assert driver.updates == {}
assert driver.tasks == {}
def test_on_launch(mocker):
agent_addr = 'mock_addr:12345'
framework_id = str(uuid.uuid4())
executor_id = str(uuid.uuid4())
env = {
'MESOS_LOCAL': 'true',
'MESOS_AGENT_ENDPOINT': agent_addr,
'MESOS_FRAMEWORK_ID': framework_id,
'MESOS_EXECUTOR_ID': executor_id,
}
mocker.patch('os.environ', env)
exc = mocker.Mock()
driver = MesosExecutorDriver(exc)
driver._started = True
task_id = str(uuid.uuid4())
framework_info = {'id': {'value': framework_id}}
task_info = {
"task_id": {
"value": task_id
},
}
event = {
'type': 'LAUNCH',
'launch': {
'framework_info': framework_info,
'task': task_info
}
}
driver.on_event(event)
exc.launchTask.assert_called_once_with(driver, task_info)
def test_on_subscribed(mocker):
agent_addr = 'mock_addr:12345'
framework_id = str(uuid.uuid4())
executor_id = str(uuid.uuid4())
env = {
'MESOS_LOCAL': 'true',
'MESOS_AGENT_ENDPOINT': agent_addr,
'MESOS_FRAMEWORK_ID': framework_id,
'MESOS_EXECUTOR_ID': executor_id,
}
mocker.patch('os.environ', env)
exc = mocker.Mock()
driver = MesosExecutorDriver(exc)
driver._started = True
executor_info = {
'executor_id': {
'value': executor_id
},
'framework_id': {
'value': framework_id
}
}
framework_info = {'id': {'value': framework_id}}
event = {
'type': 'SUBSCRIBED',
'subscribed': {
'executor_info': executor_info,
'framework_info': framework_info,
'agent_info': {}
}
}
driver.on_event(event)
exc.registered.assert_called_once_with(driver, executor_info,
framework_info, {})
def test_on_shutdown(mocker):
agent_addr = 'mock_addr:12345'
framework_id = str(uuid.uuid4())
executor_id = str(uuid.uuid4())
env = {
'MESOS_LOCAL': 'true',
'MESOS_AGENT_ENDPOINT': agent_addr,
'MESOS_FRAMEWORK_ID': framework_id,
'MESOS_EXECUTOR_ID': executor_id,
}
mocker.patch('os.environ', env)
exc = mocker.Mock()
driver = MesosExecutorDriver(exc)
event = {'type': 'shutdown'}
driver.on_event(event)
exc.shutdown.assert_called_once_with(driver)
def test_on_kill(mocker):
agent_addr = 'mock_addr:12345'
framework_id = str(uuid.uuid4())
executor_id = str(uuid.uuid4())
env = {
'MESOS_LOCAL': 'true',
'MESOS_AGENT_ENDPOINT': agent_addr,
'MESOS_FRAMEWORK_ID': framework_id,
'MESOS_EXECUTOR_ID': executor_id,
}
mocker.patch('os.environ', env)
exc = mocker.Mock()
driver = MesosExecutorDriver(exc)
task_id = {'value': str(uuid.uuid4())}
event = {'type': 'KILL', 'kill': {'task_id': task_id}}
driver.on_event(event)
exc.killTask.assert_called_once_with(driver, task_id)
def test_on_error(mocker):
agent_addr = 'mock_addr:12345'
framework_id = str(uuid.uuid4())
executor_id = str(uuid.uuid4())
env = {
'MESOS_LOCAL': 'true',
'MESOS_AGENT_ENDPOINT': agent_addr,
'MESOS_FRAMEWORK_ID': framework_id,
'MESOS_EXECUTOR_ID': executor_id,
}
mocker.patch('os.environ', env)
exc = mocker.Mock()
driver = MesosExecutorDriver(exc)
message = ''.join(
random.choice(string.printable) for _ in range(random.randint(1, 100)))
event = {'type': 'ERROR', 'error': {'message': message}}
driver.on_event(event)
exc.error.assert_called_once_with(driver, message)
def main():
logging.basicConfig(level=logging.DEBUG)
log.debug("Starting executor")
if not os.environ.has_key("MESOS_AGENT_ENDPOINT"):
# Some Mesos setups in our tests somehow lack this variable. Provide a
# fake one to maybe convince the executor driver to work.
os.environ["MESOS_AGENT_ENDPOINT"] = os.environ.get(
"MESOS_SLAVE_ENDPOINT", "127.0.0.1:5051")
log.warning("Had to fake MESOS_AGENT_ENDPOINT as %s" %
os.environ["MESOS_AGENT_ENDPOINT"])
try:
urlopen("http://%s/logging/toggle?level=1&duration=15mins" %
os.environ["MESOS_AGENT_ENDPOINT"]).read()
log.debug("Toggled agent log level")
except Exception as e:
log.debug("Failed to toggle agent log level")
# Parse the agent state
agent_state = json.loads(
urlopen("http://%s/state" % os.environ["MESOS_AGENT_ENDPOINT"]).read())
if agent_state.has_key('completed_frameworks'):
# Drop the completed frameworks whichg grow over time
del agent_state['completed_frameworks']
log.debug("Agent state: %s", str(agent_state))
log.debug("Virtual memory info in executor: %s" %
repr(psutil.virtual_memory()))
if os.path.exists('/sys/fs/cgroup/memory'):
# Mesos can limit memory with a cgroup, so we should report on that.
for (dirpath, dirnames, filenames) in os.walk('/sys/fs/cgroup/memory',
followlinks=True):
for filename in filenames:
if 'limit_in_bytes' not in filename:
continue
log.debug('cgroup memory info from %s:' %
os.path.join(dirpath, filename))
try:
for line in open(os.path.join(dirpath, filename)):
log.debug(line.rstrip())
except Exception as e:
log.debug("Failed to read file")
# Mesos can also impose rlimit limits, including on things that really
# ought to not be limited, like virtual address space size.
log.debug('DATA rlimit: %s', str(resource.getrlimit(resource.RLIMIT_DATA)))
log.debug('STACK rlimit: %s',
str(resource.getrlimit(resource.RLIMIT_STACK)))
log.debug('RSS rlimit: %s', str(resource.getrlimit(resource.RLIMIT_RSS)))
log.debug('AS rlimit: %s', str(resource.getrlimit(resource.RLIMIT_AS)))
executor = MesosExecutor()
log.debug('Made executor')
driver = MesosExecutorDriver(executor, use_addict=True)
old_on_event = driver.on_event
def patched_on_event(event):
"""
Intercept and log all pymesos events.
"""
log.debug("Event: %s", repr(event))
old_on_event(event)
driver.on_event = patched_on_event
log.debug('Made driver')
driver.start()
log.debug('Started driver')
driver_result = driver.join()
log.debug('Joined driver')
# Tolerate a None in addition to the code the docs suggest we should receive from join()
exit_value = 0 if (driver_result is None
or driver_result == 'DRIVER_STOPPED') else 1
assert len(executor.runningTasks) == 0
sys.exit(exit_value)
