def getExecutorInfo(self):
"""
Build an executor request structure
"""
uri_value = "%s/resources/nebula_worker.egg" % (self.master_url)
logging.info("in getExecutorInfo, setting execPath = " + uri_value)
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = "nebula_worker"
uri = executor.command.uris.add()
uri.value = uri_value
uri.executable = True
cmd = "python ./nebula_worker.egg -w %s " % (self.docstore.get_url())
executor.command.value = cmd
logging.info("Executor Command: %s" % cmd)
env_path = executor.command.environment.variables.add()
env_path.name = "PATH"
env_path.value = os.environ['PATH']
executor.name = "nebula_worker"
executor.source = "nebula_farm"
return executor
def test_scheduler_event_handlers(mocker):
sched = mocker.Mock()
driver = mocker.Mock()
proxy = SchedulerProxy(sched)
proxy.executorLost(driver, mesos_pb2.ExecutorInfo(),
mesos_pb2.ExecutorID(), 1)
proxy.frameworkMessage(driver, mesos_pb2.ExecutorID(),
mesos_pb2.SlaveID(), 'message')
proxy.offerRescinded(driver, mesos_pb2.OfferID())
proxy.registered(driver, mesos_pb2.FrameworkID(), mesos_pb2.MasterInfo())
proxy.resourceOffers(driver, [mesos_pb2.Offer(), mesos_pb2.Offer()])
proxy.slaveLost(driver, mesos_pb2.SlaveID())
proxy.statusUpdate(driver, mesos_pb2.TaskStatus())
proxy.reregistered(driver, mesos_pb2.MasterInfo())
proxy.error(driver, 'message')
proxy.disconnected(driver)
sched.on_executor_lost.assert_called_once()
sched.on_message.assert_called_once()
sched.on_rescinded.assert_called_once()
sched.on_registered.assert_called_once()
sched.on_offers.assert_called_once()
sched.on_slave_lost.assert_called_once()
sched.on_update.assert_called_once()
sched.on_reregistered.assert_called_once()
sched.on_error.assert_called_once()
sched.on_disconnected.assert_called_once()
def __init__(self, cfg):
executor = MesosPb2.ExecutorInfo()
executor.executor_id.value = 'default'
executor.name = cfg.mesos.executor_name
self.cfg = cfg
self.job_number = cfg.starting_job_number
connection_parms = pika.connection.URLParameters(
cfg.message.service_string)
self.msg_connection = pika.BlockingConnection(connection_parms)
self.queue_properties = pika.BasicProperties(delivery_mode=2)
self.msg_work_channel = self.msg_connection.channel()
self.msg_work_channel.queue_declare(queue=cfg.message.work_queue,
durable=True)
self.msg_status_channel = self.msg_connection.channel()
self.msg_status_channel.queue_declare(queue=cfg.message.status_queue,
durable=True)
self.states = WorkStateInfo(success='SUCCESS',
error='ERROR',
failed='FAILED',
lost='LOST',
killed='KILLED',
queued='QUEUED',
running='RUNNING')
super(OurJobScheduler, self).__init__(cfg.mesos.imp_ack, executor, cfg)
def handle(self, **options):
'''See :meth:`django.core.management.base.BaseCommand.handle`.
This method starts the scheduler.
'''
# Register a listener to handle clean shutdowns
signal.signal(signal.SIGTERM, self._onsigterm)
# TODO: clean this up
mesos_master = options.get('master')
logger.info(u'Command starting: scale_scheduler')
logger.info(u' - Master: %s', mesos_master)
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = 'scale'
executor.command.value = '%s %s scale_executor' % (
settings.PYTHON_EXECUTABLE, settings.MANAGE_FILE)
executor.name = 'Scale Executor (Python)'
try:
scheduler_zk = settings.SCHEDULER_ZK
except:
scheduler_zk = None
if scheduler_zk is not None:
import socket
from scheduler import cluster_utils
my_id = socket.gethostname()
cluster_utils.wait_for_leader(scheduler_zk, my_id,
self.run_scheduler, mesos_master,
executor)
else:
# leader election is disabled
self.run_scheduler(mesos_master, executor)
def getExecutorInfo(self, framework_id):
execInfo = mesos_pb2.ExecutorInfo()
execInfo.executor_id.value = "multiframework"
execInfo.command.value = '%s %s' % (
sys.executable, # /usr/bin/python.exe or .../python
os.path.abspath(
os.path.join(os.path.dirname(__file__), 'simpleexecutor.py')))
v = execInfo.command.environment.variables.add()
v.name = 'UID'
v.value = str(os.getuid())
v = execInfo.command.environment.variables.add()
v.name = 'GID'
v.value = str(os.getgid())
if hasattr(execInfo, 'framework_id'):
execInfo.framework_id.value = str(framework_id)
Script = os.path.realpath(sys.argv[0])
if hasattr(execInfo, 'name'):
execInfo.name = Script
execInfo.data = marshal.dumps(
(Script, os.getcwd(), sys.path, dict(os.environ),
self.task_per_node, env.environ))
return execInfo
def getExecutorInfo(self):
frameworkDir = os.path.abspath(os.path.dirname(sys.argv[0]))
executorPath = os.path.join(frameworkDir, "executor.py")
execInfo = mesos_pb2.ExecutorInfo()
execInfo.executor_id.value = "default"
execInfo.command.value = executorPath
v = execInfo.command.environment.variables.add()
v.name = 'UID'
v.value = str(os.getuid())
v = execInfo.command.environment.variables.add()
v.name = 'GID'
v.value = str(os.getgid())
if hasattr(execInfo, 'framework_id'):
execInfo.framework_id.value = str(self.framework_id)
if self.options.image and hasattr(execInfo, 'container'):
execInfo.container.type = mesos_pb2.ContainerInfo.DOCKER
execInfo.container.docker.image = self.options.image
for path in ['/etc/passwd', '/etc/group']:
v = execInfo.container.volumes.add()
v.host_path = v.container_path = path
v.mode = mesos_pb2.Volume.RO
for path in conf.MOOSEFS_MOUNT_POINTS:
v = execInfo.container.volumes.add()
v.host_path = v.container_path = path
v.mode = mesos_pb2.Volume.RW
if self.options.volumes:
for volume in self.options.volumes.split(','):
fields = volume.split(':')
if len(fields) == 3:
host_path, container_path, mode = fields
mode = mesos_pb2.Volume.RO if mode.lower() == 'ro' else mesos_pb2.Volume.RW
elif len(fields) == 2:
host_path, container_path = fields
mode = mesos_pb2.Volume.RW
elif len(fields) == 1:
container_path, = fields
host_path = ''
mode = mesos_pb2.Volume.RW
else:
raise Exception("cannot parse volume %s", volume)
try:
os.makedirs(host_path)
except OSError as e:
pass
v = execInfo.container.volumes.add()
v.container_path = container_path
v.mode = mode
if host_path:
v.host_path = host_path
return execInfo
def create():
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = "mydocker"
executor.name = "My docker example executor"
implicitAcknowledgements = 1
mesosScheduler = MyMesosScheduler(implicitAcknowledgements, executor)
return mesosScheduler
def main(master):
logging.basicConfig(level=logging.INFO,
format='[%(asctime)s %(levelname)s] %(message)s')
# Create a new executor
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = 'ExampleExecutor'
executor.name = executor.executor_id.value
executor.command.value = os.path.abspath('./executor-skeleton.py')
# Create a new framework
framework = mesos_pb2.FrameworkInfo()
framework.user = '' # the current user
framework.name = 'ExampleFramework'
framework.checkpoint = True
implicitAcknowledgements = 1
if os.getenv('EXAMPLE_AUTHENTICATE'):
logging.info('Enabling framework authentication')
credential = mesos_pb2.Credential()
credential.principal = os.getenv('EXAMPLE_PRINCIPAL')
credential.secret = os.getenv('EXAMPLE_SECRET')
framework.principal = os.getenv('EXAMPLE_PRINCIPAL')
driver = MesosSchedulerDriver(ExampleScheduler(executor), framework,
master, implicitAcknowledgements,
credential)
else:
framework.principal = framework.name
driver = MesosSchedulerDriver(ExampleScheduler(executor), framework,
master, implicitAcknowledgements)
def signal_handler(signal, frame):
logging.info('Shutting down')
driver.stop()
# driver.run() blocks, so we run it in a separate thread.
# This way, we can catch a SIGINT to kill the framework.
def run_driver_thread():
status = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1
driver.stop() # Ensure the driver process terminates
sys.exit(status)
driver_thread = Thread(target=run_driver_thread, args=())
driver_thread.start()
logging.info('Scheduler running, Ctrl-C to exit')
signal.signal(signal.SIGINT, signal_handler)
# Block the main thread while the driver thread is alive
while driver_thread.is_alive():
time.sleep(1)
logging.info('Framework finished.')
sys.exit(0)
def as_new_mesos_task(self):
"""
Take the information stored in this Task object and fill a
mesos task.
"""
assert self.task_id, "Calico task must be assigned a task_id"
assert self.slave_id, "Calico task must be assigned a slave_id"
task = mesos_pb2.TaskInfo()
task.name = repr(self)
task.task_id.value = self.task_id
task.slave_id.value = self.slave_id
cpus = task.resources.add()
cpus.name = "cpus"
cpus.type = mesos_pb2.Value.SCALAR
cpus.scalar.value = TASK_CPUS
mem = task.resources.add()
mem.name = "mem"
mem.type = mesos_pb2.Value.SCALAR
mem.scalar.value = TASK_MEM
if not self.default_executor:
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = "execute Task %s" % self.task_id
executor.command.value = "python %s" % self.executor_script
executor.name = "Test Executor for Task %s" % self.task_id
executor.source = "python_test"
executor.container.type = mesos_pb2.ContainerInfo.MESOS
task.executor.MergeFrom(executor)
self.executor_id = executor.executor_id.value
else:
task.container.type = mesos_pb2.ContainerInfo.MESOS
if self.calico:
if not self.default_executor:
network_info = task.executor.container.network_infos.add()
else:
network_info = task.container.network_infos.add()
for netgroup in self.netgroups:
network_info.groups.append(netgroup)
for ip in self.requested_ips:
network_info.ip_addresses.add().ip_address = ip
for _ in range(self.auto_ipv4):
network_info.ip_addresses.add().protocol = \
mesos_pb2.NetworkInfo.IPv4
for _ in range(self.auto_ipv6):
network_info.ip_addresses.add().protocol = \
mesos_pb2.NetworkInfo.IPv4
return task
def __init__(self, baseExecutor):
self.baseExecutor = baseExecutor
self.freeExecutors = collections.deque()
self.busyExecutors = {}
# Create EXECUTOR_COUNT executors.
for idx in range(EXECUTOR_COUNT):
e = mesos_pb2.ExecutorInfo()
e.CopyFrom(baseExecutor)
e.executor_id.value += "-{}".format(idx)
self.freeExecutors.append(e)
def _buildExecutor(self):
"""
Creates and returns an ExecutorInfo instance representing our executor implementation.
"""
# The executor program is installed as a setuptools entry point by setup.py
info = mesos_pb2.ExecutorInfo()
info.name = "toil"
info.command.value = resolveEntryPoint('_toil_mesos_executor')
info.executor_id.value = "toil-%i" % os.getpid()
info.source = pwd.getpwuid(os.getuid()).pw_name
return info
def _init_executor(self):
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = 'default'
executor.command.value = '%s -m %s.executor' % (sys.executable,
__package__)
if 'PYTHONPATH' in os.environ:
var = executor.command.environment.variables.add()
var.name = 'PYTHONPATH'
var.value = os.environ['PYTHONPATH']
executor.framework_id.value = str(self.framework_id.value)
return executor
def test_registered(self):
driver = ExecutorDriver()
executor_info = mesos_pb2.ExecutorInfo()
framework_info = mesos_pb2.FrameworkInfo()
slave_info = mesos_pb2.SlaveInfo()
self.executor_base.registered(driver, executor_info, framework_info,
slave_info)
assert self.executor_base._driver == driver
assert self.executor_base._executor_info == executor_info
assert self.executor_base._framework_info == framework_info
assert self.executor_base._slave_info == slave_info
def start_framework(master_uri, exe_path, cpu_alloc, mem_alloc,
refuse_seconds):
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = "default"
executor.command.value = exe_path
executor.name = "QoSon Executor (Python)"
executor.source = "network monitoring"
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = "QoSon Framework (Python)"
implicitAcknowledgements = 1
if os.getenv("MESOS_EXPLICIT_ACKNOWLEDGEMENTS"):
print "Enabling explicit status update acknowledgements"
implicitAcknowledgements = 0
# create a scheduler and capture the command line options
sched = NetmonScheduler(implicitAcknowledgements, executor, cpu_alloc,
mem_alloc, refuse_seconds)
if os.getenv("MESOS_CHECKPOINT"):
print "Enabling checkpoint for the framework"
framework.checkpoint = True
if os.getenv("MESOS_AUTHENTICATE"):
print "Enabling authentication for the framework"
if not os.getenv("DEFAULT_PRINCIPAL"):
print "Expecting authentication principal in the environment"
sys.exit(1)
if not os.getenv("DEFAULT_SECRET"):
print "Expecting authentication secret in the environment"
sys.exit(1)
credential = mesos_pb2.Credential()
credential.principal = os.getenv("DEFAULT_PRINCIPAL")
credential.secret = os.getenv("DEFAULT_SECRET")
framework.principal = os.getenv("DEFAULT_PRINCIPAL")
driver = mesos.native.MesosSchedulerDriver(
sched, framework, master_uri, implicitAcknowledgements,
credential)
else:
framework.principal = "test-framework-python"
driver = mesos.native.MesosSchedulerDriver(
sched, framework, master_uri, implicitAcknowledgements)
return driver, sched
def setUp(self):
self.framework_id = mesos_pb2.FrameworkID(value=self.FRAMEWORK_ID)
self.framework_info = mesos_pb2.FrameworkInfo(
user='******',
name='fake_framework_name',
)
self.command_info = mesos_pb2.CommandInfo(value='fake-command')
self.executor_id = mesos_pb2.ExecutorID(value='fake-executor-id')
self.executor_info = mesos_pb2.ExecutorInfo(
executor_id=self.executor_id,
framework_id=self.framework_id,
command=self.command_info,
)
self.slave_id = mesos_pb2.SlaveID(value='fake-slave-id')
self.offer_id = mesos_pb2.OfferID(value='1')
def buildExecutor(self):
"""
Creates and returns an ExecutorInfo instance representing our executor implementation.
"""
def scriptPath(executorClass):
path = sys.modules[executorClass.__module__].__file__
if path.endswith('.pyc'):
path = path[:-1]
return path
executorInfo = mesos_pb2.ExecutorInfo()
# The executor program is installed as a setuptools entry point by setup.py
executorInfo.command.value = "toil-mesos-executor"
executorInfo.executor_id.value = "toilExecutor"
executorInfo.name = "Test Executor (Python)"
executorInfo.source = "python_test"
return executorInfo
def main(master):
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = 'MinimalExecutor'
executor.name = executor.executor_id.value
executor.command.value = os.path.abspath('./executor-minimal.py')
framework = mesos_pb2.FrameworkInfo()
framework.user = '' # the current user
framework.name = 'MinimalFramework'
framework.checkpoint = True
framework.principal = framework.name
implicitAcknowledgements = 1
driver = MesosSchedulerDriver(
MinimalScheduler(executor),
framework,
master,
implicitAcknowledgements
)
def signal_handler(signal, frame):
driver.stop()
def run_driver_thread():
status = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1
driver.stop()
sys.exit(status)
driver_thread = Thread(target=run_driver_thread, args=())
driver_thread.start()
print('Scheduler running, Ctrl-C to quit.')
signal.signal(signal.SIGINT, signal_handler)
while driver_thread.is_alive():
time.sleep(1)
sys.exit(0)
def test_executor_event_handlers(mocker):
executor = mocker.Mock()
driver = mocker.Mock()
proxy = ExecutorProxy(executor)
proxy.registered(driver, mesos_pb2.ExecutorInfo(),
mesos_pb2.FrameworkInfo(), mesos_pb2.SlaveInfo())
proxy.reregistered(driver, mesos_pb2.SlaveInfo())
proxy.disconnected(driver)
proxy.launchTask(driver, mesos_pb2.TaskInfo())
proxy.killTask(driver, mesos_pb2.TaskID())
proxy.frameworkMessage(driver, 'message')
proxy.shutdown(driver)
proxy.error(driver, 'message')
executor.on_registered.assert_called_once()
executor.on_reregistered.assert_called_once()
executor.on_disconnected.assert_called_once()
executor.on_launch.assert_called_once()
executor.on_kill.assert_called_once()
executor.on_message.assert_called_once()
executor.on_shutdown.assert_called_once()
executor.on_error.assert_called_once()
def _init_executor(self):
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = 'default'
executor.command.value = '%s -m %s.executor' % (
sys.executable, __package__)
mem = executor.resources.add()
mem.name = 'mem'
mem.type = mesos_pb2.Value.SCALAR
mem.scalar.value = MIN_MEMORY
cpus = executor.resources.add()
cpus.name = 'cpus'
cpus.type = mesos_pb2.Value.SCALAR
cpus.scalar.value = MIN_CPUS
if 'PYTHONPATH' in os.environ:
var = executor.command.environment.variables.add()
var.name = 'PYTHONPATH'
var.value = os.environ['PYTHONPATH']
executor.framework_id.value = str(self.framework_id.value)
return executor
def buildExecutor(self, bad):
"""
Creates and returns an ExecutorInfo instance representing either the regular or the "bad" test executor.
"""
# FIXME: This isn't going to work since toil might be installed in a different location on the slaves.
# FIXME: ... setup.py should install the executor as an entry point which will place it on PATH
def scriptPath(executorClass):
path = sys.modules[executorClass.__module__].__file__
if path.endswith('.pyc'):
path = path[:-1]
return path
executorInfo = mesos_pb2.ExecutorInfo()
if bad:
executorInfo.command.value = scriptPath(BadMesosExecutor)
executorInfo.executor_id.value = "badExecutor"
else:
executorInfo.command.value = scriptPath(MesosExecutor)
executorInfo.executor_id.value = "toilExecutor"
executorInfo.name = "Test Executor (Python)"
executorInfo.source = "python_test"
return executorInfo
def main(args):
# Get the path to our executor
executor_path = os.path.abspath("./executor")
# Then, boot up all the Mesos crap and get us registered with the framework.
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = "test-case-executor"
executor.command.value = executor_path
executor.name = "Test case repeater"
cpus = executor.resources.add()
mem = executor.resources.add()
cpus.name = "cpus"
cpus.type = mesos_pb2.Value.SCALAR
cpus.scalar.value = 0
mem.name = "mem"
mem.type = mesos_pb2.Value.SCALAR
mem.scalar.value = 128
#executor.container.MergeFrom(container)
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = "Test case repeater framework"
framework.checkpoint = True
framework.principal = "test-case-repeater"
driver = MesosSchedulerDriver(
TestCaseScheduler(args.database, executor, args.batch), framework,
args.controller, 1)
status = None
if driver.run() == mesos_pb2.DRIVER_STOPPED:
status = 0
else:
status = 1
return status
def getExecutorInfo(self, framework_id):
info = mesos_pb2.ExecutorInfo()
if hasattr(info, 'framework_id'):
info.framework_id.value = framework_id
if self.use_self_as_exec:
info.command.value = os.path.abspath(sys.argv[0])
info.executor_id.value = sys.argv[0]
else:
info.command.value = '%s %s' % (
sys.executable,
os.path.abspath(
os.path.join(os.path.dirname(__file__), 'executor.py')))
info.executor_id.value = "default"
v = info.command.environment.variables.add()
v.name = 'UID'
v.value = str(os.getuid())
v = info.command.environment.variables.add()
v.name = 'GID'
v.value = str(os.getgid())
if self.options.image and hasattr(info, 'container'):
info.container.type = mesos_pb2.ContainerInfo.DOCKER
info.container.docker.image = self.options.image
for path in ['/etc/passwd', '/etc/group']:
v = info.container.volumes.add()
v.host_path = v.container_path = path
v.mode = mesos_pb2.Volume.RO
for path in conf.MOOSEFS_MOUNT_POINTS:
v = info.container.volumes.add()
v.host_path = v.container_path = path
v.mode = mesos_pb2.Volume.RW
for path in conf.DPARK_WORK_DIR.split(','):
v = info.container.volumes.add()
v.host_path = v.container_path = path
v.mode = mesos_pb2.Volume.RW
if self.options.volumes:
for volume in self.options.volumes.split(','):
fields = volume.split(':')
if len(fields) == 3:
host_path, container_path, mode = fields
mode = mesos_pb2.Volume.RO if mode.lower(
) == 'ro' else mesos_pb2.Volume.RW
elif len(fields) == 2:
host_path, container_path = fields
mode = mesos_pb2.Volume.RW
elif len(fields) == 1:
container_path, = fields
host_path = ''
mode = mesos_pb2.Volume.RW
else:
raise Exception("cannot parse volume %s", volume)
mkdir_p(host_path)
v = info.container.volumes.add()
v.container_path = container_path
v.mode = mode
if host_path:
v.host_path = host_path
mem = info.resources.add()
mem.name = 'mem'
mem.type = 0 #mesos_pb2.Value.SCALAR
mem.scalar.value = EXECUTOR_MEMORY
Script = os.path.realpath(sys.argv[0])
if hasattr(info, 'name'):
info.name = Script
info.data = marshal.dumps(
(Script, os.getcwd(), sys.path, dict(os.environ),
self.task_per_node, self.out_logger, self.err_logger,
self.logLevel, env.environ))
return info
"Task %s is in state %s" %
(update.task_id.value, mesos_pb2.TaskState.Name(update.state)))
def shutdown(signal, frame):
logging.info("Shutdown signal")
driver.stop()
sys.exit(0)
if __name__ == '__main__':
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = "hello-world"
helloWorldExecutor = mesos_pb2.ExecutorInfo()
helloWorldExecutor.executor_id.value = "HelloWorld-executor"
helloWorldExecutor.command.value = "while [ true ]; do echo hello world && sleep 1 ; done"
helloWorldExecutor.name = "HelloWorld"
helloWorldScheduler = HelloWorldScheduler(helloWorldExecutor)
driver = MesosSchedulerDriver(
helloWorldScheduler,
framework,
"zk://localhost:2181/mesos" # assumes running on the master
)
driver.start()
logging.info("Listening for Ctrl-C")
signal.signal(signal.SIGINT, shutdown)
while True:
#
if __name__ == "__main__":
if len(sys.argv) < 3 or len(sys.argv) > 4:
print "Usage: %s seedUrl mesosMasterUrl [maxRenderTasks]" % sys.argv[0]
sys.exit(1)
baseURI = "/home/vagrant/hostfiles"
suffixURI = "python"
uris = [
"crawl_executor.py", "export_dot.py", "render_executor.py",
"results.py", "task_state.py"
]
uris = [os.path.join(baseURI, suffixURI, uri) for uri in uris]
uris.append(os.path.join(baseURI, "render.js"))
crawlExecutor = mesos_pb2.ExecutorInfo()
crawlExecutor.executor_id.value = "crawl-executor"
crawlExecutor.command.value = "python crawl_executor.py"
for uri in uris:
uri_proto = crawlExecutor.command.uris.add()
uri_proto.value = uri
uri_proto.extract = False
crawlExecutor.name = "Crawler"
renderExecutor = mesos_pb2.ExecutorInfo()
renderExecutor.executor_id.value = "render-executor"
renderExecutor.command.value = "python render_executor.py --local"
for uri in uris:
#
if __name__ == "__main__":
if len(sys.argv) < 3:
print "Usage: {} master_ip:port stats-output [credentials]". \
format(sys.argv[0])
sys.exit(1)
baseURI = os.path.dirname(os.path.abspath(__file__))
uris = ["pareto_executor.py", "task_state.py"]
uris = [os.path.join(baseURI, uri) for uri in uris]
framework = mesos_pb2.FrameworkInfo()
framework.user = ""
framework.name = "VilfredoMesos"
paretoExecutor = mesos_pb2.ExecutorInfo()
paretoExecutor.executor_id.value = "pareto-executor"
paretoExecutor.name = "Pareto simulator"
paretoExecutor.command.value = "python pareto_executor.py"
for uri in uris:
uri_proto = paretoExecutor.command.uris.add()
uri_proto.value = uri
uri_proto.extract = False
vilfredo = VilfredoMesosScheduler(paretoExecutor, sys.argv[2])
# Use credentials for authentication if they are provided.
credential = None
if len(sys.argv) > 3:
credentialsPath = sys.argv[3]
with open(credentialsPath) as f:
def run_forever(conf):
"""Entrypoint to keep the framework running until terminated."""
logger.info('******************Start************')
logger.debug('DB connection: %s', conf.l2_db_con)
logger.debug("Minimum Senes Per Seg: %s", conf.minscenesperseg)
logger.debug('Segment query: %s', conf.segment_query)
global shutdown
db.reset_records(db.connect(conf.l2_db_con))
# Establish framework, executor, and authentication credentials
framework = mesos_pb2.FrameworkInfo()
framework.user = conf.framework_user
framework.name = "ARD Tile Framework"
framework.principal = conf.mesos_principal
framework.role = conf.mesos_role
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = "default"
executor.name = "ARD Tile executor"
implicit_acks = 1
scheduler = ArdTileScheduler(implicit_acks, executor, conf)
if not conf.disable_creds:
logger.info(" MESOS creds ENABLED")
credential = mesos_pb2.Credential()
credential.principal = conf.mesos_principal
credential.secret = conf.mesos_secret
driver = mesos.native.MesosSchedulerDriver(scheduler, framework,
conf.master, implicit_acks,
credential)
else:
logger.info(" MESOS creds disabled")
driver = mesos.native.MesosSchedulerDriver(scheduler, framework,
conf.master, implicit_acks)
shutdown = Shutdown()
def run_driver_async():
"""Thread for async communication with Mesos offers."""
# driver.run() blocks, so run it in a separate thread.
status = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1
driver.stop()
sys.exit(status)
framework_thread = Thread(target=run_driver_async, args=())
framework_thread.start()
while framework_thread.is_alive():
# If a shutdown has been requested, suppress offers and wait for the
# framework thread to complete.
if shutdown.flag:
logger.info("Shutdown requested....")
driver.suppressOffers()
while framework_thread.is_alive():
logger.debug("Thread alive, sleep 5....")
time.sleep(5)
break
# If the job queue is empty, get work.
if (not scheduler.jobs
and queue_segments(scheduler.jobs, conf,
db.connect(conf.l2_db_con)) == ERROR):
driver.stop(True)
sys.exit(1)
# If there's no new work to be done or the max number of jobs are
# already running, suppress offers and wait for some jobs to finish.
if (not scheduler.jobs or not scheduler.scheduling_allowed()):
logger.info("No jobs or scheduling not allowed....")
driver.suppressOffers()
while not scheduler.scheduling_allowed():
logger.debug("Scheduling not alive, sleep 20....")
time.sleep(20)
while not scheduler.jobs:
if queue_segments(scheduler.jobs, conf,
db.connect(conf.l2_db_con)) == ERROR:
driver.stop(True)
sys.exit(1)
time.sleep(20)
driver.reviveOffers()
util.rm_host_from_cluster(host.strip())
util.rm_from_cluster_conf(host)
util.rm_from_hosts(host)
self.slaves.pop(host)
elif update.state == mesos_pb2.TASK_LOST \
or update.state == mesos_pb2.TASK_KILLED \
or update.state == mesos_pb2.TASK_FAILED:
driver.abort()
util.show_openlava_state()
if __name__ == '__main__':
LOG.setLevel(level='DEBUG')
EXECUTOR = mesos_pb2.ExecutorInfo()
EXECUTOR.executor_id.value = "default"
EXECUTOR.command.value = os.path.abspath("/tmp/openlava_node.sh")
EXECUTOR.name = "OpenLava executor"
EXECUTOR.source = "openlava_test"
FRAMEWORK = mesos_pb2.FrameworkInfo()
FRAMEWORK.user = ''
FRAMEWORK.name = 'OpenLava'
FRAMEWORK.webui_url = 'http://%s:9876' % ui.web.get_hostname()
# Setup the loggers
LOGGERS = (__name__, 'mesos')
for log in LOGGERS:
logging.getLogger(log).setLevel(logging.DEBUG)
time.sleep(1)
if helloWorldScheduler.runningTasks > 0:
print "Shutdown by timeout, %d task(s) have not completed" % helloWorldScheduler.runningTasks
hard_shutdown()
def hard_shutdown():
logging.info("Hard shutdown signal")
driver.stop()
sys.exit(0)
if __name__ == '__main__':
hello_executor = mesos_pb2.ExecutorInfo()
hello_executor.executor_id.value = "hello-executor"
hello_executor.name = "Hello"
hello_executor.command.value = "python hello_executor.py"
uri_proto = hello_executor.command.uris.add()
uri_proto.value = "http://kit-mesos-master:9000/hello_executor.py"
uri_proto.extract = False
goodbye_executor = mesos_pb2.ExecutorInfo()
goodbye_executor.executor_id.value = "goodbye-executor"
goodbye_executor.name = "GoodBye"
goodbye_executor.command.value = "python goodbye_executor.py"
uri_proto = goodbye_executor.command.uris.add()
uri_proto.value = "http://kit-mesos-master:9000/goodbye_executor.py"
parser.add_argument("-m", "--master", required=True, type=str,
help="IP:Port of mesos master")
parser.add_argument("--CPU", required=True, default=1, type=int,
help="Number of CPUs requried to launch this job (default: 1)")
parser.add_argument("--MEM", required=True, default=512, type=int,
help="Total memory size requried to launch this job in MB (default: 512)")
parser.add_argument("--PORT", required=True, type=int,
help="Required Port in container")
parser.add_argument("--url", required=True, type=str,
help="URL for downloading the workspace folder content")
parser.add_argument("--mode", required=True, type=str,
help="Running mode(1 or 2): 1 for train or 2 for product")
args = parser.parse_args()
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = "singa_executor"
executor.name = "Singa Executor in Docker"
framework = mesos_pb2.FrameworkInfo()
framework.user = ""
framework.name = "Singa Framework"
framework.checkpoint = False
implicitAcknowledgements = 1
if os.getenv("MESOS_EXPLICIT_ACKNOWLEDGEMENTS"):
print "Enabling explicit status update acknowledgements"
implicitAcknowledgements = 0
framework.principal = "docker-mesos-singa-framework"
driver = mesos.native.MesosSchedulerDriver(
def getExecutorInfo(self):
execInfo = mesos_pb2.ExecutorInfo()
execInfo.executor_id.value = "default"
execInfo.command.value = '%s %s' % (
sys.executable, # .../python.exe
os.path.abspath(
os.path.join(os.path.dirname(__file__), 'simpleexecutor.py')))
v = execInfo.command.environment.variables.add()
v.name = 'UID'
v.value = str(os.getuid())
v = execInfo.command.environment.variables.add()
v.name = 'GID'
v.value = str(os.getgid())
if hasattr(execInfo, 'framework_id'):
execInfo.framework_id.value = str(self.framework_id)
if self.options.image and hasattr(execInfo, 'container'):
logger.debug("container will be run in Docker")
execInfo.container.type = mesos_pb2.ContainerInfo.DOCKER
execInfo.container.docker.image = self.options.image
execInfo.command.value = '%s %s' % (
sys.executable, # /usr/bin/python.exe or .../python
os.path.abspath(
os.path.join(os.path.dirname(__file__),
'simpleexecutor.py')))
for path in ['/etc/passwd', '/etc/group']:
v = execInfo.container.volumes.add()
v.host_path = v.container_path = path
v.mode = mesos_pb2.Volume.RO
if self.options.volumes:
for volume in self.options.volumes.split(','):
fields = volume.split(':')
if len(fields) == 3:
host_path, container_path, mode = fields
mode = mesos_pb2.Volume.RO if mode.lower(
) == 'ro' else mesos_pb2.Volume.RW
elif len(fields) == 2:
host_path, container_path = fields
mode = mesos_pb2.Volume.RW
elif len(fields) == 1:
container_path, = fields
host_path = ''
mode = mesos_pb2.Volume.RW
else:
raise Exception("cannot parse volume %s", volume)
mkdir_p(host_path)
v = execInfo.container.volumes.add()
v.container_path = container_path
v.mode = mode
if host_path:
v.host_path = host_path
Script = os.path.realpath(sys.argv[0])
if hasattr(execInfo, 'name'):
execInfo.name = Script
execInfo.data = marshal.dumps(
(Script, os.getcwd(), sys.path, dict(os.environ),
self.options.task_per_node, env.environ))
return execInfo
