class MesosExecutor(TaskExecutor):
def __init__(
self,
role: str,
callbacks: MesosExecutorCallbacks,
pool=None,
principal='taskproc',
secret=None,
mesos_address='127.0.0.1:5050',
initial_decline_delay=1.0,
framework_name='taskproc-default',
framework_staging_timeout=240,
framework_id=None,
failover=False,
) -> None:
"""
Constructs the instance of a task execution, encapsulating all state
required to run, monitor and stop the job.
TODO param docstrings
"""
self.logger = logging.getLogger(__name__)
self.role = role
self.failover = failover
self.execution_framework = ExecutionFramework(
role=role,
pool=pool,
name=framework_name,
callbacks=callbacks,
task_staging_timeout_s=framework_staging_timeout,
initial_decline_delay=initial_decline_delay,
framework_id=framework_id,
)
# TODO: Get mesos master ips from smartstack
self.driver = MesosSchedulerDriver(
sched=self.execution_framework,
framework=self.execution_framework.framework_info,
use_addict=True,
master_uri=mesos_address,
implicit_acknowledgements=False,
principal=principal,
secret=secret,
failover=failover,
)
# start driver thread immediately
self.stopping = False
self.driver_thread = threading.Thread(target=self._run_driver, args=())
self.driver_thread.daemon = True
self.driver_thread.start()
def _run_driver(self):
while not self.stopping:
self.driver.run()
self.logger.warning('Driver stopped, starting again')
def run(self, task_config):
self.execution_framework.enqueue_task(task_config)
def reconcile(self, task_config):
self.execution_framework.reconcile_task(task_config)
def kill(self, task_id):
return self.execution_framework.kill_task(task_id)
def stop(self):
self.stopping = True
self.execution_framework.stop()
self.driver.stop(failover=self.failover)
self.driver.join()
def get_event_queue(self):
return self.execution_framework.event_queue
rmparam.value = "true"
cpus = task.resources.add()
cpus.name = "cpus"
cpus.type = mesos_pb2.Value.SCALAR
cpus.scalar.value = 2
cpus.role = "*"
mem = task.resources.add()
mem.name = "mem"
mem.type = mesos_pb2.Value.SCALAR
mem.scalar.value = 2927
mem.role = "*"
logging.info("Created a Mesos task for %s", self.number)
self.number += 1
driver.launchTasks(offer.id, [task], filters=mesos_pb2.Filters())
framework = mesos_pb2.FrameworkInfo()
framework.id.value = "mesostest-master"
framework.user = "******"
framework.name = "mesostest-master"
framework.role = "*"
framework.checkpoint = True
scheduler = Scheduler()
driver = MesosSchedulerDriver(scheduler, framework, "zk://zk:2181/mesostest")
driver.framework.failover_timeout = 604800
driver.run()