def fake_runner(fake_executor): runner = Subscription( executor=fake_executor, queue=mock.Mock(), ) yield runner runner.stop()
def get_runner(self, mesos_address, queue): if not self.enabled: log.info('Mesos is disabled, not creating a framework.') return None if self.runner and not self.runner.stopping: log.info('Already have a running framework, not creating one.') return self.runner framework_name = 'tron-{}'.format(socket.gethostname()) executor = self.processor.executor_from_config( provider='mesos_task', provider_config={ 'secret': self.secret, 'principal': self.principal, 'mesos_address': get_mesos_leader(mesos_address, self.mesos_master_port), 'role': self.mesos_role, 'framework_name': framework_name, 'framework_id': self.framework_id, 'failover': True, } ) def log_output(task_id, message, stream): logger = logging.getLogger( '{}.{}.{}'.format( TASK_OUTPUT_LOGGER, task_id, stream, ) ) logger.info(message) logging_executor = self.processor.executor_from_config( provider='logging', provider_config={ 'downstream_executor': executor, 'handler': log_output, 'format_string': '{line}', }, ) return Subscription(logging_executor, queue)
def main(): mesos_address = os.environ['MESOS'] with open('./examples/cluster/secret') as f: secret = f.read().strip() processor = TaskProcessor() processor.load_plugin(provider_module='task_processing.plugins.mesos') executor = processor.executor_from_config(provider='mesos_task', provider_config={ 'secret': secret, 'mesos_address': mesos_address, 'role': 'taskproc', }) queue = Queue(100) runner = Subscription(executor, queue) tasks = set() TaskConfig = executor.TASK_CONFIG_INTERFACE for _ in range(2): task_config = TaskConfig(image='busybox', cmd='/bin/true') tasks.add(task_config.task_id) runner.run(task_config) print('Running {} tasks: {}'.format(len(tasks), tasks)) while len(tasks) > 0: try: event = queue.get(block=True, timeout=10) except Empty: event = None if event is None: print('Timeout while waiting for {}'.format(tasks)) break else: if event.terminal: tasks.discard(event.task_id) runner.stop() return 0 if len(tasks) == 0 else 1