def main(): args = parse_args() processor = TaskProcessor() processor.load_plugin(provider_module='task_processing.plugins.mesos') mesos_executor = processor.executor_from_config(provider='mesos_task', provider_config={ 'secret': args.secret, 'mesos_address': args.master, 'pool': args.pool, 'role': args.role, }) executor = processor.executor_from_config(provider='timeout', provider_config={ 'downstream_executor': mesos_executor, }) TaskConfig = mesos_executor.TASK_CONFIG_INTERFACE runner = Sync(executor=executor) task_config = TaskConfig(image='docker-dev.yelpcorp.com/dumb-busybox', cmd='exec dumb-init /bin/sleep 30', timeout=10) result = runner.run(task_config) print(result) runner.stop()
def main(): args = parse_args() processor = TaskProcessor() processor.load_plugin(provider_module='task_processing.plugins.mesos') mesos_executor = processor.executor_from_config(provider='mesos_task', provider_config={ 'secret': args.secret, 'mesos_address': args.master, 'pool': args.pool, 'role': args.role, }) executor = processor.executor_from_config(provider='logging', provider_config={ 'downstream_executor': mesos_executor, }) TaskConfig = mesos_executor.TASK_CONFIG_INTERFACE runner = Sync(executor=executor) task_config = TaskConfig( image="ubuntu:14.04", cmd="bash -c 'for i in $(seq 1 5); do echo $i&&sleep 10; done'") result = runner.run(task_config) print(result) runner.stop()
def main(): mesos_address = os.getenv('MESOS', 'mesosmaster:5050') with open('./examples/cluster/secret') as f: secret = f.read().strip() processor = TaskProcessor() for p in ['mesos', 'stateful']: processor.load_plugin(provider_module='task_processing.plugins.' + p) mesos_executor = processor.executor_from_config(provider='mesos_task', provider_config={ 'secret': secret, 'mesos_address': mesos_address, 'role': 'taskproc', }) executor = processor.executor_from_config( provider='stateful', provider_config={ 'downstream_executor': mesos_executor, 'persister': FilePersistence(output_file='/tmp/foo') }) runner = Sync(executor=executor) tasks = set() TaskConfig = mesos_executor.TASK_CONFIG_INTERFACE for _ in range(1, 2): task_config = TaskConfig(image='busybox', cmd='/bin/true') tasks.add(task_config.task_id) runner.run(task_config) print(executor.status(task_config.task_id))
def test_executor_from_config(): tp = TaskProcessor() tp.load_plugin('tests.mock_plugin') executor = tp.executor_from_config(provider='dummy', provider_config={'arg': 'foobar'}) assert executor.arg == 'foobar' executor.run(None) executor.kill(None) with pytest.raises(ValueError): tp.executor_from_config('lol')
def main(): args = parse_args() processor = TaskProcessor() processor.load_plugin(provider_module='task_processing.plugins.mesos') executor = processor.executor_from_config(provider='mesos_task', provider_config={ 'secret': args.secret, 'mesos_address': args.master, 'pool': args.pool, 'role': args.role, }) TaskConfig = executor.TASK_CONFIG_INTERFACE task_config = TaskConfig(image="busybox", cmd='/bin/true') # This only works on agents that have added mesos as a containerizer # task_config = TaskConfig(containerizer='MESOS', cmd='/bin/true') with ThreadPoolExecutor(max_workers=2) as futures_executor: runner = Promise(executor, futures_executor) future = runner.run(task_config) wait([future]) result = future.result() print(result) print(result.raw) runner.stop() return 0 if result.success else 1
def main(): args = parse_args() processor = TaskProcessor() processor.load_plugin(provider_module='task_processing.plugins.mesos') executor = processor.executor_from_config(provider='mesos_task', provider_config={ 'secret': args.secret, 'mesos_address': args.master, 'role': args.role, }) counter = Counter() runner = Async(executor, [ EventHandler(predicate=lambda x: x.terminal, cb=counter.process_event) ]) TaskConfig = executor.TASK_CONFIG_INTERFACE tasks_to_launch = 2 for _ in range(tasks_to_launch): task_config = TaskConfig(image='busybox', cmd='/bin/true') runner.run(task_config) for _ in range(5): print('terminated {} tasks'.format(counter.terminated)) if counter.terminated >= tasks_to_launch: break time.sleep(2) runner.stop() return 0 if counter.terminated >= tasks_to_launch else 1
def main(): mesos_address = os.getenv('MESOS', 'mesosmaster:5050') with open('./examples/cluster/secret') as f: secret = f.read().strip() processor = TaskProcessor() for p in ['mesos', 'stateful']: processor.load_plugin(provider_module='task_processing.plugins.' + p) mesos_executor = processor.executor_from_config(provider='mesos_task', provider_config={ 'secret': secret, 'mesos_address': mesos_address, 'role': 'taskproc', }) s = session.Session(region_name='foo', aws_access_key_id='foo', aws_secret_access_key='bar') dynamo_address = os.getenv('DYNAMO', 'http://dynamodb:5050') client = s.client( service_name='dynamodb', endpoint_url=dynamo_address, ) try: create_table(client) except ClientError: pass executor = processor.executor_from_config( provider='stateful', provider_config={ 'downstream_executor': mesos_executor, 'persister': DynamoDBPersister(table_name='events', endpoint_url=dynamo_address, session=s) }) runner = Sync(executor=executor) tasks = set() TaskConfig = mesos_executor.TASK_CONFIG_INTERFACE for _ in range(1, 2): task_config = TaskConfig(image='ubuntu:14.04', cmd='/bin/sleep 2') tasks.add(task_config.task_id) runner.run(task_config) print(executor.status(task_config.task_id))
def main(): # get address of the Mesos cluster mesos_address = os.getenv('MESOS', 'mesosmaster:5050') # read in secret, this is used to authenticate the taskproc scheduler with # Mesos with open('./examples/cluster/secret') as f: secret = f.read().strip() # create a processor instance processor = TaskProcessor() # configure plugins processor.load_plugin(provider_module='task_processing.plugins.mesos') # create an executor (taskproc executor NOT to be confused with a Mesos # executor) using this defined configuration. this config can also be used # to specify other Mesos properties, such as which role to use executor = processor.executor_from_config( provider='mesos_task', provider_config={ 'secret': secret, 'mesos_address': mesos_address, 'role': 'taskproc', } ) # creates a new Sync runner that will synchronously execute tasks # (i.e. block until completion) runner = Sync(executor) # next, create a TaskConfig to run # this is where properties of the Mesos task can be specified in this # example, we use the busybox Docker image and just echo "hello world" TaskConfig = executor.TASK_CONFIG_INTERFACE task_config = TaskConfig(image="busybox", cmd='echo "hello world"') # run our task and print the result result = runner.run(task_config) print(result) # this stops the taskproc framework and unregisters it from Mesos runner.stop() return 0 if result.success else 1
def main(): mesos_address = os.environ['MESOS'] with open('./examples/cluster/secret') as f: secret = f.read().strip() processor = TaskProcessor() processor.load_plugin(provider_module='task_processing.plugins.mesos') executor = processor.executor_from_config(provider='mesos_task', provider_config={ 'secret': secret, 'mesos_address': mesos_address, 'role': 'taskproc', }) queue = Queue(100) runner = Subscription(executor, queue) tasks = set() TaskConfig = executor.TASK_CONFIG_INTERFACE for _ in range(2): task_config = TaskConfig(image='busybox', cmd='/bin/true') tasks.add(task_config.task_id) runner.run(task_config) print('Running {} tasks: {}'.format(len(tasks), tasks)) while len(tasks) > 0: try: event = queue.get(block=True, timeout=10) except Empty: event = None if event is None: print('Timeout while waiting for {}'.format(tasks)) break else: if event.terminal: tasks.discard(event.task_id) runner.stop() return 0 if len(tasks) == 0 else 1
def main(): c = Counter() args = parse_args() processor = TaskProcessor() processor.load_plugin(provider_module='task_processing.plugins.mesos') mesos_executor = processor.executor_from_config(provider='mesos_task', provider_config={ 'secret': args.secret, 'mesos_address': args.master, 'pool': args.pool, 'role': args.role, }) TaskConfig = mesos_executor.TASK_CONFIG_INTERFACE runner = Async( mesos_executor, [EventHandler( predicate=lambda x: x.terminal, cb=c.process_event, )]) timeout_task_config = TaskConfig( image='busybox', cmd='exec /bin/sleep 100', offer_timeout=5.0, cpus=20, mem=2048, disk=2000, ) runner.run(timeout_task_config) for _ in range(50): if c.terminated >= 1: break print("waiting for task %s to finish" % (timeout_task_config.task_id)) time.sleep(2) runner.stop() return 0
class MesosCluster: def __init__( self, mesos_address, mesos_master_port=None, secret=None, principal=None, mesos_role=None, framework_id=None, enabled=True, default_volumes=None, dockercfg_location=None, offer_timeout=None, ): self.mesos_address = mesos_address self.mesos_master_port = mesos_master_port self.secret = secret self.principal = principal self.mesos_role = mesos_role self.enabled = enabled self.default_volumes = default_volumes or [] self.dockercfg_location = dockercfg_location self.offer_timeout = offer_timeout self.framework_id = framework_id self.processor = TaskProcessor() self.queue = PyDeferredQueue() self.deferred = None self.runner = None self.tasks = {} self.processor.load_plugin( provider_module='task_processing.plugins.mesos') self.connect() def set_enabled(self, is_enabled): self.enabled = is_enabled if is_enabled: self.connect() else: self.stop(fail_tasks=True) def configure_tasks( self, default_volumes, dockercfg_location, offer_timeout, ): self.default_volumes = default_volumes self.dockercfg_location = dockercfg_location self.offer_timeout = offer_timeout def connect(self): self.runner = self.get_runner(self.mesos_address, self.queue) self.handle_next_event() def handle_next_event(self, deferred_result=None): if self.deferred and not self.deferred.called: log.warning( 'Already have handlers waiting for next event in queue, ' 'not adding more') return self.deferred = self.queue.get() self.deferred.addCallback(self._process_event) self.deferred.addCallback(self.handle_next_event) self.deferred.addErrback(logError) self.deferred.addErrback(self.handle_next_event) def _check_connection(self): if self.runner.stopping: # Last framework was terminated for some reason, re-connect. log.info('Last framework stopped, re-connecting') self.connect() elif self.deferred.called: # Just in case callbacks are missing, re-add. self.handle_next_event() def submit(self, task): if not task: return if not self.enabled: task.log.info('Task failed to start, Mesos is disabled.') task.exited(1) return self._check_connection() mesos_task_id = task.get_mesos_id() self.tasks[mesos_task_id] = task env = task.get_config()['environment'] clusterman_resource_str = env.get('CLUSTERMAN_RESOURCES') clusterman_metrics = get_clusterman_metrics() if clusterman_resource_str and clusterman_metrics: clusterman_resources = json.loads(clusterman_resource_str) cluster = env.get('EXECUTOR_CLUSTER', env.get('PAASTA_CLUSTER')) pool = env.get('EXECUTOR_POOL', env.get('PAASTA_POOL')) aws_region = staticconf.read(f'clusters.{cluster}.aws_region', namespace='clusterman') metrics_client = clusterman_metrics.ClustermanMetricsBotoClient( region_name=aws_region, app_identifier=pool, ) with metrics_client.get_writer( clusterman_metrics.APP_METRICS, aggregate_meteorite_dims=True) as writer: for metric_key, metric_value in clusterman_resources.items(): writer.send((metric_key, int(time.time()), metric_value)) self.runner.run(task.get_config()) log.info( 'Submitting task {} to {}'.format( mesos_task_id, self.mesos_address, ), ) task.report_resources() def recover(self, task): if not task: return if not self.enabled: task.log.info('Could not recover task, Mesos is disabled.') task.exited(None) return self._check_connection() mesos_task_id = task.get_mesos_id() self.tasks[mesos_task_id] = task task.log.info( 'TRON RESTARTED! Starting recovery procedure by reconciling state for this task from Mesos' ) task.started() self.runner.reconcile(task.get_config()) task.report_resources() def create_task( self, action_run_id, command, cpus, mem, disk, constraints, docker_image, docker_parameters, env, extra_volumes, serializer, task_id=None, ): if not self.runner: return None uris = [self.dockercfg_location] if self.dockercfg_location else [] volumes = combine_volumes(self.default_volumes, extra_volumes) task_kwargs = { 'name': action_run_id, 'cmd': command, 'cpus': cpus, 'mem': mem, 'disk': disk, 'constraints': constraints, 'image': docker_image, 'docker_parameters': docker_parameters, 'environment': env, 'volumes': volumes, 'uris': uris, 'offer_timeout': self.offer_timeout, } task_config = self.runner.TASK_CONFIG_INTERFACE(**task_kwargs) if task_id is not None: try: task_config = task_config.set_task_id(task_id) except ValueError: log.error(f'Invalid {task_id} for {action_run_id}') return return MesosTask(action_run_id, task_config, serializer) def get_runner(self, mesos_address, queue): if not self.enabled: log.info('Mesos is disabled, not creating a framework.') return None if self.runner and not self.runner.stopping: log.info('Already have a running framework, not creating one.') return self.runner framework_name = 'tron-{}'.format(socket.gethostname()) executor = self.processor.executor_from_config( provider='mesos_task', provider_config={ 'secret': self.secret, 'principal': self.principal, 'mesos_address': get_mesos_leader(mesos_address, self.mesos_master_port), 'role': self.mesos_role, 'framework_name': framework_name, 'framework_id': self.framework_id, 'failover': True, }) def log_output(task_id, message, stream): logger = logging.getLogger('{}.{}.{}'.format( TASK_OUTPUT_LOGGER, task_id, stream, )) logger.info(message) logging_executor = self.processor.executor_from_config( provider='logging', provider_config={ 'downstream_executor': executor, 'handler': log_output, 'format_string': '{line}', }, ) return Subscription(logging_executor, queue) def _process_event(self, event): if event.kind == 'control': message = getattr(event, 'message', None) if message == 'stop': # Framework has been removed, stop it. log.warning('Framework has been stopped: {}'.format(event.raw)) self.stop() MesosClusterRepository.remove(self.mesos_address) elif message == 'unknown': log.warning('Unknown error from Mesos master: {}'.format( event.raw)) elif message == 'registered': framework_id = event.raw['framework_id']['value'] MesosClusterRepository.save(self.mesos_address, framework_id) else: log.warning('Unknown type of control event: {}'.format(event)) elif event.kind == 'task': if not hasattr(event, 'task_id'): log.warning('Task event missing task_id: {}'.format(event)) return if event.task_id not in self.tasks: log.warning( 'Received event for unknown task {}: {}'.format( event.task_id, event, ), ) return task = self.tasks[event.task_id] task.handle_event(event) if task.is_done: del self.tasks[event.task_id] else: log.warning('Unknown type of event: {}'.format(event)) def stop(self, fail_tasks=False): self.framework_id = None if self.runner: self.runner.stop() # Clear message queue if self.deferred: self.deferred.cancel() self.deferred = None self.queue = PyDeferredQueue() if fail_tasks: for key, task in list(self.tasks.items()): task.exited(None) del self.tasks[key] def kill(self, task_id): return self.runner.kill(task_id)
class MesosCluster: def __init__( self, mesos_address, mesos_master_port=None, secret=None, principal=None, mesos_role=None, framework_id=None, enabled=True, default_volumes=None, dockercfg_location=None, offer_timeout=None, ): self.mesos_address = mesos_address self.mesos_master_port = mesos_master_port self.secret = secret self.principal = principal self.mesos_role = mesos_role self.enabled = enabled self.default_volumes = default_volumes or [] self.dockercfg_location = dockercfg_location self.offer_timeout = offer_timeout self.framework_id = framework_id self.processor = TaskProcessor() self.queue = PyDeferredQueue() self.deferred = None self.runner = None self.tasks = {} self.processor.load_plugin( provider_module='task_processing.plugins.mesos' ) self.connect() def set_enabled(self, is_enabled): self.enabled = is_enabled if is_enabled: self.connect() else: self.stop(fail_tasks=True) def configure_tasks( self, default_volumes, dockercfg_location, offer_timeout, ): self.default_volumes = default_volumes self.dockercfg_location = dockercfg_location self.offer_timeout = offer_timeout def connect(self): self.runner = self.get_runner(self.mesos_address, self.queue) self.handle_next_event() def handle_next_event(self, deferred_result=None): if self.deferred and not self.deferred.called: log.warning( 'Already have handlers waiting for next event in queue, ' 'not adding more' ) return self.deferred = self.queue.get() self.deferred.addCallback(self._process_event) self.deferred.addCallback(self.handle_next_event) self.deferred.addErrback(logError) self.deferred.addErrback(self.handle_next_event) def _check_connection(self): if self.runner.stopping: # Last framework was terminated for some reason, re-connect. log.info('Last framework stopped, re-connecting') self.connect() elif self.deferred.called: # Just in case callbacks are missing, re-add. self.handle_next_event() def submit(self, task): if not task: return if not self.enabled: task.log.info('Task failed to start, Mesos is disabled.') task.exited(1) return self._check_connection() mesos_task_id = task.get_mesos_id() self.tasks[mesos_task_id] = task self.runner.run(task.get_config()) log.info( 'Submitting task {} to {}'.format( mesos_task_id, self.mesos_address, ), ) task.report_resources() def recover(self, task): if not task: return if not self.enabled: task.log.info('Could not recover task, Mesos is disabled.') task.exited(None) return self._check_connection() mesos_task_id = task.get_mesos_id() self.tasks[mesos_task_id] = task task.log.info('Reconciling state for this task from Mesos') task.started() self.runner.reconcile(task.get_config()) task.report_resources() def create_task( self, action_run_id, command, cpus, mem, disk, constraints, docker_image, docker_parameters, env, extra_volumes, serializer, task_id=None, ): if not self.runner: return None uris = [self.dockercfg_location] if self.dockercfg_location else [] volumes = combine_volumes(self.default_volumes, extra_volumes) task_kwargs = { 'name': action_run_id, 'cmd': command, 'cpus': cpus, 'mem': mem, 'disk': disk, 'constraints': constraints, 'image': docker_image, 'docker_parameters': docker_parameters, 'environment': env, 'volumes': volumes, 'uris': uris, 'offer_timeout': self.offer_timeout, } task_config = self.runner.TASK_CONFIG_INTERFACE(**task_kwargs) if task_id is not None: try: task_config = task_config.set_task_id(task_id) except ValueError: log.error(f'Invalid {task_id} for {action_run_id}') return return MesosTask(action_run_id, task_config, serializer) def get_runner(self, mesos_address, queue): if not self.enabled: log.info('Mesos is disabled, not creating a framework.') return None if self.runner and not self.runner.stopping: log.info('Already have a running framework, not creating one.') return self.runner framework_name = 'tron-{}'.format(socket.gethostname()) executor = self.processor.executor_from_config( provider='mesos_task', provider_config={ 'secret': self.secret, 'principal': self.principal, 'mesos_address': get_mesos_leader(mesos_address, self.mesos_master_port), 'role': self.mesos_role, 'framework_name': framework_name, 'framework_id': self.framework_id, 'failover': True, } ) def log_output(task_id, message, stream): logger = logging.getLogger( '{}.{}.{}'.format( TASK_OUTPUT_LOGGER, task_id, stream, ) ) logger.info(message) logging_executor = self.processor.executor_from_config( provider='logging', provider_config={ 'downstream_executor': executor, 'handler': log_output, 'format_string': '{line}', }, ) return Subscription(logging_executor, queue) def _process_event(self, event): if event.kind == 'control': message = getattr(event, 'message', None) if message == 'stop': # Framework has been removed, stop it. log.warning('Framework has been stopped: {}'.format(event.raw)) self.stop() MesosClusterRepository.remove(self.mesos_address) elif message == 'unknown': log.warning( 'Unknown error from Mesos master: {}'.format(event.raw) ) elif message == 'registered': framework_id = event.raw['framework_id']['value'] MesosClusterRepository.save(self.mesos_address, framework_id) else: log.warning('Unknown type of control event: {}'.format(event)) elif event.kind == 'task': if not hasattr(event, 'task_id'): log.warning('Task event missing task_id: {}'.format(event)) return if event.task_id not in self.tasks: log.warning( 'Received event for unknown task {}: {}'.format( event.task_id, event, ), ) return task = self.tasks[event.task_id] task.handle_event(event) if task.is_done: del self.tasks[event.task_id] else: log.warning('Unknown type of event: {}'.format(event)) def stop(self, fail_tasks=False): self.framework_id = None if self.runner: self.runner.stop() # Clear message queue if self.deferred: self.deferred.cancel() self.deferred = None self.queue = PyDeferredQueue() if fail_tasks: for key, task in list(self.tasks.items()): task.exited(None) del self.tasks[key] def kill(self, task_id): return self.runner.kill(task_id)