def create_task(self, offer, t): task = Dict() task.task_id.value = '%d-%d' % (t.id, t.tried) task.agent_id.value = offer.agent_id.value task.name = 'task %s/%d' % (t.id, self.options.tasks) task.executor = self.executor env = dict(os.environ) env['DRUN_RANK'] = str(t.id) env['DRUN_SIZE'] = str(self.options.tasks) command = self.command[:] if self.options.expand: for i, x in enumerate(command): command[i] = x % {'RANK': t.id, 'SIZE': self.options.tasks} task.data = encode_data(pickle.dumps([ os.getcwd(), command, env, self.options.shell, self.std_port, self.err_port, None ])) task.resources = resources = [] cpu = Dict() resources.append(cpu) cpu.name = 'cpus' cpu.type = 'SCALAR' cpu.scalar.value = self.cpus mem = Dict() resources.append(mem) mem.name = 'mem' mem.type = 'SCALAR' mem.scalar.value = self.mem return task
def create_task(self, offer, t): task = Dict() task.task_id.value = '%d-%d' % (t.id, t.tried) task.agent_id.value = offer.agent_id.value task.name = 'task %s/%d' % (t.id, self.options.tasks) task.executor = self.executor env = dict(os.environ) env['DRUN_RANK'] = str(t.id) env['DRUN_SIZE'] = str(self.options.tasks) command = self.command[:] if self.options.expand: for i, x in enumerate(command): command[i] = x % {'RANK': t.id, 'SIZE': self.options.tasks} task.data = encode_data( pickle.dumps([ os.getcwd(), command, env, self.options.shell, self.stdout_port, self.stderr_port, None ])) task.resources = resources = [] cpu = Dict() resources.append(cpu) cpu.name = 'cpus' cpu.type = 'SCALAR' cpu.scalar.value = self.cpus mem = Dict() resources.append(mem) mem.name = 'mem' mem.type = 'SCALAR' mem.scalar.value = self.mem return task
def send_message(driver, message, max_message_length): """Sends the message, if it is smaller than the max length, using the driver. Parameters ---------- driver: MesosExecutorDriver The driver to send the message to. message: object The raw message to send. max_message_length: int The allowed max message length after encoding. Returns ------- whether the message was successfully sent """ logging.debug('Sending framework message {}'.format(message)) message_string = str(message).encode('utf8') if len(message_string) < max_message_length: encoded_message = encode_data(message_string) driver.sendFrameworkMessage(encoded_message) return True else: log_message_template = 'Unable to send message {} as it exceeds allowed max length of {}' logging.warning( log_message_template.format(message, max_message_length)) return False
def send_message(driver, error_handler, message): """Sends the message, if it is smaller than the max length, using the driver. Note: This function must rethrow any OSError exceptions that it encounters. Parameters ---------- driver: MesosExecutorDriver The driver to send the message to. error_handler: fn(os_error) OSError exception handler for out of memory situations. message: dictionary The raw message to send. Returns ------- whether the message was successfully sent """ try: logging.info('Sending framework message {}'.format(message)) message_string = json.dumps(message).encode('utf8') encoded_message = pm.encode_data(message_string) driver.sendFrameworkMessage(encoded_message) return True except Exception as exception: if cu.is_out_of_memory_error(exception): error_handler(exception) else: logging.exception( 'Exception while sending message {}'.format(message)) return False
def test_acknowledge_operation_status_update(mocker): ID = str(uuid.uuid4()) sched = mocker.Mock() framework = {'id': {'value': ID}} master = mocker.Mock() driver = MesosSchedulerDriver(sched, framework, master) driver._send = mocker.Mock() agent_id = dict(value=str(uuid.uuid4())) operation_id = dict(value=str(uuid.uuid4())) uid = encode_data(uuid.uuid4().bytes) status = { 'agent_id': agent_id, 'operation_id': operation_id, 'uuid': uid } driver.acknowledgeOperationStatusUpdate(status) driver._send.assert_not_called() driver._stream_id = 'a-stream-id' driver.acknowledgeOperationStatusUpdate(status) driver._send.assert_called_once_with({ 'type': 'ACKNOWLEDGE_OPERATION_STATUS', 'framework_id': { 'value': ID }, 'acknowledge_operation_status': { 'agent_id': agent_id, 'operation_id': operation_id, 'uuid': uid, } })
def test_send_framework_message(mocker): ID = str(uuid.uuid4()) sched = mocker.Mock() framework = {'id': {'value': ID}} master = mocker.Mock() driver = MesosSchedulerDriver(sched, framework, master) driver._send = mocker.Mock() executor_id = {'value': str(uuid.uuid4())} agent_id = {'value': str(uuid.uuid4())} message = ''.join( random.choice(string.printable) for _ in range(random.randint(1, 100))) message = encode_data(message.encode('utf-8')) driver.sendFrameworkMessage(executor_id, agent_id, message) driver._send.assert_not_called() driver._stream_id = 'a-stream-id' driver.sendFrameworkMessage(executor_id, agent_id, message) driver._send.assert_called_once_with({ 'type': 'MESSAGE', 'framework_id': { 'value': ID }, 'message': { 'agent_id': agent_id, 'executor_id': executor_id, 'data': message, } })
def test_acknowledge_operation_status_update(mocker): ID = str(uuid.uuid4()) sched = mocker.Mock() framework = {'id': {'value': ID}} master = mocker.Mock() driver = MesosSchedulerDriver(sched, framework, master) driver._send = mocker.Mock() agent_id = dict(value=str(uuid.uuid4())) operation_id = dict(value=str(uuid.uuid4())) uid = encode_data(uuid.uuid4().bytes) status = {'agent_id': agent_id, 'operation_id': operation_id, 'uuid': uid} driver.acknowledgeOperationStatusUpdate(status) driver._send.assert_not_called() driver._stream_id = 'a-stream-id' driver.acknowledgeOperationStatusUpdate(status) driver._send.assert_called_once_with({ 'type': 'ACKNOWLEDGE_OPERATION_STATUS', 'framework_id': { 'value': ID }, 'acknowledge_operation_status': { 'agent_id': agent_id, 'operation_id': operation_id, 'uuid': uid, } })
def createTask(self, o, job, t): task = Dict() tid = '%s:%s:%s' % (job.id, t.id, t.tried) task.name = 'task %s' % tid task.task_id.value = tid task.agent_id.value = o.agent_id.value task.data = encode_data( compress(six.moves.cPickle.dumps((t, t.tried), -1))) task.executor = self.executor if len(task.data) > 1000 * 1024: logger.warning('task too large: %s %d', t, len(task.data)) resources = task.resources = [] cpu = Dict() resources.append(cpu) cpu.name = 'cpus' cpu.type = 'SCALAR' cpu.scalar.value = t.cpus mem = Dict() resources.append(mem) mem.name = 'mem' mem.type = 'SCALAR' mem.scalar.value = t.mem return task
def createTask(self, o, job, t): task = Dict() tid = '%s:%s:%s' % (job.id, t.id, t.tried) task.name = 'task %s' % tid task.task_id.value = tid task.agent_id.value = o.agent_id.value task.data = encode_data( compress(cPickle.dumps((t, t.tried), -1)) ) task.executor = self.executor if len(task.data) > 1000 * 1024: logger.warning('task too large: %s %d', t, len(task.data)) resources = task.resources = [] cpu = Dict() resources.append(cpu) cpu.name = 'cpus' cpu.type = 'SCALAR' cpu.scalar.value = t.cpus mem = Dict() resources.append(mem) mem.name = 'mem' mem.type = 'SCALAR' mem.scalar.value = t.mem return task
def create_task(self, offer, t, k): task = Dict() task.task_id.value = '%s-%s' % (t.id, t.tried) task.agent_id.value = offer.agent_id.value task.name = 'task %s' % t.id task.executor = self.executor env = dict(os.environ) task.data = encode_data(pickle.dumps([ os.getcwd(), None, env, self.options.shell, self.stdout_port, self.stderr_port, self.publisher_port ])) task.resources = resources = [] cpu = Dict() resources.append(cpu) cpu.name = 'cpus' cpu.type = 'SCALAR' cpu.scalar.value = self.cpus * k mem = Dict() resources.append(mem) mem.name = 'mem' mem.type = 'SCALAR' mem.scalar.value = self.mem * k if self.gpus > 0: gpu = Dict() resources.append(gpu) gpu.name = 'gpus' gpu.type = 'SCALAR' gpu.scalar.value = self.gpus * k return task
def test_acknowledge_status_update(mocker): ID = str(uuid.uuid4()) sched = mocker.Mock() framework = {'id': {'value': ID}} master = mocker.Mock() driver = MesosSchedulerDriver(sched, framework, master) driver._send = mocker.Mock() agent_id = dict(value=str(uuid.uuid4())) task_id = dict(value=str(uuid.uuid4())) uid = encode_data(uuid.uuid4().bytes) status = { 'agent_id': agent_id, 'task_id': task_id, 'uuid': uid } driver.acknowledgeStatusUpdate(status) driver._send.assert_called_once_with({ 'type': 'ACKNOWLEDGE', 'framework_id': { 'value': ID }, 'acknowledge': { 'agent_id': agent_id, 'task_id': task_id, 'uuid': uid } })
def createTask(self, o, t): task = Dict() tid = t.try_id task.name = 'task %s' % tid task.task_id.value = tid task.agent_id.value = o.agent_id.value task.data = encode_data(compress(cPickle.dumps((t, tid), -1))) task.executor = self.executor if len(task.data) > 1000 * 1024: logger.warning('task too large: %s %d', t, len(task.data)) assert len(task.data) < (50 << 20), \ 'Task data too large: %s' % (len(task.data),) resources = task.resources = [] cpu = Dict() resources.append(cpu) cpu.name = 'cpus' cpu.type = 'SCALAR' cpu.scalar.value = t.cpus mem = Dict() resources.append(mem) mem.name = 'mem' mem.type = 'SCALAR' mem.scalar.value = t.mem cpu = Dict() resources.append(cpu) cpu.name = 'gpus' cpu.type = 'SCALAR' cpu.scalar.value = t.gpus return task
def test_on_acknowledged(mocker): agent_addr = 'mock_addr:12345' framework_id = str(uuid.uuid4()) executor_id = str(uuid.uuid4()) env = { 'MESOS_LOCAL': 'true', 'MESOS_AGENT_ENDPOINT': agent_addr, 'MESOS_FRAMEWORK_ID': framework_id, 'MESOS_EXECUTOR_ID': executor_id, } mocker.patch('os.environ', env) exc = mocker.Mock() driver = MesosExecutorDriver(exc) driver._started = True assert driver.updates == {} assert driver.tasks == {} tid = str(uuid.uuid4()) uid = uuid.uuid4() driver.updates[uid] = mocker.Mock() driver.tasks[tid] = mocker.Mock() event = { 'type': 'ACKNOWLEDGED', 'acknowledged': { 'task_id': { 'value': tid }, 'uuid': encode_data(uid.bytes) } } driver.on_event(event) assert driver.updates == {} assert driver.tasks == {}
def resourceOffers(self, driver, offers): filters = {'refuse_seconds': 5} for offer in offers: if self.left >= self.right - 1e-16: break cpus = self.getResource(offer.resources, 'cpus') mem = self.getResource(offer.resources, 'mem') if cpus < TASK_CPU or mem < TASK_MEM: continue task = Dict() task_id = str(uuid.uuid4()) task.task_id.value = task_id task.agent_id.value = offer.agent_id.value task.name = 'task {}'.format(task_id) task.executor = self.executor task.data = encode_data(self.fun + '!' \ + repr(self.left) + '!' \ + repr(min(self.right, self.left + self.step * self.taskN)) + '!' \ + repr(self.step)) task.resources = [ dict(name='cpus', type='SCALAR', scalar={'value': TASK_CPU}), dict(name='mem', type='SCALAR', scalar={'value': TASK_MEM}), ] driver.launchTasks(offer.id, [task], filters) self.left = self.left + self.step * self.taskN
def test_send_framework_message(mocker): ID = str(uuid.uuid4()) sched = mocker.Mock() framework = {'id': {'value': ID}} master = mocker.Mock() driver = MesosSchedulerDriver(sched, framework, master) driver._send = mocker.Mock() executor_id = {'value': str(uuid.uuid4())} agent_id = {'value': str(uuid.uuid4())} message = ''.join(random.choice(string.printable) for _ in range(random.randint(1, 100))) message = encode_data(message.encode('utf-8')) driver.sendFrameworkMessage(executor_id, agent_id, message) driver._send.assert_not_called() driver._stream_id = 'a-stream-id' driver.sendFrameworkMessage(executor_id, agent_id, message) driver._send.assert_called_once_with({ 'type': 'MESSAGE', 'framework_id': { 'value': ID }, 'message': { 'agent_id': agent_id, 'executor_id': executor_id, 'data': message, } })
def manage_task_runner(self, command, assertions_fn, stop_signal=Event()): driver = FakeMesosExecutorDriver() task_id = get_random_task_id() task = {'task_id': {'value': task_id}, 'data': encode_data(json.dumps({'command': command}).encode('utf8'))} stdout_name = 'build/stdout.' + str(task_id) stderr_name = 'build/stderr.' + str(task_id) completed_signal = Event() max_message_length = 300 progress_sample_interval_ms = 100 sandbox_location = '/location/to/task/sandbox/{}'.format(task_id) progress_output_name = stdout_name progress_regex_string = '\^\^\^\^JOB-PROGRESS: (\d*)(?: )?(.*)' config = cc.ExecutorConfig(max_message_length=max_message_length, progress_output_name=progress_output_name, progress_regex_string=progress_regex_string, progress_sample_interval_ms=progress_sample_interval_ms, sandbox_location=sandbox_location) try: ce.manage_task(driver, task, stop_signal, completed_signal, config, stdout_name, stderr_name) self.assertTrue(completed_signal.isSet()) assertions_fn(driver, task_id, sandbox_location) finally: cleanup_output(stdout_name, stderr_name)
def run_task(task): update = Dict() update.task_id.value = task.task_id.value update.state = 'TASK_RUNNING' update.timestamp = time.time() driver.sendStatusUpdate(update) # 保留以作测试用 print(decode_data(task.data), file=sys.stderr) cnt = 0 N = 2000000 for i in range(N): x = random() y = random() if (x * x + y * y) < 1: cnt += 1 vPi = 4.0 * cnt / N print(vPi) driver.sendFrameworkMessage(encode_data(str(vPi))) time.sleep(30) update = Dict() update.task_id.value = task.task_id.value update.state = 'TASK_FINISHED' update.timestamp = time.time() driver.sendStatusUpdate(update)
def resourceOffers(self, driver, offers): if self.i >= self.count: return None filters = {'refuse_seconds': 5} for offer in offers: if self.i >= self.count: break cpus = self.getResource(offer.resources, 'cpus') mem = self.getResource(offer.resources, 'mem') if cpus < TASK_CPU or mem < TASK_MEM: continue task = Dict() task_id = str(uuid.uuid4()) task.task_id.value = task_id task.agent_id.value = offer.agent_id.value task.name = 'task {}'.format(task_id) task.executor = self.executor # 保留以作测试用 :) task.data = encode_data('Hello from task {}!'.format(task_id)) task.resources = [ dict(name='cpus', type='SCALAR', scalar={'value': TASK_CPU}), dict(name='mem', type='SCALAR', scalar={'value': TASK_MEM}), ] driver.launchTasks(offer.id, [task], filters) self.i = self.i + 1
def resourceOffers(self, driver, offers): filters = {'refuse_seconds': 5} for offer in offers: if self.launched_task == TASK_NUM: return cpus = self.getResource(offer.resources, 'cpus') mem = self.getResource(offer.resources, 'mem') if cpus < TASK_CPU or mem < TASK_MEM: continue task = Dict() task_id = str(uuid.uuid4()) task.task_id.value = task_id task.agent_id.value = offer.agent_id.value task.name = 'task {}'.format(task_id) task.executor = self.executor # send data to executor task.data = encode_data(self.data_split[self.launched_task]) task.resources = [ dict(name='cpus', type='SCALAR', scalar={'value': TASK_CPU}), dict(name='mem', type='SCALAR', scalar={'value': TASK_MEM}), ] self.launched_task += 1 driver.launchTasks(offer.id, [task], filters)
def resourceOffers(self, driver, offers): filters = {'refuse_seconds': 5} # if all tasks have been launched, return directly if self.task_launched == TASK_NUM: return # for every offer for offer in offers: # check if the offer satisfy the requirments cpus = self.getResource(offer.resources, 'cpus') mem = self.getResource(offer.resources, 'mem') if cpus < TASK_CPU or mem < TASK_MEM: continue # config a new task task = Dict() task_id = str(uuid.uuid4()) task.task_id.value = task_id task.agent_id.value = offer.agent_id.value task.name = 'task {}'.format(task_id) task.executor = self.executor #task.data = encode_data(bytes('Hello from task {}!'.format(task_id), 'utf-8')) task.data = encode_data(bytes('ThisIsASeparator'.join(self.datas[self.task_launched]), 'utf-8')) task.resources = [ dict(name='cpus', type='SCALAR', scalar={'value': TASK_CPU}), dict(name='mem', type='SCALAR', scalar={'value': TASK_MEM}), ] # launch task driver.launchTasks(offer.id, [task], filters) self.task_launched += 1
def run_task(task): update = Dict() update.task_id.value = task.task_id.value update.state = 'TASK_RUNNING' update.timestamp = time.time() driver.sendStatusUpdate(update) tmp = decode_data(task.data).split('!') fun = tmp[0] left = float(tmp[1]) right = float(tmp[2]) step = float(tmp[3]) res_tot = 0 x = left while x < right - 1e-16: exec(fun) in globals(), locals() res_tot = res_tot + res x = x + step driver.sendFrameworkMessage(encode_data(repr(step * res_tot))) update = Dict() update.task_id.value = task.task_id.value update.state = 'TASK_FINISHED' update.timestamp = time.time() driver.sendStatusUpdate(update)
def test_send_message(mocker): agent_addr = 'mock_addr:12345' framework_id = str(uuid.uuid4()) executor_id = str(uuid.uuid4()) env = { 'MESOS_LOCAL': 'true', 'MESOS_AGENT_ENDPOINT': agent_addr, 'MESOS_FRAMEWORK_ID': framework_id, 'MESOS_EXECUTOR_ID': executor_id, } mocker.patch('os.environ', env) exc = mocker.Mock() driver = MesosExecutorDriver(exc) driver._send = mocker.Mock() message = ''.join( random.choice(string.printable) for _ in range(random.randint(1, 100))) message = encode_data(message.encode('utf8')) driver.sendFrameworkMessage(message) driver._send.assert_called_once_with({ 'type': 'MESSAGE', 'framework_id': { 'value': framework_id, }, 'executor_id': { 'value': executor_id, }, 'message': { 'data': message, } })
def reply_status(driver, task_id, state, data=None): status = Dict() status.task_id = task_id status.state = state status.timestamp = time.time() if data is not None: status.data = encode_data(data) driver.sendStatusUpdate(status)
def test_launch_task_no_command(self): task_id = get_random_task_id() task = {'task_id': {'value': task_id}, 'data': encode_data(json.dumps({'command': ''}).encode('utf8'))} stdout_name = '' stderr_name = '' result = ce.launch_task(task, stdout_name, stderr_name) self.assertIsNone(result)
def reply_status(driver, task_id, state, reason=None, msg=None, data=None): status = Dict() status.task_id = task_id status.state = state if reason is not None: status.message = '{}:{}'.format(reason, msg) status.timestamp = time.time() if data is not None: status.data = encode_data(data) driver.sendStatusUpdate(status)
def manage_task_runner(self, command, assertions_fn, stop_signal=None, task_id=None, config=None, driver=None): if driver is None: driver = tu.FakeMesosExecutorDriver() if stop_signal is None: stop_signal = Event() if task_id is None: task_id = tu.get_random_task_id() task = { 'task_id': { 'value': task_id }, 'data': pm.encode_data(json.dumps({ 'command': command }).encode('utf8')) } stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) completed_signal = Event() if config is None: sandbox_directory = '/location/to/task/sandbox/{}'.format(task_id) config = cc.ExecutorConfig( max_message_length=300, progress_output_name=stdout_name, progress_regex_string= '\^\^\^\^JOB-PROGRESS:\s+([0-9]*\.?[0-9]+)($|\s+.*)', progress_sample_interval_ms=100, sandbox_directory=sandbox_directory) else: sandbox_directory = config.sandbox_directory try: ce.manage_task(driver, task, stop_signal, completed_signal, config) self.assertTrue(completed_signal.isSet()) assertions_fn(driver, task_id, sandbox_directory) finally: tu.cleanup_output(stdout_name, stderr_name)
def test_launch_task_no_command(self): task_id = tu.get_random_task_id() task = { 'task_id': { 'value': task_id }, 'data': pm.encode_data(json.dumps({ 'command': '' }).encode('utf8')) } process = ce.launch_task(task, os.environ) self.assertIsNone(process)
def test_launch_task_interactive_output(self): task_id = tu.get_random_task_id() command = 'echo "Start"; echo "Hello"; sleep 100; echo "World"; echo "Done"; ' task = { 'task_id': { 'value': task_id }, 'data': pm.encode_data(json.dumps({ 'command': command }).encode('utf8')) } stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) try: process = ce.launch_task(task, os.environ) self.assertIsNotNone(process) # let the process run for up to 50 seconds for _ in range(5000): if cs.is_process_running(process): time.sleep(0.01) with open(stdout_name) as f: stdout_content = f.read() if 'Start' in stdout_content and 'Hello' in stdout_content: break try: with open(stdout_name) as f: stdout_content = f.read() logging.info( 'Contents of stdout: {}'.format(stdout_content)) self.assertTrue("Start" in stdout_content) self.assertTrue("Hello" in stdout_content) self.assertFalse("World" in stdout_content) self.assertFalse("Done" in stdout_content) finally: if process.poll() is None: logging.info('Killing launched process') process.kill() finally: tu.cleanup_output(stdout_name, stderr_name)
def test_launch_task(self): task_id = get_random_task_id() command = 'echo "Hello World"; echo "Error Message" >&2' task = { 'task_id': { 'value': task_id }, 'data': encode_data(json.dumps({ 'command': command }).encode('utf8')) } stdout_name = ensure_directory('build/stdout.' + str(task_id)) stderr_name = ensure_directory('build/stderr.' + str(task_id)) if not os.path.isdir("build"): os.mkdir("build") try: process, stdout, stderr = ce.launch_task(task, stdout_name, stderr_name) self.assertIsNotNone(process) for i in range(100): if process.poll() is None: time.sleep(0.01) stdout.close() stderr.close() if process.poll() is None: process.kill() self.assertEqual(0, process.poll()) with open(stdout_name) as f: stdout_content = f.read() self.assertEqual("Hello World\n", stdout_content) with open(stderr_name) as f: stderr_content = f.read() self.assertEqual("Error Message\n", stderr_content) finally: cleanup_output(stdout_name, stderr_name)
def _newMesosTask(self, job, offer): """ Build the Mesos task object for a given the Toil job and Mesos offer """ task = addict.Dict() task.task_id.value = str(job.jobID) task.agent_id.value = offer.agent_id.value task.name = job.name task.data = encode_data(pickle.dumps(job)) task.executor = addict.Dict(self.executor) task.resources = [] task.resources.append(addict.Dict()) cpus = task.resources[-1] cpus.name = 'cpus' cpus.type = 'SCALAR' cpus.scalar.value = job.resources.cores task.resources.append(addict.Dict()) disk = task.resources[-1] disk.name = 'disk' disk.type = 'SCALAR' if toMiB(job.resources.disk) > 1: disk.scalar.value = toMiB(job.resources.disk) else: log.warning( "Job %s uses less disk than Mesos requires. Rounding %s up to 1 MiB.", job.jobID, job.resources.disk) disk.scalar.value = 1 task.resources.append(addict.Dict()) mem = task.resources[-1] mem.name = 'mem' mem.type = 'SCALAR' if toMiB(job.resources.memory) > 1: mem.scalar.value = toMiB(job.resources.memory) else: log.warning( "Job %s uses less memory than Mesos requires. Rounding %s up to 1 MiB.", job.jobID, job.resources.memory) mem.scalar.value = 1 return task
def test_launch_task(self): task_id = tu.get_random_task_id() command = 'echo "Hello World"; echo "Error Message" >&2' task = { 'task_id': { 'value': task_id }, 'data': pm.encode_data(json.dumps({ 'command': command }).encode('utf8')) } stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) try: process = ce.launch_task(task, os.environ) self.assertIsNotNone(process) for _ in range(100): if cs.is_process_running(process): time.sleep(0.01) if process.poll() is None: process.kill() tu.close_sys_outputs() self.assertEqual(0, process.poll()) with open(stdout_name) as f: stdout_content = f.read() self.assertTrue("Hello World\n" in stdout_content) with open(stderr_name) as f: stderr_content = f.read() self.assertTrue("Error Message\n" in stderr_content) finally: tu.cleanup_output(stdout_name, stderr_name)
def run_task(task): update = Dict() update.task_id.value = task.task_id.value update.state = 'TASK_RUNNING' update.timestamp = time.time() driver.sendStatusUpdate(update) tmp = decode_data(task.data).split(' ') left = int(tmp[0]) right = int(tmp[1]) res = 0 for i in xrange(left, right): res = res + i driver.sendFrameworkMessage(encode_data(str(res))) update = Dict() update.task_id.value = task.task_id.value update.state = 'TASK_FINISHED' update.timestamp = time.time() driver.sendStatusUpdate(update)
def _sendFrameworkMessage(self, driver): message = None while True: # The psutil documentation recommends that we ignore the value returned by the first # invocation of cpu_percent(). However, we do want to send a sign of life early after # starting (e.g. to unblock the provisioner waiting for an instance to come up) so # we call it once and discard the value. if message is None: message = Expando(address=self.address) psutil.cpu_percent() else: message.nodeInfo = dict(coresUsed=float(psutil.cpu_percent()) * .01, memoryUsed=float(psutil.virtual_memory().percent) * .01, coresTotal=psutil.cpu_count(), memoryTotal=psutil.virtual_memory().total, workers=len(self.runningTasks)) log.debug("Send framework message: %s", message) driver.sendFrameworkMessage(encode_data(repr(message))) # Prevent workers launched together from repeatedly hitting the leader at the same time time.sleep(random.randint(45, 75))
def run_task(task): update = Dict() update.task_id.value = task.task_id.value update.state = 'TASK_RUNNING' update.timestamp = time.time() driver.sendStatusUpdate(update) data = decode_data(task.data).split(' ') result = 0 for x in data: if x != '': result += int(x) # send the result to the scheduler driver.sendFrameworkMessage(encode_data(str(result))) update = Dict() update.task_id.value = task.task_id.value update.state = 'TASK_FINISHED' update.timestamp = time.time() driver.sendStatusUpdate(update)
def resourceOffers(self, driver, offers): filters = {'refuse_seconds': 5} for offer in offers: cpus = self.getResource(offer.resources, 'cpus') mem = self.getResource(offer.resources, 'mem') if cpus < TASK_CPU or mem < TASK_MEM: continue task = Dict() task_id = str(uuid.uuid4()) task.task_id.value = task_id task.agent_id.value = offer.agent_id.value task.name = 'task {}'.format(task_id) task.executor = self.executor task.data = encode_data('Hello from task {}!'.format(task_id)) task.resources = [ dict(name='cpus', type='SCALAR', scalar={'value': TASK_CPU}), dict(name='mem', type='SCALAR', scalar={'value': TASK_MEM}), ] driver.launchTasks(offer.id, [task], filters)
def test_on_message(mocker): ID = str(uuid.uuid4()) sched = mocker.Mock() framework = {'id': {'value': ID}} master = mocker.Mock() driver = MesosSchedulerDriver(sched, framework, master) driver._started = True executor_id = {'value': str(uuid.uuid4())} agent_id = {'value': str(uuid.uuid4())} message = ''.join(random.choice(string.printable) for _ in range(random.randint(1, 100))) data = encode_data(message.encode('utf8')) event = { 'type': 'MESSAGE', 'message': { 'executor_id': executor_id, 'agent_id': agent_id, 'data': data } } driver.on_event(event) sched.frameworkMessage.assert_called_once_with(driver, executor_id, agent_id, data)
def getExecutorInfo(self, framework_id): info = Dict() info.framework_id.value = framework_id if self.use_self_as_exec: info.command.value = os.path.abspath(sys.argv[0]) info.executor_id.value = sys.argv[0] else: info.command.value = '%s %s' % ( sys.executable, os.path.abspath( os.path.join( os.path.dirname(__file__), 'executor.py')) ) info.executor_id.value = 'default' info.command.environment.variables = variables = [] v = Dict() variables.append(v) v.name = 'UID' v.value = str(os.getuid()) v = Dict() variables.append(v) v.name = 'GID' v.value = str(os.getgid()) if self.options.image: info.container.type = 'DOCKER' info.container.docker.image = self.options.image info.container.volumes = volumes = [] for path in ['/etc/passwd', '/etc/group']: v = Dict() volumes.append(v) v.host_path = v.container_path = path v.mode = 'RO' for path in conf.MOOSEFS_MOUNT_POINTS: v = Dict() volumes.append(v) v.host_path = v.container_path = path v.mode = 'RW' for path in conf.DPARK_WORK_DIR.split(','): v = Dict() volumes.append(v) v.host_path = v.container_path = path v.mode = 'RW' if self.options.volumes: for volume in self.options.volumes.split(','): fields = volume.split(':') if len(fields) == 3: host_path, container_path, mode = fields mode = mode.upper() assert mode in ('RO', 'RW') elif len(fields) == 2: host_path, container_path = fields mode = 'RW' elif len(fields) == 1: container_path, = fields host_path = '' mode = 'RW' else: raise Exception('cannot parse volume %s', volume) mkdir_p(host_path) v = Dict() volumes.append(v) v.container_path = container_path v.mode = mode if host_path: v.host_path = host_path info.resources = resources = [] mem = Dict() resources.append(mem) mem.name = 'mem' mem.type = 'SCALAR' mem.scalar.value = EXECUTOR_MEMORY cpus = Dict() resources.append(cpus) cpus.name = 'cpus' cpus.type = 'SCALAR' cpus.scalar.value = EXECUTOR_CPUS Script = os.path.realpath(sys.argv[0]) info.name = Script info.data = encode_data(marshal.dumps( ( Script, os.getcwd(), sys.path, dict(os.environ), self.task_per_node, self.out_logger, self.err_logger, self.logLevel, env.environ ) )) return info