Example #1
0
    def create_task(self, offer, t):
        task = Dict()
        task.task_id.value = '%d-%d' % (t.id, t.tried)
        task.agent_id.value = offer.agent_id.value
        task.name = 'task %s/%d' % (t.id, self.options.tasks)
        task.executor = self.executor
        env = dict(os.environ)
        env['DRUN_RANK'] = str(t.id)
        env['DRUN_SIZE'] = str(self.options.tasks)
        command = self.command[:]
        if self.options.expand:
            for i, x in enumerate(command):
                command[i] = x % {'RANK': t.id, 'SIZE': self.options.tasks}

        task.data = encode_data(pickle.dumps([
            os.getcwd(), command, env, self.options.shell,
            self.std_port, self.err_port, None
        ]))

        task.resources = resources = []

        cpu = Dict()
        resources.append(cpu)
        cpu.name = 'cpus'
        cpu.type = 'SCALAR'
        cpu.scalar.value = self.cpus

        mem = Dict()
        resources.append(mem)
        mem.name = 'mem'
        mem.type = 'SCALAR'
        mem.scalar.value = self.mem
        return task
Example #2
0
    def create_task(self, offer, t):
        task = Dict()
        task.task_id.value = '%d-%d' % (t.id, t.tried)
        task.agent_id.value = offer.agent_id.value
        task.name = 'task %s/%d' % (t.id, self.options.tasks)
        task.executor = self.executor
        env = dict(os.environ)
        env['DRUN_RANK'] = str(t.id)
        env['DRUN_SIZE'] = str(self.options.tasks)
        command = self.command[:]
        if self.options.expand:
            for i, x in enumerate(command):
                command[i] = x % {'RANK': t.id, 'SIZE': self.options.tasks}

        task.data = encode_data(
            pickle.dumps([
                os.getcwd(), command, env, self.options.shell,
                self.stdout_port, self.stderr_port, None
            ]))

        task.resources = resources = []

        cpu = Dict()
        resources.append(cpu)
        cpu.name = 'cpus'
        cpu.type = 'SCALAR'
        cpu.scalar.value = self.cpus

        mem = Dict()
        resources.append(mem)
        mem.name = 'mem'
        mem.type = 'SCALAR'
        mem.scalar.value = self.mem
        return task
Example #3
0
def send_message(driver, message, max_message_length):
    """Sends the message, if it is smaller than the max length, using the driver.

    Parameters
    ----------
    driver: MesosExecutorDriver
        The driver to send the message to.
    message: object
        The raw message to send.
    max_message_length: int
        The allowed max message length after encoding.

    Returns
    -------
    whether the message was successfully sent
    """
    logging.debug('Sending framework message {}'.format(message))
    message_string = str(message).encode('utf8')
    if len(message_string) < max_message_length:
        encoded_message = encode_data(message_string)
        driver.sendFrameworkMessage(encoded_message)
        return True
    else:
        log_message_template = 'Unable to send message {} as it exceeds allowed max length of {}'
        logging.warning(
            log_message_template.format(message, max_message_length))
        return False
Example #4
0
def send_message(driver, error_handler, message):
    """Sends the message, if it is smaller than the max length, using the driver.

    Note: This function must rethrow any OSError exceptions that it encounters.

    Parameters
    ----------
    driver: MesosExecutorDriver
        The driver to send the message to.
    error_handler: fn(os_error)
        OSError exception handler for out of memory situations.
    message: dictionary
        The raw message to send.

    Returns
    -------
    whether the message was successfully sent
    """
    try:
        logging.info('Sending framework message {}'.format(message))
        message_string = json.dumps(message).encode('utf8')
        encoded_message = pm.encode_data(message_string)
        driver.sendFrameworkMessage(encoded_message)
        return True
    except Exception as exception:
        if cu.is_out_of_memory_error(exception):
            error_handler(exception)
        else:
            logging.exception(
                'Exception while sending message {}'.format(message))
        return False
Example #5
0
def test_acknowledge_operation_status_update(mocker):
    ID = str(uuid.uuid4())
    sched = mocker.Mock()
    framework = {'id': {'value': ID}}
    master = mocker.Mock()
    driver = MesosSchedulerDriver(sched, framework, master)
    driver._send = mocker.Mock()
    agent_id = dict(value=str(uuid.uuid4()))
    operation_id = dict(value=str(uuid.uuid4()))
    uid = encode_data(uuid.uuid4().bytes)
    status = {
        'agent_id': agent_id,
        'operation_id': operation_id,
        'uuid': uid
    }
    driver.acknowledgeOperationStatusUpdate(status)
    driver._send.assert_not_called()
    driver._stream_id = 'a-stream-id'
    driver.acknowledgeOperationStatusUpdate(status)
    driver._send.assert_called_once_with({
        'type': 'ACKNOWLEDGE_OPERATION_STATUS',
        'framework_id': {
            'value': ID
        },
        'acknowledge_operation_status': {
            'agent_id': agent_id,
            'operation_id': operation_id,
            'uuid': uid,
        }
    })
def test_send_framework_message(mocker):
    ID = str(uuid.uuid4())
    sched = mocker.Mock()
    framework = {'id': {'value': ID}}
    master = mocker.Mock()
    driver = MesosSchedulerDriver(sched, framework, master)
    driver._send = mocker.Mock()
    executor_id = {'value': str(uuid.uuid4())}
    agent_id = {'value': str(uuid.uuid4())}
    message = ''.join(
        random.choice(string.printable) for _ in range(random.randint(1, 100)))
    message = encode_data(message.encode('utf-8'))
    driver.sendFrameworkMessage(executor_id, agent_id, message)
    driver._send.assert_not_called()
    driver._stream_id = 'a-stream-id'
    driver.sendFrameworkMessage(executor_id, agent_id, message)
    driver._send.assert_called_once_with({
        'type': 'MESSAGE',
        'framework_id': {
            'value': ID
        },
        'message': {
            'agent_id': agent_id,
            'executor_id': executor_id,
            'data': message,
        }
    })
def test_acknowledge_operation_status_update(mocker):
    ID = str(uuid.uuid4())
    sched = mocker.Mock()
    framework = {'id': {'value': ID}}
    master = mocker.Mock()
    driver = MesosSchedulerDriver(sched, framework, master)
    driver._send = mocker.Mock()
    agent_id = dict(value=str(uuid.uuid4()))
    operation_id = dict(value=str(uuid.uuid4()))
    uid = encode_data(uuid.uuid4().bytes)
    status = {'agent_id': agent_id, 'operation_id': operation_id, 'uuid': uid}
    driver.acknowledgeOperationStatusUpdate(status)
    driver._send.assert_not_called()
    driver._stream_id = 'a-stream-id'
    driver.acknowledgeOperationStatusUpdate(status)
    driver._send.assert_called_once_with({
        'type': 'ACKNOWLEDGE_OPERATION_STATUS',
        'framework_id': {
            'value': ID
        },
        'acknowledge_operation_status': {
            'agent_id': agent_id,
            'operation_id': operation_id,
            'uuid': uid,
        }
    })
Example #8
0
    def createTask(self, o, job, t):
        task = Dict()
        tid = '%s:%s:%s' % (job.id, t.id, t.tried)
        task.name = 'task %s' % tid
        task.task_id.value = tid
        task.agent_id.value = o.agent_id.value
        task.data = encode_data(
            compress(six.moves.cPickle.dumps((t, t.tried), -1)))
        task.executor = self.executor
        if len(task.data) > 1000 * 1024:
            logger.warning('task too large: %s %d', t, len(task.data))

        resources = task.resources = []

        cpu = Dict()
        resources.append(cpu)
        cpu.name = 'cpus'
        cpu.type = 'SCALAR'
        cpu.scalar.value = t.cpus

        mem = Dict()
        resources.append(mem)
        mem.name = 'mem'
        mem.type = 'SCALAR'
        mem.scalar.value = t.mem
        return task
Example #9
0
    def createTask(self, o, job, t):
        task = Dict()
        tid = '%s:%s:%s' % (job.id, t.id, t.tried)
        task.name = 'task %s' % tid
        task.task_id.value = tid
        task.agent_id.value = o.agent_id.value
        task.data = encode_data(
            compress(cPickle.dumps((t, t.tried), -1))
        )
        task.executor = self.executor
        if len(task.data) > 1000 * 1024:
            logger.warning('task too large: %s %d',
                           t, len(task.data))

        resources = task.resources = []

        cpu = Dict()
        resources.append(cpu)
        cpu.name = 'cpus'
        cpu.type = 'SCALAR'
        cpu.scalar.value = t.cpus

        mem = Dict()
        resources.append(mem)
        mem.name = 'mem'
        mem.type = 'SCALAR'
        mem.scalar.value = t.mem
        return task
Example #10
0
    def create_task(self, offer, t, k):
        task = Dict()
        task.task_id.value = '%s-%s' % (t.id, t.tried)
        task.agent_id.value = offer.agent_id.value
        task.name = 'task %s' % t.id
        task.executor = self.executor
        env = dict(os.environ)
        task.data = encode_data(pickle.dumps([
            os.getcwd(), None, env, self.options.shell,
            self.stdout_port, self.stderr_port, self.publisher_port
        ]))

        task.resources = resources = []
        cpu = Dict()
        resources.append(cpu)
        cpu.name = 'cpus'
        cpu.type = 'SCALAR'
        cpu.scalar.value = self.cpus * k

        mem = Dict()
        resources.append(mem)
        mem.name = 'mem'
        mem.type = 'SCALAR'
        mem.scalar.value = self.mem * k

        if self.gpus > 0:
            gpu = Dict()
            resources.append(gpu)
            gpu.name = 'gpus'
            gpu.type = 'SCALAR'
            gpu.scalar.value = self.gpus * k

        return task
Example #11
0
def test_acknowledge_status_update(mocker):
    ID = str(uuid.uuid4())
    sched = mocker.Mock()
    framework = {'id': {'value': ID}}
    master = mocker.Mock()
    driver = MesosSchedulerDriver(sched, framework, master)
    driver._send = mocker.Mock()
    agent_id = dict(value=str(uuid.uuid4()))
    task_id = dict(value=str(uuid.uuid4()))
    uid = encode_data(uuid.uuid4().bytes)
    status = {
        'agent_id': agent_id,
        'task_id': task_id,
        'uuid': uid
    }
    driver.acknowledgeStatusUpdate(status)
    driver._send.assert_called_once_with({
        'type': 'ACKNOWLEDGE',
        'framework_id': {
            'value': ID
        },
        'acknowledge': {
            'agent_id': agent_id,
            'task_id': task_id,
            'uuid': uid
        }
    })
Example #12
0
    def createTask(self, o, t):
        task = Dict()
        tid = t.try_id
        task.name = 'task %s' % tid
        task.task_id.value = tid
        task.agent_id.value = o.agent_id.value
        task.data = encode_data(compress(cPickle.dumps((t, tid), -1)))
        task.executor = self.executor
        if len(task.data) > 1000 * 1024:
            logger.warning('task too large: %s %d', t, len(task.data))

        assert len(task.data) < (50 << 20), \
            'Task data too large: %s' % (len(task.data),)

        resources = task.resources = []

        cpu = Dict()
        resources.append(cpu)
        cpu.name = 'cpus'
        cpu.type = 'SCALAR'
        cpu.scalar.value = t.cpus

        mem = Dict()
        resources.append(mem)
        mem.name = 'mem'
        mem.type = 'SCALAR'
        mem.scalar.value = t.mem

        cpu = Dict()
        resources.append(cpu)
        cpu.name = 'gpus'
        cpu.type = 'SCALAR'
        cpu.scalar.value = t.gpus

        return task
Example #13
0
def test_on_acknowledged(mocker):
    agent_addr = 'mock_addr:12345'
    framework_id = str(uuid.uuid4())
    executor_id = str(uuid.uuid4())
    env = {
        'MESOS_LOCAL': 'true',
        'MESOS_AGENT_ENDPOINT': agent_addr,
        'MESOS_FRAMEWORK_ID': framework_id,
        'MESOS_EXECUTOR_ID': executor_id,
    }
    mocker.patch('os.environ', env)
    exc = mocker.Mock()
    driver = MesosExecutorDriver(exc)
    driver._started = True
    assert driver.updates == {}
    assert driver.tasks == {}
    tid = str(uuid.uuid4())
    uid = uuid.uuid4()
    driver.updates[uid] = mocker.Mock()
    driver.tasks[tid] = mocker.Mock()
    event = {
        'type': 'ACKNOWLEDGED',
        'acknowledged': {
            'task_id': {
                'value': tid
            },
            'uuid': encode_data(uid.bytes)
        }
    }
    driver.on_event(event)
    assert driver.updates == {}
    assert driver.tasks == {}
Example #14
0
    def resourceOffers(self, driver, offers):
        filters = {'refuse_seconds': 5}

        for offer in offers:
            if self.left >= self.right - 1e-16:
                break
            cpus = self.getResource(offer.resources, 'cpus')
            mem = self.getResource(offer.resources, 'mem')
            if cpus < TASK_CPU or mem < TASK_MEM:
                continue

            task = Dict()
            task_id = str(uuid.uuid4())
            task.task_id.value = task_id
            task.agent_id.value = offer.agent_id.value
            task.name = 'task {}'.format(task_id)
            task.executor = self.executor
            task.data = encode_data(self.fun + '!' \
              + repr(self.left) + '!' \
              + repr(min(self.right, self.left + self.step * self.taskN)) + '!' \
              + repr(self.step))

            task.resources = [
                dict(name='cpus', type='SCALAR', scalar={'value': TASK_CPU}),
                dict(name='mem', type='SCALAR', scalar={'value': TASK_MEM}),
            ]

            driver.launchTasks(offer.id, [task], filters)
            self.left = self.left + self.step * self.taskN
Example #15
0
    def create_task(self, offer, t, k):
        task = Dict()
        task.task_id.value = '%s-%s' % (t.id, t.tried)
        task.agent_id.value = offer.agent_id.value
        task.name = 'task %s' % t.id
        task.executor = self.executor
        env = dict(os.environ)
        task.data = encode_data(pickle.dumps([
            os.getcwd(), None, env, self.options.shell,
            self.stdout_port, self.stderr_port, self.publisher_port
        ]))

        task.resources = resources = []
        cpu = Dict()
        resources.append(cpu)
        cpu.name = 'cpus'
        cpu.type = 'SCALAR'
        cpu.scalar.value = self.cpus * k

        mem = Dict()
        resources.append(mem)
        mem.name = 'mem'
        mem.type = 'SCALAR'
        mem.scalar.value = self.mem * k

        if self.gpus > 0:
            gpu = Dict()
            resources.append(gpu)
            gpu.name = 'gpus'
            gpu.type = 'SCALAR'
            gpu.scalar.value = self.gpus * k


        return task
Example #16
0
def test_send_framework_message(mocker):
    ID = str(uuid.uuid4())
    sched = mocker.Mock()
    framework = {'id': {'value': ID}}
    master = mocker.Mock()
    driver = MesosSchedulerDriver(sched, framework, master)
    driver._send = mocker.Mock()
    executor_id = {'value': str(uuid.uuid4())}
    agent_id = {'value': str(uuid.uuid4())}
    message = ''.join(random.choice(string.printable)
                      for _ in range(random.randint(1, 100)))
    message = encode_data(message.encode('utf-8'))
    driver.sendFrameworkMessage(executor_id, agent_id, message)
    driver._send.assert_not_called()
    driver._stream_id = 'a-stream-id'
    driver.sendFrameworkMessage(executor_id, agent_id, message)
    driver._send.assert_called_once_with({
        'type': 'MESSAGE',
        'framework_id': {
            'value': ID
        },
        'message': {
            'agent_id': agent_id,
            'executor_id': executor_id,
            'data': message,
        }
    })
Example #17
0
    def manage_task_runner(self, command, assertions_fn, stop_signal=Event()):
        driver = FakeMesosExecutorDriver()
        task_id = get_random_task_id()
        task = {'task_id': {'value': task_id},
                'data': encode_data(json.dumps({'command': command}).encode('utf8'))}

        stdout_name = 'build/stdout.' + str(task_id)
        stderr_name = 'build/stderr.' + str(task_id)

        completed_signal = Event()
        max_message_length = 300
        progress_sample_interval_ms = 100
        sandbox_location = '/location/to/task/sandbox/{}'.format(task_id)
        progress_output_name = stdout_name
        progress_regex_string = '\^\^\^\^JOB-PROGRESS: (\d*)(?: )?(.*)'
        config = cc.ExecutorConfig(max_message_length=max_message_length,
                                   progress_output_name=progress_output_name,
                                   progress_regex_string=progress_regex_string,
                                   progress_sample_interval_ms=progress_sample_interval_ms,
                                   sandbox_location=sandbox_location)

        try:

            ce.manage_task(driver, task, stop_signal, completed_signal, config, stdout_name, stderr_name)

            self.assertTrue(completed_signal.isSet())
            assertions_fn(driver, task_id, sandbox_location)

        finally:
            cleanup_output(stdout_name, stderr_name)
Example #18
0
        def run_task(task):
            update = Dict()
            update.task_id.value = task.task_id.value
            update.state = 'TASK_RUNNING'
            update.timestamp = time.time()
            driver.sendStatusUpdate(update)

            # äæē•™ä»„ä½œęµ‹čƕē”Ø
            print(decode_data(task.data), file=sys.stderr)
            cnt = 0
            N = 2000000
            for i in range(N):
                x = random()
                y = random()
                if (x * x + y * y) < 1:
                    cnt += 1
            vPi = 4.0 * cnt / N
            print(vPi)
            driver.sendFrameworkMessage(encode_data(str(vPi)))

            time.sleep(30)

            update = Dict()
            update.task_id.value = task.task_id.value
            update.state = 'TASK_FINISHED'
            update.timestamp = time.time()
            driver.sendStatusUpdate(update)
Example #19
0
    def resourceOffers(self, driver, offers):
        if self.i >= self.count:
            return None
        filters = {'refuse_seconds': 5}

        for offer in offers:
            if self.i >= self.count:
                break
            cpus = self.getResource(offer.resources, 'cpus')
            mem = self.getResource(offer.resources, 'mem')
            if cpus < TASK_CPU or mem < TASK_MEM:
                continue

            task = Dict()
            task_id = str(uuid.uuid4())
            task.task_id.value = task_id
            task.agent_id.value = offer.agent_id.value
            task.name = 'task {}'.format(task_id)
            task.executor = self.executor
            # äæē•™ä»„ä½œęµ‹čƕē”Ø ļ¼šļ¼‰
            task.data = encode_data('Hello from task {}!'.format(task_id))

            task.resources = [
                dict(name='cpus', type='SCALAR', scalar={'value': TASK_CPU}),
                dict(name='mem', type='SCALAR', scalar={'value': TASK_MEM}),
            ]

            driver.launchTasks(offer.id, [task], filters)
            self.i = self.i + 1
Example #20
0
    def resourceOffers(self, driver, offers):
        filters = {'refuse_seconds': 5}

        for offer in offers:

            if self.launched_task == TASK_NUM:
                return

            cpus = self.getResource(offer.resources, 'cpus')
            mem = self.getResource(offer.resources, 'mem')
            if cpus < TASK_CPU or mem < TASK_MEM:
                continue

            task = Dict()
            task_id = str(uuid.uuid4())
            task.task_id.value = task_id
            task.agent_id.value = offer.agent_id.value
            task.name = 'task {}'.format(task_id)
            task.executor = self.executor
            # send data to executor
            task.data = encode_data(self.data_split[self.launched_task])

            task.resources = [
                dict(name='cpus', type='SCALAR', scalar={'value': TASK_CPU}),
                dict(name='mem', type='SCALAR', scalar={'value': TASK_MEM}),
            ]

            self.launched_task += 1
            driver.launchTasks(offer.id, [task], filters)
Example #21
0
    def resourceOffers(self, driver, offers):
        filters = {'refuse_seconds': 5}

        # if all tasks have been launched, return directly 
        if self.task_launched == TASK_NUM:
            return 

        # for every offer
        for offer in offers:
            # check if the offer satisfy the requirments
            cpus = self.getResource(offer.resources, 'cpus')
            mem = self.getResource(offer.resources, 'mem')
            if cpus < TASK_CPU or mem < TASK_MEM:
                continue

            # config a new task
            task = Dict()
            task_id = str(uuid.uuid4())
            task.task_id.value = task_id
            task.agent_id.value = offer.agent_id.value
            task.name = 'task {}'.format(task_id)
            task.executor = self.executor
            #task.data = encode_data(bytes('Hello from task {}!'.format(task_id), 'utf-8'))
            task.data = encode_data(bytes('ThisIsASeparator'.join(self.datas[self.task_launched]), 'utf-8'))

            task.resources = [
                dict(name='cpus', type='SCALAR', scalar={'value': TASK_CPU}),
                dict(name='mem', type='SCALAR', scalar={'value': TASK_MEM}),
            ]

            # launch task
            driver.launchTasks(offer.id, [task], filters)

            self.task_launched += 1
Example #22
0
        def run_task(task):
            update = Dict()
            update.task_id.value = task.task_id.value
            update.state = 'TASK_RUNNING'
            update.timestamp = time.time()
            driver.sendStatusUpdate(update)

            tmp = decode_data(task.data).split('!')
            fun = tmp[0]
            left = float(tmp[1])
            right = float(tmp[2])
            step = float(tmp[3])
            res_tot = 0
            x = left
            while x < right - 1e-16:
                exec(fun) in globals(), locals()
                res_tot = res_tot + res
                x = x + step
            driver.sendFrameworkMessage(encode_data(repr(step * res_tot)))

            update = Dict()
            update.task_id.value = task.task_id.value
            update.state = 'TASK_FINISHED'
            update.timestamp = time.time()
            driver.sendStatusUpdate(update)
Example #23
0
def test_send_message(mocker):
    agent_addr = 'mock_addr:12345'
    framework_id = str(uuid.uuid4())
    executor_id = str(uuid.uuid4())
    env = {
        'MESOS_LOCAL': 'true',
        'MESOS_AGENT_ENDPOINT': agent_addr,
        'MESOS_FRAMEWORK_ID': framework_id,
        'MESOS_EXECUTOR_ID': executor_id,
    }
    mocker.patch('os.environ', env)
    exc = mocker.Mock()
    driver = MesosExecutorDriver(exc)
    driver._send = mocker.Mock()
    message = ''.join(
        random.choice(string.printable) for _ in range(random.randint(1, 100)))
    message = encode_data(message.encode('utf8'))
    driver.sendFrameworkMessage(message)
    driver._send.assert_called_once_with({
        'type': 'MESSAGE',
        'framework_id': {
            'value': framework_id,
        },
        'executor_id': {
            'value': executor_id,
        },
        'message': {
            'data': message,
        }
    })
Example #24
0
def reply_status(driver, task_id, state, data=None):
    status = Dict()
    status.task_id = task_id
    status.state = state
    status.timestamp = time.time()
    if data is not None:
        status.data = encode_data(data)
    driver.sendStatusUpdate(status)
Example #25
0
def reply_status(driver, task_id, state, data=None):
    status = Dict()
    status.task_id = task_id
    status.state = state
    status.timestamp = time.time()
    if data is not None:
        status.data = encode_data(data)
    driver.sendStatusUpdate(status)
Example #26
0
    def test_launch_task_no_command(self):
        task_id = get_random_task_id()
        task = {'task_id': {'value': task_id},
                'data': encode_data(json.dumps({'command': ''}).encode('utf8'))}
        stdout_name = ''
        stderr_name = ''

        result = ce.launch_task(task, stdout_name, stderr_name)

        self.assertIsNone(result)
Example #27
0
def reply_status(driver, task_id, state, reason=None, msg=None, data=None):
    status = Dict()
    status.task_id = task_id
    status.state = state
    if reason is not None:
        status.message = '{}:{}'.format(reason, msg)
    status.timestamp = time.time()
    if data is not None:
        status.data = encode_data(data)
    driver.sendStatusUpdate(status)
Example #28
0
def reply_status(driver, task_id, state, reason=None, msg=None, data=None):
    status = Dict()
    status.task_id = task_id
    status.state = state
    if reason is not None:
        status.message = '{}:{}'.format(reason, msg)
    status.timestamp = time.time()
    if data is not None:
        status.data = encode_data(data)
    driver.sendStatusUpdate(status)
Example #29
0
    def manage_task_runner(self,
                           command,
                           assertions_fn,
                           stop_signal=None,
                           task_id=None,
                           config=None,
                           driver=None):

        if driver is None:
            driver = tu.FakeMesosExecutorDriver()
        if stop_signal is None:
            stop_signal = Event()
        if task_id is None:
            task_id = tu.get_random_task_id()

        task = {
            'task_id': {
                'value': task_id
            },
            'data':
            pm.encode_data(json.dumps({
                'command': command
            }).encode('utf8'))
        }

        stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
        stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))

        tu.redirect_stdout_to_file(stdout_name)
        tu.redirect_stderr_to_file(stderr_name)

        completed_signal = Event()
        if config is None:
            sandbox_directory = '/location/to/task/sandbox/{}'.format(task_id)
            config = cc.ExecutorConfig(
                max_message_length=300,
                progress_output_name=stdout_name,
                progress_regex_string=
                '\^\^\^\^JOB-PROGRESS:\s+([0-9]*\.?[0-9]+)($|\s+.*)',
                progress_sample_interval_ms=100,
                sandbox_directory=sandbox_directory)
        else:
            sandbox_directory = config.sandbox_directory

        try:

            ce.manage_task(driver, task, stop_signal, completed_signal, config)

            self.assertTrue(completed_signal.isSet())
            assertions_fn(driver, task_id, sandbox_directory)

        finally:
            tu.cleanup_output(stdout_name, stderr_name)
Example #30
0
    def test_launch_task_no_command(self):
        task_id = tu.get_random_task_id()
        task = {
            'task_id': {
                'value': task_id
            },
            'data': pm.encode_data(json.dumps({
                'command': ''
            }).encode('utf8'))
        }

        process = ce.launch_task(task, os.environ)

        self.assertIsNone(process)
Example #31
0
    def test_launch_task_interactive_output(self):
        task_id = tu.get_random_task_id()
        command = 'echo "Start"; echo "Hello"; sleep 100; echo "World"; echo "Done"; '
        task = {
            'task_id': {
                'value': task_id
            },
            'data':
            pm.encode_data(json.dumps({
                'command': command
            }).encode('utf8'))
        }

        stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
        stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))

        tu.redirect_stdout_to_file(stdout_name)
        tu.redirect_stderr_to_file(stderr_name)

        try:
            process = ce.launch_task(task, os.environ)

            self.assertIsNotNone(process)

            # let the process run for up to 50 seconds
            for _ in range(5000):
                if cs.is_process_running(process):
                    time.sleep(0.01)
                    with open(stdout_name) as f:
                        stdout_content = f.read()
                        if 'Start' in stdout_content and 'Hello' in stdout_content:
                            break

            try:
                with open(stdout_name) as f:
                    stdout_content = f.read()
                    logging.info(
                        'Contents of stdout: {}'.format(stdout_content))
                    self.assertTrue("Start" in stdout_content)
                    self.assertTrue("Hello" in stdout_content)
                    self.assertFalse("World" in stdout_content)
                    self.assertFalse("Done" in stdout_content)
            finally:
                if process.poll() is None:
                    logging.info('Killing launched process')
                    process.kill()

        finally:
            tu.cleanup_output(stdout_name, stderr_name)
Example #32
0
    def test_launch_task(self):
        task_id = get_random_task_id()
        command = 'echo "Hello World"; echo "Error Message" >&2'
        task = {
            'task_id': {
                'value': task_id
            },
            'data':
            encode_data(json.dumps({
                'command': command
            }).encode('utf8'))
        }
        stdout_name = ensure_directory('build/stdout.' + str(task_id))
        stderr_name = ensure_directory('build/stderr.' + str(task_id))

        if not os.path.isdir("build"):
            os.mkdir("build")

        try:
            process, stdout, stderr = ce.launch_task(task, stdout_name,
                                                     stderr_name)

            self.assertIsNotNone(process)
            for i in range(100):
                if process.poll() is None:
                    time.sleep(0.01)

            stdout.close()
            stderr.close()

            if process.poll() is None:
                process.kill()

            self.assertEqual(0, process.poll())

            with open(stdout_name) as f:
                stdout_content = f.read()
                self.assertEqual("Hello World\n", stdout_content)

            with open(stderr_name) as f:
                stderr_content = f.read()
                self.assertEqual("Error Message\n", stderr_content)

        finally:
            cleanup_output(stdout_name, stderr_name)
Example #33
0
    def _newMesosTask(self, job, offer):
        """
        Build the Mesos task object for a given the Toil job and Mesos offer
        """
        task = addict.Dict()
        task.task_id.value = str(job.jobID)
        task.agent_id.value = offer.agent_id.value
        task.name = job.name
        task.data = encode_data(pickle.dumps(job))
        task.executor = addict.Dict(self.executor)

        task.resources = []

        task.resources.append(addict.Dict())
        cpus = task.resources[-1]
        cpus.name = 'cpus'
        cpus.type = 'SCALAR'
        cpus.scalar.value = job.resources.cores

        task.resources.append(addict.Dict())
        disk = task.resources[-1]
        disk.name = 'disk'
        disk.type = 'SCALAR'
        if toMiB(job.resources.disk) > 1:
            disk.scalar.value = toMiB(job.resources.disk)
        else:
            log.warning(
                "Job %s uses less disk than Mesos requires. Rounding %s up to 1 MiB.",
                job.jobID, job.resources.disk)
            disk.scalar.value = 1

        task.resources.append(addict.Dict())
        mem = task.resources[-1]
        mem.name = 'mem'
        mem.type = 'SCALAR'
        if toMiB(job.resources.memory) > 1:
            mem.scalar.value = toMiB(job.resources.memory)
        else:
            log.warning(
                "Job %s uses less memory than Mesos requires. Rounding %s up to 1 MiB.",
                job.jobID, job.resources.memory)
            mem.scalar.value = 1
        return task
Example #34
0
    def test_launch_task(self):
        task_id = tu.get_random_task_id()
        command = 'echo "Hello World"; echo "Error Message" >&2'
        task = {
            'task_id': {
                'value': task_id
            },
            'data':
            pm.encode_data(json.dumps({
                'command': command
            }).encode('utf8'))
        }

        stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
        stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))

        tu.redirect_stdout_to_file(stdout_name)
        tu.redirect_stderr_to_file(stderr_name)

        try:
            process = ce.launch_task(task, os.environ)

            self.assertIsNotNone(process)

            for _ in range(100):
                if cs.is_process_running(process):
                    time.sleep(0.01)

            if process.poll() is None:
                process.kill()
            tu.close_sys_outputs()

            self.assertEqual(0, process.poll())

            with open(stdout_name) as f:
                stdout_content = f.read()
                self.assertTrue("Hello World\n" in stdout_content)

            with open(stderr_name) as f:
                stderr_content = f.read()
                self.assertTrue("Error Message\n" in stderr_content)
        finally:
            tu.cleanup_output(stdout_name, stderr_name)
Example #35
0
        def run_task(task):
            update = Dict()
            update.task_id.value = task.task_id.value
            update.state = 'TASK_RUNNING'
            update.timestamp = time.time()
            driver.sendStatusUpdate(update)

            tmp = decode_data(task.data).split(' ')
            left = int(tmp[0])
            right = int(tmp[1])
            res = 0
            for i in xrange(left, right):
                res = res + i
            driver.sendFrameworkMessage(encode_data(str(res)))

            update = Dict()
            update.task_id.value = task.task_id.value
            update.state = 'TASK_FINISHED'
            update.timestamp = time.time()
            driver.sendStatusUpdate(update)
Example #36
0
 def _sendFrameworkMessage(self, driver):
     message = None
     while True:
         # The psutil documentation recommends that we ignore the value returned by the first
         # invocation of cpu_percent(). However, we do want to send a sign of life early after
         # starting (e.g. to unblock the provisioner waiting for an instance to come up) so
         # we call it once and discard the value.
         if message is None:
             message = Expando(address=self.address)
             psutil.cpu_percent()
         else:
             message.nodeInfo = dict(coresUsed=float(psutil.cpu_percent()) * .01,
                                     memoryUsed=float(psutil.virtual_memory().percent) * .01,
                                     coresTotal=psutil.cpu_count(),
                                     memoryTotal=psutil.virtual_memory().total,
                                     workers=len(self.runningTasks))
         log.debug("Send framework message: %s", message)
         driver.sendFrameworkMessage(encode_data(repr(message)))
         # Prevent workers launched together from repeatedly hitting the leader at the same time
         time.sleep(random.randint(45, 75))
Example #37
0
        def run_task(task):
            update = Dict()
            update.task_id.value = task.task_id.value
            update.state = 'TASK_RUNNING'
            update.timestamp = time.time()
            driver.sendStatusUpdate(update)

            data = decode_data(task.data).split(' ')
            result = 0
            for x in data:
                if x != '':
                    result += int(x)

            # send the result to the scheduler
            driver.sendFrameworkMessage(encode_data(str(result)))

            update = Dict()
            update.task_id.value = task.task_id.value
            update.state = 'TASK_FINISHED'
            update.timestamp = time.time()
            driver.sendStatusUpdate(update)
Example #38
0
    def resourceOffers(self, driver, offers):
        filters = {'refuse_seconds': 5}

        for offer in offers:
            cpus = self.getResource(offer.resources, 'cpus')
            mem = self.getResource(offer.resources, 'mem')
            if cpus < TASK_CPU or mem < TASK_MEM:
                continue

            task = Dict()
            task_id = str(uuid.uuid4())
            task.task_id.value = task_id
            task.agent_id.value = offer.agent_id.value
            task.name = 'task {}'.format(task_id)
            task.executor = self.executor
            task.data = encode_data('Hello from task {}!'.format(task_id))

            task.resources = [
                dict(name='cpus', type='SCALAR', scalar={'value': TASK_CPU}),
                dict(name='mem', type='SCALAR', scalar={'value': TASK_MEM}),
            ]

            driver.launchTasks(offer.id, [task], filters)
Example #39
0
def test_on_message(mocker):
    ID = str(uuid.uuid4())
    sched = mocker.Mock()
    framework = {'id': {'value': ID}}
    master = mocker.Mock()
    driver = MesosSchedulerDriver(sched, framework, master)
    driver._started = True
    executor_id = {'value': str(uuid.uuid4())}
    agent_id = {'value': str(uuid.uuid4())}
    message = ''.join(random.choice(string.printable)
                      for _ in range(random.randint(1, 100)))
    data = encode_data(message.encode('utf8'))

    event = {
        'type': 'MESSAGE',
        'message': {
            'executor_id': executor_id,
            'agent_id': agent_id,
            'data': data
        }
    }
    driver.on_event(event)
    sched.frameworkMessage.assert_called_once_with(driver, executor_id,
                                                   agent_id, data)
Example #40
0
    def getExecutorInfo(self, framework_id):
        info = Dict()
        info.framework_id.value = framework_id

        if self.use_self_as_exec:
            info.command.value = os.path.abspath(sys.argv[0])
            info.executor_id.value = sys.argv[0]
        else:
            info.command.value = '%s %s' % (
                sys.executable,
                os.path.abspath(
                    os.path.join(
                        os.path.dirname(__file__),
                        'executor.py'))
            )
            info.executor_id.value = 'default'

        info.command.environment.variables = variables = []

        v = Dict()
        variables.append(v)
        v.name = 'UID'
        v.value = str(os.getuid())

        v = Dict()
        variables.append(v)
        v.name = 'GID'
        v.value = str(os.getgid())

        if self.options.image:
            info.container.type = 'DOCKER'
            info.container.docker.image = self.options.image

            info.container.volumes = volumes = []
            for path in ['/etc/passwd', '/etc/group']:
                v = Dict()
                volumes.append(v)
                v.host_path = v.container_path = path
                v.mode = 'RO'

            for path in conf.MOOSEFS_MOUNT_POINTS:
                v = Dict()
                volumes.append(v)
                v.host_path = v.container_path = path
                v.mode = 'RW'

            for path in conf.DPARK_WORK_DIR.split(','):
                v = Dict()
                volumes.append(v)
                v.host_path = v.container_path = path
                v.mode = 'RW'

            if self.options.volumes:
                for volume in self.options.volumes.split(','):
                    fields = volume.split(':')
                    if len(fields) == 3:
                        host_path, container_path, mode = fields
                        mode = mode.upper()
                        assert mode in ('RO', 'RW')
                    elif len(fields) == 2:
                        host_path, container_path = fields
                        mode = 'RW'
                    elif len(fields) == 1:
                        container_path, = fields
                        host_path = ''
                        mode = 'RW'
                    else:
                        raise Exception('cannot parse volume %s', volume)

                    mkdir_p(host_path)

                v = Dict()
                volumes.append(v)
                v.container_path = container_path
                v.mode = mode
                if host_path:
                    v.host_path = host_path

        info.resources = resources = []

        mem = Dict()
        resources.append(mem)
        mem.name = 'mem'
        mem.type = 'SCALAR'
        mem.scalar.value = EXECUTOR_MEMORY

        cpus = Dict()
        resources.append(cpus)
        cpus.name = 'cpus'
        cpus.type = 'SCALAR'
        cpus.scalar.value = EXECUTOR_CPUS

        Script = os.path.realpath(sys.argv[0])
        info.name = Script

        info.data = encode_data(marshal.dumps(
            (
                Script, os.getcwd(), sys.path, dict(os.environ),
                self.task_per_node, self.out_logger, self.err_logger,
                self.logLevel, env.environ
            )
        ))
        return info