def test_python_task_encode():
fn, args, kwargs = sum, [range(5)], {}
data = (fn, args, kwargs)
dumped = cloudpickle.dumps(data)
task = PythonTask(fn=fn,
args=args,
kwargs=kwargs,
id='test-id',
envs={'TEST': 'value'},
uris=['test_dependency'])
proto = encode(task)
assert isinstance(proto, mesos_pb2.TaskInfo)
assert proto.data == dumped
assert proto.task_id.value == 'test-id'
assert proto.executor.command.uris[0].value == 'test_dependency'
assert proto.executor.command.environment.variables[0].name == 'TEST'
assert proto.executor.command.environment.variables[0].value == 'value'
task = PythonTask(id=TaskID(value='test-id'))
task.data = data
proto = encode(task)
assert isinstance(proto, mesos_pb2.TaskInfo)
assert proto.data == dumped
assert proto.task_id.value == 'test-id'
def test_python_task_execution():
fn, args, kwargs = sum, [range(5)], {}
task = PythonTask(fn=fn, args=args, kwargs=kwargs,
id='test-id')
task = decode(encode(task))
assert task() == 10
def fn(lst1, lst2):
return sum(lst1) - sum(lst2)
args = [range(5), range(3)]
task = PythonTask(fn=fn, args=args, id='test-id')
task = decode(encode(task))
assert task() == 7
def test_python_task_execution():
fn, args, kwargs = sum, [range(5)], {}
task = PythonTask(fn=fn, args=args, kwargs=kwargs, id='test-id')
task = decode(encode(task))
assert task() == 10
def fn(lst1, lst2):
return sum(lst1) - sum(lst2)
args = [range(5), range(3)]
task = PythonTask(fn=fn, args=args, id='test-id')
task = decode(encode(task))
assert task() == 7
def test_encode_resources():
pb = encode(Cpus(0.1))
assert pb.scalar.value == 0.1
assert pb.name == 'cpus'
assert pb.type == mesos_pb2.Value.SCALAR
pb = encode(Mem(16))
assert pb.scalar.value == 16
assert pb.name == 'mem'
assert pb.type == mesos_pb2.Value.SCALAR
pb = encode(Disk(256))
assert pb.scalar.value == 256
assert pb.name == 'disk'
assert pb.type == mesos_pb2.Value.SCALAR
def test_encode_resources():
pb = encode(Cpus(0.1))
assert pb.scalar.value == 0.1
assert pb.name == 'cpus'
assert pb.type == mesos_pb2.Value.SCALAR
pb = encode(Mem(16))
assert pb.scalar.value == 16
assert pb.name == 'mem'
assert pb.type == mesos_pb2.Value.SCALAR
pb = encode(Disk(256))
assert pb.scalar.value == 256
assert pb.name == 'disk'
assert pb.type == mesos_pb2.Value.SCALAR
def test_python_task_encode():
fn, args, kwargs = sum, [range(5)], {}
data = (fn, args, kwargs)
dumped = cloudpickle.dumps(data)
task = PythonTask(fn=fn, args=args, kwargs=kwargs, id={'value': 'test-id'})
proto = encode(task)
assert isinstance(proto, mesos_pb2.TaskInfo)
assert proto.data == dumped
assert proto.task_id.value == 'test-id'
task = PythonTask(id=TaskID(value='test-id'))
task.data = data
proto = encode(task)
assert isinstance(proto, mesos_pb2.TaskInfo)
assert proto.data == dumped
assert proto.task_id.value == 'test-id'
def test_python_task_status_encode():
data = {'arbitrary': 'data', 'value': 5}
dumped = cloudpickle.dumps(data)
status = PythonTaskStatus(task_id='test-id', state='TASK_STAGING',
data=data)
proto = encode(status)
assert isinstance(proto, mesos_pb2.TaskStatus)
assert proto.data == dumped
assert proto.task_id.value == 'test-id'
assert proto.state == mesos_pb2.TASK_STAGING
status = PythonTaskStatus(task_id='test-id', state='TASK_RUNNING')
status.data = data
proto = encode(status)
assert isinstance(proto, mesos_pb2.TaskStatus)
assert proto.data == dumped
assert proto.task_id.value == 'test-id'
assert proto.state == mesos_pb2.TASK_RUNNING
def test_python_task_encode():
fn, args, kwargs = sum, [range(5)], {}
data = (fn, args, kwargs)
dumped = cloudpickle.dumps(data)
task = PythonTask(fn=fn, args=args, kwargs=kwargs,
id={'value': 'test-id'})
proto = encode(task)
assert isinstance(proto, mesos_pb2.TaskInfo)
assert proto.data == dumped
assert proto.task_id.value == 'test-id'
task = PythonTask(id=TaskID(value='test-id'))
task.data = data
proto = encode(task)
assert isinstance(proto, mesos_pb2.TaskInfo)
assert proto.data == dumped
assert proto.task_id.value == 'test-id'
def test_python_task_status_encode():
data = {'arbitrary': 'data', 'value': 5}
dumped = cloudpickle.dumps(data)
status = PythonTaskStatus(task_id='test-id',
state='TASK_STAGING',
data=data)
proto = encode(status)
assert isinstance(proto, mesos_pb2.TaskStatus)
assert proto.data == dumped
assert proto.task_id.value == 'test-id'
assert proto.state == mesos_pb2.TASK_STAGING
status = PythonTaskStatus(task_id='test-id', state='TASK_RUNNING')
status.data = data
proto = encode(status)
assert isinstance(proto, mesos_pb2.TaskStatus)
assert proto.data == dumped
assert proto.task_id.value == 'test-id'
assert proto.state == mesos_pb2.TASK_RUNNING
def test_non_strict_encode_task_info():
t = TaskInfo(name='test-task',
id=TaskID(value='test-task-id'),
resources=[Cpus(0.1), Mem(16)],
command=CommandInfo(value='echo 100'))
t.result = 'some binary data'
t.status = TaskStatus()
p = encode(t)
assert isinstance(p, mesos_pb2.TaskInfo)
assert p.command.value == 'echo 100'
with pytest.raises(AttributeError):
p.status
def test_status_in_task_info():
t = TaskInfo(name='test-task',
id=TaskID(value='test-task-id'),
resources=[Cpus(0.1), Mem(16)],
command=CommandInfo(value='echo 100'))
assert isinstance(t.status, TaskStatus)
assert t.status.state == 'TASK_STAGING'
p = encode(t)
assert isinstance(p, mesos_pb2.TaskInfo)
with pytest.raises(AttributeError):
p.status
def test_encode_task_info_resources():
task = TaskInfo(name='test-task',
id=TaskID(value='test-task-id'),
resources=[Cpus(0.1), Mem(16)],
command=CommandInfo(value='testcmd'))
pb = encode(task)
assert pb.name == 'test-task'
assert pb.task_id.value == 'test-task-id'
assert pb.resources[0].name == 'cpus'
assert pb.resources[0].scalar.value == 0.1
assert pb.resources[1].name == 'mem'
assert pb.resources[1].scalar.value == 16
assert pb.command.value == 'testcmd'
def test_non_strict_encode_task_info():
t = TaskInfo(name='test-task',
id=TaskID(value='test-task-id'),
resources=[Cpus(0.1), Mem(16)],
command=CommandInfo(value='echo 100'))
t.result = 'some binary data'
t.status = TaskStatus()
p = encode(t)
assert isinstance(p, mesos_pb2.TaskInfo)
assert p.command.value == 'echo 100'
with pytest.raises(AttributeError):
p.status
def test_encode_task_info():
t = TaskInfo(name='test-task',
id=TaskID(value='test-task-id'),
resources=[Cpus(0.1), Mem(16)],
command=CommandInfo(value='echo 100'))
p = encode(t)
assert isinstance(p, mesos_pb2.TaskInfo)
assert p.command.value == 'echo 100'
assert p.name == 'test-task'
assert p.resources[0].name == 'cpus'
assert p.resources[0].scalar.value == 0.1
assert p.task_id.value == 'test-task-id'
def test_encode_task_info_resources():
task = TaskInfo(name='test-task',
id=TaskID(value='test-task-id'),
resources=[Cpus(0.1), Mem(16)],
command=CommandInfo(value='testcmd'))
pb = encode(task)
assert pb.name == 'test-task'
assert pb.task_id.value == 'test-task-id'
assert pb.resources[0].name == 'cpus'
assert pb.resources[0].scalar.value == 0.1
assert pb.resources[1].name == 'mem'
assert pb.resources[1].scalar.value == 16
assert pb.command.value == 'testcmd'
def test_encode_task_info():
t = TaskInfo(name='test-task',
id=TaskID(value='test-task-id'),
resources=[Cpus(0.1), Mem(16)],
command=CommandInfo(value='echo 100'))
p = encode(t)
assert isinstance(p, mesos_pb2.TaskInfo)
assert p.command.value == 'echo 100'
assert p.name == 'test-task'
assert p.resources[0].name == 'cpus'
assert p.resources[0].scalar.value == 0.1
assert p.task_id.value == 'test-task-id'
def test_python_task_encode():
fn, args, kwargs = sum, [range(5)], {}
data = (fn, args, kwargs)
dumped = cloudpickle.dumps(data)
task = PythonTask(fn=fn, args=args, kwargs=kwargs,
id='test-id',
envs={'TEST': 'value'},
uris=['test_dependency'])
proto = encode(task)
assert isinstance(proto, mesos_pb2.TaskInfo)
assert proto.data == dumped
assert proto.task_id.value == 'test-id'
assert proto.executor.command.uris[0].value == 'test_dependency'
assert proto.executor.command.environment.variables[0].name == 'TEST'
assert proto.executor.command.environment.variables[0].value == 'value'
task = PythonTask(id=TaskID(value='test-id'))
task.data = data
proto = encode(task)
assert isinstance(proto, mesos_pb2.TaskInfo)
assert proto.data == dumped
assert proto.task_id.value == 'test-id'