def test_validate_dependency_result(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code3 = Code(add3)
code1 >> code3
code2 >> code3
code3.receive('b') << code1
code3.receive('c') << code2
code1(1, 2)
code2(3, 4)
code2.service['delivery'].cancel(code2.serial_number)
ret = code3(a=3)
assert ret is None
assert code3.get_state() == 'ERROR'
code2(3, 4)
code2.send_result(item=1, timestamp=datetime.now().timestamp() + 1)
ret = code3(a=3)
assert ret is 7
assert code3.get_state() == 'TERMINATED'
code2(3, 4)
snapshots = code3.dependency.load_snapshot()
assert len(snapshots) == 2
assert code3.dependency.check_delivery() is True
snapshot_dict = {x['_id']: x for x in snapshots}
code2_state_info = snapshot_dict.pop(code2.serial_number)
assert code3.dependency.validate(snapshot=snapshot_dict.values()) == 'WAITING'
snapshot_dict[code2.serial_number] = code2_state_info
assert code3.dependency.validate(snapshot=snapshot_dict.values()) == 'READY'
code2.service['delivery'].cancel(code2.serial_number)
assert code3.dependency.validate(snapshot=snapshot_dict.values()) == 'ERROR'
code2.service['delivery'].send(id='dummy', serial_number=code2.serial_number, item=123)
assert code3.dependency.validate(snapshot=snapshot_dict.values()) == 'READY'
def test_memory_worker_run_snapshot_with_nothing():
worker = Default.get_employee('worker')
assert isinstance(worker.messenger, MemoryMessenger)
code = Code(add2)
assert code.get_state() == 'UNKNOWN'
sn = worker.run_snapshot(code.to_snapshot(kwargs={'a': 3, 'b': 5}))
assert sn == code.serial_number
assert code.get_state() == 'TERMINATED'
assert code.get_result() == 8
worker.stop()
def test_dependency_error_propagation(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code3 = Code(add3)
code1 >> code3
code2 >> code3
assert code3.get_state() == 'UNKNOWN'
with pytest.raises(TypeError):
code1()
assert code1.get_state() == 'ERROR'
ret = code3(1, 2, 3)
assert ret is None
assert code3.get_state() == 'WAITING'
def test_memory_supervisor_organize():
supervisor = Default.get_employee('supervisor')
assert isinstance(supervisor.messenger, MemoryMessenger)
assert supervisor.messenger.topic == 'codepack'
assert supervisor.messenger.queues['codepack'].empty()
code1 = Code(add2)
code2 = Code(mul2)
code1 >> code2
code2(3, b=5)
assert code1.get_state() == 'UNKNOWN'
assert code2.get_state() == 'WAITING'
supervisor.organize()
assert code1.get_state() == 'UNKNOWN'
assert code2.get_state() == 'WAITING'
assert supervisor.messenger.queues['codepack'].empty()
code1(a=2, b=4)
assert code1.get_state() == 'TERMINATED'
assert code2.get_state() == 'WAITING'
assert supervisor.messenger.queues['codepack'].empty()
supervisor.organize()
assert code1.get_state() == 'TERMINATED'
assert code2.get_state() == 'READY'
assert not supervisor.messenger.queues['codepack'].empty(
) and supervisor.messenger.queues['codepack'].qsize() == 1
item = supervisor.messenger.queues['codepack'].get(block=False)
item.pop('source', None)
expected_item = code2.to_snapshot(args=(3, ),
kwargs={
'b': 5
},
timestamp=item['timestamp']).to_dict()
expected_item.pop('source', None)
assert item == expected_item
supervisor.close()
def test_check_dependency_state(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code3 = Code(add3)
code1 >> code3
code2 >> code3
assert len(code3.dependency) == 2
ret = code3()
assert not ret and code3.get_state() == 'WAITING'
a = code1(a=3, b=5)
assert a == 8 and code1.get_state() == 'TERMINATED'
ret = code3()
assert not ret and code3.get_state() == 'WAITING'
b = code2(a=2, b=3)
assert b == 6 and code2.get_state() == 'TERMINATED'
try:
ret = code3()
except TypeError:
pass
assert not ret and code3.get_state() == 'ERROR'
ret = code3(a=1, b=5)
assert ret == 8 and code3.get_state() == 'TERMINATED'
code3.receive('a') << code1
code3.receive('b') << code2
ret = code3()
assert ret == 16 and code3.get_state() == 'TERMINATED'
ret = code3(a=1, b=5)
assert ret == 8 and code3.get_state() == 'TERMINATED'
def test_memory_worker_run_snapshot_with_image_and_callback(
mock_docker_client):
worker = Default.get_employee('worker')
worker.callback = dummy_callback_function
assert isinstance(worker.messenger, MemoryMessenger)
code = Code(add2, image='dummy')
assert code.get_state() == 'UNKNOWN'
sn = worker.run_snapshot(code.to_snapshot(kwargs={'a': 3, 'b': 5}))
assert sn == code.serial_number
default_config_dir = Config.get_default_config_dir()
script_dir = os.path.join(default_config_dir, 'scripts')
mock_docker_client.return_value.containers.run.assert_called_once_with(
auto_remove=True,
command=[
'python', 'run_snapshot.py',
'%s.json' % sn, '-p', '.', '-l', 'worker-logger', '-c',
'dummy_callback_function'
],
dns=['8.8.8.8'],
environment=['CODEPACK_LOGGER_LOG_DIR=/usr/logs'],
image='dummy',
name=id(worker.docker_manager),
volumes=[
'%s:/usr/src/codepack' % script_dir,
'%s:/usr/logs' % os.path.abspath(Config.get_log_dir())
],
working_dir='/usr/src/codepack')
worker.stop()
def test_default_arg(default_os_env):
code1 = Code(add2)
code2 = Code(add3)
ret = code2(a=1, b=3)
assert ret == 6 and code2.get_state() == 'TERMINATED'
code1 >> code2
code2.receive('c') << code1
ret = code2(a=1, b=3)
assert not ret and code2.get_state() == 'WAITING'
c = code1(a=3, b=5)
assert c == 8 and code1.get_state() == 'TERMINATED'
ret = code2(a=1, b=3)
assert ret == 12 and code2.get_state() == 'TERMINATED'
code1 // code2
ret = code2(a=1, b=3)
assert ret == 6 and code2.get_state() == 'TERMINATED'
def test_dependency_monitor_in_background(default_os_env):
dm = DependencyMonitor(callback=load_and_run_code_from_snapshot,
interval=0.5,
background=True)
c1 = Code(add3)
c2 = Code(mul2)
c3 = Code(combination)
c4 = Code(linear)
c5 = Code(print_x)
c1 >> c3
c2 >> c3
c3 >> [c4, c5]
c3.receive('c') << c1
c3.receive('d') << c2
c4.receive('c') << c3
c5.receive('x') << c3
c1.save()
c2.save()
c3.save()
c4.save()
c5.save()
c3(2, b=3)
c5()
c4(3, 6)
c2(3, 5)
c1(1, 2, 3)
assert c1.get_state() == 'TERMINATED'
assert c2.get_state() == 'TERMINATED'
assert c3.get_state() == 'WAITING'
assert c4.get_state() == 'WAITING'
assert c5.get_state() == 'WAITING'
dm.start()
time.sleep(1)
assert c1.get_state() == 'TERMINATED'
assert c2.get_state() == 'TERMINATED'
assert c3.get_state() == 'TERMINATED'
assert c4.get_state() == 'TERMINATED'
assert c5.get_state() == 'TERMINATED'
assert c4.get_result() == 114
dm.stop()
def test_memory_worker_run_snapshot_with_env(mock_subprocess_run):
worker = Default.get_employee('worker')
assert isinstance(worker.messenger, MemoryMessenger)
code = Code(add2, env='test_env', image='dummy')
assert code.get_state() == 'UNKNOWN'
sn = worker.run_snapshot(code.to_snapshot(kwargs={'a': 3, 'b': 5}))
assert sn == code.serial_number
default_config_dir = Config.get_default_config_dir()
script_dir = os.path.join(default_config_dir, 'scripts')
mock_subprocess_run.assert_called_once_with([
os.path.join(worker.interpreter_manager.path, 'test_env', 'bin',
'python'),
os.path.join(script_dir, 'run_snapshot.py'),
os.path.join(script_dir, '%s.json' % sn), '-p', script_dir, '-l',
'worker-logger'
])
worker.stop()
def test_code_waiting(default_os_env):
code1 = Code(add2)
code2 = Code(add3)
code1 >> code2
code2.receive('b') << code1
code2(3, c=2)
assert code2.get_state() == 'WAITING'
snapshot = code2.service['snapshot'].load(
serial_number=code2.serial_number)
assert 'args' in snapshot
assert isinstance(snapshot['args'], list)
assert len(snapshot['args']) == 1
assert snapshot['args'][0] == 3
assert 'kwargs' in snapshot
assert isinstance(snapshot['kwargs'], dict)
assert len(snapshot['kwargs']) == 1
assert 'c' in snapshot['kwargs'].keys()
assert snapshot['kwargs']['c'] == 2
def test_get_function_from_source(default_os_env):
source = """def plus1(x):\n return x + 1"""
code = Code(source=source)
ret = code(x=5)
assert ret == 6
assert code.get_state() == 'TERMINATED'
def test_code_new(default_os_env):
code1 = Code(add2)
assert code1.get_state() == 'UNKNOWN'
code2 = Code(add3, state='NEW')
assert code2.get_state() == 'NEW'
def test_dependency_monitor(default_os_env):
dm = DependencyMonitor(
callback=load_and_run_code_from_snapshot_and_raise_exception,
interval=0.5,
background=False)
c1 = Code(add3)
c2 = Code(mul2)
c3 = Code(combination)
c4 = Code(linear)
c1 >> c3
c2 >> c3
c3 >> c4
c3.receive('c') << c1
c3.receive('d') << c2
c4.receive('c') << c3
c1.save()
c2.save()
c3.save()
c4.save()
c3(2, b=3)
c4(3, 6)
c2(3, 5)
c1(1, 2, 3)
assert c1.get_state() == 'TERMINATED'
assert c2.get_state() == 'TERMINATED'
assert c3.get_state() == 'WAITING'
assert c4.get_state() == 'WAITING'
dm.start()
assert c1.get_state() == 'TERMINATED'
assert c2.get_state() == 'TERMINATED'
assert c3.get_state() == 'TERMINATED'
assert c4.get_state() == 'WAITING'
dm.start()
assert c1.get_state() == 'TERMINATED'
assert c2.get_state() == 'TERMINATED'
assert c3.get_state() == 'TERMINATED'
assert c4.get_state() == 'TERMINATED'
assert c4.get_result() == 114
dm.stop()
