def test_validate_dependency_result(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code3 = Code(add3)
code1 >> code3
code2 >> code3
code3.receive('b') << code1
code3.receive('c') << code2
code1(1, 2)
code2(3, 4)
code2.service['delivery'].cancel(code2.serial_number)
ret = code3(a=3)
assert ret is None
assert code3.get_state() == 'ERROR'
code2(3, 4)
code2.send_result(item=1, timestamp=datetime.now().timestamp() + 1)
ret = code3(a=3)
assert ret is 7
assert code3.get_state() == 'TERMINATED'
code2(3, 4)
snapshots = code3.dependency.load_snapshot()
assert len(snapshots) == 2
assert code3.dependency.check_delivery() is True
snapshot_dict = {x['_id']: x for x in snapshots}
code2_state_info = snapshot_dict.pop(code2.serial_number)
assert code3.dependency.validate(snapshot=snapshot_dict.values()) == 'WAITING'
snapshot_dict[code2.serial_number] = code2_state_info
assert code3.dependency.validate(snapshot=snapshot_dict.values()) == 'READY'
code2.service['delivery'].cancel(code2.serial_number)
assert code3.dependency.validate(snapshot=snapshot_dict.values()) == 'ERROR'
code2.service['delivery'].send(id='dummy', serial_number=code2.serial_number, item=123)
assert code3.dependency.validate(snapshot=snapshot_dict.values()) == 'READY'
def test_argpack_codepack_execution(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code1 >> code2
code2.receive('b') << code1
codepack = CodePack(id='argpack_test', code=code1, subscribe=code2)
argpack = codepack.make_argpack()
snapshot = codepack.to_snapshot(argpack=argpack)
assert isinstance(snapshot.argpack, dict)
assert snapshot.argpack['add2']['a'] is None
assert snapshot.argpack['add2']['b'] is None
assert snapshot.argpack['mul2']['a'] is None
assert 'b' not in snapshot.argpack['mul2']
snapshot2 = CodePackSnapshot.from_dict(snapshot.to_dict())
assert isinstance(snapshot2.argpack, dict)
assert snapshot2.argpack['add2']['a'] is None
assert snapshot2.argpack['add2']['b'] is None
assert snapshot2.argpack['mul2']['a'] is None
assert 'b' not in snapshot2.argpack['mul2']
argpack['add2'](a=3, b=2)
argpack['mul2'](a=2)
snapshot3 = codepack.to_snapshot(argpack=argpack)
assert snapshot3.argpack['add2']['a'] == 3
assert snapshot3.argpack['add2']['b'] == 2
assert snapshot3.argpack['mul2']['a'] == 2
snapshot4 = CodePackSnapshot.from_dict(snapshot3.to_dict())
assert snapshot4.argpack['add2']['a'] == 3
assert snapshot4.argpack['add2']['b'] == 2
assert snapshot4.argpack['mul2']['a'] == 2
codepack2 = CodePack.from_snapshot(snapshot4)
assert codepack2(argpack=snapshot4.argpack) == 10
def test_check_dependency_state(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code3 = Code(add3)
code1 >> code3
code2 >> code3
assert len(code3.dependency) == 2
ret = code3()
assert not ret and code3.get_state() == 'WAITING'
a = code1(a=3, b=5)
assert a == 8 and code1.get_state() == 'TERMINATED'
ret = code3()
assert not ret and code3.get_state() == 'WAITING'
b = code2(a=2, b=3)
assert b == 6 and code2.get_state() == 'TERMINATED'
try:
ret = code3()
except TypeError:
pass
assert not ret and code3.get_state() == 'ERROR'
ret = code3(a=1, b=5)
assert ret == 8 and code3.get_state() == 'TERMINATED'
code3.receive('a') << code1
code3.receive('b') << code2
ret = code3()
assert ret == 16 and code3.get_state() == 'TERMINATED'
ret = code3(a=1, b=5)
assert ret == 8 and code3.get_state() == 'TERMINATED'
def test_print_info(default_os_env):
code1 = Code(add2, env='test-env')
code2 = Code(add3, image='test-image', owner='admin')
code1 >> code2
code2.receive('b') << code1
assert code1.get_info() == "Code(id: add2, function: add2, args: (a, b), receive: {}," \
" env: test-env, state: UNKNOWN)"
assert code2.get_info() == "Code(id: add3, function: add3, args: (a, b, c=2), receive: {'b': 'add2'}," \
" image: test-image, owner: admin, state: UNKNOWN)"
assert code1.get_info(state=False) == "Code(id: add2, function: add2, args: (a, b), receive: {}, env: test-env)"
assert code2.get_info(state=False) == "Code(id: add3, function: add3, args: (a, b, c=2), receive: {'b': 'add2'}," \
" image: test-image, owner: admin)"
code1.image = 'test-image2'
code1.owner = 'admin2'
assert code1.get_info() == "Code(id: add2, function: add2, args: (a, b), receive: {}," \
" env: test-env, image: test-image2, owner: admin2, state: UNKNOWN)"
assert code1.get_info(state=False) == "Code(id: add2, function: add2, args: (a, b), receive: {}," \
" env: test-env, image: test-image2, owner: admin2)"
code2.owner = None
assert code2.get_info() == "Code(id: add3, function: add3, args: (a, b, c=2), receive: {'b': 'add2'}," \
" image: test-image, state: UNKNOWN)"
assert code2.get_info(state=False) == "Code(id: add3, function: add3, args: (a, b, c=2), receive: {'b': 'add2'}," \
" image: test-image)"
code2.image = None
assert code2.get_info() == "Code(id: add3, function: add3, args: (a, b, c=2), receive: {'b': 'add2'}," \
" state: UNKNOWN)"
assert code2.get_info(
state=False) == "Code(id: add3, function: add3, args: (a, b, c=2), receive: {'b': 'add2'})"
def test_check_dependency_linkage(default_os_env):
code1 = Code(add2)
code2 = Code(print_x)
code3 = Code(add3)
code4 = Code(linear)
code1 >> [code2, code3]
code3 >> code4
assert code3.serial_number in code4.dependency
assert code3.id in code4.parents
assert code4.id in code3.children
assert set(code2.dependency.keys()) == set(code3.dependency.keys())
assert not code4.dependency[code3.serial_number].arg
code4.receive('c') << code3
assert code4.dependency[code3.serial_number].arg == 'c'
code3.receive('b') << code1
assert code3.dependency[code1.serial_number].arg == 'b'
assert not code2.dependency[code1.serial_number].arg
code3 // code4
assert len(set(code4.dependency)) == 0
assert code3.id not in code4.parents
assert code4.id not in code3.children
code1 // [code2, code3]
assert len(code2.dependency.keys()) == len(code3.dependency.keys()) == 0
assert len(code1.children) == 0
assert len(code2.parents) == 0 and len(code3.parents) == 0
def test_argpack_input(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code1 >> code2
code2.receive('b') << code1
codepack = CodePack(id='argpack_test', code=code1, subscribe=code2)
argpack = codepack.make_argpack()
argpack['add2'](a=3, b=2)
with pytest.raises(TypeError):
argpack['mul2'](c=5)
argpack['mul2'](a=2)
assert argpack['add2']['a'] == 3
assert argpack['add2']['b'] == 2
assert argpack['mul2']['a'] == 2
assert codepack(argpack=argpack) == 10
def test_default_arg(default_os_env):
code1 = Code(add2)
code2 = Code(add3)
ret = code2(a=1, b=3)
assert ret == 6 and code2.get_state() == 'TERMINATED'
code1 >> code2
code2.receive('c') << code1
ret = code2(a=1, b=3)
assert not ret and code2.get_state() == 'WAITING'
c = code1(a=3, b=5)
assert c == 8 and code1.get_state() == 'TERMINATED'
ret = code2(a=1, b=3)
assert ret == 12 and code2.get_state() == 'TERMINATED'
code1 // code2
ret = code2(a=1, b=3)
assert ret == 6 and code2.get_state() == 'TERMINATED'
def test_code_waiting(default_os_env):
code1 = Code(add2)
code2 = Code(add3)
code1 >> code2
code2.receive('b') << code1
code2(3, c=2)
assert code2.get_state() == 'WAITING'
snapshot = code2.service['snapshot'].load(
serial_number=code2.serial_number)
assert 'args' in snapshot
assert isinstance(snapshot['args'], list)
assert len(snapshot['args']) == 1
assert snapshot['args'][0] == 3
assert 'kwargs' in snapshot
assert isinstance(snapshot['kwargs'], dict)
assert len(snapshot['kwargs']) == 1
assert 'c' in snapshot['kwargs'].keys()
assert snapshot['kwargs']['c'] == 2
def test_code_snapshot_diff(default_os_env):
timestamp = datetime.now().timestamp()
code1 = Code(mul2)
code2 = Code(add2)
code3 = Code(add3)
code1 >> code3
code2 >> code3
code3.receive('c') << code2
code3(1, b=2)
snapshot1 = CodeSnapshot(code3, args=(1, ), kwargs={'b': 2}, timestamp=timestamp)
snapshot2 = CodeSnapshot.from_dict(snapshot1.to_dict())
snapshot3 = CodeSnapshot(code3, args=(1, ), kwargs={'b': 3}, timestamp=timestamp + 1)
assert snapshot1.diff(snapshot2) == dict()
assert snapshot2.diff(snapshot1) == dict()
diff = snapshot1.diff(snapshot3)
assert set(diff.keys()) == {'kwargs', 'timestamp'}
assert diff['kwargs'] == {'b': 3}
assert diff['timestamp'] == timestamp + 1
def test_argpack_str(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code1 >> code2
code2.receive('b') << code1
codepack = CodePack(id='argpack_test', code=code1, subscribe=code2)
argpack = codepack.make_argpack()
assert argpack.__str__(
) == 'ArgPack(id: argpack_test, args: {add2(a=None, b=None), mul2(a=None)})'
argpack['add2'](a=3, b=2)
with pytest.raises(TypeError):
argpack['mul2'](c=5)
argpack['mul2'](a=2)
assert argpack['add2']['a'] == 3
assert argpack['add2']['b'] == 2
assert argpack['mul2']['a'] == 2
assert argpack.__str__(
) == 'ArgPack(id: argpack_test, args: {add2(a=3, b=2), mul2(a=2)})'
assert codepack(argpack=argpack) == 10
def test_update_serial_number(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code3 = Code(add3)
code4 = Code(linear)
code1 >> code2
code2 >> [code3, code4]
code2.receive('a') << code1
code3.receive('b') << code2
code4.receive('c') << code2
old_serial_number = code2.serial_number
new_serial_number = '1234'
code2.update_serial_number(new_serial_number)
assert code2.serial_number == new_serial_number
assert code2.id in code1.children
assert code1.children[code2.id].serial_number == new_serial_number
assert code2.id in code3.parents
assert code3.parents[code2.id].serial_number == new_serial_number
assert old_serial_number not in code3.dependency
assert new_serial_number in code3.dependency
assert code3.dependency[new_serial_number].arg == 'b'
assert code3.dependency[new_serial_number].code == code3
assert code3.dependency[new_serial_number].id == code2.id
assert code3.dependency[new_serial_number].serial_number == new_serial_number
assert code2.id in code4.parents
assert code4.parents[code2.id].serial_number == new_serial_number
assert old_serial_number not in code4.dependency
assert new_serial_number in code4.dependency
assert code4.dependency[new_serial_number].arg == 'c'
assert code4.dependency[new_serial_number].code == code4
assert code4.dependency[new_serial_number].id == code2.id
assert code4.dependency[new_serial_number].serial_number == new_serial_number
def test_sync_codepack(default_os_env):
c1 = Code(add3)
c2 = Code(mul2)
c3 = Code(combination)
c4 = Code(linear)
c5 = Code(print_x)
c1 >> c3
c2 >> c3
c3 >> [c4, c5]
c3.receive('c') << c1
c3.receive('d') << c2
c4.receive('c') << c3
c5.receive('x') << c3
cp = CodePack(id='test_codepack', code=c1, subscribe=c4)
argpack = cp.make_argpack()
argpack['add3']['a'] = 1
argpack['add3']['b'] = 2
argpack['add3']['c'] = 3
argpack['mul2']['a'] = 1
argpack['mul2']['b'] = 2
argpack['combination']['a'] = 2
argpack['combination']['b'] = 5
argpack['linear']['b'] = 7
argpack['linear']['a'] = 5
ret = cp(argpack)
assert ret == 57
def test_dependency_monitor(default_os_env):
dm = DependencyMonitor(
callback=load_and_run_code_from_snapshot_and_raise_exception,
interval=0.5,
background=False)
c1 = Code(add3)
c2 = Code(mul2)
c3 = Code(combination)
c4 = Code(linear)
c1 >> c3
c2 >> c3
c3 >> c4
c3.receive('c') << c1
c3.receive('d') << c2
c4.receive('c') << c3
c1.save()
c2.save()
c3.save()
c4.save()
c3(2, b=3)
c4(3, 6)
c2(3, 5)
c1(1, 2, 3)
assert c1.get_state() == 'TERMINATED'
assert c2.get_state() == 'TERMINATED'
assert c3.get_state() == 'WAITING'
assert c4.get_state() == 'WAITING'
dm.start()
assert c1.get_state() == 'TERMINATED'
assert c2.get_state() == 'TERMINATED'
assert c3.get_state() == 'TERMINATED'
assert c4.get_state() == 'WAITING'
dm.start()
assert c1.get_state() == 'TERMINATED'
assert c2.get_state() == 'TERMINATED'
assert c3.get_state() == 'TERMINATED'
assert c4.get_state() == 'TERMINATED'
assert c4.get_result() == 114
dm.stop()
def test_memory_supervisor_run_codepack():
supervisor = Default.get_employee('supervisor')
assert isinstance(supervisor.messenger, MemoryMessenger)
assert supervisor.messenger.topic == 'codepack'
assert supervisor.messenger.queues['codepack'].empty()
code1 = Code(add2)
code2 = Code(mul2)
code3 = Code(combination)
code1 >> code3
code2 >> code3
code3.receive('c') << code1
code3.receive('d') << code2
codepack = CodePack('test', code=code1, subscribe=code3)
argpack = codepack.make_argpack()
argpack['add2'](a=3, b=5)
argpack['mul2'](a=2, b=3)
argpack['combination'](a=2, b=4)
sn = supervisor.run_codepack(codepack=codepack, argpack=argpack)
assert sn == codepack.serial_number
assert not supervisor.messenger.queues['codepack'].empty(
) and supervisor.messenger.queues['codepack'].qsize() == 3
items = [
supervisor.messenger.queues['codepack'].get(block=False)
for _ in range(3)
]
snapshots = [
c.to_snapshot(kwargs=argpack[i].to_dict()).to_dict()
for i, c in codepack.codes.items()
]
for item in items:
item.pop('timestamp', None)
for snapshot in snapshots:
snapshot.pop('timestamp', None)
assert sorted(items,
key=lambda x: x['id']) == sorted(snapshots,
key=lambda x: x['id'])
supervisor.close()
def test_code_snapshot_to_dict_and_from_dict(default_os_env):
code1 = Code(mul2)
code2 = Code(add2)
code3 = Code(add3, image='test-image')
code1 >> code3
code2 >> code3
code3.receive('c') << code2
code3(1, b=2)
snapshot1 = CodeSnapshot(code3, args=(1, ), kwargs={'b': 2})
snapshot_dict1 = snapshot1.to_dict()
assert 'state' in snapshot_dict1
assert snapshot_dict1['state'] == 'WAITING'
assert 'source' in snapshot_dict1
assert snapshot_dict1['source'] == code3.source
assert 'dependency' in snapshot_dict1
for dependency in snapshot_dict1['dependency']:
code3_dependency = code3.dependency[dependency['serial_number']]
assert set(dependency.keys()) == {'id', 'serial_number', 'arg'}
assert dependency['id'] == code3_dependency.id
assert dependency['serial_number'] == code3_dependency.serial_number
assert dependency['arg'] == code3_dependency.arg
assert 'image' in snapshot_dict1
assert snapshot_dict1['image'] is 'test-image'
assert 'env' in snapshot_dict1
assert snapshot_dict1['env'] is None
assert 'owner' in snapshot_dict1
assert snapshot_dict1['owner'] is None
snapshot2 = CodeSnapshot.from_dict(snapshot_dict1)
assert snapshot1.id == snapshot2.id
assert snapshot1.serial_number == snapshot2.serial_number
assert snapshot1.timestamp == snapshot2.timestamp
assert snapshot1.args == snapshot2.args
assert snapshot1.kwargs == snapshot2.kwargs
assert snapshot1.source == snapshot2.source
assert snapshot1.dependency == snapshot2.dependency
assert snapshot1.image == snapshot2.image
assert snapshot1.owner == snapshot2.owner
def test_dependency_monitor_in_background(default_os_env):
dm = DependencyMonitor(callback=load_and_run_code_from_snapshot,
interval=0.5,
background=True)
c1 = Code(add3)
c2 = Code(mul2)
c3 = Code(combination)
c4 = Code(linear)
c5 = Code(print_x)
c1 >> c3
c2 >> c3
c3 >> [c4, c5]
c3.receive('c') << c1
c3.receive('d') << c2
c4.receive('c') << c3
c5.receive('x') << c3
c1.save()
c2.save()
c3.save()
c4.save()
c5.save()
c3(2, b=3)
c5()
c4(3, 6)
c2(3, 5)
c1(1, 2, 3)
assert c1.get_state() == 'TERMINATED'
assert c2.get_state() == 'TERMINATED'
assert c3.get_state() == 'WAITING'
assert c4.get_state() == 'WAITING'
assert c5.get_state() == 'WAITING'
dm.start()
time.sleep(1)
assert c1.get_state() == 'TERMINATED'
assert c2.get_state() == 'TERMINATED'
assert c3.get_state() == 'TERMINATED'
assert c4.get_state() == 'TERMINATED'
assert c5.get_state() == 'TERMINATED'
assert c4.get_result() == 114
dm.stop()
def test_codepack_to_snapshot_and_from_snapshot(default_os_env):
c1 = Code(add3)
c2 = Code(mul2)
c3 = Code(combination)
c4 = Code(linear)
c5 = Code(print_x)
c1 >> c3
c2 >> c3
c3 >> [c4, c5]
c3.receive('c') << c1
c4.receive('c') << c3
c5.receive('x') << c3
cp1 = CodePack(id='test_codepack', code=c1, subscribe=c4)
argpack = cp1.make_argpack()
argpack['add3']['a'] = 1
argpack['add3']['b'] = 2
argpack['add3']['c'] = 3
argpack['mul2']['a'] = 2
argpack['mul2']['b'] = 1
argpack['combination']['a'] = 2
argpack['combination']['b'] = 5
argpack['linear']['b'] = 7
argpack['linear']['a'] = 5
ret = None
with pytest.raises(TypeError):
ret = cp1(argpack)
assert ret is None
assert cp1.get_state() == 'ERROR'
snapshot1 = cp1.to_snapshot(argpack=argpack)
assert snapshot1.owner is None
snapshot1['owner'] = 'codepack'
assert snapshot1.owner == 'codepack'
cp2 = CodePack.from_snapshot(snapshot1)
assert cp2.owner == 'codepack'
assert cp1.id == cp2.id
assert cp1.serial_number == cp2.serial_number
assert cp1.get_state() == cp2.get_state()
cp1_source = cp1.get_source()
cp2_source = cp2.get_source()
assert cp1_source.keys() == cp2_source.keys()
for id in cp1_source.keys():
assert cp1_source[id].strip() == cp2_source[id].strip()
# assert cp1.get_structure() == cp2.get_structure()
assert cp1.subscribe == cp2.subscribe
assert set(cp1.codes.keys()) == set(cp2.codes.keys())
assert cp1.owner is None
assert cp2.owner == 'codepack'
for code_id in cp1.codes.keys():
code1 = cp1.codes[code_id]
code2 = cp2.codes[code_id]
assert code1.serial_number == code2.serial_number
assert code1.parents.keys() == code2.parents.keys()
for id in code1.parents.keys():
assert code1.parents[id].serial_number == code2.parents[id].serial_number
assert code1.children.keys() == code2.children.keys()
for id in code1.children.keys():
assert code1.children[id].serial_number == code2.children[id].serial_number
assert code1.get_state() == code2.get_state()
assert code1.dependency.keys() == code2.dependency.keys()
for serial_number in code1.dependency.keys():
assert code1.dependency[serial_number].id == code2.dependency[serial_number].id
assert code1.dependency[serial_number].serial_number == code2.dependency[serial_number].serial_number
assert code1.dependency[serial_number].arg == code2.dependency[serial_number].arg