def test_check_dependency_state(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code3 = Code(add3)
code1 >> code3
code2 >> code3
assert len(code3.dependency) == 2
ret = code3()
assert not ret and code3.get_state() == 'WAITING'
a = code1(a=3, b=5)
assert a == 8 and code1.get_state() == 'TERMINATED'
ret = code3()
assert not ret and code3.get_state() == 'WAITING'
b = code2(a=2, b=3)
assert b == 6 and code2.get_state() == 'TERMINATED'
try:
ret = code3()
except TypeError:
pass
assert not ret and code3.get_state() == 'ERROR'
ret = code3(a=1, b=5)
assert ret == 8 and code3.get_state() == 'TERMINATED'
code3.receive('a') << code1
code3.receive('b') << code2
ret = code3()
assert ret == 16 and code3.get_state() == 'TERMINATED'
ret = code3(a=1, b=5)
assert ret == 8 and code3.get_state() == 'TERMINATED'
def test_validate_dependency_result(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code3 = Code(add3)
code1 >> code3
code2 >> code3
code3.receive('b') << code1
code3.receive('c') << code2
code1(1, 2)
code2(3, 4)
code2.service['delivery'].cancel(code2.serial_number)
ret = code3(a=3)
assert ret is None
assert code3.get_state() == 'ERROR'
code2(3, 4)
code2.send_result(item=1, timestamp=datetime.now().timestamp() + 1)
ret = code3(a=3)
assert ret is 7
assert code3.get_state() == 'TERMINATED'
code2(3, 4)
snapshots = code3.dependency.load_snapshot()
assert len(snapshots) == 2
assert code3.dependency.check_delivery() is True
snapshot_dict = {x['_id']: x for x in snapshots}
code2_state_info = snapshot_dict.pop(code2.serial_number)
assert code3.dependency.validate(snapshot=snapshot_dict.values()) == 'WAITING'
snapshot_dict[code2.serial_number] = code2_state_info
assert code3.dependency.validate(snapshot=snapshot_dict.values()) == 'READY'
code2.service['delivery'].cancel(code2.serial_number)
assert code3.dependency.validate(snapshot=snapshot_dict.values()) == 'ERROR'
code2.service['delivery'].send(id='dummy', serial_number=code2.serial_number, item=123)
assert code3.dependency.validate(snapshot=snapshot_dict.values()) == 'READY'
def test_check_dependency_linkage(default_os_env):
code1 = Code(add2)
code2 = Code(print_x)
code3 = Code(add3)
code4 = Code(linear)
code1 >> [code2, code3]
code3 >> code4
assert code3.serial_number in code4.dependency
assert code3.id in code4.parents
assert code4.id in code3.children
assert set(code2.dependency.keys()) == set(code3.dependency.keys())
assert not code4.dependency[code3.serial_number].arg
code4.receive('c') << code3
assert code4.dependency[code3.serial_number].arg == 'c'
code3.receive('b') << code1
assert code3.dependency[code1.serial_number].arg == 'b'
assert not code2.dependency[code1.serial_number].arg
code3 // code4
assert len(set(code4.dependency)) == 0
assert code3.id not in code4.parents
assert code4.id not in code3.children
code1 // [code2, code3]
assert len(code2.dependency.keys()) == len(code3.dependency.keys()) == 0
assert len(code1.children) == 0
assert len(code2.parents) == 0 and len(code3.parents) == 0
def test_update_serial_number(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code3 = Code(add3)
code4 = Code(linear)
code1 >> code2
code2 >> [code3, code4]
code2.receive('a') << code1
code3.receive('b') << code2
code4.receive('c') << code2
old_serial_number = code2.serial_number
new_serial_number = '1234'
code2.update_serial_number(new_serial_number)
assert code2.serial_number == new_serial_number
assert code2.id in code1.children
assert code1.children[code2.id].serial_number == new_serial_number
assert code2.id in code3.parents
assert code3.parents[code2.id].serial_number == new_serial_number
assert old_serial_number not in code3.dependency
assert new_serial_number in code3.dependency
assert code3.dependency[new_serial_number].arg == 'b'
assert code3.dependency[new_serial_number].code == code3
assert code3.dependency[new_serial_number].id == code2.id
assert code3.dependency[new_serial_number].serial_number == new_serial_number
assert code2.id in code4.parents
assert code4.parents[code2.id].serial_number == new_serial_number
assert old_serial_number not in code4.dependency
assert new_serial_number in code4.dependency
assert code4.dependency[new_serial_number].arg == 'c'
assert code4.dependency[new_serial_number].code == code4
assert code4.dependency[new_serial_number].id == code2.id
assert code4.dependency[new_serial_number].serial_number == new_serial_number
def test_mongo_storage_service_check(default_os_env, fake_mongodb):
db = 'test'
collection = 'codes'
storage = MongoStorage(item_type=Code, mongodb=fake_mongodb, db=db, collection=collection)
assert storage.key == 'serial_number'
mss = StorageService(storage=storage)
assert storage.key == 'id'
code1 = Code(hello, id='hello1', storage_service=mss)
code2 = Code(hello, id='hello2', storage_service=mss)
code3 = Code(hello, id='hello3', storage_service=mss)
for i in ['hello1', 'hello2', 'hello3']:
try:
mss.remove(id=i)
except Exception:
continue
code1.save()
code2.save()
check = mss.check([code1.id, code2.id, code3.id])
assert isinstance(check, list)
assert check[0] and check[1] and not check[2]
mss.remove(id=code1.id)
code3.save()
check = mss.check([code1.id, code2.id, code3.id])
assert isinstance(check, list)
assert not check[0] and check[1] and check[2]
assert len(check) == 3
def test_argpack_codepack_execution(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code1 >> code2
code2.receive('b') << code1
codepack = CodePack(id='argpack_test', code=code1, subscribe=code2)
argpack = codepack.make_argpack()
snapshot = codepack.to_snapshot(argpack=argpack)
assert isinstance(snapshot.argpack, dict)
assert snapshot.argpack['add2']['a'] is None
assert snapshot.argpack['add2']['b'] is None
assert snapshot.argpack['mul2']['a'] is None
assert 'b' not in snapshot.argpack['mul2']
snapshot2 = CodePackSnapshot.from_dict(snapshot.to_dict())
assert isinstance(snapshot2.argpack, dict)
assert snapshot2.argpack['add2']['a'] is None
assert snapshot2.argpack['add2']['b'] is None
assert snapshot2.argpack['mul2']['a'] is None
assert 'b' not in snapshot2.argpack['mul2']
argpack['add2'](a=3, b=2)
argpack['mul2'](a=2)
snapshot3 = codepack.to_snapshot(argpack=argpack)
assert snapshot3.argpack['add2']['a'] == 3
assert snapshot3.argpack['add2']['b'] == 2
assert snapshot3.argpack['mul2']['a'] == 2
snapshot4 = CodePackSnapshot.from_dict(snapshot3.to_dict())
assert snapshot4.argpack['add2']['a'] == 3
assert snapshot4.argpack['add2']['b'] == 2
assert snapshot4.argpack['mul2']['a'] == 2
codepack2 = CodePack.from_snapshot(snapshot4)
assert codepack2(argpack=snapshot4.argpack) == 10
def test_sync_codepack(default_os_env):
c1 = Code(add3)
c2 = Code(mul2)
c3 = Code(combination)
c4 = Code(linear)
c5 = Code(print_x)
c1 >> c3
c2 >> c3
c3 >> [c4, c5]
c3.receive('c') << c1
c3.receive('d') << c2
c4.receive('c') << c3
c5.receive('x') << c3
cp = CodePack(id='test_codepack', code=c1, subscribe=c4)
argpack = cp.make_argpack()
argpack['add3']['a'] = 1
argpack['add3']['b'] = 2
argpack['add3']['c'] = 3
argpack['mul2']['a'] = 1
argpack['mul2']['b'] = 2
argpack['combination']['a'] = 2
argpack['combination']['b'] = 5
argpack['linear']['b'] = 7
argpack['linear']['a'] = 5
ret = cp(argpack)
assert ret == 57
def test_print_info(default_os_env):
code1 = Code(add2, env='test-env')
code2 = Code(add3, image='test-image', owner='admin')
code1 >> code2
code2.receive('b') << code1
assert code1.get_info() == "Code(id: add2, function: add2, args: (a, b), receive: {}," \
" env: test-env, state: UNKNOWN)"
assert code2.get_info() == "Code(id: add3, function: add3, args: (a, b, c=2), receive: {'b': 'add2'}," \
" image: test-image, owner: admin, state: UNKNOWN)"
assert code1.get_info(state=False) == "Code(id: add2, function: add2, args: (a, b), receive: {}, env: test-env)"
assert code2.get_info(state=False) == "Code(id: add3, function: add3, args: (a, b, c=2), receive: {'b': 'add2'}," \
" image: test-image, owner: admin)"
code1.image = 'test-image2'
code1.owner = 'admin2'
assert code1.get_info() == "Code(id: add2, function: add2, args: (a, b), receive: {}," \
" env: test-env, image: test-image2, owner: admin2, state: UNKNOWN)"
assert code1.get_info(state=False) == "Code(id: add2, function: add2, args: (a, b), receive: {}," \
" env: test-env, image: test-image2, owner: admin2)"
code2.owner = None
assert code2.get_info() == "Code(id: add3, function: add3, args: (a, b, c=2), receive: {'b': 'add2'}," \
" image: test-image, state: UNKNOWN)"
assert code2.get_info(state=False) == "Code(id: add3, function: add3, args: (a, b, c=2), receive: {'b': 'add2'}," \
" image: test-image)"
code2.image = None
assert code2.get_info() == "Code(id: add3, function: add3, args: (a, b, c=2), receive: {'b': 'add2'}," \
" state: UNKNOWN)"
assert code2.get_info(
state=False) == "Code(id: add3, function: add3, args: (a, b, c=2), receive: {'b': 'add2'})"
def test_memory_supervisor_organize():
supervisor = Default.get_employee('supervisor')
assert isinstance(supervisor.messenger, MemoryMessenger)
assert supervisor.messenger.topic == 'codepack'
assert supervisor.messenger.queues['codepack'].empty()
code1 = Code(add2)
code2 = Code(mul2)
code1 >> code2
code2(3, b=5)
assert code1.get_state() == 'UNKNOWN'
assert code2.get_state() == 'WAITING'
supervisor.organize()
assert code1.get_state() == 'UNKNOWN'
assert code2.get_state() == 'WAITING'
assert supervisor.messenger.queues['codepack'].empty()
code1(a=2, b=4)
assert code1.get_state() == 'TERMINATED'
assert code2.get_state() == 'WAITING'
assert supervisor.messenger.queues['codepack'].empty()
supervisor.organize()
assert code1.get_state() == 'TERMINATED'
assert code2.get_state() == 'READY'
assert not supervisor.messenger.queues['codepack'].empty(
) and supervisor.messenger.queues['codepack'].qsize() == 1
item = supervisor.messenger.queues['codepack'].get(block=False)
item.pop('source', None)
expected_item = code2.to_snapshot(args=(3, ),
kwargs={
'b': 5
},
timestamp=item['timestamp']).to_dict()
expected_item.pop('source', None)
assert item == expected_item
supervisor.close()
def test_load_code_from_storage_service_with_id(default_os_env):
code1 = Code(add2)
code1.save()
code2 = Code(id='add2', serial_number='1234')
assert code1.source == code2.source
assert code1(1, 2) == code2(1, 2)
assert code1.serial_number != code2.serial_number
assert code2.serial_number == '1234'
def test_dependency_error_propagation(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code3 = Code(add3)
code1 >> code3
code2 >> code3
assert code3.get_state() == 'UNKNOWN'
with pytest.raises(TypeError):
code1()
assert code1.get_state() == 'ERROR'
ret = code3(1, 2, 3)
assert ret is None
assert code3.get_state() == 'WAITING'
def test_argpack_input(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code1 >> code2
code2.receive('b') << code1
codepack = CodePack(id='argpack_test', code=code1, subscribe=code2)
argpack = codepack.make_argpack()
argpack['add2'](a=3, b=2)
with pytest.raises(TypeError):
argpack['mul2'](c=5)
argpack['mul2'](a=2)
assert argpack['add2']['a'] == 3
assert argpack['add2']['b'] == 2
assert argpack['mul2']['a'] == 2
assert codepack(argpack=argpack) == 10
def test_memory_storage_service(default_os_env):
storage = MemoryStorage(item_type=Code)
mss = StorageService(storage=storage)
mss.storage.init()
code1 = Code(hello, storage_service=mss)
code2 = Code(add2, storage_service=mss)
code1.save()
assert len(storage.memory) == 1
assert code1.id in storage.memory
assert code2.id not in storage.memory
code3 = mss.load(code1.id)
assert code1.id == code3.id
assert code1.source == code3.source
assert code1("CodePack") == code3("CodePack")
mss.remove(code3.id)
assert len(storage.memory) == 0
def test_default_arg(default_os_env):
code1 = Code(add2)
code2 = Code(add3)
ret = code2(a=1, b=3)
assert ret == 6 and code2.get_state() == 'TERMINATED'
code1 >> code2
code2.receive('c') << code1
ret = code2(a=1, b=3)
assert not ret and code2.get_state() == 'WAITING'
c = code1(a=3, b=5)
assert c == 8 and code1.get_state() == 'TERMINATED'
ret = code2(a=1, b=3)
assert ret == 12 and code2.get_state() == 'TERMINATED'
code1 // code2
ret = code2(a=1, b=3)
assert ret == 6 and code2.get_state() == 'TERMINATED'
def test_file_storage_jobstore_codepack_snapshot(testdir):
code = Code(hello)
codepack = CodePack('codepack_test', code=code, subscribe=code)
argpack = codepack.make_argpack()
argpack['hello']['name'] = 'CodePack'
job_id = 'job_test'
storage = FileStorage(item_type=StorableJob,
key='id',
path='testdir/scheduler/')
jobstore = JobStore(storage=storage)
scheduler = Scheduler(jobstore=jobstore, blocking=False)
scheduler.add_codepack(job_id=job_id,
codepack=codepack,
argpack=argpack,
trigger='interval',
seconds=30)
scheduler.start()
assert scheduler.is_running()
assert not storage.exist(key=codepack.id)
assert storage.exist(key=job_id)
job = storage.load(key=job_id, to_dict=True)
for k in [
'_id', 'trigger', 'codepack', 'snapshot', 'last_run_time',
'next_run_time'
]:
assert k in job
assert job['_id'] == job_id
assert job['trigger'] == 'interval[0:00:30]'
assert job['codepack'] == codepack.id
assert job['snapshot'] == codepack.serial_number
scheduler.remove_job(job_id)
assert len(storage.list_all()) == 0
scheduler.stop()
def test_memory_worker_run_snapshot_with_image_and_callback(
mock_docker_client):
worker = Default.get_employee('worker')
worker.callback = dummy_callback_function
assert isinstance(worker.messenger, MemoryMessenger)
code = Code(add2, image='dummy')
assert code.get_state() == 'UNKNOWN'
sn = worker.run_snapshot(code.to_snapshot(kwargs={'a': 3, 'b': 5}))
assert sn == code.serial_number
default_config_dir = Config.get_default_config_dir()
script_dir = os.path.join(default_config_dir, 'scripts')
mock_docker_client.return_value.containers.run.assert_called_once_with(
auto_remove=True,
command=[
'python', 'run_snapshot.py',
'%s.json' % sn, '-p', '.', '-l', 'worker-logger', '-c',
'dummy_callback_function'
],
dns=['8.8.8.8'],
environment=['CODEPACK_LOGGER_LOG_DIR=/usr/logs'],
image='dummy',
name=id(worker.docker_manager),
volumes=[
'%s:/usr/src/codepack' % script_dir,
'%s:/usr/logs' % os.path.abspath(Config.get_log_dir())
],
working_dir='/usr/src/codepack')
worker.stop()
def test_memory_storage_jobstore_codepack_snapshot():
code = Code(hello)
codepack = CodePack('codepack_test', code=code, subscribe=code)
argpack = codepack.make_argpack()
argpack['hello']['name'] = 'CodePack'
job_id = 'job_test'
storage = MemoryStorage(item_type=StorableJob, key='id')
jobstore = JobStore(storage=storage)
scheduler = Scheduler(jobstore=jobstore, blocking=False)
scheduler.add_codepack(job_id=job_id,
codepack=codepack,
argpack=argpack,
trigger='interval',
seconds=30)
scheduler.start()
assert scheduler.is_running()
assert codepack.id not in storage.memory
assert job_id in storage.memory
job = storage.memory[job_id].to_dict()
for k in [
'_id', 'trigger', 'codepack', 'snapshot', 'last_run_time',
'next_run_time'
]:
assert k in job
assert job['_id'] == job_id
assert job['trigger'] == 'interval[0:00:30]'
assert job['codepack'] == codepack.id
assert job['snapshot'] == codepack.serial_number
scheduler.remove_job(job_id)
assert len(storage.memory) == 0
scheduler.stop()
def test_file_storage_service(default_os_env, testdir_storage_service):
filepath = testdir_storage_service
storage = FileStorage(item_type=Code, path=filepath)
fss = StorageService(storage=storage)
assert storage.path == filepath
code1 = Code(hello, storage_service=fss)
code2 = Code(add2, storage_service=fss)
code1.save()
assert not os.path.isfile(Code.get_path(key=code1.id))
assert os.path.isfile(Code.get_path(key=code1.id, path=filepath))
assert not os.path.isfile(Code.get_path(key=code2.id, path=filepath))
code3 = fss.load(code1.id)
assert code1.id == code3.id
assert code1.source.strip() == code3.source.strip()
assert code1("CodePack") == code3("CodePack")
fss.remove(code3.id)
assert not os.path.isfile(Code.get_path(key=code1.id, path=filepath))
def test_add_dependency(default_os_env):
dependency = [{'id': 'test1', 'serial_number': '1234', 'arg': None}, {'id': 'test2', 'serial_number': '5678', 'arg': 'a'}]
code = Code(add2, dependency=dependency)
assert '1234' in code.dependency and '5678' in code.dependency
assert code.dependency['5678'].arg == 'a'
assert not code.dependency['1234'].arg
tmp = code.dependency.get_args()
assert len(tmp) == 1 and 'a' in tmp
def test_mongo_code_snapshot_service_search_and_remove(default_os_env,
fake_mongodb):
db = 'test'
collection = 'snapshot'
storage = MongoStorage(item_type=CodeSnapshot,
mongodb=fake_mongodb,
db=db,
collection=collection)
mss = SnapshotService(storage=storage)
code1 = Code(add2)
code2 = Code(add3)
code1 >> code2
code2(1, 2, 3)
snapshot1 = CodeSnapshot(code1)
snapshot2 = CodeSnapshot(code2)
mss.save(snapshot=snapshot1)
mss.save(snapshot=snapshot2)
assert fake_mongodb[db][collection].count_documents({}) == 2
loaded = mss.load(
[snapshot1.serial_number, snapshot2.serial_number, '1234'])
assert len(loaded) == 2
assert loaded[0]['state'] == 'UNKNOWN'
assert loaded[1]['state'] == 'WAITING'
loaded = mss.load(
[snapshot1.serial_number, snapshot2.serial_number, '1234'],
projection=['state'])
for x in loaded:
assert set(x.keys()) == {'state', 'serial_number'}
search_result = mss.search(key='state', value='WAITING')
assert len(search_result) == 1
assert search_result[0] == snapshot2.to_dict()
search_result = mss.search(key='state',
value='WAITING',
projection=['state'])
assert len(search_result) == 1
assert len(search_result[0].keys()) == 2
assert search_result[0]['state'] == 'WAITING'
assert search_result[0]['serial_number'] == code2.serial_number
mss.remove(snapshot2.serial_number)
loaded = mss.load(
[snapshot1.serial_number, snapshot2.serial_number, '1234'])
assert len(loaded) == 1
search_result = mss.search(key='state',
value='WAITING',
projection=['state'])
assert len(search_result) == 0
def test_code_snapshot_diff(default_os_env):
timestamp = datetime.now().timestamp()
code1 = Code(mul2)
code2 = Code(add2)
code3 = Code(add3)
code1 >> code3
code2 >> code3
code3.receive('c') << code2
code3(1, b=2)
snapshot1 = CodeSnapshot(code3, args=(1, ), kwargs={'b': 2}, timestamp=timestamp)
snapshot2 = CodeSnapshot.from_dict(snapshot1.to_dict())
snapshot3 = CodeSnapshot(code3, args=(1, ), kwargs={'b': 3}, timestamp=timestamp + 1)
assert snapshot1.diff(snapshot2) == dict()
assert snapshot2.diff(snapshot1) == dict()
diff = snapshot1.diff(snapshot3)
assert set(diff.keys()) == {'kwargs', 'timestamp'}
assert diff['kwargs'] == {'b': 3}
assert diff['timestamp'] == timestamp + 1
def test_mongo_storage_service(default_os_env, fake_mongodb):
db = 'test'
collection = 'codes'
storage = MongoStorage(item_type=Code, mongodb=fake_mongodb, db=db, collection=collection)
mss = StorageService(storage=storage)
assert mss.storage.item_type == Code
code1 = Code(hello, storage_service=mss)
code2 = Code(add2, storage_service=mss)
code1.save()
assert storage.mongodb[db][collection].count_documents({'_id': code1.id}) == 1
assert storage.mongodb[db][collection].count_documents({'_id': code2.id}) == 0
assert storage.item_type == Code
code3 = mss.load(code1.id)
assert code1.id == code3.id
assert code1.source.strip() == code3.source.strip()
assert code1("CodePack") == code3("CodePack")
mss.remove(code3.id)
assert storage.mongodb[db][collection].count_documents({'_id': code1.id}) == 0
def test_code_waiting(default_os_env):
code1 = Code(add2)
code2 = Code(add3)
code1 >> code2
code2.receive('b') << code1
code2(3, c=2)
assert code2.get_state() == 'WAITING'
snapshot = code2.service['snapshot'].load(
serial_number=code2.serial_number)
assert 'args' in snapshot
assert isinstance(snapshot['args'], list)
assert len(snapshot['args']) == 1
assert snapshot['args'][0] == 3
assert 'kwargs' in snapshot
assert isinstance(snapshot['kwargs'], dict)
assert len(snapshot['kwargs']) == 1
assert 'c' in snapshot['kwargs'].keys()
assert snapshot['kwargs']['c'] == 2
def test_memory_worker_run_snapshot_with_nothing():
worker = Default.get_employee('worker')
assert isinstance(worker.messenger, MemoryMessenger)
code = Code(add2)
assert code.get_state() == 'UNKNOWN'
sn = worker.run_snapshot(code.to_snapshot(kwargs={'a': 3, 'b': 5}))
assert sn == code.serial_number
assert code.get_state() == 'TERMINATED'
assert code.get_result() == 8
worker.stop()
def test_argpack_str(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code1 >> code2
code2.receive('b') << code1
codepack = CodePack(id='argpack_test', code=code1, subscribe=code2)
argpack = codepack.make_argpack()
assert argpack.__str__(
) == 'ArgPack(id: argpack_test, args: {add2(a=None, b=None), mul2(a=None)})'
argpack['add2'](a=3, b=2)
with pytest.raises(TypeError):
argpack['mul2'](c=5)
argpack['mul2'](a=2)
assert argpack['add2']['a'] == 3
assert argpack['add2']['b'] == 2
assert argpack['mul2']['a'] == 2
assert argpack.__str__(
) == 'ArgPack(id: argpack_test, args: {add2(a=3, b=2), mul2(a=2)})'
assert codepack(argpack=argpack) == 10
def test_file_code_snapshot_service_search_and_remove(
default_os_env, testdir_snapshot_service):
storage = FileStorage(item_type=CodeSnapshot,
path=testdir_snapshot_service)
fss = SnapshotService(storage=storage)
code1 = Code(add2)
code2 = Code(add3)
code1 >> code2
code2(1, 2, 3)
snapshot1 = CodeSnapshot(code1)
snapshot2 = CodeSnapshot(code2)
fss.save(snapshot=snapshot1)
fss.save(snapshot=snapshot2)
assert len(os.listdir(testdir_snapshot_service)) == 2
loaded = fss.load(
[snapshot1.serial_number, snapshot2.serial_number, '1234'])
assert len(loaded) == 2
assert loaded[0]['state'] == 'UNKNOWN'
assert loaded[1]['state'] == 'WAITING'
loaded = fss.load(
[snapshot1.serial_number, snapshot2.serial_number, '1234'],
projection=['state'])
for x in loaded:
assert set(x.keys()) == {'state', 'serial_number'}
search_result = fss.search(key='state', value='WAITING')
assert len(search_result) == 1
assert search_result[0] == snapshot2.to_dict()
search_result = fss.search(key='state',
value='WAITING',
projection=['state'])
assert len(search_result) == 1
assert len(search_result[0].keys()) == 2
assert search_result[0]['state'] == 'WAITING'
assert search_result[0]['serial_number'] == code2.serial_number
fss.remove(snapshot2.serial_number)
loaded = fss.load(
[snapshot1.serial_number, snapshot2.serial_number, '1234'])
assert len(loaded) == 1
search_result = fss.search(key='state',
value='WAITING',
projection=['state'])
assert len(search_result) == 0
def test_no_config():
code = Code(add2)
assert 'delivery' in code.service and isinstance(code.service['delivery'],
DeliveryService)
assert 'snapshot' in code.service and isinstance(code.service['snapshot'],
SnapshotService)
assert 'storage' in code.service and isinstance(code.service['storage'],
StorageService)
assert isinstance(code.service['delivery'].storage, MemoryStorage)
assert isinstance(code.service['snapshot'].storage, MemoryStorage)
assert isinstance(code.service['storage'].storage, MemoryStorage)
def test_memory_storage_service_check(default_os_env):
storage = MemoryStorage(item_type=Code)
assert storage.key == 'serial_number'
mss = StorageService(storage=storage)
assert storage.key == 'id'
mss.storage.init()
code1 = Code(hello, id='hello1', storage_service=mss)
code2 = Code(hello, id='hello2', storage_service=mss)
code3 = Code(hello, id='hello3', storage_service=mss)
code1.save()
assert isinstance(mss.check(id=code1.id), bool)
assert code1.id in storage.memory
mss.remove(id=code1.id)
code2.save()
code3.save()
assert len(storage.memory) == 2
check = mss.check([code1.id, code2.id, code3.id])
assert isinstance(check, list)
assert len(check) == 3
assert not check[0] and check[1] and check[2]
def test_dependency_monitor_in_background(default_os_env):
dm = DependencyMonitor(callback=load_and_run_code_from_snapshot,
interval=0.5,
background=True)
c1 = Code(add3)
c2 = Code(mul2)
c3 = Code(combination)
c4 = Code(linear)
c5 = Code(print_x)
c1 >> c3
c2 >> c3
c3 >> [c4, c5]
c3.receive('c') << c1
c3.receive('d') << c2
c4.receive('c') << c3
c5.receive('x') << c3
c1.save()
c2.save()
c3.save()
c4.save()
c5.save()
c3(2, b=3)
c5()
c4(3, 6)
c2(3, 5)
c1(1, 2, 3)
assert c1.get_state() == 'TERMINATED'
assert c2.get_state() == 'TERMINATED'
assert c3.get_state() == 'WAITING'
assert c4.get_state() == 'WAITING'
assert c5.get_state() == 'WAITING'
dm.start()
time.sleep(1)
assert c1.get_state() == 'TERMINATED'
assert c2.get_state() == 'TERMINATED'
assert c3.get_state() == 'TERMINATED'
assert c4.get_state() == 'TERMINATED'
assert c5.get_state() == 'TERMINATED'
assert c4.get_result() == 114
dm.stop()
def test_if_default_services_have_single_instance_for_each_service(
testdir_snapshot_service):
os.environ['CODEPACK_DELIVERY_SOURCE'] = 'memory'
os.environ['CODEPACK_CODESNAPSHOT_SOURCE'] = 'file'
os.environ['CODEPACK_CODESNAPSHOT_PATH'] = testdir_snapshot_service
os.environ['CODEPACK_CODE_SOURCE'] = 'mongodb'
os.environ['CODEPACK_CODE_DB'] = 'test'
os.environ['CODEPACK_CODE_COLLECTION'] = 'codes'
try:
code1 = Code(add2)
code2 = Code(add3)
assert isinstance(code1.service['delivery'], DeliveryService)
assert isinstance(code1.service['snapshot'], SnapshotService)
assert isinstance(code1.service['storage'], StorageService)
assert isinstance(code2.service['delivery'], DeliveryService)
assert isinstance(code2.service['snapshot'], SnapshotService)
assert isinstance(code2.service['storage'], StorageService)
assert isinstance(code1.service['delivery'].storage, MemoryStorage)
assert isinstance(code1.service['snapshot'].storage, FileStorage)
assert isinstance(code1.service['storage'].storage, MongoStorage)
assert isinstance(code2.service['delivery'].storage, MemoryStorage)
assert isinstance(code2.service['snapshot'].storage, FileStorage)
assert isinstance(code2.service['storage'].storage, MongoStorage)
assert (code1.service['delivery']) == (code2.service['delivery'])
assert (code1.service['snapshot']) == (code2.service['snapshot'])
assert (code1.service['storage']) == (code2.service['storage'])
assert id(code1.service['delivery']) == id(code2.service['delivery'])
assert id(code1.service['snapshot']) == id(code2.service['snapshot'])
assert id(code1.service['storage']) == id(code2.service['storage'])
finally:
for env in [
'CODEPACK_DELIVERY_SOURCE', 'CODEPACK_CODESNAPSHOT_SOURCE',
'CODEPACK_CODESNAPSHOT_PATH', 'CODEPACK_CODE_SOURCE',
'CODEPACK_CODE_DB', 'CODEPACK_CODE_COLLECTION'
]:
os.environ.pop(env, None)
