def test_file_storage_jobstore_codepack_snapshot(testdir):
code = Code(hello)
codepack = CodePack('codepack_test', code=code, subscribe=code)
argpack = codepack.make_argpack()
argpack['hello']['name'] = 'CodePack'
job_id = 'job_test'
storage = FileStorage(item_type=StorableJob,
key='id',
path='testdir/scheduler/')
jobstore = JobStore(storage=storage)
scheduler = Scheduler(jobstore=jobstore, blocking=False)
scheduler.add_codepack(job_id=job_id,
codepack=codepack,
argpack=argpack,
trigger='interval',
seconds=30)
scheduler.start()
assert scheduler.is_running()
assert not storage.exist(key=codepack.id)
assert storage.exist(key=job_id)
job = storage.load(key=job_id, to_dict=True)
for k in [
'_id', 'trigger', 'codepack', 'snapshot', 'last_run_time',
'next_run_time'
]:
assert k in job
assert job['_id'] == job_id
assert job['trigger'] == 'interval[0:00:30]'
assert job['codepack'] == codepack.id
assert job['snapshot'] == codepack.serial_number
scheduler.remove_job(job_id)
assert len(storage.list_all()) == 0
scheduler.stop()
def test_memory_storage_jobstore_codepack_snapshot():
code = Code(hello)
codepack = CodePack('codepack_test', code=code, subscribe=code)
argpack = codepack.make_argpack()
argpack['hello']['name'] = 'CodePack'
job_id = 'job_test'
storage = MemoryStorage(item_type=StorableJob, key='id')
jobstore = JobStore(storage=storage)
scheduler = Scheduler(jobstore=jobstore, blocking=False)
scheduler.add_codepack(job_id=job_id,
codepack=codepack,
argpack=argpack,
trigger='interval',
seconds=30)
scheduler.start()
assert scheduler.is_running()
assert codepack.id not in storage.memory
assert job_id in storage.memory
job = storage.memory[job_id].to_dict()
for k in [
'_id', 'trigger', 'codepack', 'snapshot', 'last_run_time',
'next_run_time'
]:
assert k in job
assert job['_id'] == job_id
assert job['trigger'] == 'interval[0:00:30]'
assert job['codepack'] == codepack.id
assert job['snapshot'] == codepack.serial_number
scheduler.remove_job(job_id)
assert len(storage.memory) == 0
scheduler.stop()
def test_sync_codepack(default_os_env):
c1 = Code(add3)
c2 = Code(mul2)
c3 = Code(combination)
c4 = Code(linear)
c5 = Code(print_x)
c1 >> c3
c2 >> c3
c3 >> [c4, c5]
c3.receive('c') << c1
c3.receive('d') << c2
c4.receive('c') << c3
c5.receive('x') << c3
cp = CodePack(id='test_codepack', code=c1, subscribe=c4)
argpack = cp.make_argpack()
argpack['add3']['a'] = 1
argpack['add3']['b'] = 2
argpack['add3']['c'] = 3
argpack['mul2']['a'] = 1
argpack['mul2']['b'] = 2
argpack['combination']['a'] = 2
argpack['combination']['b'] = 5
argpack['linear']['b'] = 7
argpack['linear']['a'] = 5
ret = cp(argpack)
assert ret == 57
def test_argpack_input(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code1 >> code2
code2.receive('b') << code1
codepack = CodePack(id='argpack_test', code=code1, subscribe=code2)
argpack = codepack.make_argpack()
argpack['add2'](a=3, b=2)
with pytest.raises(TypeError):
argpack['mul2'](c=5)
argpack['mul2'](a=2)
assert argpack['add2']['a'] == 3
assert argpack['add2']['b'] == 2
assert argpack['mul2']['a'] == 2
assert codepack(argpack=argpack) == 10
async def register(params: CodePackJSONJob):
codepack = CodePack.from_json(params.codepack)
argpack = ArgPack.from_json(params.argpack)
common.scheduler.add_codepack(codepack=codepack,
argpack=argpack,
job_id=params.job_id,
trigger=params.trigger,
**params.trigger_config)
return {'serial_number': codepack.serial_number}
def test_argpack_str(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code1 >> code2
code2.receive('b') << code1
codepack = CodePack(id='argpack_test', code=code1, subscribe=code2)
argpack = codepack.make_argpack()
assert argpack.__str__(
) == 'ArgPack(id: argpack_test, args: {add2(a=None, b=None), mul2(a=None)})'
argpack['add2'](a=3, b=2)
with pytest.raises(TypeError):
argpack['mul2'](c=5)
argpack['mul2'](a=2)
assert argpack['add2']['a'] == 3
assert argpack['add2']['b'] == 2
assert argpack['mul2']['a'] == 2
assert argpack.__str__(
) == 'ArgPack(id: argpack_test, args: {add2(a=3, b=2), mul2(a=2)})'
assert codepack(argpack=argpack) == 10
async def result(serial_number: str):
snapshot_service = Default.get_service('codepack_snapshot',
'snapshot_service')
snapshot = snapshot_service.load(serial_number=serial_number)
_result = None
if snapshot and snapshot['subscribe']:
codepack_snapshot = CodePackSnapshot.from_dict(snapshot)
codepack = CodePack.from_snapshot(codepack_snapshot)
_result = codepack.get_result()
return {'serial_number': serial_number, 'result': _result}
def test_argpack_codepack_execution(default_os_env):
code1 = Code(add2)
code2 = Code(mul2)
code1 >> code2
code2.receive('b') << code1
codepack = CodePack(id='argpack_test', code=code1, subscribe=code2)
argpack = codepack.make_argpack()
snapshot = codepack.to_snapshot(argpack=argpack)
assert isinstance(snapshot.argpack, dict)
assert snapshot.argpack['add2']['a'] is None
assert snapshot.argpack['add2']['b'] is None
assert snapshot.argpack['mul2']['a'] is None
assert 'b' not in snapshot.argpack['mul2']
snapshot2 = CodePackSnapshot.from_dict(snapshot.to_dict())
assert isinstance(snapshot2.argpack, dict)
assert snapshot2.argpack['add2']['a'] is None
assert snapshot2.argpack['add2']['b'] is None
assert snapshot2.argpack['mul2']['a'] is None
assert 'b' not in snapshot2.argpack['mul2']
argpack['add2'](a=3, b=2)
argpack['mul2'](a=2)
snapshot3 = codepack.to_snapshot(argpack=argpack)
assert snapshot3.argpack['add2']['a'] == 3
assert snapshot3.argpack['add2']['b'] == 2
assert snapshot3.argpack['mul2']['a'] == 2
snapshot4 = CodePackSnapshot.from_dict(snapshot3.to_dict())
assert snapshot4.argpack['add2']['a'] == 3
assert snapshot4.argpack['add2']['b'] == 2
assert snapshot4.argpack['mul2']['a'] == 2
codepack2 = CodePack.from_snapshot(snapshot4)
assert codepack2(argpack=snapshot4.argpack) == 10
def test_mongo_storage_jobstore_codepack_snapshot(fake_mongodb):
code = Code(hello)
codepack = CodePack('codepack_test', code=code, subscribe=code)
argpack = codepack.make_argpack()
argpack['hello']['name'] = 'CodePack'
job_id = 'job_test'
db = 'test'
collection = 'scheduler'
storage = MongoStorage(item_type=StorableJob,
key='id',
mongodb=fake_mongodb,
db=db,
collection=collection)
jobstore = JobStore(storage=storage)
scheduler = Scheduler(jobstore=jobstore, blocking=False)
scheduler.add_codepack(job_id=job_id,
codepack=codepack,
argpack=argpack,
trigger='interval',
seconds=30)
scheduler.start()
assert scheduler.is_running()
assert fake_mongodb[db][collection].count_documents({'_id':
codepack.id}) == 0
assert fake_mongodb[db][collection].count_documents({'_id': job_id}) == 1
document = fake_mongodb[db][collection].find_one({'_id': job_id})
for k in [
'_id', 'trigger', 'codepack', 'snapshot', 'last_run_time',
'next_run_time'
]:
assert k in document
assert document['_id'] == job_id
assert document['trigger'] == 'interval[0:00:30]'
assert document['codepack'] == codepack.id
assert document['snapshot'] == codepack.serial_number
scheduler.remove_job(job_id)
assert fake_mongodb[db][collection].count_documents({'_id': job_id}) == 0
scheduler.stop()
def test_memory_supervisor_run_codepack():
supervisor = Default.get_employee('supervisor')
assert isinstance(supervisor.messenger, MemoryMessenger)
assert supervisor.messenger.topic == 'codepack'
assert supervisor.messenger.queues['codepack'].empty()
code1 = Code(add2)
code2 = Code(mul2)
code3 = Code(combination)
code1 >> code3
code2 >> code3
code3.receive('c') << code1
code3.receive('d') << code2
codepack = CodePack('test', code=code1, subscribe=code3)
argpack = codepack.make_argpack()
argpack['add2'](a=3, b=5)
argpack['mul2'](a=2, b=3)
argpack['combination'](a=2, b=4)
sn = supervisor.run_codepack(codepack=codepack, argpack=argpack)
assert sn == codepack.serial_number
assert not supervisor.messenger.queues['codepack'].empty(
) and supervisor.messenger.queues['codepack'].qsize() == 3
items = [
supervisor.messenger.queues['codepack'].get(block=False)
for _ in range(3)
]
snapshots = [
c.to_snapshot(kwargs=argpack[i].to_dict()).to_dict()
for i, c in codepack.codes.items()
]
for item in items:
item.pop('timestamp', None)
for snapshot in snapshots:
snapshot.pop('timestamp', None)
assert sorted(items,
key=lambda x: x['id']) == sorted(snapshots,
key=lambda x: x['id'])
supervisor.close()
async def state(serial_number: str):
snapshot_service = Default.get_service('codepack_snapshot',
'snapshot_service')
tmp = snapshot_service.load(serial_number=serial_number)
ret = {'serial_number': serial_number}
if tmp:
codepack_snapshot = CodePackSnapshot.from_dict(tmp)
codepack = CodePack.from_snapshot(codepack_snapshot)
_state = codepack.get_state()
if _state == 'ERROR':
ret['message'] = codepack.get_message()
else:
_state = 'UNKNOWN'
ret['state'] = _state
return ret
def test_codepack_to_snapshot_and_from_snapshot(default_os_env):
c1 = Code(add3)
c2 = Code(mul2)
c3 = Code(combination)
c4 = Code(linear)
c5 = Code(print_x)
c1 >> c3
c2 >> c3
c3 >> [c4, c5]
c3.receive('c') << c1
c4.receive('c') << c3
c5.receive('x') << c3
cp1 = CodePack(id='test_codepack', code=c1, subscribe=c4)
argpack = cp1.make_argpack()
argpack['add3']['a'] = 1
argpack['add3']['b'] = 2
argpack['add3']['c'] = 3
argpack['mul2']['a'] = 2
argpack['mul2']['b'] = 1
argpack['combination']['a'] = 2
argpack['combination']['b'] = 5
argpack['linear']['b'] = 7
argpack['linear']['a'] = 5
ret = None
with pytest.raises(TypeError):
ret = cp1(argpack)
assert ret is None
assert cp1.get_state() == 'ERROR'
snapshot1 = cp1.to_snapshot(argpack=argpack)
assert snapshot1.owner is None
snapshot1['owner'] = 'codepack'
assert snapshot1.owner == 'codepack'
cp2 = CodePack.from_snapshot(snapshot1)
assert cp2.owner == 'codepack'
assert cp1.id == cp2.id
assert cp1.serial_number == cp2.serial_number
assert cp1.get_state() == cp2.get_state()
cp1_source = cp1.get_source()
cp2_source = cp2.get_source()
assert cp1_source.keys() == cp2_source.keys()
for id in cp1_source.keys():
assert cp1_source[id].strip() == cp2_source[id].strip()
# assert cp1.get_structure() == cp2.get_structure()
assert cp1.subscribe == cp2.subscribe
assert set(cp1.codes.keys()) == set(cp2.codes.keys())
assert cp1.owner is None
assert cp2.owner == 'codepack'
for code_id in cp1.codes.keys():
code1 = cp1.codes[code_id]
code2 = cp2.codes[code_id]
assert code1.serial_number == code2.serial_number
assert code1.parents.keys() == code2.parents.keys()
for id in code1.parents.keys():
assert code1.parents[id].serial_number == code2.parents[id].serial_number
assert code1.children.keys() == code2.children.keys()
for id in code1.children.keys():
assert code1.children[id].serial_number == code2.children[id].serial_number
assert code1.get_state() == code2.get_state()
assert code1.dependency.keys() == code2.dependency.keys()
for serial_number in code1.dependency.keys():
assert code1.dependency[serial_number].id == code2.dependency[serial_number].id
assert code1.dependency[serial_number].serial_number == code2.dependency[serial_number].serial_number
assert code1.dependency[serial_number].arg == code2.dependency[serial_number].arg
async def update(codepack: CodePackJSON):
tmp = CodePack.from_json(codepack.codepack)
tmp.save(update=True)
return {'id': tmp.id}
async def save(codepack: CodePackJSON):
tmp = CodePack.from_json(codepack.codepack)
tmp.save()
return {'id': tmp.id}
async def run_by_snapshot(params: SnapshotJSON):
snapshot = CodePackSnapshot.from_json(params.snapshot)
codepack = CodePack.from_snapshot(snapshot)
argpack = ArgPack.from_dict(snapshot.argpack)
common.supervisor.run_codepack(codepack=codepack, argpack=argpack)
return {'serial_number': codepack.serial_number}
async def run(params: CodePackJSON):
codepack = CodePack.from_json(params.codepack)
argpack = ArgPack.from_json(params.argpack)
common.supervisor.run_codepack(codepack=codepack, argpack=argpack)
return {'serial_number': codepack.serial_number}
