def test_file_delivery_service_check(testdir_delivery_service):
storage = FileStorage(item_type=Delivery, path=testdir_delivery_service)
fds = DeliveryService(storage=storage)
sender = 'sender'
serial_number = '1234'
serial_numbers = ['123', '456', '789']
try:
fds.cancel(serial_number=serial_number)
except Exception:
pass
for i in serial_numbers:
try:
fds.cancel(serial_number=i)
except Exception:
continue
item = 'Hello, World!'
fds.send(id=sender, serial_number=serial_number, item=item)
check = fds.check(serial_number=serial_number)
assert isinstance(check, bool)
assert check is True
fds.cancel(serial_number=serial_number)
check = fds.check(serial_number=serial_number)
assert check is False
fds.send(id=sender, serial_number=serial_numbers[0], item=item)
fds.send(id=sender, serial_number=serial_numbers[1], item=item)
check = fds.check(serial_number=serial_numbers)
assert isinstance(check, bool)
assert check is False
fds.send(id=sender, serial_number=serial_numbers[2], item=item)
check = fds.check(serial_number=serial_numbers)
assert check is True
def test_file_storage_list_all(testdir_file_storage, dummy_deliveries):
storage = FileStorage(item_type=Delivery,
key='id',
path=testdir_file_storage)
assert storage.key == 'id'
storage.save(item=dummy_deliveries)
tmp = sorted(os.listdir(testdir_file_storage))
assert tmp == sorted(['%s.json' % d.id for d in dummy_deliveries])
dummy_keys = [d.id for d in dummy_deliveries]
all_keys = storage.list_all()
assert sorted(all_keys) == sorted(dummy_keys)
all_items = storage.load(key=all_keys)
assert type(all_items) == list and len(all_items) == 3
assert sorted([d.id for d in all_items]) == sorted(all_keys)
storage.remove(key=all_keys)
assert len(os.listdir(testdir_file_storage)) == len(
storage.list_all()) == 0
def test_file_storage_jobstore_codepack_snapshot(testdir):
code = Code(hello)
codepack = CodePack('codepack_test', code=code, subscribe=code)
argpack = codepack.make_argpack()
argpack['hello']['name'] = 'CodePack'
job_id = 'job_test'
storage = FileStorage(item_type=StorableJob,
key='id',
path='testdir/scheduler/')
jobstore = JobStore(storage=storage)
scheduler = Scheduler(jobstore=jobstore, blocking=False)
scheduler.add_codepack(job_id=job_id,
codepack=codepack,
argpack=argpack,
trigger='interval',
seconds=30)
scheduler.start()
assert scheduler.is_running()
assert not storage.exist(key=codepack.id)
assert storage.exist(key=job_id)
job = storage.load(key=job_id, to_dict=True)
for k in [
'_id', 'trigger', 'codepack', 'snapshot', 'last_run_time',
'next_run_time'
]:
assert k in job
assert job['_id'] == job_id
assert job['trigger'] == 'interval[0:00:30]'
assert job['codepack'] == codepack.id
assert job['snapshot'] == codepack.serial_number
scheduler.remove_job(job_id)
assert len(storage.list_all()) == 0
scheduler.stop()
def test_file_code_snapshot_service_save_and_load(default_os_env,
testdir_snapshot_service):
storage = FileStorage(item_type=CodeSnapshot,
path=testdir_snapshot_service)
fss = SnapshotService(storage=storage)
code = Code(add2)
snapshot = CodeSnapshot(code)
fss.save(snapshot=snapshot)
assert len(os.listdir(testdir_snapshot_service)) == 1
assert fss.load(snapshot.serial_number) == snapshot.to_dict()
loaded = fss.load(snapshot.serial_number, projection=['state'])
assert set(loaded.keys()) == {'serial_number', 'state'}
assert 'state' in loaded
assert loaded['state'] == 'UNKNOWN'
assert 'serial_number' in loaded
assert loaded['serial_number'] == snapshot.serial_number
def test_file_snapshot_service_update(testdir_snapshot_service):
storage = FileStorage(item_type=Snapshot, path=testdir_snapshot_service)
fss = SnapshotService(storage=storage)
timestamp = datetime.now().timestamp()
snapshot1 = Snapshot(id='1234', serial_number='5678', timestamp=timestamp)
snapshot2 = Snapshot(id='1234',
serial_number='5678',
timestamp=timestamp + 1)
fss.save(snapshot=snapshot1)
fss.save(snapshot=snapshot2)
assert len(os.listdir(testdir_snapshot_service)) == 1
search_result = fss.search(key='id',
value='1234',
projection=['timestamp'])
assert len(search_result) == 1
assert search_result[0]['timestamp'] == timestamp + 1
def test_file_delivery_service(testdir_delivery_service):
storage = FileStorage(item_type=Delivery, path=testdir_delivery_service)
fds = DeliveryService(storage=storage)
sender = 'sender'
serial_number = '1234'
item = 'Hello, World!'
fds.send(id=sender, serial_number=serial_number, item=item)
assert os.path.isfile(
Delivery.get_path(key=serial_number,
path=storage.path)), "'send' failed"
check = fds.check(serial_number=serial_number)
receive = fds.receive(serial_number=serial_number)
assert check is True
assert receive == item, "'receive' error"
fds.cancel(serial_number=serial_number)
check2 = fds.check(serial_number=serial_number)
assert check2 is False, "'cancel' error"
def test_file_storage_service(default_os_env, testdir_storage_service):
filepath = testdir_storage_service
storage = FileStorage(item_type=Code, path=filepath)
fss = StorageService(storage=storage)
assert storage.path == filepath
code1 = Code(hello, storage_service=fss)
code2 = Code(add2, storage_service=fss)
code1.save()
assert not os.path.isfile(Code.get_path(key=code1.id))
assert os.path.isfile(Code.get_path(key=code1.id, path=filepath))
assert not os.path.isfile(Code.get_path(key=code2.id, path=filepath))
code3 = fss.load(code1.id)
assert code1.id == code3.id
assert code1.source.strip() == code3.source.strip()
assert code1("CodePack") == code3("CodePack")
fss.remove(code3.id)
assert not os.path.isfile(Code.get_path(key=code1.id, path=filepath))
def test_file_code_snapshot_service_search_and_remove(
default_os_env, testdir_snapshot_service):
storage = FileStorage(item_type=CodeSnapshot,
path=testdir_snapshot_service)
fss = SnapshotService(storage=storage)
code1 = Code(add2)
code2 = Code(add3)
code1 >> code2
code2(1, 2, 3)
snapshot1 = CodeSnapshot(code1)
snapshot2 = CodeSnapshot(code2)
fss.save(snapshot=snapshot1)
fss.save(snapshot=snapshot2)
assert len(os.listdir(testdir_snapshot_service)) == 2
loaded = fss.load(
[snapshot1.serial_number, snapshot2.serial_number, '1234'])
assert len(loaded) == 2
assert loaded[0]['state'] == 'UNKNOWN'
assert loaded[1]['state'] == 'WAITING'
loaded = fss.load(
[snapshot1.serial_number, snapshot2.serial_number, '1234'],
projection=['state'])
for x in loaded:
assert set(x.keys()) == {'state', 'serial_number'}
search_result = fss.search(key='state', value='WAITING')
assert len(search_result) == 1
assert search_result[0] == snapshot2.to_dict()
search_result = fss.search(key='state',
value='WAITING',
projection=['state'])
assert len(search_result) == 1
assert len(search_result[0].keys()) == 2
assert search_result[0]['state'] == 'WAITING'
assert search_result[0]['serial_number'] == code2.serial_number
fss.remove(snapshot2.serial_number)
loaded = fss.load(
[snapshot1.serial_number, snapshot2.serial_number, '1234'])
assert len(loaded) == 1
search_result = fss.search(key='state',
value='WAITING',
projection=['state'])
assert len(search_result) == 0
def test_file_storage_service_check(default_os_env, testdir_storage_service):
storage = FileStorage(item_type=Code, path=testdir_storage_service)
assert storage.key == 'serial_number'
fss = StorageService(storage=storage)
assert storage.key == 'id'
code1 = Code(hello, id='hello1', storage_service=fss)
code2 = Code(hello, id='hello2', storage_service=fss)
code3 = Code(hello, id='hello3', storage_service=fss)
for i in ['hello1', 'hello2', 'hello3']:
try:
fss.remove(id=i)
except Exception:
continue
code1.save()
code2.save()
check = fss.check([code1.id, code2.id, code3.id])
assert isinstance(check, list)
assert check[0] and check[1] and not check[2]
fss.remove(id=code1.id)
code3.save()
check = fss.check([code1.id, code2.id, code3.id])
assert isinstance(check, list)
assert not check[0] and check[1] and check[2]
assert len(check) == 3
'--callback',
action='append',
help='callback functions for the snapshot')
parser.add_argument(
'-p',
'--path',
help='path to the directory including callback functions')
parser.add_argument('-l', '--logger', help='a logger name to write logs')
args = parser.parse_args()
if args.logger:
logger = Default.get_logger(args.logger)
sys.stdout.write = partial(Worker.log, logger.info)
else:
logger = None
try:
callback_storage = FileStorage(item_type=Callback,
path=args.path if args.path else '.')
callback_service = CallbackService(storage=callback_storage)
code_snapshot = CodeSnapshot.from_file(args.snapshot)
code_args = code_snapshot.args
code_kwargs = code_snapshot.kwargs
code = Code.from_snapshot(code_snapshot)
if args.callback:
callbacks = list()
for name in args.callback:
cb = callback_service.pull(name)
callbacks.append(cb)
code.register_callback(callbacks)
code(*code_args, **code_kwargs)
except Exception as e:
if logger:
logger.error(e)
def test_file_storage(testdir_file_storage, dummy_deliveries):
storage = FileStorage(item_type=Delivery,
key='id',
path=testdir_file_storage)
assert storage.key == 'id'
storage.save(item=dummy_deliveries)
assert sorted(os.listdir(testdir_file_storage)) == sorted(
['%s.json' % d.id for d in dummy_deliveries])
with pytest.raises(ValueError):
storage.save(item=dummy_deliveries[0])
assert dummy_deliveries[0].item == 'x'
dummy_deliveries[0].send(item='z')
storage.save(item=dummy_deliveries[0], update=True)
storage.remove(key=[dummy_deliveries[1].id, dummy_deliveries[2].id])
assert storage.exist(key=[d.id for d in dummy_deliveries]) == [
True, False, False
]
assert storage.exist(key=[d.id for d in dummy_deliveries],
summary='and') is False
assert storage.exist(key=[d.id
for d in dummy_deliveries], summary='or') is True
storage.save(item=[dummy_deliveries[1], dummy_deliveries[2]])
assert storage.exist(key=[d.id for d in dummy_deliveries]) == [
True, True, True
]
assert storage.exist(key=[d.id for d in dummy_deliveries],
summary='and') is True
assert storage.exist(key=[d.id
for d in dummy_deliveries], summary='or') is True
search_result = storage.search(key='item', value=json.dumps('?'))
assert type(search_result) == list and len(search_result) == 0
ref = sorted([dummy_deliveries[1].id, dummy_deliveries[2].id])
search_result = storage.search(key='item', value=json.dumps('z'))
assert len(
search_result) == 1 and search_result[0].id == dummy_deliveries[0].id
search_result = storage.search(key='item', value=json.dumps('y'))
assert len(search_result) == 2
assert sorted([s.id for s in search_result]) == ref
search_result = storage.search(key='item',
value=json.dumps('y'),
to_dict=True)
assert len(search_result) == 2
assert sorted([s['id'] for s in search_result]) == ref
assert type(search_result[0]) == dict and type(search_result[1]) == dict
assert set(search_result[0].keys()) == {
'item', 'id', '_id', 'timestamp', 'serial_number'
}
assert set(search_result[1].keys()) == {
'item', 'id', '_id', 'timestamp', 'serial_number'
}
search_result = storage.search(key='item',
value=json.dumps('y'),
projection=['item'])
assert len(search_result) == 2
assert type(search_result[0]) == dict and type(search_result[1]) == dict
assert set(search_result[0].keys()) == {'item', 'id'}
assert set(search_result[1].keys()) == {'item', 'id'}
load_result = storage.load(key='???')
assert load_result is None
load_result = storage.load(key=['!??', '?!?', '??!'])
assert type(load_result) == list and len(load_result) == 0
load_result = storage.load(key=[d.id for d in dummy_deliveries])
assert type(load_result) == list and len(load_result) == len(
dummy_deliveries)
for i in range(len(load_result)):
assert load_result[i].id == dummy_deliveries[i].id
assert load_result[i].item == dummy_deliveries[i].item
assert load_result[i].serial_number == dummy_deliveries[
i].serial_number
load_result = storage.load(key=dummy_deliveries[1].id)
assert isinstance(load_result, Delivery)
load_result = storage.load(
key=[dummy_deliveries[1].id, dummy_deliveries[2].id, '???'],
to_dict=True)
assert type(load_result) == list and len(load_result) == 2
assert type(load_result[0]) == dict and type(load_result[1]) == dict
assert set(load_result[0].keys()) == {
'item', 'id', '_id', 'timestamp', 'serial_number'
}
assert set(load_result[1].keys()) == {
'item', 'id', '_id', 'timestamp', 'serial_number'
}
load_result = storage.load(
key=[dummy_deliveries[1].id, dummy_deliveries[2].id],
projection=['timestamp', '_id'])
assert type(load_result) == list
assert type(load_result[0]) == dict and type(load_result[1]) == dict
assert set(load_result[0].keys()) == {'id', '_id', 'timestamp'}
assert set(load_result[1].keys()) == {'id', '_id', 'timestamp'}
storage.new_path = True
storage.close()
assert not os.path.exists(testdir_file_storage)