def test_md_magic():
d = MergeDict()
repr(d)
dir(d)
d.copy()
with pytest.raises(AttributeError):
d.a
d(a=1)
assert d.a == 1
def setup(cls, app, conf):
context = Context({}, loop=app.loop)
context.app = app
conf = MergeDict(conf)
conf.name = 'mailer'
m = cls(conf, context=context, loop=app.loop)
async def start(app):
await m.init()
await m.start()
app.on_startup.append(start)
app.on_shutdown.append(lambda x: m.stop())
async def test_key(loop):
with tempfile.TemporaryDirectory() as d:
config = MergeDict(
name='',
path=d,
executor=None,
)
context = Context({}, loop=loop)
storage = FileSystemStorage(config, context=context, loop=loop)
await storage.init()
assert str(storage.raw_key(('1', '3', ('4', )))).endswith('1/3/4')
assert str(storage.raw_key(
(PurePath('1'), '3', ('4', )))).endswith('1/3/4')
with pytest.raises(TypeError):
storage.raw_key(1)
with pytest.raises(ValueError):
storage.raw_key('../..')
with pytest.raises(ValueError):
storage.raw_key('/abs/path')
with pytest.raises(ValueError):
storage.raw_key(PurePath('/abs/path'))
async def test_set_get(loop):
key = ('4423', '123')
data = b'234'
with tempfile.TemporaryDirectory() as d:
config = MergeDict(
name='',
path=d,
executor=None,
)
context = Context({}, loop=loop)
storage = FileSystemStorage(config, context=context, loop=loop)
await storage.init()
await storage.set(('empty', 'value', '1'), None)
assert not await storage.get('empty value 2')
await storage.set(key, data)
await storage.set(key, None) # del file
assert not await storage.get(key)
await storage.set(key[0], None) # del dir
await storage.set(key, data)
d = await asyncio.gather(storage.get(key), storage.get(key), loop=loop)
for j in d:
assert data == j
async def test_context_create(loop):
conf = Config()
conf.update(
MergeDict({
'q.cls': 'aioworkers.queue.timeout.TimestampQueue',
'f.e': 1,
}))
c = Context(conf, loop=loop)
await c.init()
await c.start()
assert c.config.f.e == 1
with pytest.raises(AttributeError):
c.r
with pytest.raises(KeyError):
c['r']
async def handler(context):
pass
c.on_stop.append(handler)
async def handler():
raise ValueError
c.on_stop.append(handler)
c.on_stop.append(handler())
c.on_stop.append(1)
await c.stop()
async def test_copy(loop):
key1 = '1'
key2 = '2'
key3 = '3'
key4 = ('4', '5')
data = b'234'
with tempfile.TemporaryDirectory() as d:
config = MergeDict(
name='',
path=d,
executor=None,
)
context = Context({}, loop=loop)
storage = FileSystemStorage(config, context=context, loop=loop)
await storage.init()
await storage.set(key1, data)
await storage.copy(key1, storage, key3)
assert data == await storage.get(key3)
await storage.copy(key2, storage, key3)
await storage.move(key2, storage, key4)
assert not await storage.get(key4)
fstor = FutureStorage(mock.Mock(name=''), loop=loop)
await fstor.init()
await storage.copy(key2, fstor, key3)
await storage.move(key2, fstor, key4)
async def test_func(loop):
config = MergeDict(now={
'func': 'time.monotonic',
})
context = Context(config, loop=loop)
async with context:
assert isinstance(context.now, float)
async def test_ts_zqueue(loop, mocker):
config = MergeDict(
key=str(uuid.uuid4()),
format='str',
timeout=10,
)
config['app.redis_pool'] = await aioredis.create_pool(('localhost', 6379),
loop=loop)
context = config
q = TimestampZQueue(config, context=context, loop=loop)
await q.init()
async def breaker(*args, **kwargs):
q._lock.release()
raise InterruptedError
with pytest.raises(InterruptedError):
with mock.patch('asyncio.sleep', breaker):
await q.get()
await q.put('c', time.time() + 4)
await q.put('a', 4)
assert 2 == await q.length()
assert ['a', 'c'] == await q.list()
assert 'a' == await q.get()
assert 1 == await q.length()
assert ['c'] == await q.list()
with pytest.raises(InterruptedError):
with mock.patch('asyncio.sleep', breaker):
await q.get()
async def test_zqueue(loop, mocker):
config = MergeDict(
key=str(uuid.uuid4()),
format='str',
timeout=0,
)
config['app.redis_pool'] = await aioredis.create_pool(('localhost', 6379),
loop=loop)
context = config
q = RedisZQueue(config, context=context, loop=loop)
await q.init()
await q.put('a', 4)
await q.put('c', 3)
await q.put('b', 2)
await q.put('a', 1)
assert 3 == await q.length()
assert ['a', 'b', 'c'] == await q.list()
assert 3 == await q.length()
assert 'a' == await q.get()
assert ['b', 'c'] == await q.list()
assert 2 == await q.length()
assert 'b' == await q.get()
assert ['c'] == await q.list()
assert 1 == await q.length()
assert 'c' == await q.get()
assert [] == await q.list()
assert not await q.length()
with pytest.raises(TypeError):
with mocker.patch('asyncio.sleep'):
await q.get()
async def test_1(loop):
config = MergeDict(
name='',
autorun=False,
)
context = Context(config, loop=loop)
worker = BaseUpdater(config, context=context, loop=loop)
await worker.init()
def test_merge():
d = MergeDict(f=4)
d(dict(g=3))
assert d.g == 3, d
assert d.f == 4, d
d(dict(f=dict(g=2)), dict(f=4))
assert d.g == 3, d
assert d.f == 4, d
async def test_autorun(loop):
config = MergeDict(name='', autorun=True)
context = Context(config, loop=loop)
worker = Worker(config, context=context, loop=loop)
await worker.init()
await context.start()
await worker._future
assert worker._started_at
assert not worker.running()
async def test_format(loop):
config = MergeDict(
name='',
semaphore=1,
format='bytes',
)
context = Context(config=config, loop=loop)
await context.init()
storage = Storage(config, context=context, loop=loop)
await storage.init()
assert isinstance(storage.raw_key('test'), URL)
await storage.stop()
async def test_score_time(loop):
q = ScoreQueue(MergeDict(default_score='time.time', ), loop=loop)
await q.init()
await q.put(2)
assert 2 == await q.get()
await q.put(3)
await q.put(4)
a1, t1 = await q.get(score=True)
a2, t2 = await q.get(score=True)
assert a1 == 3
assert a2 == 4
assert t1 < t2
assert not q
async def test_set_get(loop, test_client):
app = web.Application()
app.router.add_get(
'/test/1',
lambda x: web.json_response(["Python"]))
client = await test_client(app)
url = client.make_url('/')
data = 'Python'
config = MergeDict(
storage=MergeDict(
cls='aioworkers.storage.http.Storage',
prefix=str(url),
semaphore=1,
format='json',
),
)
async with Context(config=config, loop=loop) as context:
storage = context.storage
assert data in await storage.get('test/1')
with pytest.raises(StorageError):
await storage.set('test/1', data)
async def test_crontab(loop, mocker):
config = MergeDict(
name='',
autorun=True,
persist=True,
crontab='*/1 * * * *',
)
context = Context(config, loop=loop)
worker = Worker(config, context=context, loop=loop)
await worker.init()
await context.start()
await asyncio.sleep(0.1, loop=loop)
await worker.stop()
assert not worker.running()
async def test_pickle(loop):
key = '4423'
data = {'f': 3}
with tempfile.TemporaryDirectory() as d:
config = MergeDict(
name='',
path=d,
format='pickle',
executor=None,
)
context = Context({}, loop=loop)
storage = HashFileSystemStorage(config, context=context, loop=loop)
await storage.init()
await storage.set(key, data)
assert data == await storage.get(key)
async def test_queue(loop):
config = MergeDict(key=str(uuid.uuid4()))
config['app.redis_pool'] = await aioredis.create_pool(('localhost', 6379),
loop=loop)
context = config
q = RedisQueue(config, context=context, loop=loop)
await q.init()
await q.put(3)
assert 1 == await q.length()
assert [b'3'] == await q.list()
assert b'3' == await q.get()
await q.put(3)
assert 1 == await q.length()
await q.clear()
assert not await q.length()
async def test_stop(loop, mocker):
config = MergeDict(
name='',
autorun=True,
persist=True,
sleep=0.01,
sleep_start=0.01,
)
context = Context(config, loop=loop)
worker = Worker(config, context=context, loop=loop)
await worker.init()
await context.start()
await asyncio.sleep(0.1, loop=loop)
await worker.stop()
assert not worker.running()
assert isinstance(await worker.status(), dict)
def test_run(loop):
ns = Namespace()
config = MergeDict()
config['a.b'] = 2
config['c'] = 'a.b'
config['d'] = 's'
with Context(config, loop=loop) as ctx:
run('time.time', ctx, ns=ns, argv=[])
assert run('a.b', ctx, ns=ns, argv=[]) == 2
assert run('c', ctx, ns=ns, argv=[]) == 2
assert run('tests.test_command.coro', ctx, ns=ns, argv=[]) == 2
assert run('d', ctx, ns=ns, argv=[]) == 's'
with pytest.raises(CommandNotFound):
run('not found', ctx, ns=ns, argv=[])
with pytest.raises(CommandNotFound):
run('time.time2', ctx, ns=ns, argv=[])
async def test_chunk(loop):
key4 = ('4', '5')
data = b'234'
with tempfile.TemporaryDirectory() as d:
config = MergeDict(
name='',
path=d,
executor=None,
)
context = Context({}, loop=loop)
storage = FileSystemStorage(config, context=context, loop=loop)
await storage.init()
f = await storage._open(key4, 'wb')
await storage._write_chunk(f, data)
await storage._close(f)
assert data == await storage.get(key4)
async def test_storage(loop):
config = MergeDict(
name='1',
prefix=str(uuid.uuid4()),
format='json',
)
config['app.redis_pool'] = await aioredis.create_pool(('localhost', 6379),
loop=loop)
context = config
q = RedisStorage(config, context=context, loop=loop)
await q.init()
await q.set('g', {'f': 3})
assert {'f': 3} == await q.get('g')
assert 1 == await q.length()
assert ['g'] == await q.list()
await q.set('g', None)
assert not await q.length()
async def test_queue_json(loop):
config = MergeDict(
key=str(uuid.uuid4()),
format='json',
)
config['app.redis_pool'] = await aioredis.create_pool(('localhost', 6379),
loop=loop)
context = config
q = RedisQueue(config, context=context, loop=loop)
await q.init()
await q.put({'f': 3})
assert 1 == await q.length()
assert [{'f': 3}] == await q.list()
assert {'f': 3} == await q.get()
await q.put({'f': 3})
assert 1 == await q.length()
await q.clear()
assert not await q.length()
async def test_autorun(loop):
config = MergeDict(
name='',
autorun=True,
children=1,
child={
'cls': 'aioworkers.worker.base.Worker',
},
)
context = Context(config, loop=loop)
worker = Supervisor(config, context=context, loop=loop)
await worker.init()
await context.start()
await worker._future
assert worker._started_at
assert not worker.running()
await worker.stop()
assert await worker.status()
async def test_coro_run(loop, mocker):
f = loop.create_future()
async def myrun(*args, **kwargs):
f.set_result(10)
config = MergeDict(
name='',
autorun=True,
run='mocked.run',
)
context = Context(config, loop=loop)
worker = Worker(config, context=context, loop=loop)
mocker.patch('aioworkers.worker.base.import_name',
lambda x: myrun)
await worker.init()
await context.start()
assert worker._started_at
await worker._future
assert worker._stopped_at
assert f.result() == 10
async def test_field_storage(loop):
key = ('5', '6')
data = {'f': 3, 'g': 4, 'h': 5}
fields = ['f', 'g']
with tempfile.TemporaryDirectory() as d:
config = MergeDict(
name='',
path=d,
format='json',
executor=None,
)
context = Context({}, loop=loop)
storage = Store(config, context=context, loop=loop)
await storage.init()
await storage.set(key, data)
assert data == await storage.get(key)
assert 5 == await storage.get(key, field='h')
await storage.set(key, 6, field='h')
assert {'f': 3, 'g': 4} == await storage.get(key, fields=fields)
await storage.set(key, {'z': 1, 'y': 6}, fields=['z'])
assert {'f': 3, 'g': 4, 'h': 6, 'z': 1} == await storage.get(key)
await storage.set(key, None)
async def test_freespace(loop):
key1 = '1'
key2 = '2'
data = b'000'
with tempfile.TemporaryDirectory() as d:
config = MergeDict(
name='',
path=d,
executor=None,
)
context = Context({}, loop=loop)
storage = FileSystemStorage(config, context=context, loop=loop)
await storage.init()
with mock.patch.object(storage, 'get_free_space',
asyncio.coroutine(lambda: 1)):
assert 1 == await storage.get_free_space()
storage.config.limit_free_space = 2
f = asyncio.ensure_future(storage.set(key1, data), loop=loop)
await asyncio.sleep(0, loop=loop)
await storage.set(key2, data)
await storage.set(key1, None)
assert not storage._space_waiters
await f
def test_dict_set4():
d = MergeDict(f=3, d=dict(g=dict()))
d['d.g'] = dict(r=4)
assert d.d.g.r == 4, d
def test_dict_set3():
d = MergeDict(f=3, d=dict(g=1))
d.d = dict(r=4)
assert d.d.r == 4, d
assert d.d.g == 1, d
def test_dict_replace3():
d = MergeDict(f=3, d=dict(g=1))
d['d!'] = dict(r=4)
assert d.d.r == 4, d
