def test_md_magic(): d = MergeDict() repr(d) dir(d) d.copy() with pytest.raises(AttributeError): d.a d(a=1) assert d.a == 1
def setup(cls, app, conf): context = Context({}, loop=app.loop) context.app = app conf = MergeDict(conf) conf.name = 'mailer' m = cls(conf, context=context, loop=app.loop) async def start(app): await m.init() await m.start() app.on_startup.append(start) app.on_shutdown.append(lambda x: m.stop())
async def test_key(loop): with tempfile.TemporaryDirectory() as d: config = MergeDict( name='', path=d, executor=None, ) context = Context({}, loop=loop) storage = FileSystemStorage(config, context=context, loop=loop) await storage.init() assert str(storage.raw_key(('1', '3', ('4', )))).endswith('1/3/4') assert str(storage.raw_key( (PurePath('1'), '3', ('4', )))).endswith('1/3/4') with pytest.raises(TypeError): storage.raw_key(1) with pytest.raises(ValueError): storage.raw_key('../..') with pytest.raises(ValueError): storage.raw_key('/abs/path') with pytest.raises(ValueError): storage.raw_key(PurePath('/abs/path'))
async def test_set_get(loop): key = ('4423', '123') data = b'234' with tempfile.TemporaryDirectory() as d: config = MergeDict( name='', path=d, executor=None, ) context = Context({}, loop=loop) storage = FileSystemStorage(config, context=context, loop=loop) await storage.init() await storage.set(('empty', 'value', '1'), None) assert not await storage.get('empty value 2') await storage.set(key, data) await storage.set(key, None) # del file assert not await storage.get(key) await storage.set(key[0], None) # del dir await storage.set(key, data) d = await asyncio.gather(storage.get(key), storage.get(key), loop=loop) for j in d: assert data == j
async def test_context_create(loop): conf = Config() conf.update( MergeDict({ 'q.cls': 'aioworkers.queue.timeout.TimestampQueue', 'f.e': 1, })) c = Context(conf, loop=loop) await c.init() await c.start() assert c.config.f.e == 1 with pytest.raises(AttributeError): c.r with pytest.raises(KeyError): c['r'] async def handler(context): pass c.on_stop.append(handler) async def handler(): raise ValueError c.on_stop.append(handler) c.on_stop.append(handler()) c.on_stop.append(1) await c.stop()
async def test_copy(loop): key1 = '1' key2 = '2' key3 = '3' key4 = ('4', '5') data = b'234' with tempfile.TemporaryDirectory() as d: config = MergeDict( name='', path=d, executor=None, ) context = Context({}, loop=loop) storage = FileSystemStorage(config, context=context, loop=loop) await storage.init() await storage.set(key1, data) await storage.copy(key1, storage, key3) assert data == await storage.get(key3) await storage.copy(key2, storage, key3) await storage.move(key2, storage, key4) assert not await storage.get(key4) fstor = FutureStorage(mock.Mock(name=''), loop=loop) await fstor.init() await storage.copy(key2, fstor, key3) await storage.move(key2, fstor, key4)
async def test_func(loop): config = MergeDict(now={ 'func': 'time.monotonic', }) context = Context(config, loop=loop) async with context: assert isinstance(context.now, float)
async def test_ts_zqueue(loop, mocker): config = MergeDict( key=str(uuid.uuid4()), format='str', timeout=10, ) config['app.redis_pool'] = await aioredis.create_pool(('localhost', 6379), loop=loop) context = config q = TimestampZQueue(config, context=context, loop=loop) await q.init() async def breaker(*args, **kwargs): q._lock.release() raise InterruptedError with pytest.raises(InterruptedError): with mock.patch('asyncio.sleep', breaker): await q.get() await q.put('c', time.time() + 4) await q.put('a', 4) assert 2 == await q.length() assert ['a', 'c'] == await q.list() assert 'a' == await q.get() assert 1 == await q.length() assert ['c'] == await q.list() with pytest.raises(InterruptedError): with mock.patch('asyncio.sleep', breaker): await q.get()
async def test_zqueue(loop, mocker): config = MergeDict( key=str(uuid.uuid4()), format='str', timeout=0, ) config['app.redis_pool'] = await aioredis.create_pool(('localhost', 6379), loop=loop) context = config q = RedisZQueue(config, context=context, loop=loop) await q.init() await q.put('a', 4) await q.put('c', 3) await q.put('b', 2) await q.put('a', 1) assert 3 == await q.length() assert ['a', 'b', 'c'] == await q.list() assert 3 == await q.length() assert 'a' == await q.get() assert ['b', 'c'] == await q.list() assert 2 == await q.length() assert 'b' == await q.get() assert ['c'] == await q.list() assert 1 == await q.length() assert 'c' == await q.get() assert [] == await q.list() assert not await q.length() with pytest.raises(TypeError): with mocker.patch('asyncio.sleep'): await q.get()
async def test_1(loop): config = MergeDict( name='', autorun=False, ) context = Context(config, loop=loop) worker = BaseUpdater(config, context=context, loop=loop) await worker.init()
def test_merge(): d = MergeDict(f=4) d(dict(g=3)) assert d.g == 3, d assert d.f == 4, d d(dict(f=dict(g=2)), dict(f=4)) assert d.g == 3, d assert d.f == 4, d
async def test_autorun(loop): config = MergeDict(name='', autorun=True) context = Context(config, loop=loop) worker = Worker(config, context=context, loop=loop) await worker.init() await context.start() await worker._future assert worker._started_at assert not worker.running()
async def test_format(loop): config = MergeDict( name='', semaphore=1, format='bytes', ) context = Context(config=config, loop=loop) await context.init() storage = Storage(config, context=context, loop=loop) await storage.init() assert isinstance(storage.raw_key('test'), URL) await storage.stop()
async def test_score_time(loop): q = ScoreQueue(MergeDict(default_score='time.time', ), loop=loop) await q.init() await q.put(2) assert 2 == await q.get() await q.put(3) await q.put(4) a1, t1 = await q.get(score=True) a2, t2 = await q.get(score=True) assert a1 == 3 assert a2 == 4 assert t1 < t2 assert not q
async def test_set_get(loop, test_client): app = web.Application() app.router.add_get( '/test/1', lambda x: web.json_response(["Python"])) client = await test_client(app) url = client.make_url('/') data = 'Python' config = MergeDict( storage=MergeDict( cls='aioworkers.storage.http.Storage', prefix=str(url), semaphore=1, format='json', ), ) async with Context(config=config, loop=loop) as context: storage = context.storage assert data in await storage.get('test/1') with pytest.raises(StorageError): await storage.set('test/1', data)
async def test_crontab(loop, mocker): config = MergeDict( name='', autorun=True, persist=True, crontab='*/1 * * * *', ) context = Context(config, loop=loop) worker = Worker(config, context=context, loop=loop) await worker.init() await context.start() await asyncio.sleep(0.1, loop=loop) await worker.stop() assert not worker.running()
async def test_pickle(loop): key = '4423' data = {'f': 3} with tempfile.TemporaryDirectory() as d: config = MergeDict( name='', path=d, format='pickle', executor=None, ) context = Context({}, loop=loop) storage = HashFileSystemStorage(config, context=context, loop=loop) await storage.init() await storage.set(key, data) assert data == await storage.get(key)
async def test_queue(loop): config = MergeDict(key=str(uuid.uuid4())) config['app.redis_pool'] = await aioredis.create_pool(('localhost', 6379), loop=loop) context = config q = RedisQueue(config, context=context, loop=loop) await q.init() await q.put(3) assert 1 == await q.length() assert [b'3'] == await q.list() assert b'3' == await q.get() await q.put(3) assert 1 == await q.length() await q.clear() assert not await q.length()
async def test_stop(loop, mocker): config = MergeDict( name='', autorun=True, persist=True, sleep=0.01, sleep_start=0.01, ) context = Context(config, loop=loop) worker = Worker(config, context=context, loop=loop) await worker.init() await context.start() await asyncio.sleep(0.1, loop=loop) await worker.stop() assert not worker.running() assert isinstance(await worker.status(), dict)
def test_run(loop): ns = Namespace() config = MergeDict() config['a.b'] = 2 config['c'] = 'a.b' config['d'] = 's' with Context(config, loop=loop) as ctx: run('time.time', ctx, ns=ns, argv=[]) assert run('a.b', ctx, ns=ns, argv=[]) == 2 assert run('c', ctx, ns=ns, argv=[]) == 2 assert run('tests.test_command.coro', ctx, ns=ns, argv=[]) == 2 assert run('d', ctx, ns=ns, argv=[]) == 's' with pytest.raises(CommandNotFound): run('not found', ctx, ns=ns, argv=[]) with pytest.raises(CommandNotFound): run('time.time2', ctx, ns=ns, argv=[])
async def test_chunk(loop): key4 = ('4', '5') data = b'234' with tempfile.TemporaryDirectory() as d: config = MergeDict( name='', path=d, executor=None, ) context = Context({}, loop=loop) storage = FileSystemStorage(config, context=context, loop=loop) await storage.init() f = await storage._open(key4, 'wb') await storage._write_chunk(f, data) await storage._close(f) assert data == await storage.get(key4)
async def test_storage(loop): config = MergeDict( name='1', prefix=str(uuid.uuid4()), format='json', ) config['app.redis_pool'] = await aioredis.create_pool(('localhost', 6379), loop=loop) context = config q = RedisStorage(config, context=context, loop=loop) await q.init() await q.set('g', {'f': 3}) assert {'f': 3} == await q.get('g') assert 1 == await q.length() assert ['g'] == await q.list() await q.set('g', None) assert not await q.length()
async def test_queue_json(loop): config = MergeDict( key=str(uuid.uuid4()), format='json', ) config['app.redis_pool'] = await aioredis.create_pool(('localhost', 6379), loop=loop) context = config q = RedisQueue(config, context=context, loop=loop) await q.init() await q.put({'f': 3}) assert 1 == await q.length() assert [{'f': 3}] == await q.list() assert {'f': 3} == await q.get() await q.put({'f': 3}) assert 1 == await q.length() await q.clear() assert not await q.length()
async def test_autorun(loop): config = MergeDict( name='', autorun=True, children=1, child={ 'cls': 'aioworkers.worker.base.Worker', }, ) context = Context(config, loop=loop) worker = Supervisor(config, context=context, loop=loop) await worker.init() await context.start() await worker._future assert worker._started_at assert not worker.running() await worker.stop() assert await worker.status()
async def test_coro_run(loop, mocker): f = loop.create_future() async def myrun(*args, **kwargs): f.set_result(10) config = MergeDict( name='', autorun=True, run='mocked.run', ) context = Context(config, loop=loop) worker = Worker(config, context=context, loop=loop) mocker.patch('aioworkers.worker.base.import_name', lambda x: myrun) await worker.init() await context.start() assert worker._started_at await worker._future assert worker._stopped_at assert f.result() == 10
async def test_field_storage(loop): key = ('5', '6') data = {'f': 3, 'g': 4, 'h': 5} fields = ['f', 'g'] with tempfile.TemporaryDirectory() as d: config = MergeDict( name='', path=d, format='json', executor=None, ) context = Context({}, loop=loop) storage = Store(config, context=context, loop=loop) await storage.init() await storage.set(key, data) assert data == await storage.get(key) assert 5 == await storage.get(key, field='h') await storage.set(key, 6, field='h') assert {'f': 3, 'g': 4} == await storage.get(key, fields=fields) await storage.set(key, {'z': 1, 'y': 6}, fields=['z']) assert {'f': 3, 'g': 4, 'h': 6, 'z': 1} == await storage.get(key) await storage.set(key, None)
async def test_freespace(loop): key1 = '1' key2 = '2' data = b'000' with tempfile.TemporaryDirectory() as d: config = MergeDict( name='', path=d, executor=None, ) context = Context({}, loop=loop) storage = FileSystemStorage(config, context=context, loop=loop) await storage.init() with mock.patch.object(storage, 'get_free_space', asyncio.coroutine(lambda: 1)): assert 1 == await storage.get_free_space() storage.config.limit_free_space = 2 f = asyncio.ensure_future(storage.set(key1, data), loop=loop) await asyncio.sleep(0, loop=loop) await storage.set(key2, data) await storage.set(key1, None) assert not storage._space_waiters await f
def test_dict_set4(): d = MergeDict(f=3, d=dict(g=dict())) d['d.g'] = dict(r=4) assert d.d.g.r == 4, d
def test_dict_set3(): d = MergeDict(f=3, d=dict(g=1)) d.d = dict(r=4) assert d.d.r == 4, d assert d.d.g == 1, d
def test_dict_replace3(): d = MergeDict(f=3, d=dict(g=1)) d['d!'] = dict(r=4) assert d.d.r == 4, d