async def test_standalone(tmp_dir):
s = FileSystemStorage(path=tmp_dir, format='json')
await s.set('a', 1)
assert 1 == await s.get('a')
c = Context()
c.storage = s
assert s.context is c
async with c:
assert 1 == await c.storage.get('a')
def setup(cls, app, conf):
context = Context({}, loop=app.loop)
context.app = app
conf = MergeDict(conf)
conf.name = 'mailer'
m = cls(conf, context=context, loop=app.loop)
async def start(app):
await m.init()
await m.start()
app.on_startup.append(start)
app.on_shutdown.append(lambda x: m.stop())
async def test_context_items(loop):
f = Context({}, loop=loop)
f.r = 1
assert f['r'] == 1
f['g'] = 2
assert f.g == 2
f['y.t'] = 3
assert f.y.t == 3
f['d.w.f'] = 4
assert dir(f)
assert repr(f)
await f.stop()
async def test_autorun(loop):
async with Context(config.autorun, loop=loop) as ctx:
await ctx.sv._future
assert ctx.sv._started_at
assert not ctx.sv.running()
await ctx.sv.stop()
assert await ctx.sv.status()
async def test_autorun(loop):
config = Config(w=dict(cls='aioworkers.worker.base.Worker', autorun=True))
async with Context(config, loop=loop) as context:
worker = context.w
await worker._future
assert worker._started_at
assert not worker.running()
async def test_key(loop):
with tempfile.TemporaryDirectory() as d:
config = MergeDict(
name='',
path=d,
executor=None,
)
context = Context({}, loop=loop)
storage = FileSystemStorage(config, context=context, loop=loop)
await storage.init()
assert str(storage.raw_key(('1', '3', ('4', )))).endswith('1/3/4')
assert str(storage.raw_key(
(PurePath('1'), '3', ('4', )))).endswith('1/3/4')
with pytest.raises(TypeError):
storage.raw_key(1)
with pytest.raises(ValueError):
storage.raw_key('../..')
with pytest.raises(ValueError):
storage.raw_key('/abs/path')
with pytest.raises(ValueError):
storage.raw_key(PurePath('/abs/path'))
async def test_copy(loop):
key1 = '1'
key2 = '2'
key3 = '3'
key4 = ('4', '5')
data = b'234'
with tempfile.TemporaryDirectory() as d:
config = MergeDict(
name='',
path=d,
executor=None,
)
context = Context({}, loop=loop)
storage = FileSystemStorage(config, context=context, loop=loop)
await storage.init()
await storage.set(key1, data)
await storage.copy(key1, storage, key3)
assert data == await storage.get(key3)
await storage.copy(key2, storage, key3)
await storage.move(key2, storage, key4)
assert not await storage.get(key4)
fstor = FutureStorage(mock.Mock(name=''), loop=loop)
await fstor.init()
await storage.copy(key2, fstor, key3)
await storage.move(key2, fstor, key4)
async def test_context_create(loop):
conf = Config()
conf.update(
MergeDict({
'q.cls': 'aioworkers.queue.timeout.TimestampQueue',
'f.e': 1,
}))
c = Context(conf, loop=loop)
await c.init()
await c.start()
assert c.config.f.e == 1
with pytest.raises(AttributeError):
c.r
with pytest.raises(KeyError):
c['r']
async def handler(context):
pass
c.on_stop.append(handler)
async def handler():
raise ValueError
c.on_stop.append(handler)
c.on_stop.append(handler())
c.on_stop.append(1)
await c.stop()
async def test_func(loop):
config = MergeDict(now={
'func': 'time.monotonic',
})
context = Context(config, loop=loop)
async with context:
assert isinstance(context.now, float)
async def test_set_get(loop):
key = ('4423', '123')
data = b'234'
with tempfile.TemporaryDirectory() as d:
config = MergeDict(
name='',
path=d,
executor=None,
)
context = Context({}, loop=loop)
storage = FileSystemStorage(config, context=context, loop=loop)
await storage.init()
await storage.set(('empty', 'value', '1'), None)
assert not await storage.get('empty value 2')
await storage.set(key, data)
await storage.set(key, None) # del file
assert not await storage.get(key)
await storage.set(key[0], None) # del dir
await storage.set(key, data)
d = await asyncio.gather(storage.get(key), storage.get(key), loop=loop)
for j in d:
assert data == j
async def test_set_context(loop):
worker = Worker()
config = Config(name='')
context = Context(loop=loop)
worker.set_config(config)
worker.set_context(context)
with pytest.raises(RuntimeError):
worker.set_context(context)
def context(loop, config):
with Context(
config,
loop=loop,
# Run all groups
group_resolver=GroupResolver(
include=['web', 'workers', 'engine'])) as ctx:
yield ctx
async def test_q(loop):
conf = Config()
conf.update({'q.cls': utils.import_uri(proxy.ProxyQueue)})
async with Context(conf, loop=loop) as ctx:
ctx.q.set_queue(Queue())
await ctx.q.put(1)
assert 1 == await ctx.q.get()
async def test_format(loop):
config = Config(storage=dict(
cls='aioworkers.storage.http.Storage',
format='bytes',
))
async with Context(config=config, loop=loop) as context:
storage = context.storage
assert isinstance(storage.raw_key('test'), URL)
async def test_super_queue(loop):
async with Context(config.super.queue, loop=loop) as ctx:
await ctx.q1.put(1)
result = await ctx.q2.get()
assert result == 1
await ctx.sv(2)
result = await ctx.q2.get()
assert result == 2
async def test_http():
q = Queue()
a = AsgiMiddleware(app, context=Context())
await a({'type': 'http'}, q.get, q.put)
assert {
"type": "http.response.start",
"status": 404,
} == await q.get()
async def test_signal(loop):
gr = GroupResolver()
context = Context({}, loop=loop, group_resolver=gr)
s = Signal(context)
s.append(1, ('1', ))
s.append(1)
await s.send(gr)
await s.send(GroupResolver(all_groups=True))
async def test_1(loop):
config = MergeDict(
name='',
autorun=False,
)
context = Context(config, loop=loop)
worker = BaseUpdater(config, context=context, loop=loop)
await worker.init()
def context(loop, groups, config):
from aioworkers.core.context import Context, GroupResolver
gr = GroupResolver(
include=groups.include,
exclude=groups.exclude,
)
with Context(config, group_resolver=gr, loop=loop) as ctx:
yield ctx
async def test_autorun(loop):
config = MergeDict(name='', autorun=True)
context = Context(config, loop=loop)
worker = Worker(config, context=context, loop=loop)
await worker.init()
await context.start()
await worker._future
assert worker._started_at
assert not worker.running()
async def test_crontab(loop):
config = Config(w=dict(
cls='aioworkers.worker.base.Worker',
persist=True,
crontab='*/1 * * * *',
))
async with Context(config, loop=loop) as context:
worker = context.w
await asyncio.sleep(0.1, loop=loop)
await worker.stop()
assert not worker.running()
async def test_pool_close(loop, dsn):
conf = Config(
db={
"cls": "aioworkers_pg.base.Connector",
"dsn": dsn,
},
)
async with Context(conf, loop=loop) as c:
assert c.db._pool is not None
assert not c.db._pool._closed
assert c.db._pool is None
async def test_format(loop):
config = MergeDict(
name='',
semaphore=1,
format='bytes',
)
context = Context(config=config, loop=loop)
await context.init()
storage = Storage(config, context=context, loop=loop)
await storage.init()
assert isinstance(storage.raw_key('test'), URL)
await storage.stop()
async def test_stop(loop):
config = Config(w=dict(
cls='aioworkers.worker.base.Worker',
autorun=True,
persist=True,
sleep=0.01,
sleep_start=0.01,
))
async with Context(config, loop=loop) as context:
worker = context.w
await asyncio.sleep(0.1, loop=loop)
await worker.stop()
assert not worker.running()
assert isinstance(await worker.status(), dict)
async def test_crontab(loop, mocker):
config = MergeDict(
name='',
autorun=True,
persist=True,
crontab='*/1 * * * *',
)
context = Context(config, loop=loop)
worker = Worker(config, context=context, loop=loop)
await worker.init()
await context.start()
await asyncio.sleep(0.1, loop=loop)
await worker.stop()
assert not worker.running()
async def test_plq(loop):
conf = Config()
conf.update({
'q.cls': utils.import_uri(proxy.PipeLineQueue),
'q.format': 'newline:str',
})
async with Context(conf, loop=loop) as ctx:
fin = io.BytesIO(b'123\n')
fout = io.BytesIO()
ctx.q.set_reader(fin)
ctx.q.set_writer(fout)
assert '123' == await ctx.q.get()
await ctx.q.put('1')
assert b'1\n' == fout.getvalue()
async def test_pickle(loop):
key = '4423'
data = {'f': 3}
with tempfile.TemporaryDirectory() as d:
config = MergeDict(
name='',
path=d,
format='pickle',
executor=None,
)
context = Context({}, loop=loop)
storage = HashFileSystemStorage(config, context=context, loop=loop)
await storage.init()
await storage.set(key, data)
assert data == await storage.get(key)
async def test_stop(loop, mocker):
config = MergeDict(
name='',
autorun=True,
persist=True,
sleep=0.01,
sleep_start=0.01,
)
context = Context(config, loop=loop)
worker = Worker(config, context=context, loop=loop)
await worker.init()
await context.start()
await asyncio.sleep(0.1, loop=loop)
await worker.stop()
assert not worker.running()
assert isinstance(await worker.status(), dict)
def test_run(loop):
ns = Namespace()
config = MergeDict()
config['a.b'] = 2
config['c'] = 'a.b'
config['d'] = 's'
with Context(config, loop=loop) as ctx:
run('time.time', ctx, ns=ns, argv=[])
assert run('a.b', ctx, ns=ns, argv=[]) == 2
assert run('c', ctx, ns=ns, argv=[]) == 2
assert run('tests.test_command.coro', ctx, ns=ns, argv=[]) == 2
assert run('d', ctx, ns=ns, argv=[]) == 's'
with pytest.raises(CommandNotFound):
run('not found', ctx, ns=ns, argv=[])
with pytest.raises(CommandNotFound):
run('time.time2', ctx, ns=ns, argv=[])
async def test_chunk(loop):
key4 = ('4', '5')
data = b'234'
with tempfile.TemporaryDirectory() as d:
config = MergeDict(
name='',
path=d,
executor=None,
)
context = Context({}, loop=loop)
storage = FileSystemStorage(config, context=context, loop=loop)
await storage.init()
f = await storage._open(key4, 'wb')
await storage._write_chunk(f, data)
await storage._close(f)
assert data == await storage.get(key4)
