def test_regen_gen(self):
g = regen(iter(list(range(10))))
self.assertEqual(g[7], 7)
self.assertEqual(g[6], 6)
self.assertEqual(g[5], 5)
self.assertEqual(g[4], 4)
self.assertEqual(g[3], 3)
self.assertEqual(g[2], 2)
self.assertEqual(g[1], 1)
self.assertEqual(g[0], 0)
self.assertEqual(g.data, list(range(10)))
self.assertEqual(g[8], 8)
self.assertEqual(g[0], 0)
g = regen(iter(list(range(10))))
self.assertEqual(g[0], 0)
self.assertEqual(g[1], 1)
self.assertEqual(g.data, list(range(10)))
g = regen(iter([1]))
self.assertEqual(g[0], 1)
with self.assertRaises(IndexError):
g[1]
self.assertEqual(g.data, [1])
g = regen(iter(list(range(10))))
self.assertEqual(g[-1], 9)
self.assertEqual(g[-2], 8)
self.assertEqual(g[-3], 7)
self.assertEqual(g[-4], 6)
self.assertEqual(g[-5], 5)
self.assertEqual(g[5], 5)
self.assertEqual(g.data, list(range(10)))
self.assertListEqual(list(iter(g)), list(range(10)))
def test_gen(self):
g = regen(iter(list(range(10))))
assert g[7] == 7
assert g[6] == 6
assert g[5] == 5
assert g[4] == 4
assert g[3] == 3
assert g[2] == 2
assert g[1] == 1
assert g[0] == 0
assert g.data, list(range(10))
assert g[8] == 8
assert g[0] == 0
g = regen(iter(list(range(10))))
assert g[0] == 0
assert g[1] == 1
assert g.data == list(range(10))
g = regen(iter([1]))
assert g[0] == 1
with pytest.raises(IndexError):
g[1]
assert g.data == [1]
g = regen(iter(list(range(10))))
assert g[-1] == 9
assert g[-2] == 8
assert g[-3] == 7
assert g[-4] == 6
assert g[-5] == 5
assert g[5] == 5
assert g.data == list(range(10))
assert list(iter(g)) == list(range(10))
def test_regen_gen(self):
g = regen(iter(list(range(10))))
self.assertEqual(g[7], 7)
self.assertEqual(g[6], 6)
self.assertEqual(g[5], 5)
self.assertEqual(g[4], 4)
self.assertEqual(g[3], 3)
self.assertEqual(g[2], 2)
self.assertEqual(g[1], 1)
self.assertEqual(g[0], 0)
self.assertEqual(g.data, list(range(10)))
self.assertEqual(g[8], 8)
self.assertEqual(g[0], 0)
g = regen(iter(list(range(10))))
self.assertEqual(g[0], 0)
self.assertEqual(g[1], 1)
self.assertEqual(g.data, list(range(10)))
g = regen(iter([1]))
self.assertEqual(g[0], 1)
with self.assertRaises(IndexError):
g[1]
self.assertEqual(g.data, [1])
g = regen(iter(list(range(10))))
self.assertEqual(g[-1], 9)
self.assertEqual(g[-2], 8)
self.assertEqual(g[-3], 7)
self.assertEqual(g[-4], 6)
self.assertEqual(g[-5], 5)
self.assertEqual(g[5], 5)
self.assertEqual(g.data, list(range(10)))
self.assertListEqual(list(iter(g)), list(range(10)))
def test_regen_list(self):
l = [1, 2]
r = regen(iter(l))
self.assertIs(regen(l), l)
self.assertEqual(r, l)
self.assertEqual(r, l)
self.assertEqual(r.__length_hint__(), 0)
fun, args = r.__reduce__()
self.assertEqual(fun(*args), l)
def test_regen_list(self):
l = [1, 2]
r = regen(iter(l))
self.assertIs(regen(l), l)
self.assertEqual(r, l)
self.assertEqual(r, l)
self.assertEqual(r.__length_hint__(), 0)
fun, args = r.__reduce__()
self.assertEqual(fun(*args), l)
def test_list(self):
l = [1, 2]
r = regen(iter(l))
assert regen(l) is l
assert r == l
assert r == l # again
assert r.__length_hint__() == 0
fun, args = r.__reduce__()
assert fun(*args) == l
def __init__(self, *tasks, **options):
tasks = (regen(tasks[0]) if len(tasks) == 1 and is_list(tasks[0])
else tasks)
Signature.__init__(
self, 'celery.chain', (), {'tasks': tasks}, **options
)
self.subtask_type = 'chain'
def __init__(self, *tasks, **options):
tasks = (regen(tasks[0])
if len(tasks) == 1 and is_list(tasks[0]) else tasks)
Signature.__init__(self, 'celery.chain', (), {'tasks': tasks},
**options)
self.tasks = tasks
self.subtask_type = 'chain'
def test_partial_reconcretisation(self):
class WeirdIterator():
def __init__(self, iter_):
self.iter_ = iter_
self._errored = False
def __iter__(self):
yield from self.iter_
if not self._errored:
try:
# This should stop the regen instance from marking
# itself as being done
raise AssertionError("Iterator errored")
finally:
self._errored = True
original_list = list(range(42))
g = regen(WeirdIterator(original_list))
iter_g = iter(g)
for e in original_list:
assert e == next(iter_g)
with pytest.raises(AssertionError, match="Iterator errored"):
next(iter_g)
# The following checks are for the known "misbehaviour"
assert getattr(g, "_regen__done") is False
# If the `regen()` instance doesn't think it's done then it'll dupe the
# elements from the underlying iterator if it can be re-used
iter_g = iter(g)
for e in original_list * 2:
assert next(iter_g) == e
with pytest.raises(StopIteration):
next(iter_g)
assert getattr(g, "_regen__done") is True
# Finally we xfail this test to keep track of it
raise pytest.xfail(reason="#6794")
def test_repr(self):
def die():
raise AssertionError("Generator died")
yield None
# Confirm that `regen()` instances are not concretised when represented
g = regen(die())
assert "..." in repr(g)
def _maybe_group(tasks):
if isinstance(tasks, group):
tasks = list(tasks.tasks)
elif isinstance(tasks, abstract.CallableSignature):
tasks = [tasks]
else:
tasks = [signature(t) for t in regen(tasks)]
return tasks
def __init__(self, *tasks, **options):
tasks = (regen(tasks[0])
if len(tasks) == 1 and is_list(tasks[0]) else tasks)
Signature.__init__(self, 'celery.chain', (), {'tasks': tasks},
**options)
self._use_link = options.pop('use_link', None)
self.subtask_type = 'chain'
self._frozen = None
def test_deque(self):
original_list = [42]
d = collections.deque(original_list)
# Confirm that concretising a `regen()` instance repeatedly for an
# equality check always returns the original list
g = regen(d)
assert g == original_list
assert g == original_list
def __init__(self, task, it, **options):
Signature.__init__(self,
self._task_name, (), {
'task': task,
'it': regen(it)
},
immutable=True,
**options)
def _maybe_group(tasks):
if isinstance(tasks, group):
tasks = list(tasks.tasks)
elif isinstance(tasks, Signature):
tasks = [tasks]
else:
tasks = regen(tasks)
return tasks
def _maybe_group(tasks):
if isinstance(tasks, group):
tasks = list(tasks.tasks)
elif isinstance(tasks, abstract.CallableSignature):
tasks = [tasks]
else:
tasks = [signature(t) for t in regen(tasks)]
return tasks
def _maybe_group(tasks):
if isinstance(tasks, group):
tasks = list(tasks.tasks)
elif isinstance(tasks, Signature):
tasks = [tasks]
else:
tasks = regen(tasks)
return tasks
def test_nonzero__does_not_consume_more_than_first_item(self):
def build_generator():
yield 1
pytest.fail("generator should not consume past first item")
yield 2
g = regen(build_generator())
assert bool(g)
assert g[0] == 1
def __init__(self, *tasks, **options):
tasks = (regen(tasks[0]) if len(tasks) == 1 and is_list(tasks[0])
else tasks)
Signature.__init__(
self, 'celery.chain', (), {'tasks': tasks}, **options
)
self._use_link = options.pop('use_link', None)
self.subtask_type = 'chain'
self._frozen = None
def __init__(self, task, it, n, **options):
Signature.__init__(self,
'celery.chunks', (), {
'task': task,
'it': regen(it),
'n': n
},
immutable=True,
**options)
def __init__(self, *tasks, **options):
if len(tasks) == 1:
tasks = tasks[0]
if isinstance(tasks, group):
tasks = tasks.tasks
if not isinstance(tasks, _regen):
tasks = regen(tasks)
Signature.__init__(self, 'celery.group', (), {'tasks': tasks},
**options)
self.subtask_type = 'group'
def _maybe_group(tasks, app):
if isinstance(tasks, dict):
tasks = signature(tasks, app=app)
if isinstance(tasks, group):
tasks = tasks.tasks
elif isinstance(tasks, abstract.CallableSignature):
tasks = [tasks]
else:
tasks = [signature(t, app=app) for t in regen(tasks)]
return tasks
def test_nonzero__does_not_consume_more_than_first_item(self):
def build_generator():
yield 1
self.consumed_second_item = True
yield 2
self.consumed_second_item = False
g = regen(build_generator())
assert bool(g)
assert g[0] == 1
assert not self.consumed_second_item
def _maybe_group(tasks, app):
if isinstance(tasks, dict):
tasks = signature(tasks, app=app)
if isinstance(tasks, group):
tasks = tasks.tasks
elif isinstance(tasks, abstract.CallableSignature):
tasks = [tasks]
else:
tasks = [signature(t, app=app) for t in regen(tasks)]
return tasks
def __init__(self, *tasks, **options):
if len(tasks) == 1:
tasks = tasks[0]
if isinstance(tasks, group):
tasks = tasks.tasks
if not isinstance(tasks, _regen):
tasks = regen(tasks)
Signature.__init__(
self, 'celery.group', (), {'tasks': tasks}, **options
)
self.subtask_type = 'group'
def test_lookahead_consume(self, subtests):
"""
Confirm that regen looks ahead by a single item as expected.
"""
def g():
yield from ["foo", "bar"]
raise pytest.fail("This should never be reached")
with subtests.test(msg="bool does not overconsume"):
assert bool(regen(g()))
with subtests.test(msg="getitem 0th does not overconsume"):
assert regen(g())[0] == "foo"
with subtests.test(msg="single iter does not overconsume"):
assert next(iter(regen(g()))) == "foo"
class ExpectedException(BaseException):
pass
def g2():
yield from ["foo", "bar"]
raise ExpectedException()
with subtests.test(msg="getitem 1th does overconsume"):
r = regen(g2())
with pytest.raises(ExpectedException):
r[1]
# Confirm that the item was concretised anyway
assert r[1] == "bar"
with subtests.test(msg="full iter does overconsume"):
r = regen(g2())
with pytest.raises(ExpectedException):
for _ in r:
pass
# Confirm that the items were concretised anyway
assert r == ["foo", "bar"]
with subtests.test(msg="data access does overconsume"):
r = regen(g2())
with pytest.raises(ExpectedException):
r.data
# Confirm that the items were concretised anyway
assert r == ["foo", "bar"]
def __init__(self, task, it, n, **options):
Signature.__init__(self, "celery.chunks", (), {
"task": task,
"it": regen(it),
"n": n
}, **options)
def __init__(self, task, it, **options):
Signature.__init__(self, self._task_name, (), {
"task": task,
"it": regen(it)
}, **options)
def __init__(self, header, body=None, **options):
Signature.__init__(self, 'celery.chord', (), {
'header': regen(header),
'body': maybe_subtask(body)
}, options)
self.subtask_type = 'chord'
def _maybe_group(tasks):
if isinstance(tasks, group):
tasks = list(tasks.tasks)
else:
tasks = regen(tasks if is_list(tasks) else tasks)
return tasks
def g(self):
return regen(iter(list(range(10))))
def test_nonzero__empty_iter(self):
assert not regen(iter([]))
def __init__(self, task, it, n, **options):
Signature.__init__(
self, "celery.chunks", (), {"task": task, "it": regen(it), "n": n}, immutable=True, **options
)
def __init__(self, task, it, **options):
Signature.__init__(self, self._task_name, (), {"task": task, "it": regen(it)}, immutable=True, **options)
def __init__(self, *tasks, **options):
tasks = regen(
tasks[0] if len(tasks) == 1 and is_list(tasks[0]) else tasks)
Signature.__init__(self, "celery.group", (), {"tasks": tasks}, options)
self.tasks, self.subtask_type = tasks, "group"
def __init__(self, header, body=None, **options):
Signature.__init__(self, "celery.chord", (), {
"header": regen(header),
"body": maybe_subtask(body)
}, options)
self.subtask_type = "chord"
def _maybe_group(tasks):
if isinstance(tasks, group):
tasks = list(tasks.tasks)
else:
tasks = regen(tasks if is_list(tasks) else tasks)
return tasks
def __init__(self, *tasks, **options):
tasks = regen(tasks[0] if len(tasks) == 1 and is_list(tasks[0])
else tasks)
Signature.__init__(self, "celery.group", (), {"tasks": tasks}, options)
self.tasks, self.subtask_type = tasks, "group"
def __init__(self, task, it, **options):
Signature.__init__(
self, self._task_name, (),
{'task': task, 'it': regen(it)}, immutable=True, **options
)
def __init__(self, task, it, n, **options):
Signature.__init__(
self, 'celery.chunks', (),
{'task': task, 'it': regen(it), 'n': n},
immutable=True, **options
)
def __init__(self, header, body=None, **options):
Signature.__init__(self, "celery.chord", (),
{"header": regen(header),
"body": maybe_subtask(body)}, options)
self.subtask_type = "chord"
def __init__(self, *tasks, **options):
tasks = regen(tasks[0]) if len(tasks) == 1 and is_list(tasks[0]) else tasks
Signature.__init__(self, "celery.chain", (), {"tasks": tasks}, **options)
self.subtask_type = "chain"
def __init__(self, *tasks, **options):
tasks = regen(tasks[0]) if len(tasks) == 1 and is_list(tasks[0]) else tasks
Signature.__init__(self, "celery.chain", (), {"tasks": tasks}, **options)
self._use_link = options.pop("use_link", None)
self.subtask_type = "chain"
self._frozen = None