async def test_simplest_path():
ti = OrderedTaskPreparation(TwoPrereqs, identity, lambda x: x - 1)
ti.set_finished_dependency(3)
assert ti.num_unready() == 0
assert ti.num_ready() == 0
assert ti.num_tasks() == 0
ti.register_tasks((4, ))
assert ti.num_unready() == 1
assert ti.num_ready() == 0
assert ti.num_tasks() == 1
ti.finish_prereq(TwoPrereqs.PREREQ1, (4, ))
assert ti.num_unready() == 1
assert ti.num_ready() == 0
assert ti.num_tasks() == 1
ti.finish_prereq(TwoPrereqs.PREREQ2, (4, ))
assert ti.num_unready() == 0
assert ti.num_ready() == 1
assert ti.num_tasks() == 1
ready = await wait(ti.ready_tasks())
assert ti.num_unready() == 0
assert ti.num_ready() == 0
assert ti.num_tasks() == 0
assert ready == (4, )
def test_finish_same_task_twice():
ti = OrderedTaskPreparation(TwoPrereqs, identity, lambda x: x - 1)
ti.set_finished_dependency(1)
ti.register_tasks((2, ))
ti.finish_prereq(TwoPrereqs.Prereq1, (2, ))
with pytest.raises(ValidationError):
ti.finish_prereq(TwoPrereqs.Prereq1, (2, ))
def __init__(self,
chain: BaseAsyncChain,
db: BaseAsyncChainDB,
peer_pool: ETHPeerPool,
header_syncer: HeaderSyncerAPI,
block_importer: BaseBlockImporter,
token: CancelToken = None) -> None:
super().__init__(chain, db, peer_pool, header_syncer, token)
# track when block bodies are downloaded, so that blocks can be imported
self._block_import_tracker = OrderedTaskPreparation(
BlockImportPrereqs,
id_extractor=attrgetter('hash'),
# make sure that a block is not imported until the parent block is imported
dependency_extractor=attrgetter('parent_hash'),
)
self._block_importer = block_importer
# Track if any headers have been received yet
self._got_first_header = asyncio.Event()
# Rate limit the block import logs
self._import_log_limiter = TokenBucket(
0.33, # show about one log per 3 seconds
5, # burst up to 5 logs after a lag
)
async def test_random_pruning(ignore_duplicates, recomplete_idx, batch_size,
task_series, prune_depth):
ti = OrderedTaskPreparation(
NoPrerequisites,
identity,
lambda x: x - 1,
accept_dangling_tasks=True,
max_depth=prune_depth,
)
ti.set_finished_dependency(task_series[0])
for idx, task_batch in enumerate(partition_all(batch_size, task_series)):
if ignore_duplicates:
registerable_tasks = task_batch
else:
registerable_tasks = set(task_batch)
if idx == recomplete_idx:
task_to_mark_finished = task_batch[0] - 1
if task_to_mark_finished not in ti._tasks:
ti.set_finished_dependency(task_to_mark_finished)
try:
ti.register_tasks(registerable_tasks,
ignore_duplicates=ignore_duplicates)
except DuplicateTasks:
if ignore_duplicates:
raise
else:
continue
if ti.has_ready_tasks():
await wait(ti.ready_tasks())
def __init__(self,
chain: BaseAsyncChain,
db: BaseAsyncHeaderDB,
peer_pool: BaseChainPeerPool,
token: CancelToken = None) -> None:
super().__init__(token)
self._db = db
self._chain = chain
self._peer_pool = peer_pool
self._tip_monitor = self.tip_monitor_class(peer_pool,
token=self.cancel_token)
self._skeleton: SkeletonSyncer[TChainPeer] = None
# stitch together headers as they come in
self._stitcher = OrderedTaskPreparation(
# we don't have to do any prep work on the headers, just linearize them, so empty enum
OrderedTaskPreparation.NoPrerequisites,
id_extractor=attrgetter('hash'),
# make sure that a header is not returned in new_sync_headers until its parent has been
dependency_extractor=attrgetter('parent_hash'),
# headers will come in out of order
accept_dangling_tasks=True,
)
# When downloading the headers into the gaps left by the syncer, they must be linearized
# by the stitcher
self._meat = HeaderMeatSyncer(chain, peer_pool, self._stitcher, token)
self._last_target_header_hash: Hash32 = None
async def test_pruning():
# make a number task depend on the mod10, so 4 and 14 both depend on task 3
ti = OrderedTaskPreparation(OnePrereq, identity, lambda x: (x % 10) - 1, max_depth=2)
ti.set_finished_dependency(3)
ti.register_tasks((4, 5, 6))
ti.finish_prereq(OnePrereq.one, (4, 5, 6))
# it's fine to prepare a task that depends up to two back in history
# this depends on 5
ti.register_tasks((16, ))
# this depends on 4
ti.register_tasks((15, ))
# but depending 3 back in history should raise a validation error, because it's pruned
with pytest.raises(MissingDependency):
# this depends on 3
ti.register_tasks((14, ))
# test the same concept, but after pruning more than just the starting task...
ti.register_tasks((7, ))
ti.finish_prereq(OnePrereq.one, (7, ))
ti.register_tasks((26, ))
ti.register_tasks((27, ))
with pytest.raises(MissingDependency):
ti.register_tasks((25, ))
def __init__(self,
chain: AsyncChainAPI,
db: BaseAsyncChainDB,
peer_pool: ETHPeerPool,
header_syncer: HeaderSyncerAPI,
block_importer: BaseBlockImporter,
token: CancelToken = None) -> None:
super().__init__(chain, db, peer_pool, header_syncer, token)
# track when block bodies are downloaded, so that blocks can be imported
self._block_import_tracker = OrderedTaskPreparation(
BlockImportPrereqs,
id_extractor=attrgetter('hash'),
# make sure that a block is not imported until the parent block is imported
dependency_extractor=attrgetter('parent_hash'),
# Avoid problems by keeping twice as much data as the import queue size
max_depth=BLOCK_IMPORT_QUEUE_SIZE * 2,
)
self._block_importer = block_importer
# Track if any headers have been received yet
self._got_first_header = asyncio.Event()
# Rate limit the block import logs
self._import_log_limiter = TokenBucket(
0.33, # show about one log per 3 seconds
5, # burst up to 5 logs after a lag
)
# the queue of blocks that are downloaded and ready to be imported
self._import_queue: 'asyncio.Queue[BlockAPI]' = asyncio.Queue(BLOCK_IMPORT_QUEUE_SIZE)
self._import_active = asyncio.Lock()
async def test_wait_if_too_many_ready_tasks():
ti = OrderedTaskPreparation(OnePrereq,
identity,
lambda x: x - 1,
max_tasks=1)
ti.set_finished_dependency(3)
ti.register_tasks((4, ))
# This should raise a timeout error because it gets locked waiting for the task to be finished
with pytest.raises(asyncio.TimeoutError):
await wait(ti.wait_add_tasks((5, )))
ti.finish_prereq(OnePrereq.ONE, (4, ))
# This should raise a timeout error because it gets locked waiting for the task to be picked up
with pytest.raises(asyncio.TimeoutError):
await wait(ti.wait_add_tasks((5, )))
completed = await wait(ti.ready_tasks())
assert completed == (4, )
# Now we can add the other task
await wait(ti.wait_add_tasks((5, )))
# ... and finish & pick up the completed one
ti.finish_prereq(OnePrereq.ONE, (5, ))
completed = await wait(ti.ready_tasks())
assert completed == (5, )
def __init__(self,
chain: BaseAsyncChain,
db: BaseAsyncChainDB,
peer_pool: ETHPeerPool,
header_syncer: HeaderSyncerAPI,
token: CancelToken = None) -> None:
super().__init__(chain, db, peer_pool, token)
# queue up any idle peers, in order of how fast they return receipts
self._receipt_peers: WaitingPeers[ETHPeer] = WaitingPeers(
commands.Receipts)
self._header_syncer = header_syncer
# Track receipt download tasks
# - arbitrarily allow several requests-worth of headers queued up
# - try to get receipts from lower block numbers first
buffer_size = MAX_RECEIPTS_FETCH * REQUEST_BUFFER_MULTIPLIER
self._receipt_tasks = TaskQueue(buffer_size,
attrgetter('block_number'))
# track when both bodies and receipts are collected, so that blocks can be persisted
self._block_persist_tracker = OrderedTaskPreparation(
BlockPersistPrereqs,
id_extractor=attrgetter('hash'),
# make sure that a block is not persisted until the parent block is persisted
dependency_extractor=attrgetter('parent_hash'),
)
# Track whether the fast chain syncer completed its goal
self.is_complete = False
async def test_simplest_path():
ti = OrderedTaskPreparation(TwoPrereqs, identity, lambda x: x - 1)
ti.set_finished_dependency(3)
ti.register_tasks((4, ))
ti.finish_prereq(TwoPrereqs.Prereq1, (4, ))
ti.finish_prereq(TwoPrereqs.Prereq2, (4, ))
ready = await wait(ti.ready_tasks())
assert ready == (4, )
async def test_no_prereq_tasks():
ti = OrderedTaskPreparation(NoPrerequisites, identity, lambda x: x - 1)
ti.set_finished_dependency(1)
ti.register_tasks((2, 3))
# with no prerequisites, tasks are *immediately* finished, as long as they are in order
finished = await wait(ti.ready_tasks())
assert finished == (2, 3)
async def test_wait_forever():
ti = OrderedTaskPreparation(OnePrereq, identity, lambda x: x - 1)
try:
finished = await wait(ti.ready_tasks())
except asyncio.TimeoutError:
pass
else:
assert False, f"No steps should complete, but got {finished!r}"
async def test_two_steps_simultaneous_complete():
ti = OrderedTaskPreparation(OnePrereq, identity, lambda x: x - 1)
ti.set_finished_dependency(3)
ti.register_tasks((4, 5))
ti.finish_prereq(OnePrereq.one, (4, ))
ti.finish_prereq(OnePrereq.one, (5, ))
completed = await wait(ti.ready_tasks())
assert completed == (4, 5)
async def test_ignore_duplicates():
ti = OrderedTaskPreparation(NoPrerequisites, identity, lambda x: x - 1)
ti.set_finished_dependency(1)
ti.register_tasks((2, ))
# this will ignore the 2 task:
ti.register_tasks((2, 3), ignore_duplicates=True)
# this will be completely ignored:
ti.register_tasks((2, 3), ignore_duplicates=True)
# with no prerequisites, tasks are *immediately* finished, as long as they are in order
finished = await wait(ti.ready_tasks())
assert finished == (2, 3)
async def test_register_out_of_order():
ti = OrderedTaskPreparation(OnePrereq, identity, lambda x: x - 1, accept_dangling_tasks=True)
ti.set_finished_dependency(1)
ti.register_tasks((4, 5))
ti.finish_prereq(OnePrereq.one, (4, 5))
await assert_nothing_ready(ti)
ti.register_tasks((2, 3))
ti.finish_prereq(OnePrereq.one, (2, 3))
finished = await wait(ti.ready_tasks())
assert finished == (2, 3, 4, 5)
async def test_forked_pruning():
ti = OrderedTaskPreparation(
NoPrerequisites,
task_id,
fork_prereq,
max_depth=2,
)
ti.set_finished_dependency(Task(0, 0, 0))
ti.register_tasks((
Task(1, 0, 0),
Task(2, 0, 0),
Task(2, 1, 0),
))
ti.register_tasks((
Task(3, 0, 0),
Task(3, 1, 1),
))
ti.register_tasks((
Task(4, 0, 0),
Task(4, 1, 1),
))
ti.register_tasks((
Task(5, 0, 0),
Task(6, 0, 0),
Task(7, 0, 0),
Task(8, 0, 0),
Task(9, 0, 0),
Task(10, 0, 0),
))
ti.register_tasks((
Task(5, 1, 1),
))
finished = await ti.ready_tasks()
assert len(finished) == 14
# nothing should be pruned, because caller is still "acting" on all tasks
assert TaskID(1, 0) in ti._tasks
# caller indicates that she is done working on tasks by calling ready_tasks() again
await assert_nothing_ready(ti)
assert TaskID(1, 0) not in ti._tasks
assert TaskID(2, 0) not in ti._tasks
assert TaskID(3, 0) not in ti._tasks
assert TaskID(4, 0) not in ti._tasks
assert TaskID(5, 0) not in ti._tasks
assert TaskID(2, 1) not in ti._tasks
assert TaskID(10, 0) in ti._tasks
def test_re_fork_at_prune_boundary():
def task_id(task):
return TaskID(task.idx, task.fork)
def fork_prereq(task):
# allow tasks to fork for a few in a row
return TaskID(task.idx - 1, task.parent_fork)
ti = OrderedTaskPreparation(
NoPrerequisites,
task_id,
fork_prereq,
max_depth=2,
)
ti.set_finished_dependency(Task(0, 0, 0))
ti.register_tasks((
Task(1, 0, 0),
Task(2, 0, 0),
Task(2, 1, 0),
))
ti.register_tasks((
Task(3, 0, 0),
Task(3, 1, 1),
))
ti.register_tasks((
Task(4, 0, 0),
Task(4, 1, 1),
Task(4, 2, 1),
))
ti.register_tasks((
Task(5, 0, 0),
Task(6, 0, 0),
Task(7, 0, 0),
Task(8, 0, 0),
Task(9, 0, 0),
Task(10, 0, 0),
))
ti.register_tasks((
Task(5, 1, 1),
Task(5, 2, 2),
Task(5, 3, 2),
))
ti.register_tasks((
Task(6, 3, 3),
Task(7, 3, 3),
Task(8, 3, 3),
Task(9, 3, 3),
))
async def test_register_out_of_order():
ti = OrderedTaskPreparation(OnePrereq, identity, lambda x: x - 1, accept_dangling_tasks=True)
ti.set_finished_dependency(1)
ti.register_tasks((4, 5))
ti.finish_prereq(OnePrereq.one, (4, 5))
try:
finished = await wait(ti.ready_tasks())
except asyncio.TimeoutError:
pass
else:
assert False, f"No steps should be ready, but got {finished!r}"
ti.register_tasks((2, 3))
ti.finish_prereq(OnePrereq.one, (2, 3))
finished = await wait(ti.ready_tasks())
assert finished == (2, 3, 4, 5)
def __init__(self,
chain: BaseAsyncChain,
db: BaseAsyncChainDB,
peer_pool: ETHPeerPool,
header_syncer: HeaderSyncerAPI,
token: CancelToken = None) -> None:
super().__init__(chain, db, peer_pool, token)
self._header_syncer = header_syncer
# track when block bodies are downloaded, so that blocks can be imported
self._block_import_tracker = OrderedTaskPreparation(
BlockImportPrereqs,
id_extractor=attrgetter('hash'),
# make sure that a block is not imported until the parent block is imported
dependency_extractor=attrgetter('parent_hash'),
)
async def test_no_prereq_tasks_out_of_order():
ti = OrderedTaskPreparation(
NoPrerequisites,
identity,
lambda x: x - 1,
accept_dangling_tasks=True,
)
ti.set_finished_dependency(1)
ti.register_tasks((4, 5))
await assert_nothing_ready(ti)
ti.register_tasks((2, 3))
# with no prerequisites, tasks are *immediately* finished, as long as they are in order
finished = await wait(ti.ready_tasks())
assert finished == (2, 3, 4, 5)
async def test_finish_different_entry_at_same_step():
def previous_even_number(num):
return ((num - 1) // 2) * 2
ti = OrderedTaskPreparation(OnePrereq, identity, previous_even_number)
ti.set_finished_dependency(2)
ti.register_tasks((3, 4))
# depends on 2
ti.finish_prereq(OnePrereq.one, (3, ))
# also depends on 2
ti.finish_prereq(OnePrereq.one, (4, ))
completed = await wait(ti.ready_tasks())
assert completed == (3, 4)
async def test_pruning_consecutive_finished_deps():
ti = OrderedTaskPreparation(NoPrerequisites, identity, lambda x: x - 1, max_depth=2)
ti.set_finished_dependency(3)
ti.set_finished_dependency(4)
ti.register_tasks((5, 6))
assert 3 in ti._tasks
assert 4 in ti._tasks
# trigger pruning by requesting the ready tasks through 6, then "finishing" them
# by requesting the next batch of ready tasks (7)
completed = await wait(ti.ready_tasks())
assert completed == (5, 6)
ti.register_tasks((7, ))
completed = await wait(ti.ready_tasks())
assert completed == (7, )
assert 3 not in ti._tasks
assert 4 in ti._tasks
async def test_return_original_entry():
# for no particular reason, the id is 3 before the number
ti = OrderedTaskPreparation(OnePrereq, lambda x: x - 3, lambda x: x - 4)
# translates to id -1
ti.set_finished_dependency(2)
ti.register_tasks((3, 4))
# translates to id 0
ti.finish_prereq(OnePrereq.one, (3, ))
# translates to id 1
ti.finish_prereq(OnePrereq.one, (4, ))
entries = await wait(ti.ready_tasks())
# make sure that the original task is returned, not the id
assert entries == (3, 4)
async def test_pruning_speed():
length = 10000
ti = OrderedTaskPreparation(
NoPrerequisites,
identity,
lambda x: x - 1,
max_depth=length,
)
ti.set_finished_dependency(-1)
ti.register_tasks(range(length))
finished = await ti.ready_tasks()
assert len(finished) == length
await assert_nothing_ready(ti)
# nothing should be pruned, because the max depth hasn't been reached
assert -1 in ti._tasks
ti.register_tasks((length, ))
finished = await ti.ready_tasks()
assert finished == (length, )
# nothing should be pruned, because caller is still "acting" on task 10000
# give ready_tasks something to respond with at next call, so we don't wait for the timeout
ti.register_tasks((length + 1, ))
# start timer to measure pruning speed
start = time.perf_counter()
# caller indicates that she is done working on task 10000 by calling ready_tasks() again
finished = await ti.ready_tasks()
assert finished == (length + 1, )
# make sure pruning actually happened
assert -1 not in ti._tasks
# but not too much pruning
assert 0 in ti._tasks
# make sure pruning was fast enough
duration = time.perf_counter() - start
assert duration < 0.0001
async def test_wait_to_prune_until_yielded():
"""
We need to be able to mark dependencies as finished, after task completion
"""
ti = OrderedTaskPreparation(NoPrerequisites, identity, lambda x: x - 1, max_depth=2)
ti.set_finished_dependency(-1)
ti.register_tasks(range(10))
# the old tasks aren't pruned yet, so duplicates with known parents are fine
ti.register_tasks((3, ), ignore_duplicates=True)
ready = await wait(ti.ready_tasks())
assert ready == tuple(range(10))
# old tasks STILL aren't pruned, until we indicate that we are finished processing
# them by calling ready_tasks on the *next* batch
ti.register_tasks((10, ))
ready = await wait(ti.ready_tasks())
assert ready == (10, )
# now old tasks are pruned
with pytest.raises(MissingDependency):
ti.register_tasks((3, ), ignore_duplicates=True)
def _reset_buffer(self) -> None:
# stitch together headers as they come in
self._stitcher = OrderedTaskPreparation(
# we don't have to do any prep work on the headers, just linearize them, so empty enum
OrderedTaskPreparation.NoPrerequisites,
id_extractor=attrgetter('hash'),
# make sure that a header is not returned in new_sync_headers until its parent has been
dependency_extractor=attrgetter('parent_hash'),
# headers will come in out of order
accept_dangling_tasks=True,
)
# When downloading the headers into the gaps left by the syncer, they must be linearized
# by the stitcher
self._meat = HeaderMeatSyncer(
self._chain,
self._peer_pool,
self._stitcher,
)
# Queue has reset, so always start with capacity
self._buffer_capacity.set()
async def test_pruning():
# make a number task depend on the mod10, so 4 and 14 both depend on task 3
ti = OrderedTaskPreparation(OnePrereq,
identity,
lambda x: (x % 10) - 1,
max_depth=2)
ti.set_finished_dependency(3)
ti.register_tasks((4, 5, 6, 7, 8))
ti.finish_prereq(OnePrereq.one, (4, 5, 6))
# trigger pruning by requesting the ready tasks through 6, then "finishing" them
# by requesting the next batch of ready tasks (7)
completed = await wait(ti.ready_tasks())
assert completed == (4, 5, 6)
ti.finish_prereq(OnePrereq.one, (7, ))
completed = await wait(ti.ready_tasks())
assert completed == (7, )
# it's fine to prepare a task that depends up to two back in history
# this depends on 5
ti.register_tasks((16, ))
# this depends on 4
ti.register_tasks((15, ))
# but depending 3 back in history should raise a validation error, because it's pruned
with pytest.raises(MissingDependency):
# this depends on 3
ti.register_tasks((14, ))
# test the same concept, but after pruning tasks that weren't the starting tasks
# trigger pruning from the head at 7 by completing the one *after* 7
ti.finish_prereq(OnePrereq.one, (8, ))
completed = await wait(ti.ready_tasks())
assert completed == (8, )
ti.register_tasks((26, ))
ti.register_tasks((27, ))
with pytest.raises(MissingDependency):
ti.register_tasks((25, ))
async def test_no_prereq_tasks_out_of_order():
ti = OrderedTaskPreparation(
NoPrerequisites,
identity,
lambda x: x - 1,
accept_dangling_tasks=True,
)
ti.set_finished_dependency(1)
ti.register_tasks((4, 5))
try:
finished = await wait(ti.ready_tasks())
except asyncio.TimeoutError:
pass
else:
assert False, f"No steps should be ready, but got {finished!r}"
ti.register_tasks((2, 3))
# with no prerequisites, tasks are *immediately* finished, as long as they are in order
finished = await wait(ti.ready_tasks())
assert finished == (2, 3, 4, 5)
async def test_finished_dependency_midstream():
"""
We need to be able to mark dependencies as finished, after task completion
"""
ti = OrderedTaskPreparation(TwoPrereqs, identity, lambda x: x - 1)
ti.set_finished_dependency(3)
ti.register_tasks((4, ))
ti.finish_prereq(TwoPrereqs.Prereq1, (4, ))
ti.finish_prereq(TwoPrereqs.Prereq2, (4, ))
ready = await wait(ti.ready_tasks())
assert ready == (4, )
# now start in a discontinuous series of tasks
with pytest.raises(MissingDependency):
ti.register_tasks((6, ))
ti.set_finished_dependency(5)
ti.register_tasks((6, ))
ti.finish_prereq(TwoPrereqs.Prereq1, (6, ))
ti.finish_prereq(TwoPrereqs.Prereq2, (6, ))
ready = await wait(ti.ready_tasks())
assert ready == (6, )
async def test_dangled_pruning():
# make a number task depend on the mod10, so 4 and 14 both depend on task 3
ti = OrderedTaskPreparation(
NoPrerequisites,
identity,
lambda x: (x % 10) - 1,
max_depth=2,
accept_dangling_tasks=True,
)
ti.set_finished_dependency(3)
ti.register_tasks((4, 5, 6))
finished = await ti.ready_tasks()
assert finished == (4, 5, 6)
# No obvious way to check which tasks are pruned when accepting dangling tasks,
# so use an internal API until a better option is found:
# Nothing should be pruned yet, because caller is still "acting" on (4, 5, 6)
assert 3 in ti._tasks
# caller indicates that she is done working on (4, 5, 6) by calling ready_tasks() again
await assert_nothing_ready(ti)
# 3 should be pruned now
assert 3 not in ti._tasks
assert 4 in ti._tasks
ti.register_tasks((7, ))
finished = await ti.ready_tasks()
assert finished == (7, )
# 4 still shouldn't be pruned, because caller is "acting" on 7
assert 4 in ti._tasks
# caller indicates that she is done working on (7, ) by calling ready_tasks() again
await assert_nothing_ready(ti)
# 4 should be pruned now
assert 4 not in ti._tasks
