async def run(self):
"""
Pool periodically checks the current demand and provided drones.
If demand is higher than the current level, the pool takes care of
initialising new drones. Otherwise drones get removed.
"""
async with Scope() as scope:
await self.init_pool(scope=scope, init=self._level)
async for _ in interval(1):
drones_required = min(self._demand,
self._capacity) - self._level
while drones_required > 0:
drones_required -= 1
# start a new drone
drone = self.make_drone(10)
scope.do(drone.run())
self._drones.append(drone)
self._level += 1
if drones_required < 0:
for drone in self.drones:
if drone.jobs == 0:
drones_required += 1
self._level -= 1
self._drones.remove(drone)
scope.do(drone.shutdown())
if drones_required == 0:
break
async def transfer(self, file: RequestedFile):
"""
Every time a file is requested from this kind of storage, `_hitrate` percent
of the file are found on and transferred from this storage.
1 - `_hitrate` percent of the file are transferred from the remote storage
associated to the hitrate storage.
:param file:
"""
hitrate_size = self._hitrate * file.filesize
async with Scope() as scope:
logging.getLogger("implementation").warning(
"{} {} @ {} in {}".format(
hitrate_size,
file.filesize - hitrate_size,
time.now,
file.filename[-30:],
)
)
scope.do(self.connection.transfer(total=hitrate_size))
scope.do(
self.remote_storage.connection.transfer(
total=file.filesize - hitrate_size
)
)
return TransferStatistics(
bytes_from_remote=file.filesize - hitrate_size,
bytes_from_cache=hitrate_size,
)
async def run(self):
"""
Pool runs forever and does not check if number of drones needs to be adapted.
"""
async with Scope() as scope:
await self.init_pool(scope=scope, init=self._level)
await eternity
async def transfer(self, file: RequestedFile, job_repr=None):
"""
Every time a file is requested from this kind of storage, `_hitrate` percent
of the file are found on and transferred from this storage.
1 - `_hitrate` percent of the file are transferred from the remote storage
associated to the hitrate storage.
:param file:
:param job_repr:
"""
async with Scope() as scope:
logging.getLogger("implementation").warning(
"{} {} @ {} in {}".format(
self._hitrate * file.filesize,
(1 - self._hitrate) * file.filesize,
time.now,
file.filename[-30:],
)
)
scope.do(self.connection.transfer(total=self._hitrate * file.filesize))
scope.do(
self.remote_storage.connection.transfer(
total=(1 - self._hitrate) * file.filesize
)
)
async def main():
pipe = MonitoredPipe(128)
async with Scope() as scope:
scope.do(report_load(pipe), volatile=True)
scope.do(perform_load(pipe, 0, 512))
scope.do(perform_load(pipe, 4, 1024))
scope.do(perform_load(pipe, 6, 128))
scope.do(perform_load(pipe, 12, 1024))
async def test_do(self):
async def payload():
return 2
async with Scope() as scope:
activity = scope.do(payload())
assert await activity == 2
async def bare_scope():
await instant # allow the task to postpone before we cancel it
# a bare Scope postpones twice:
# 1) signalling body done
# 2) waiting for children (even if there are none)
async with Scope():
...
await eternity # receive late cancellations
async def test_run_job(self):
drone = DummyDrone()
job = Job(resources={"walltime": 50}, used_resources={"walltime": 10})
assert float("inf") == job.waiting_time
async with Scope() as scope:
scope.do(job.run(drone))
assert 10 == time
assert 0 == job.waiting_time
assert job.successful
async def test_after_and_at(self):
async def payload():
return 2
async with Scope() as scope:
_payload = payload()
with pytest.raises(AssertionError):
scope.do(_payload, after=1, at=1)
_payload.close()
async def test_explicit(self):
def record():
pass
record.created = pytime()
filter = SimulationTimeFilter()
async with Scope() as _:
filter.filter(record)
assert record.created == 0
async def test_teardown_late(self):
"""Test that the scope may receive failures during shutdown"""
async def fail_late(scope):
await scope
raise KeyError
with pytest.raises(Concurrent[KeyError]):
async with Scope() as scope:
scope.do(fail_late(scope))
async def test_spawn_late(self):
"""Test spawning during graceful shutdown"""
async def spawn_late(scope):
await scope
scope.do(time + 10)
async with Scope() as scope:
scope.do(spawn_late(scope))
assert time.now == 10
async def test_order():
async def add_char(position: int, target: list):
target.append(chr(ord('a') + position))
result = []
async with Scope() as scope:
for i in range(5):
scope.do(add_char(i, result))
assert "".join(result) == "abcde"
async def test_state_success(self):
async with Scope() as scope:
activity = scope.do(sleep(20))
assert activity.status == TaskState.CREATED
await instant
assert activity.status == TaskState.RUNNING
await activity.done
assert activity.status == TaskState.SUCCESS
assert activity.status & TaskState.FINISHED
async def test_transfer_inexact(self):
# Values adapted from MatterMiners/lapis#61
# Need to advance the simulation time to have a lower
# time resolution. This makes it more likely to round
# down the calculated transfer time.
await (time + 100)
pipe = Pipe(throughput=10)
async with Scope() as scope:
for _ in range(6):
scope.do(pipe.transfer(total=15))
async def test_comparison_wait(self):
value = 1137
tracked = Tracked(value)
await (tracked == value)
assert time.now == 0
async with Scope() as scope:
scope.do(tracked + 10, after=10)
scope.do(tracked + 10, after=20)
await (tracked == value + 20)
assert time.now == 20
async def test_volatile(self):
async def payload():
await eternity
return 2
async with Scope() as scope:
activity = scope.do(payload(), volatile=True)
with pytest.raises(VolatileTaskClosed):
assert await activity
assert activity.status == TaskState.CANCELLED
async def test_await(self):
async with Scope() as scope:
activity = scope.do(sleep(20))
assert time.now == 0
# await inside scope
await activity.done
assert time.now == 20
# await outside scope
await activity.done
assert time.now == 20
async def test_spawn_after_shutdown(self):
async def activity(value):
return value
async with Scope() as scope:
pass
payload = activity(3)
with pytest.raises(RuntimeError):
scope.do(payload)
assert inspect.getcoroutinestate(payload) == inspect.CORO_CLOSED
async def test_order_with_cancel():
async def add_char(position: int, target: list):
target.append(chr(ord('a') + position))
result = []
async with Scope() as scope:
for i in range(7):
activity = scope.do(add_char(i, result))
if i % 2 == 0:
activity.cancel()
assert "".join(result) == "bdf"
async def test_result(self):
async with Scope() as scope:
activity = scope.do(sleep(20))
assert time.now == 0
# await result inside scope
assert await activity == 20
# await result delayed us
assert time.now == 20
# await outside scope
assert await activity == 20
assert time.now == 20
async def test_do_at(self):
async with Scope() as scope:
apass = scope.do(async_pass(), at=10)
assert time.now == 0
await apass
assert time.now == 10
# zero delay is well-defined
apass = scope.do(async_pass(), at=10)
assert time.now == 10
await apass
assert time.now == 10
async def test_negative(self):
async def payload():
return 2
async with Scope() as scope:
_payload = payload()
with pytest.raises(AssertionError):
scope.do(_payload, after=-1)
with pytest.raises(AssertionError):
scope.do(_payload, at=-1)
_payload.close()
async def test_at(self):
async def payload(duration):
await (time + duration)
async with Scope() as scope:
activity_one = scope.do(payload(10), at=5)
activity_two = scope.do(payload(15), at=5)
await (activity_one.done | activity_two.done)
assert time.now == 15
await (activity_one.done & activity_two.done)
assert time.now == 20
async def test_after(self):
async def payload():
await (time + 10)
async with Scope() as scope:
activity = scope.do(payload(), after=5)
assert activity.status == TaskState.CREATED
await (time + 4)
assert activity.status == TaskState.RUNNING
await activity.done
assert time.now == 15
assert activity.status == TaskState.SUCCESS
async def test_contended(self):
lock = Lock()
async def mutext_sleep(delay):
async with lock:
await (time + delay)
async with Scope() as scope:
scope.do(mutext_sleep(5))
scope.do(mutext_sleep(5))
scope.do(mutext_sleep(10))
assert time == 20
async def test_state_cancel_running(self):
async with Scope() as scope:
activity = scope.do(sleep(20))
assert activity.status == TaskState.CREATED
await instant
assert activity.status == TaskState.RUNNING
activity.cancel()
# running cancellation is graceful
assert activity.status == TaskState.RUNNING
await activity.done
assert activity.status == TaskState.CANCELLED
assert activity.status & TaskState.FINISHED
async def test_state_cancel_created(self):
async with Scope() as scope:
activity = scope.do(sleep(20))
assert activity.status == TaskState.CREATED
activity.cancel()
# early cancellation does not run
assert activity.status == TaskState.CANCELLED
await instant
assert activity.status == TaskState.CANCELLED
await activity.done
assert activity.status == TaskState.CANCELLED
assert activity.status & TaskState.FINISHED
async def test_transfers(self):
pipe = UnboundedPipe()
for total in (0, 10, float('inf')):
await pipe.transfer(total=total)
assert (time == 0)
async with Scope() as scope:
for total in (0, 10, float('inf')):
scope.do(pipe.transfer(total=total))
scope.do(pipe.transfer(total=total))
scope.do(pipe.transfer(total=total))
scope.do(pipe.transfer(total=total))
assert (time == 0)
async def test_congested(self):
resources = self.resource_type(a=10, b=10)
async def borrow(duration, **amounts):
async with resources.borrow(**amounts):
await (time + duration)
assert time == 0
async with Scope() as scope:
scope.do(borrow(10, a=6, b=4))
scope.do(borrow(10, a=6, b=4)) # not compatible with 1)
scope.do(borrow(10, a=4, b=6)) # compatible with 1) and 2)
assert time == 20