async def test_do_misuse(self):
fail_pass = async_pass()
with pytest.raises(AssertionError):
async with Scope() as scope:
scope.do(fail_pass, at=-1)
with pytest.raises(AssertionError):
async with Scope() as scope:
scope.do(fail_pass, after=-1)
fail_pass.close()
async def test_inner_exit(self):
"""Inner scopes done last"""
async with Scope() as scope1:
scope1.do(time + 5)
async with Scope() as scope2:
scope2.do(time + 7)
async with Scope() as scope3:
scope3.do(time + 10)
assert time.now == 10
assert time.now == 10
assert time.now == 10
async def test_joint_exit(self):
"""Multiple scopes done at the same time"""
async with Scope() as scope1:
scope1.do(time + 10)
async with Scope() as scope2:
scope2.do(time + 10)
async with Scope() as scope3:
scope3.do(time + 10)
assert time.now == 10
assert time.now == 10
assert time.now == 10
async def test_middle_exit(self):
"""Intermediate scopes done last"""
async with Scope() as scope1:
scope1.do(time + 7)
async with Scope() as scope2:
scope2.do(time + 10)
async with Scope() as scope3:
scope3.do(time + 5)
assert time.now == 5
assert time.now == 10
assert time.now == 10
async def test_fail_privileged(self):
"""Failure inside children with privileged errors is not collapsed"""
for exc_type in (AssertionError, KeyboardInterrupt, SystemExit):
with pytest.raises(exc_type):
async with Scope() as scope:
scope.do(async_raise(IndexError(), 0))
scope.do(async_raise(TypeError(), 0))
scope.do(async_raise(KeyError(), 0))
scope.do(async_raise(exc_type(), 0))
with pytest.raises(exc_type):
async with Scope() as scope:
scope.do(async_raise(IndexError(), 0))
scope.do(async_raise(TypeError(), 0))
scope.do(async_raise(KeyError(), 0))
raise exc_type
async def test_job_in_drone(self):
scheduler = DummyScheduler()
job = CachingJob(
resources={
"walltime": 50,
"cores": 1,
"memory": 1
},
used_resources={
"walltime": 10,
"cores": 1,
"memory": 1
},
)
drone = Drone(
scheduler=scheduler,
pool_resources={
"cores": 1,
"memory": 1
},
scheduling_duration=0,
connection=Connection(throughput=1),
)
async with Scope() as scope:
scope.do(drone.run(), volatile=True)
scope.do(drone.schedule_job(job=job))
await (scheduler.statistics._available ==
scheduler.statistics.resource_type(job_succeeded=1))
assert 10 == time.now
assert 0 == job.waiting_time
assert job.successful
async def test_nonmatching_job_in_drone(self):
scheduler = DummyScheduler()
job = Job(
resources={
"walltime": 50,
"cores": 2,
"memory": 1
},
used_resources={
"walltime": 10,
"cores": 1,
"memory": 1
},
)
drone = Drone(
scheduler=scheduler,
pool_resources={
"cores": 1,
"memory": 1
},
scheduling_duration=0,
)
async with Scope() as scope:
scope.do(drone.run(), volatile=True)
scope.do(drone.schedule_job(job=job))
await (scheduler.statistics._available ==
scheduler.statistics.resource_type(job_failed=1))
assert 0 == time
assert not job.successful
assert 0 == job.waiting_time
async def run(self):
"""
Pool runs forever and does not check if number of drones needs to be adapted.
"""
async with Scope() as scope:
await self.init_pool(scope=scope, init=self._level)
await eternity
async def transfer(self, file: RequestedFile):
"""
Every time a file is requested from this kind of storage, `_hitrate` percent
of the file are found on and transferred from this storage.
1 - `_hitrate` percent of the file are transferred from the remote storage
associated to the hitrate storage.
:param file:
"""
hitrate_size = self._hitrate * file.filesize
async with Scope() as scope:
logging.getLogger("implementation").warning(
"{} {} @ {} in {}".format(
hitrate_size,
file.filesize - hitrate_size,
time.now,
file.filename[-30:],
)
)
scope.do(self.connection.transfer(total=hitrate_size))
scope.do(
self.remote_storage.connection.transfer(
total=file.filesize - hitrate_size
)
)
return TransferStatistics(
bytes_from_remote=file.filesize - hitrate_size,
bytes_from_cache=hitrate_size,
)
async def run(self):
"""
Pool periodically checks the current demand and provided drones.
If demand is higher than the current level, the pool takes care of
initialising new drones. Otherwise drones get removed.
"""
async with Scope() as scope:
await self.init_pool(scope=scope, init=self._level)
async for _ in interval(1):
drones_required = min(self._demand,
self._capacity) - self._level
while drones_required > 0:
drones_required -= 1
# start a new drone
drone = self.make_drone(10)
scope.do(drone.run())
self._drones.append(drone)
self._level += 1
if drones_required < 0:
for drone in self.drones:
if drone.jobs == 0:
drones_required += 1
self._level -= 1
self._drones.remove(drone)
scope.do(drone.shutdown())
if drones_required == 0:
break
async def run(self):
async with Scope() as scope:
scope.do(self._collect_jobs())
async for _ in interval(self.interval):
for job in self.job_queue.copy():
best_match = self._schedule_job(job)
if best_match:
await best_match.schedule_job(job)
self.job_queue.remove(job)
await sampling_required.put(self.job_queue)
await sampling_required.put(UserDemand(len(self.job_queue)))
self.unregister_drone(best_match)
left_resources = best_match.theoretical_available_resources
left_resources = {
key: value - job.resources.get(key, 0)
for key, value in left_resources.items()
}
self._add_drone(best_match, left_resources)
if (
not self._collecting
and not self.job_queue
and self._processing.levels.jobs == 0
):
break
await sampling_required.put(self)
async def transfer(self, file: RequestedFile, job_repr=None):
"""
Every time a file is requested from this kind of storage, `_hitrate` percent
of the file are found on and transferred from this storage.
1 - `_hitrate` percent of the file are transferred from the remote storage
associated to the hitrate storage.
:param file:
:param job_repr:
"""
async with Scope() as scope:
logging.getLogger("implementation").warning(
"{} {} @ {} in {}".format(
self._hitrate * file.filesize,
(1 - self._hitrate) * file.filesize,
time.now,
file.filename[-30:],
)
)
scope.do(self.connection.transfer(total=self._hitrate * file.filesize))
scope.do(
self.remote_storage.connection.transfer(
total=(1 - self._hitrate) * file.filesize
)
)
async def main():
pipe = MonitoredPipe(128)
async with Scope() as scope:
scope.do(report_load(pipe), volatile=True)
scope.do(perform_load(pipe, 0, 512))
scope.do(perform_load(pipe, 4, 1024))
scope.do(perform_load(pipe, 6, 128))
scope.do(perform_load(pipe, 12, 1024))
async def test_do(self):
async def payload():
return 2
async with Scope() as scope:
activity = scope.do(payload())
assert await activity == 2
async def bare_scope():
await instant # allow the task to postpone before we cancel it
# a bare Scope postpones twice:
# 1) signalling body done
# 2) waiting for children (even if there are none)
async with Scope():
...
await eternity # receive late cancellations
async def test_after_and_at(self):
async def payload():
return 2
async with Scope() as scope:
_payload = payload()
with pytest.raises(AssertionError):
scope.do(_payload, after=1, at=1)
_payload.close()
async def test_state_success(self):
async with Scope() as scope:
activity = scope.do(sleep(20))
assert activity.status == TaskState.CREATED
await instant
assert activity.status == TaskState.RUNNING
await activity.done
assert activity.status == TaskState.SUCCESS
assert activity.status & TaskState.FINISHED
async def test_run_job(self):
drone = DummyDrone()
job = Job(resources={"walltime": 50}, used_resources={"walltime": 10})
assert float("inf") == job.waiting_time
async with Scope() as scope:
scope.do(job.run(drone))
assert 10 == time
assert 0 == job.waiting_time
assert job.successful
async def test_order():
async def add_char(position: int, target: list):
target.append(chr(ord('a') + position))
result = []
async with Scope() as scope:
for i in range(5):
scope.do(add_char(i, result))
assert "".join(result) == "abcde"
async def test_spawn_late(self):
"""Test spawning during graceful shutdown"""
async def spawn_late(scope):
await scope
scope.do(time + 10)
async with Scope() as scope:
scope.do(spawn_late(scope))
assert time.now == 10
async def test_explicit(self):
def record():
pass
record.created = pytime()
filter = SimulationTimeFilter()
async with Scope() as _:
filter.filter(record)
assert record.created == 0
async def test_teardown_late(self):
"""Test that the scope may receive failures during shutdown"""
async def fail_late(scope):
await scope
raise KeyError
with pytest.raises(Concurrent[KeyError]):
async with Scope() as scope:
scope.do(fail_late(scope))
async def test_spawn_after_shutdown(self):
async def activity(value):
return value
async with Scope() as scope:
pass
payload = activity(3)
with pytest.raises(RuntimeError):
scope.do(payload)
assert inspect.getcoroutinestate(payload) == inspect.CORO_CLOSED
async def test_comparison_wait(self):
value = 1137
tracked = Tracked(value)
await (tracked == value)
assert time.now == 0
async with Scope() as scope:
scope.do(tracked + 10, after=10)
scope.do(tracked + 10, after=20)
await (tracked == value + 20)
assert time.now == 20
async def test_volatile(self):
async def payload():
await eternity
return 2
async with Scope() as scope:
activity = scope.do(payload(), volatile=True)
with pytest.raises(VolatileTaskClosed):
assert await activity
assert activity.status == TaskState.CANCELLED
async def test_transfer_inexact(self):
# Values adapted from MatterMiners/lapis#61
# Need to advance the simulation time to have a lower
# time resolution. This makes it more likely to round
# down the calculated transfer time.
await (time + 100)
pipe = Pipe(throughput=10)
async with Scope() as scope:
for _ in range(6):
scope.do(pipe.transfer(total=15))
async def test_await(self):
async with Scope() as scope:
activity = scope.do(sleep(20))
assert time.now == 0
# await inside scope
await activity.done
assert time.now == 20
# await outside scope
await activity.done
assert time.now == 20
async def test_do_at(self):
async with Scope() as scope:
apass = scope.do(async_pass(), at=10)
assert time.now == 0
await apass
assert time.now == 10
# zero delay is well-defined
apass = scope.do(async_pass(), at=10)
assert time.now == 10
await apass
assert time.now == 10
async def test_at(self):
async def payload(duration):
await (time + duration)
async with Scope() as scope:
activity_one = scope.do(payload(10), at=5)
activity_two = scope.do(payload(15), at=5)
await (activity_one.done | activity_two.done)
assert time.now == 15
await (activity_one.done & activity_two.done)
assert time.now == 20
async def test_result(self):
async with Scope() as scope:
activity = scope.do(sleep(20))
assert time.now == 0
# await result inside scope
assert await activity == 20
# await result delayed us
assert time.now == 20
# await outside scope
assert await activity == 20
assert time.now == 20