def test_count():
sm = SystemMonitor(n=5)
assert sm.count == 1
sm.update()
assert sm.count == 2
for i in range(10):
sm.update()
assert sm.count == 12
for v in sm.quantities.values():
assert len(v) == 5
def test_SystemMonitor():
sm = SystemMonitor()
a = sm.update()
sleep(0.01)
b = sm.update()
assert sm.cpu
assert sm.memory
assert set(a) == set(b)
assert all(rb >= 0 for rb in sm.read_bytes)
assert all(wb >= 0 for wb in sm.write_bytes)
assert all(len(q) == 2 for q in sm.quantities.values())
assert 'cpu' in repr(sm)
def test_zombie():
with tempfile.TemporaryDirectory() as folder:
sm = SystemMonitor()
sm.update()
process = subprocess.Popen(
(sys.executable, write_script(folder)),
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
sm.update()
process.terminate()
try:
sm.update()
except Exception as exc:
pytest.fail("unexpected exception: {0}".format(exc))
def test_missing_subprocess():
sm = SystemMonitor()
p = multiprocessing.Process(target=fib, args=(20, ))
p.start()
sm.update()
p.join()
try:
sm.update()
except Exception as exc:
pytest.fail("unexpected exception: {0}".format(exc))
def test_count():
sm = SystemMonitor(n=5)
assert sm.count == 1
sm.update()
assert sm.count == 2
for i in range(10):
sm.update()
assert sm.count == 12
for v in sm.quantities.values():
assert len(v) == 5
def test_subprocess():
sm = SystemMonitor()
a = sm.update()
p = multiprocessing.Process(target=fib, args=(20, ))
p.start()
# On the first iteration CPU usage of the subprocess is 0
sm.update()
b = sm.update()
p.join()
assert sm.cpu
assert sm.memory
assert a["memory"] < b["memory"]
def __init__(
self,
handlers,
blocked_handlers=None,
stream_handlers=None,
connection_limit=512,
deserialize=True,
serializers=None,
deserializers=None,
connection_args=None,
timeout=None,
io_loop=None,
):
self.handlers = {
"identity": self.identity,
"echo": self.echo,
"connection_stream": self.handle_stream,
"dump_state": self._to_dict,
}
self.handlers.update(handlers)
if blocked_handlers is None:
blocked_handlers = dask.config.get(
"distributed.%s.blocked-handlers" % type(self).__name__.lower(), []
)
self.blocked_handlers = blocked_handlers
self.stream_handlers = {}
self.stream_handlers.update(stream_handlers or {})
self.id = type(self).__name__ + "-" + str(uuid.uuid4())
self._address = None
self._listen_address = None
self._port = None
self._comms = {}
self.deserialize = deserialize
self.monitor = SystemMonitor()
self.counters = None
self.digests = None
self._ongoing_coroutines = weakref.WeakSet()
self._event_finished = asyncio.Event()
self.listeners = []
self.io_loop = io_loop or IOLoop.current()
self.loop = self.io_loop
if not hasattr(self.io_loop, "profile"):
ref = weakref.ref(self.io_loop)
def stop():
loop = ref()
return loop is None or loop.asyncio_loop.is_closed()
self.io_loop.profile = profile.watch(
omit=("profile.py", "selectors.py"),
interval=dask.config.get("distributed.worker.profile.interval"),
cycle=dask.config.get("distributed.worker.profile.cycle"),
stop=stop,
)
# Statistics counters for various events
with suppress(ImportError):
from distributed.counter import Digest
self.digests = defaultdict(partial(Digest, loop=self.io_loop))
from distributed.counter import Counter
self.counters = defaultdict(partial(Counter, loop=self.io_loop))
self.periodic_callbacks = dict()
pc = PeriodicCallback(
self.monitor.update,
parse_timedelta(
dask.config.get("distributed.admin.system-monitor.interval")
)
* 1000,
)
self.periodic_callbacks["monitor"] = pc
self._last_tick = time()
self._tick_counter = 0
self._tick_count = 0
self._tick_count_last = time()
self._tick_interval = parse_timedelta(
dask.config.get("distributed.admin.tick.interval"), default="ms"
)
self._tick_interval_observed = self._tick_interval
self.periodic_callbacks["tick"] = PeriodicCallback(
self._measure_tick, self._tick_interval * 1000
)
self.periodic_callbacks["ticks"] = PeriodicCallback(
self._cycle_ticks,
parse_timedelta(dask.config.get("distributed.admin.tick.cycle")) * 1000,
)
self.thread_id = 0
def set_thread_ident():
self.thread_id = threading.get_ident()
self.io_loop.add_callback(set_thread_ident)
self._startup_lock = asyncio.Lock()
self.status = Status.undefined
self.rpc = ConnectionPool(
limit=connection_limit,
deserialize=deserialize,
serializers=serializers,
deserializers=deserializers,
connection_args=connection_args,
timeout=timeout,
server=self,
)
self.__stopped = False
def test_range_query():
sm = SystemMonitor(n=5)
assert all(len(v) == 1 for v in sm.range_query(0).values())
assert all(len(v) == 0 for v in sm.range_query(123).values())
sm.update()
sm.update()
sm.update()
assert all(len(v) == 4 for v in sm.range_query(0).values())
assert all(len(v) == 3 for v in sm.range_query(1).values())
for i in range(10):
sm.update()
assert all(len(v) == 4 for v in sm.range_query(10).values())
assert all(len(v) == 5 for v in sm.range_query(0).values())
def test_range_query():
sm = SystemMonitor(n=5)
assert all(len(v) == 1 for v in sm.range_query(0).values())
assert all(len(v) == 0 for v in sm.range_query(123).values())
sm.update()
sm.update()
sm.update()
assert all(len(v) == 4 for v in sm.range_query(0).values())
assert all(len(v) == 3 for v in sm.range_query(1).values())
for i in range(10):
sm.update()
assert all(len(v) == 4 for v in sm.range_query(10).values())
assert all(len(v) == 5 for v in sm.range_query(0).values())