class State(object):
"""Records clusters state."""
event_count = 0
task_count = 0
def __init__(self, callback=None,
max_workers_in_memory=5000, max_tasks_in_memory=10000):
self.workers = LRUCache(limit=max_workers_in_memory)
self.tasks = LRUCache(limit=max_tasks_in_memory)
self.event_callback = callback
self.group_handlers = {'worker': self.worker_event,
'task': self.task_event}
self._mutex = threading.Lock()
def freeze_while(self, fun, *args, **kwargs):
clear_after = kwargs.pop('clear_after', False)
with self._mutex:
try:
return fun(*args, **kwargs)
finally:
if clear_after:
self._clear()
def clear_tasks(self, ready=True):
with self._mutex:
return self._clear_tasks(ready)
def _clear_tasks(self, ready=True):
if ready:
in_progress = dict((uuid, task) for uuid, task in self.itertasks()
if task.state not in states.READY_STATES)
self.tasks.clear()
self.tasks.update(in_progress)
else:
self.tasks.clear()
def _clear(self, ready=True):
self.workers.clear()
self._clear_tasks(ready)
self.event_count = 0
self.task_count = 0
def clear(self, ready=True):
with self._mutex:
return self._clear(ready)
def get_or_create_worker(self, hostname, **kwargs):
"""Get or create worker by hostname."""
try:
worker = self.workers[hostname]
worker.update(kwargs)
except KeyError:
worker = self.workers[hostname] = Worker(
hostname=hostname, **kwargs)
return worker
def get_or_create_task(self, uuid):
"""Get or create task by uuid."""
try:
return self.tasks[uuid]
except KeyError:
task = self.tasks[uuid] = Task(uuid=uuid)
return task
def worker_event(self, type, fields):
"""Process worker event."""
hostname = fields.pop('hostname', None)
if hostname:
worker = self.get_or_create_worker(hostname)
handler = getattr(worker, 'on_%s' % type, None)
if handler:
handler(**fields)
def task_event(self, type, fields):
"""Process task event."""
uuid = fields.pop('uuid')
hostname = fields.pop('hostname')
worker = self.get_or_create_worker(hostname)
task = self.get_or_create_task(uuid)
handler = getattr(task, 'on_%s' % type, None)
if type == 'received':
self.task_count += 1
if handler:
handler(**fields)
else:
task.on_unknown_event(type, **fields)
task.worker = worker
def event(self, event):
with self._mutex:
return self._dispatch_event(event)
def _dispatch_event(self, event):
self.event_count += 1
event = kwdict(event)
group, _, type = event.pop('type').partition('-')
self.group_handlers[group](type, event)
if self.event_callback:
self.event_callback(self, event)
def itertasks(self, limit=None):
for index, row in enumerate(self.tasks.iteritems()):
yield row
if limit and index + 1 >= limit:
break
def tasks_by_timestamp(self, limit=None):
"""Get tasks by timestamp.
Returns a list of `(uuid, task)` tuples.
"""
return self._sort_tasks_by_time(self.itertasks(limit))
def _sort_tasks_by_time(self, tasks):
"""Sort task items by time."""
return sorted(tasks, key=lambda t: t[1].timestamp,
reverse=True)
def tasks_by_type(self, name, limit=None):
"""Get all tasks by type.
Returns a list of `(uuid, task)` tuples.
"""
sorted_tasks = self._sort_tasks_by_time((uuid, task)
for uuid, task in self.tasks.iteritems()
if task.name == name)
return sorted_tasks[0:limit or None]
def tasks_by_worker(self, hostname, limit=None):
"""Get all tasks by worker.
Returns a list of `(uuid, task)` tuples.
"""
return self._sort_tasks_by_time((uuid, task)
for uuid, task in self.itertasks(limit)
if task.worker.hostname == hostname)
def task_types(self):
"""Returns a list of all seen task types."""
return list(sorted(set(task.name for task in self.tasks.itervalues())))
def alive_workers(self):
"""Returns a list of (seemingly) alive workers."""
return [w for w in self.workers.values() if w.alive]
def __repr__(self):
return '<ClusterState: events=%s tasks=%s>' % (self.event_count,
self.task_count)
class State(object):
"""Records clusters state."""
event_count = 0
task_count = 0
def __init__(self, callback=None, max_workers_in_memory=5000, max_tasks_in_memory=10000):
self.max_workers_in_memory = max_workers_in_memory
self.max_tasks_in_memory = max_tasks_in_memory
self.workers = LRUCache(limit=self.max_workers_in_memory)
self.tasks = LRUCache(limit=self.max_tasks_in_memory)
self._taskheap = []
self.event_callback = callback
self.group_handlers = {"worker": self.worker_event, "task": self.task_event}
self._mutex = threading.Lock()
def freeze_while(self, fun, *args, **kwargs):
clear_after = kwargs.pop("clear_after", False)
with self._mutex:
try:
return fun(*args, **kwargs)
finally:
if clear_after:
self._clear()
def clear_tasks(self, ready=True):
with self._mutex:
return self._clear_tasks(ready)
def _clear_tasks(self, ready=True):
if ready:
in_progress = dict((uuid, task) for uuid, task in self.itertasks() if task.state not in states.READY_STATES)
self.tasks.clear()
self.tasks.update(in_progress)
else:
self.tasks.clear()
self._taskheap[:] = []
def _clear(self, ready=True):
self.workers.clear()
self._clear_tasks(ready)
self.event_count = 0
self.task_count = 0
def clear(self, ready=True):
with self._mutex:
return self._clear(ready)
def get_or_create_worker(self, hostname, **kwargs):
"""Get or create worker by hostname.
Returns tuple of ``(worker, was_created)``.
"""
try:
worker = self.workers[hostname]
worker.update(kwargs)
return worker, False
except KeyError:
worker = self.workers[hostname] = Worker(hostname=hostname, **kwargs)
return worker, True
def get_or_create_task(self, uuid):
"""Get or create task by uuid."""
try:
return self.tasks[uuid], True
except KeyError:
task = self.tasks[uuid] = Task(uuid=uuid)
return task, False
def worker_event(self, type, fields):
"""Process worker event."""
try:
hostname = fields["hostname"]
except KeyError:
pass
else:
worker, created = self.get_or_create_worker(hostname)
handler = getattr(worker, "on_" + type, None)
if handler:
handler(**fields)
return worker, created
def task_event(self, type, fields):
"""Process task event."""
uuid = fields.pop("uuid")
hostname = fields.pop("hostname")
worker, _ = self.get_or_create_worker(hostname)
task, created = self.get_or_create_task(uuid)
task.worker = worker
taskheap = self._taskheap
timestamp = fields.get("timestamp") or 0
clock = 0 if type == "sent" else fields.get("clock")
heappush(taskheap, _lamportinfo(clock, timestamp, worker.id, task))
curcount = len(self.tasks)
if len(taskheap) > self.max_tasks_in_memory * 2:
taskheap[:] = taskheap[curcount:]
handler = getattr(task, "on_" + type, None)
if type == "received":
self.task_count += 1
if handler:
handler(**fields)
else:
task.on_unknown_event(type, **fields)
return created
def event(self, event):
with self._mutex:
return self._dispatch_event(event)
def _dispatch_event(self, event):
self.event_count += 1
event = kwdict(event)
group, _, subject = event.pop("type").partition("-")
self.group_handlers[group](subject, event)
if self.event_callback:
self.event_callback(self, event)
def itertasks(self, limit=None):
for index, row in enumerate(items(self.tasks)):
yield row
if limit and index + 1 >= limit:
break
def tasks_by_time(self, limit=None):
"""Generator giving tasks ordered by time,
in ``(uuid, Task)`` tuples."""
seen = set()
for evtup in islice(reversed(self._taskheap), 0, limit):
uuid = evtup[3].uuid
if uuid not in seen:
yield uuid, evtup[3]
seen.add(uuid)
tasks_by_timestamp = tasks_by_time
def tasks_by_type(self, name, limit=None):
"""Get all tasks by type.
Returns a list of ``(uuid, Task)`` tuples.
"""
return islice(((uuid, task) for uuid, task in self.tasks_by_time() if task.name == name), 0, limit)
def tasks_by_worker(self, hostname, limit=None):
"""Get all tasks by worker.
"""
return islice(
((uuid, task) for uuid, task in self.tasks_by_time() if task.worker.hostname == hostname), 0, limit
)
def task_types(self):
"""Returns a list of all seen task types."""
return list(sorted(set(task.name for task in values(self.tasks))))
def alive_workers(self):
"""Returns a list of (seemingly) alive workers."""
return [w for w in values(self.workers) if w.alive]
def __repr__(self):
return "<State: events={0.event_count} tasks={0.task_count}>".format(self)
def __getstate__(self):
d = dict(vars(self))
d.pop("_mutex")
return d
def __setstate__(self, state):
self.__dict__ = state
self._mutex = threading.Lock()
class State(object):
"""Records clusters state."""
event_count = 0
task_count = 0
def __init__(self,
callback=None,
max_workers_in_memory=5000,
max_tasks_in_memory=10000):
self.workers = LRUCache(limit=max_workers_in_memory)
self.tasks = LRUCache(limit=max_tasks_in_memory)
self.event_callback = callback
self.group_handlers = {
'worker': self.worker_event,
'task': self.task_event
}
self._mutex = Lock()
def freeze_while(self, fun, *args, **kwargs):
clear_after = kwargs.pop('clear_after', False)
with self._mutex:
try:
return fun(*args, **kwargs)
finally:
if clear_after:
self._clear()
def clear_tasks(self, ready=True):
with self._mutex:
return self._clear_tasks(ready)
def _clear_tasks(self, ready=True):
if ready:
in_progress = dict((uuid, task) for uuid, task in self.itertasks()
if task.state not in states.READY_STATES)
self.tasks.clear()
self.tasks.update(in_progress)
else:
self.tasks.clear()
def _clear(self, ready=True):
self.workers.clear()
self._clear_tasks(ready)
self.event_count = 0
self.task_count = 0
def clear(self, ready=True):
with self._mutex:
return self._clear(ready)
def get_or_create_worker(self, hostname, **kwargs):
"""Get or create worker by hostname."""
try:
worker = self.workers[hostname]
worker.update(kwargs)
except KeyError:
worker = self.workers[hostname] = Worker(hostname=hostname,
**kwargs)
return worker
def get_or_create_task(self, uuid):
"""Get or create task by uuid."""
try:
return self.tasks[uuid]
except KeyError:
task = self.tasks[uuid] = Task(uuid=uuid)
return task
def worker_event(self, type, fields):
"""Process worker event."""
hostname = fields.pop('hostname', None)
if hostname:
worker = self.get_or_create_worker(hostname)
handler = getattr(worker, 'on_%s' % type, None)
if handler:
handler(**fields)
def task_event(self, type, fields):
"""Process task event."""
uuid = fields.pop('uuid')
hostname = fields.pop('hostname')
worker = self.get_or_create_worker(hostname)
task = self.get_or_create_task(uuid)
handler = getattr(task, 'on_%s' % type, None)
if type == 'received':
self.task_count += 1
if handler:
handler(**fields)
else:
task.on_unknown_event(type, **fields)
task.worker = worker
def event(self, event):
with self._mutex:
return self._dispatch_event(event)
def _dispatch_event(self, event):
self.event_count += 1
event = kwdict(event)
group, _, type = event.pop('type').partition('-')
self.group_handlers[group](type, event)
if self.event_callback:
self.event_callback(self, event)
def itertasks(self, limit=None):
for index, row in enumerate(self.tasks.iteritems()):
yield row
if limit and index + 1 >= limit:
break
def tasks_by_timestamp(self, limit=None):
"""Get tasks by timestamp.
Returns a list of `(uuid, task)` tuples.
"""
return self._sort_tasks_by_time(self.itertasks(limit))
def _sort_tasks_by_time(self, tasks):
"""Sort task items by time."""
return sorted(tasks, key=lambda t: t[1].timestamp, reverse=True)
def tasks_by_type(self, name, limit=None):
"""Get all tasks by type.
Returns a list of `(uuid, task)` tuples.
"""
return self._sort_tasks_by_time([
(uuid, task) for uuid, task in self.itertasks(limit)
if task.name == name
])
def tasks_by_worker(self, hostname, limit=None):
"""Get all tasks by worker.
Returns a list of `(uuid, task)` tuples.
"""
return self._sort_tasks_by_time([
(uuid, task) for uuid, task in self.itertasks(limit)
if task.worker.hostname == hostname
])
def task_types(self):
"""Returns a list of all seen task types."""
return list(sorted(set(task.name for task in self.tasks.itervalues())))
def alive_workers(self):
"""Returns a list of (seemingly) alive workers."""
return [w for w in self.workers.values() if w.alive]
def __repr__(self):
return '<ClusterState: events=%s tasks=%s>' % (self.event_count,
self.task_count)
class State(object):
"""Records clusters state."""
event_count = 0
task_count = 0
def __init__(self,
callback=None,
max_workers_in_memory=5000,
max_tasks_in_memory=10000):
self.max_workers_in_memory = max_workers_in_memory
self.max_tasks_in_memory = max_tasks_in_memory
self.workers = LRUCache(limit=self.max_workers_in_memory)
self.tasks = LRUCache(limit=self.max_tasks_in_memory)
self._taskheap = []
self.event_callback = callback
self._mutex = threading.Lock()
def freeze_while(self, fun, *args, **kwargs):
clear_after = kwargs.pop('clear_after', False)
with self._mutex:
try:
return fun(*args, **kwargs)
finally:
if clear_after:
self._clear()
def clear_tasks(self, ready=True):
with self._mutex:
return self._clear_tasks(ready)
def _clear_tasks(self, ready=True):
if ready:
in_progress = dict((uuid, task) for uuid, task in self.itertasks()
if task.state not in states.READY_STATES)
self.tasks.clear()
self.tasks.update(in_progress)
else:
self.tasks.clear()
self._taskheap[:] = []
def _clear(self, ready=True):
self.workers.clear()
self._clear_tasks(ready)
self.event_count = 0
self.task_count = 0
def clear(self, ready=True):
with self._mutex:
return self._clear(ready)
def get_or_create_worker(self, hostname, **kwargs):
"""Get or create worker by hostname.
Returns tuple of ``(worker, was_created)``.
"""
try:
worker = self.workers[hostname]
worker.update(kwargs)
return worker, False
except KeyError:
worker = self.workers[hostname] = Worker(hostname=hostname,
**kwargs)
return worker, True
def get_or_create_task(self, uuid):
"""Get or create task by uuid."""
try:
return self.tasks[uuid], True
except KeyError:
task = self.tasks[uuid] = Task(uuid=uuid)
return task, False
def worker_event(self, type, fields):
"""Process worker event."""
try:
hostname = fields['hostname']
except KeyError:
pass
else:
worker, created = self.get_or_create_worker(hostname)
handler = getattr(worker, 'on_' + type, None)
if handler:
handler(**fields)
return worker, created
def task_event(self, type, fields):
"""Process task event."""
uuid = fields['uuid']
hostname = fields['hostname']
worker, _ = self.get_or_create_worker(hostname)
task, created = self.get_or_create_task(uuid)
task.worker = worker
taskheap = self._taskheap
timestamp = fields.get('timestamp') or 0
clock = 0 if type == 'sent' else fields.get('clock')
heappush(taskheap, _lamportinfo(clock, timestamp, worker.id, task))
curcount = len(self.tasks)
if len(taskheap) > self.max_tasks_in_memory * 2:
taskheap[:] = taskheap[curcount:]
handler = getattr(task, 'on_' + type, None)
if type == 'received':
self.task_count += 1
if handler:
handler(**fields)
else:
task.on_unknown_event(type, **fields)
return created
def event(self, event):
with self._mutex:
return self._dispatch_event(event)
def _dispatch_event(self, event):
self.event_count += 1
event = kwdict(event)
group, _, subject = event['type'].partition('-')
getattr(self, group + '_event')(subject, event)
if self.event_callback:
self.event_callback(self, event)
def itertasks(self, limit=None):
for index, row in enumerate(items(self.tasks)):
yield row
if limit and index + 1 >= limit:
break
def tasks_by_time(self, limit=None):
"""Generator giving tasks ordered by time,
in ``(uuid, Task)`` tuples."""
seen = set()
for evtup in islice(reversed(self._taskheap), 0, limit):
uuid = evtup[3].uuid
if uuid not in seen:
yield uuid, evtup[3]
seen.add(uuid)
tasks_by_timestamp = tasks_by_time
def tasks_by_type(self, name, limit=None):
"""Get all tasks by type.
Returns a list of ``(uuid, Task)`` tuples.
"""
return islice(
((uuid, task)
for uuid, task in self.tasks_by_time() if task.name == name),
0,
limit,
)
def tasks_by_worker(self, hostname, limit=None):
"""Get all tasks by worker.
"""
return islice(
((uuid, task) for uuid, task in self.tasks_by_time()
if task.worker.hostname == hostname),
0,
limit,
)
def task_types(self):
"""Returns a list of all seen task types."""
return list(sorted(set(task.name for task in values(self.tasks))))
def alive_workers(self):
"""Returns a list of (seemingly) alive workers."""
return [w for w in values(self.workers) if w.alive]
def __repr__(self):
return '<State: events={0.event_count} tasks={0.task_count}>' \
.format(self)
def __getstate__(self):
d = dict(vars(self))
d.pop('_mutex')
return d
def __setstate__(self, state):
self.__dict__ = state
self._mutex = threading.Lock()