def get_active_tasks(self, status=None):
if status:
for task in six.itervalues(self._status_tasks[status]):
yield task
else:
for task in six.itervalues(self._tasks):
yield task
def get_active_tasks(self, status=None):
if status:
for task in six.itervalues(self._status_tasks[status]):
yield task
else:
for task in six.itervalues(self._tasks):
yield task
def load(self):
if os.path.exists(self._state_path):
logger.info("Attempting to load state from %s", self._state_path)
try:
with open(self._state_path, 'rb') as fobj:
state = pickle.load(fobj)
except BaseException:
logger.exception("Error when loading state. Starting from clean slate.")
return
self._tasks, self._active_workers = state
self._status_tasks = collections.defaultdict(dict)
for task in six.itervalues(self._tasks):
self._status_tasks[task.status][task.id] = task
# Convert from old format
# TODO: this is really ugly, we need something more future-proof
# Every time we add an attribute to the Worker class, this code needs to be updated
for k, v in six.iteritems(self._active_workers):
if isinstance(v, float):
self._active_workers[k] = Worker(worker_id=k, last_active=v)
if any(not hasattr(w, 'tasks') for k, w in six.iteritems(self._active_workers)):
# If you load from an old format where Workers don't contain tasks.
for k, worker in six.iteritems(self._active_workers):
worker.tasks = set()
for task in six.itervalues(self._tasks):
for worker_id in task.workers:
self._active_workers[worker_id].tasks.add(task)
else:
logger.info("No prior state file exists at %s. Starting with clean slate", self._state_path)
def load(self):
if os.path.exists(self._state_path):
logger.info("Attempting to load state from %s", self._state_path)
try:
with open(self._state_path, 'rb') as fobj:
state = pickle.load(fobj)
except BaseException:
logger.exception("Error when loading state. Starting from clean slate.")
return
self._tasks, self._active_workers = state
self._status_tasks = collections.defaultdict(dict)
for task in six.itervalues(self._tasks):
self._status_tasks[task.status][task.id] = task
# Convert from old format
# TODO: this is really ugly, we need something more future-proof
# Every time we add an attribute to the Worker class, this code needs to be updated
for k, v in six.iteritems(self._active_workers):
if isinstance(v, float):
self._active_workers[k] = Worker(worker_id=k, last_active=v)
if any(not hasattr(w, 'tasks') for k, w in six.iteritems(self._active_workers)):
# If you load from an old format where Workers don't contain tasks.
for k, worker in six.iteritems(self._active_workers):
worker.tasks = set()
for task in six.itervalues(self._tasks):
for worker_id in task.workers:
self._active_workers[worker_id].tasks.add(task)
else:
logger.info("No prior state file exists at %s. Starting with clean slate", self._state_path)
def _send_email(self, fail_counts, disable_counts, scheduling_counts, fail_expls, owner):
num_failures = sum(six.itervalues(fail_counts))
num_disables = sum(six.itervalues(disable_counts))
num_scheduling_failures = sum(six.itervalues(scheduling_counts))
subject_parts = [
_plural_format('{} failure{}', num_failures),
_plural_format('{} disable{}', num_disables),
_plural_format('{} scheduling failure{}', num_scheduling_failures),
]
subject_base = ', '.join(filter(None, subject_parts))
if subject_base:
prefix = '' if owner in self._default_owner else 'Your tasks have '
subject = 'Luigi: {}{} in the last {} minutes'.format(
prefix, subject_base, self._config.email_interval)
email_body = self._email_body(fail_counts, disable_counts, scheduling_counts, fail_expls)
send_email(subject, email_body, email().sender, (owner,))
def _send_email(self, fail_counts, disable_counts, scheduling_counts, fail_expls, owner):
num_failures = sum(six.itervalues(fail_counts))
num_disables = sum(six.itervalues(disable_counts))
num_scheduling_failures = sum(six.itervalues(scheduling_counts))
subject_parts = [
_plural_format('{} failure{}', num_failures),
_plural_format('{} disable{}', num_disables),
_plural_format('{} scheduling failure{}', num_scheduling_failures),
]
subject_base = ', '.join(filter(None, subject_parts))
if subject_base:
prefix = '' if owner in self._default_owner else 'Your tasks have '
subject = 'Luigi: {}{} in the last {} minutes'.format(
prefix, subject_base, self._config.email_interval)
email_body = self._email_body(fail_counts, disable_counts, scheduling_counts, fail_expls)
send_email(subject, email_body, email().sender, (owner,))
def get_active_workers(self, last_active_lt=None, last_get_work_gt=None):
for worker in six.itervalues(self._active_workers):
if last_active_lt is not None and worker.last_active >= last_active_lt:
continue
last_get_work = getattr(worker, "last_get_work", None)
if last_get_work_gt is not None and (last_get_work is None or last_get_work <= last_get_work_gt):
continue
yield worker
def get_active_workers(self, last_active_lt=None, last_get_work_gt=None):
for worker in six.itervalues(self._active_workers):
if last_active_lt is not None and worker.last_active >= last_active_lt:
continue
last_get_work = getattr(worker, 'last_get_work', None)
if last_get_work_gt is not None and (
last_get_work is None or last_get_work <= last_get_work_gt):
continue
yield worker
def load(self):
if os.path.exists(self._state_path):
logger.info("Attempting to load state from %s", self._state_path)
try:
with open(self._state_path, 'rb') as fobj:
state = pickle.load(fobj)
except BaseException:
logger.exception("Error when loading state. Starting from empty state.")
return
self.set_state(state)
self._status_tasks = collections.defaultdict(dict)
for task in six.itervalues(self._tasks):
self._status_tasks[task.status][task.id] = task
else:
logger.info("No prior state file exists at %s. Starting with empty state", self._state_path)
def get_active_workers(self, last_active_lt=None):
for worker in six.itervalues(self._active_workers):
if last_active_lt is not None and worker.last_active >= last_active_lt:
continue
yield worker
def get_pending_tasks(self):
return itertools.chain.from_iterable(six.itervalues(self._status_tasks[status])
for status in [PENDING, RUNNING])
def get_running_tasks(self):
return six.itervalues(self._status_tasks[RUNNING])
def get_pending_tasks(self):
return itertools.chain.from_iterable(
six.itervalues(self._status_tasks[status])
for status in [PENDING, RUNNING])
def get_running_tasks(self):
return six.itervalues(self._status_tasks[RUNNING])
def get_active_workers(self, last_active_lt=None):
for worker in six.itervalues(self._active_workers):
if last_active_lt is not None and worker.last_active >= last_active_lt:
continue
yield worker
