def resubmit_task(self, user, force=False):
"""Resubmit failed/canceled top-level task."""
if not user.is_superuser:
if self.owner.username != user.username:
raise Exception("You are not task owner or superuser.")
if self.parent:
raise Exception("Task is not top-level: %s" % self.id)
if self.exclusive:
raise Exception("Cannot resubmit exclusive task: %s" % self.id)
if not force and self.state not in FAILED_STATES:
states = [ TASK_STATES.get_value(i) for i in FAILED_STATES ]
raise Exception("Task '%s' must be in: %s" % (self.id, states))
kwargs = {
"owner_name": self.owner.username,
"label": self.label,
"method": self.method,
"args": self.args,
"comment": self.comment,
"parent_id": None,
"worker_name": None,
"arch_name": self.arch.name,
"channel_name": self.channel.name,
"priority": self.priority,
"weight": self.weight,
"exclusive": self.exclusive,
"resubmitted_by": user,
"resubmitted_from": self,
}
return Task.create_task(**kwargs)
def resubmit_task(self, user, force=False, priority=None):
"""Resubmit failed/canceled top-level task."""
if not user.is_superuser:
if self.owner.username != user.username:
raise Exception("You are not task owner or superuser.")
if self.parent:
raise Exception("Task is not top-level: %s" % self.id)
if self.exclusive:
raise Exception("Cannot resubmit exclusive task: %s" % self.id)
if not force and self.state not in FAILED_STATES:
states = [TASK_STATES.get_value(i) for i in FAILED_STATES]
raise Exception("Task '%s' must be in: %s" % (self.id, states))
kwargs = {
"owner_name": self.owner.username,
"label": self.label,
"method": self.method,
"args": self.args,
"comment": self.comment,
"parent_id": None,
"worker_name": None,
"arch_name": self.arch.name,
"channel_name": self.channel.name,
"priority": priority if priority is not None else self.priority,
"weight": self.weight,
"exclusive": self.exclusive,
"resubmitted_by": user,
"resubmitted_from": self,
}
return Task.create_task(**kwargs)
def __lock(self,
worker_id,
new_state=TASK_STATES["ASSIGNED"],
initial_states=None):
"""Critical section. Ensures that only one worker takes the task."""
if type(initial_states) in (list, tuple):
# filter out invalid state codes
initial_states = [
i for i, j in TASK_STATES.get_mapping() if i in initial_states
]
if not initial_states:
# initial_states is empty
initial_states = (TASK_STATES["FREE"], )
else:
initial_states = (TASK_STATES["FREE"], )
# it is safe to pass initial_states directly to query,
# because these values are checked in the code above
query = dedent( # nosec B608
"""
UPDATE
hub_task
SET
state=%%s,
worker_id=%%s,
dt_started=%%s,
dt_finished=%%s,
waiting=%%s
WHERE
id=%%s
and state in (%(initial_states)s)
and (worker_id is null or worker_id=%%s)
""") % {
"initial_states": ",".join(
("'%s'" % i for i in initial_states))
}
dt_started = self.dt_started
if new_state == TASK_STATES["OPEN"]:
dt_started = datetime.datetime.now()
dt_finished = self.dt_finished
if new_state in FINISHED_STATES:
dt_finished = datetime.datetime.now()
new_worker_id = worker_id
if new_state == TASK_STATES["FREE"]:
new_worker_id = None
waiting = False
with transaction.atomic():
cursor = connection.cursor()
cursor.execute(query, (new_state, new_worker_id, dt_started,
dt_finished, waiting, self.id, worker_id))
if cursor.rowcount == 0:
if self.state in FINISHED_STATES:
logger.debug(
"Trying to interrupt closed task %s, ignoring.",
self.id)
return
else:
raise ObjectDoesNotExist()
if cursor.rowcount > 1:
raise MultipleObjectsReturned()
self.dt_started = dt_started
self.dt_finished = dt_finished
if new_worker_id is not None:
self.worker = Worker.objects.get(id=new_worker_id)
self.state = new_state
self.waiting = waiting
class Task(models.Model):
"""Model for hub_task table."""
archive = models.BooleanField(
default=False,
help_text=
_("When a task is archived, it disappears from admin interface and cannot be accessed by taskd.<br />Make sure that archived tasks are finished and you won't need them anymore."
))
owner = models.ForeignKey(settings.AUTH_USER_MODEL,
on_delete=models.CASCADE)
worker = models.ForeignKey(
Worker,
null=True,
blank=True,
help_text=_("A worker which has this task assigned."),
on_delete=models.CASCADE)
parent = models.ForeignKey("self",
null=True,
blank=True,
help_text=_("Parent task."),
on_delete=models.CASCADE)
state = models.PositiveIntegerField(default=TASK_STATES["FREE"],
choices=TASK_STATES.get_mapping(),
help_text=_("Current task state."))
label = models.CharField(
max_length=255,
blank=True,
help_text=_("Label, description or any reason for this task."))
exclusive = models.BooleanField(
default=False,
help_text=
_("Exclusive tasks have highest priority. They are used e.g. when shutting down a worker."
))
method = models.CharField(
max_length=255,
help_text=_("Method name represents appropriate task handler."))
args = kobo.django.fields.JSONField(
blank=True,
default={},
help_text=_("Method arguments. JSON serialized dictionary."))
result = models.TextField(
blank=True,
help_text=
_("Task result. Do not store a lot of data here (use HubProxy.upload_task_log instead)."
))
comment = models.TextField(null=True, blank=True)
arch = models.ForeignKey(Arch, on_delete=models.CASCADE)
channel = models.ForeignKey(Channel, on_delete=models.CASCADE)
timeout = models.PositiveIntegerField(
null=True,
blank=True,
help_text=_("Task timeout. Leave blank for no timeout."))
waiting = models.BooleanField(
default=False,
help_text=_("Task is waiting until some subtasks finish."))
awaited = models.BooleanField(
default=False, help_text=_("Task is awaited by another task."))
dt_created = models.DateTimeField(auto_now_add=True)
dt_started = models.DateTimeField(null=True, blank=True)
dt_finished = models.DateTimeField(null=True, blank=True)
priority = models.PositiveIntegerField(default=10,
help_text=_("Priority."))
weight = models.PositiveIntegerField(
default=1,
help_text=
_("Weight determines how many resources is used when processing the task."
))
resubmitted_by = models.ForeignKey(settings.AUTH_USER_MODEL,
null=True,
blank=True,
related_name="resubmitted_by1",
on_delete=models.CASCADE)
resubmitted_from = models.ForeignKey("self",
null=True,
blank=True,
related_name="resubmitted_from1",
on_delete=models.CASCADE)
subtask_count = models.PositiveIntegerField(
default=0,
help_text=_("Subtask count.<br />This is a generated field."))
# override default *objects* Manager
objects = TaskManager()
class Meta:
ordering = ("-id", )
permissions = (("can_see_traceback", _("Can see traceback")), )
def __init__(self, *args, **kwargs):
self.logs = TaskLogs(self)
traceback = kwargs.pop("traceback", None)
if traceback:
self.logs["traceback.log"] = traceback
stdout = kwargs.pop("stdout", None)
if stdout:
self.logs["stdout.log"] = stdout
super(Task, self).__init__(*args, **kwargs)
def __str__(self):
if self.parent:
return u"#%s [method: %s, state: %s, worker: %s, parent: #%s]" % (
self.id, self.method, self.get_state_display(), self.worker,
self.parent.id)
return u"#%s [method: %s, state: %s, worker: %s]" % (
self.id, self.method, self.get_state_display(), self.worker)
def save(self, *args, **kwargs):
# save to db to precalculate subtask counts and obtain an ID (on insert) for stdout and traceback
self.subtask_count = self.subtasks().count()
super(self.__class__, self).save()
self.logs.save()
if self.parent:
self.parent.save(*args, **kwargs)
@classmethod
def get_task_dir(cls, task_id, create=False):
'''Task files (logs, etc.) are saved in TASK_DIR in following structure based on task_id:
TASK_DIR/millions/tens_of_thousands/task_id/*
'''
task_id = int(task_id)
third = task_id
second = task_id // 10000 * 10000
first = task_id // 1000000 * 1000000
task_dir = os.path.abspath(settings.TASK_DIR)
path = os.path.join(task_dir, str(first), str(second), str(third))
path = os.path.abspath(path)
if not path.startswith(task_dir):
raise Exception('Possible hack, trying to read path "%s"' % path)
if create and not os.path.isdir(path):
os.makedirs(path, mode=0o755)
return path
def task_dir(self, create=False):
return Task.get_task_dir(self.id, create)
@classmethod
def create_task(cls,
owner_name,
label,
method,
args=None,
comment=None,
parent_id=None,
worker_name=None,
arch_name="noarch",
channel_name="default",
timeout=None,
priority=10,
weight=1,
exclusive=False,
resubmitted_by=None,
resubmitted_from=None,
state=None):
"""Create a new task."""
task = cls()
task.owner = get_user_model().objects.get(username=owner_name)
task.label = label
task.method = method
task.args = args or {}
task.comment = comment
if parent_id is not None:
task.parent = cls.objects.get(id=parent_id)
if state is not None:
task.state = state
if worker_name is not None:
task.worker = Worker.objects.get(name=worker_name)
task.state = TASK_STATES["ASSIGNED"]
task.resubmitted_by = resubmitted_by
task.resubmitted_from = resubmitted_from
task.arch = Arch.objects.get(name=arch_name)
task.channel = Channel.objects.get(name=channel_name)
task.priority = priority
task.timeout = timeout
task.weight = weight
task.exclusive = exclusive
# TODO: unsupported in Django 1.0
#task.validate()
task.save()
return task.id
@classmethod
def create_shutdown_task(cls, owner_name, worker_name, kill=False):
"""Create a new ShutdownWorker task."""
kwargs = {
"owner_name": owner_name,
"label": "Shutdown a worker.",
"method": "ShutdownWorker",
"args": {
"kill": kill,
},
"worker_name": worker_name,
"weight": 0,
"exclusive": True,
}
return cls.create_task(**kwargs)
def set_args(self, **kwargs):
"""Serialize args dictionary."""
print(
"DeprecationWarning: kobo.hub.models.Task.set_args() is deprecated. Use kobo.hub.models.Task.args instead.",
file=sys.stderr)
self.args = kwargs
def get_args(self):
"""Deserialize args dictionary."""
print(
"DeprecationWarning: kobo.hub.models.Task.get_args() is deprecated. Use kobo.hub.models.Task.args instead.",
file=sys.stderr)
return self.args.copy()
def get_args_display(self):
"""Deserialize args dictionary to human readable form"""
from collections import OrderedDict
result = OrderedDict()
for key, value in sorted(self.args.items()):
result[key] = json.dumps(value)
return result
def export(self, flat=True):
"""Export data for xml-rpc."""
result = {
"id":
self.id,
"owner":
self.owner.username,
"worker":
self.worker_id,
"parent":
self.parent_id,
"state":
self.state,
"label":
self.label,
"method":
self.method,
"args":
self.args,
"result":
self.result,
"exclusive":
self.exclusive,
"arch":
self.arch_id,
"channel":
self.channel_id,
"timeout":
self.timeout,
"waiting":
self.waiting,
"awaited":
self.awaited,
"dt_created":
datetime.datetime.strftime(self.dt_created, "%F %R:%S"),
"dt_started":
self.dt_started and datetime.datetime.strftime(
self.dt_started, "%Y-%m-%d %H:%M:%S") or None,
"dt_finished":
self.dt_finished
and datetime.datetime.strftime(self.dt_finished, "%F %R:%S")
or None,
"priority":
self.priority,
"weight":
self.weight,
"resubmitted_by":
getattr(self.resubmitted_by, "username", None),
"resubmitted_from":
getattr(self.resubmitted_from, "id", None),
# used by task watcher
"state_label":
self.get_state_display(),
"is_finished":
self.is_finished(),
"is_failed":
self.is_failed(),
}
if not flat:
result.update({
"worker": self.worker and self.worker.export() or None,
"parent": self.parent and self.parent.export() or None,
"arch": self.arch.export(),
"channel": self.channel.export(),
"subtask_id_list": [i.id for i in self.subtasks()],
})
return result
def subtasks(self):
return Task.objects.filter(parent=self)
@property
def time(self):
"""return time spent in the task"""
if not self.dt_started:
return None
elif not self.dt_finished:
return datetime.datetime.now() - self.dt_started
else:
return self.dt_finished - self.dt_started
def get_time_display(self):
"""display time in human readable form"""
if self.time is None:
return ""
time = "%02d:%02d:%02d" % (
(self.time.seconds / 60.0 / 60.0),
(self.time.seconds / 60.0 % 60), self.time.seconds % 60)
if self.time.days:
time = _("%s days, %s") % (self.time.days, time)
return time
get_time_display.short_description = "Time"
def __lock(self,
worker_id,
new_state=TASK_STATES["ASSIGNED"],
initial_states=None):
"""Critical section. Ensures that only one worker takes the task."""
if type(initial_states) in (list, tuple):
# filter out invalid state codes
initial_states = [
i for i, j in TASK_STATES.get_mapping() if i in initial_states
]
if not initial_states:
# initial_states is empty
initial_states = (TASK_STATES["FREE"], )
else:
initial_states = (TASK_STATES["FREE"], )
# it is safe to pass initial_states directly to query,
# because these values are checked in the code above
query = dedent( # nosec B608
"""
UPDATE
hub_task
SET
state=%%s,
worker_id=%%s,
dt_started=%%s,
dt_finished=%%s,
waiting=%%s
WHERE
id=%%s
and state in (%(initial_states)s)
and (worker_id is null or worker_id=%%s)
""") % {
"initial_states": ",".join(
("'%s'" % i for i in initial_states))
}
dt_started = self.dt_started
if new_state == TASK_STATES["OPEN"]:
dt_started = datetime.datetime.now()
dt_finished = self.dt_finished
if new_state in FINISHED_STATES:
dt_finished = datetime.datetime.now()
new_worker_id = worker_id
if new_state == TASK_STATES["FREE"]:
new_worker_id = None
waiting = False
with transaction.atomic():
cursor = connection.cursor()
cursor.execute(query, (new_state, new_worker_id, dt_started,
dt_finished, waiting, self.id, worker_id))
if cursor.rowcount == 0:
if self.state in FINISHED_STATES:
logger.debug(
"Trying to interrupt closed task %s, ignoring.",
self.id)
return
else:
raise ObjectDoesNotExist()
if cursor.rowcount > 1:
raise MultipleObjectsReturned()
self.dt_started = dt_started
self.dt_finished = dt_finished
if new_worker_id is not None:
self.worker = Worker.objects.get(id=new_worker_id)
self.state = new_state
self.waiting = waiting
def free_task(self):
"""Free the task."""
try:
self.__lock(self.worker_id,
new_state=TASK_STATES["FREE"],
initial_states=(TASK_STATES["FREE"],
TASK_STATES["ASSIGNED"],
TASK_STATES["CREATED"]))
except (MultipleObjectsReturned, ObjectDoesNotExist):
raise Exception("Cannot free task %d, state is %s" %
(self.id, self.get_state_display()))
def assign_task(self, worker_id=None):
"""Assign the task to a worker identified by worker_id."""
if worker_id is None:
worker_id = self.worker_id
try:
self.__lock(worker_id,
new_state=TASK_STATES["ASSIGNED"],
initial_states=(TASK_STATES["FREE"],
TASK_STATES["CREATED"]))
except (MultipleObjectsReturned, ObjectDoesNotExist):
raise Exception("Cannot assign task %d" % (self.id))
def open_task(self, worker_id=None):
"""Open the task on a worker identified by worker_id."""
if worker_id is None:
worker_id = self.worker_id
try:
self.__lock(worker_id,
new_state=TASK_STATES["OPEN"],
initial_states=(TASK_STATES["FREE"],
TASK_STATES["ASSIGNED"]))
except (MultipleObjectsReturned, ObjectDoesNotExist):
raise Exception("Cannot open task %d, state is %s" %
(self.id, self.get_state_display()))
@transaction.atomic
def close_task(self, task_result=""):
"""Close the task and save result."""
if task_result:
self.result = task_result
self.save()
try:
self.__lock(self.worker_id,
new_state=TASK_STATES["CLOSED"],
initial_states=(TASK_STATES["OPEN"], ))
except (MultipleObjectsReturned, ObjectDoesNotExist):
raise Exception("Cannot close task %d, state is %s" %
(self.id, self.get_state_display()))
self.logs.gzip_logs()
@transaction.atomic
def cancel_task(self, user=None, recursive=True):
"""Cancel the task."""
if user is not None and not user.is_superuser:
if self.owner.username != user.username:
raise Exception("You are not task owner or superuser.")
try:
self.__lock(
self.worker_id,
new_state=TASK_STATES["CANCELED"],
initial_states=(TASK_STATES["FREE"], TASK_STATES["ASSIGNED"],
TASK_STATES["OPEN"], TASK_STATES["CREATED"]))
except (MultipleObjectsReturned, ObjectDoesNotExist):
raise Exception("Cannot cancel task %d, state is %s" %
(self.id, self.get_state_display()))
if recursive:
for task in self.subtasks():
task.cancel_task(recursive=True)
self.logs.gzip_logs()
def cancel_subtasks(self):
"""Cancel all subtasks of the task."""
result = True
for task in self.subtasks():
try:
result &= task.cancel_task()
except (MultipleObjectsReturned, ObjectDoesNotExist):
result = False
return result
@transaction.atomic
def interrupt_task(self, recursive=True):
"""Set the task state to interrupted."""
try:
self.__lock(self.worker_id,
new_state=TASK_STATES["INTERRUPTED"],
initial_states=(TASK_STATES["OPEN"], ))
except (MultipleObjectsReturned, ObjectDoesNotExist):
raise Exception("Cannot interrupt task %d, state is %s" %
(self.id, self.get_state_display()))
if recursive:
for task in self.subtasks():
task.interrupt_task(recursive=True)
self.logs.gzip_logs()
@transaction.atomic
def timeout_task(self, recursive=True):
"""Set the task state to timeout."""
try:
self.__lock(self.worker_id,
new_state=TASK_STATES["TIMEOUT"],
initial_states=(TASK_STATES["OPEN"], ))
except (MultipleObjectsReturned, ObjectDoesNotExist):
raise Exception("Cannot timeout task %d, state is %s" %
(self.id, self.get_state_display()))
if recursive:
for task in self.subtasks():
task.timeout_task(recursive=True)
self.logs.gzip_logs()
@transaction.atomic
def fail_task(self, task_result=""):
"""Fail this task and save result."""
if task_result:
self.result = task_result
self.save()
try:
self.__lock(self.worker_id,
new_state=TASK_STATES["FAILED"],
initial_states=(TASK_STATES["OPEN"], ))
except (MultipleObjectsReturned, ObjectDoesNotExist):
raise Exception("Cannot fail task %i, state is %s" %
(self.id, self.get_state_display()))
self.logs.gzip_logs()
def is_finished(self):
"""Is the task finished? Task state can be one of: closed, interrupted, canceled, failed."""
return self.state in FINISHED_STATES
def is_failed(self):
"""Is the task successfuly finished? Task state must be closed."""
return self.state in FAILED_STATES
def resubmit_task(self, user, force=False, priority=None):
"""Resubmit failed/canceled top-level task."""
if not user.is_superuser:
if self.owner.username != user.username:
raise Exception("You are not task owner or superuser.")
if self.parent:
raise Exception("Task is not top-level: %s" % self.id)
if self.exclusive:
raise Exception("Cannot resubmit exclusive task: %s" % self.id)
if not force and self.state not in FAILED_STATES:
states = [TASK_STATES.get_value(i) for i in FAILED_STATES]
raise Exception("Task '%s' must be in: %s" % (self.id, states))
kwargs = {
"owner_name": self.owner.username,
"label": self.label,
"method": self.method,
"args": self.args,
"comment": self.comment,
"parent_id": None,
"worker_name": None,
"arch_name": self.arch.name,
"channel_name": self.channel.name,
"priority": priority if priority is not None else self.priority,
"weight": self.weight,
"exclusive": self.exclusive,
"resubmitted_by": user,
"resubmitted_from": self,
}
return Task.create_task(**kwargs)
def clone_task(self, user, **kwargs):
"""Clone a task, override field values by kwargs."""
if not user.is_superuser:
raise Exception("You are not superuser.")
if self.parent:
raise Exception("Task is not top-level: %s" % self.id)
kwargs.pop("resubmitted_by", None)
kwargs.pop("resubmitted_from", None)
new_kwargs = {
"owner_name": self.owner.username,
"label": self.label,
"method": self.method,
"args": self.args,
"comment": self.comment,
"parent_id": None,
"worker_name": None,
"arch_name": self.arch.name,
"channel_name": self.channel.name,
"priority": self.priority,
"weight": self.weight,
"exclusive": self.exclusive,
"resubmitted_by": user,
"resubmitted_from": self,
}
new_kwargs.update(kwargs)
return Task.create_task(**new_kwargs)
def wait(self, child_task_list=None):
"""Set this task as waiting and all subtasks in child_task_list as awaited.
If child_task_list is None, process all related subtasks.
"""
tasks = self.subtasks().filter(state__in=(TASK_STATES["FREE"],
TASK_STATES["ASSIGNED"],
TASK_STATES["OPEN"]))
if child_task_list is not None:
tasks = tasks.filter(id__in=child_task_list)
for task in tasks:
task.awaited = True
task.save()
self.waiting = True
self.save()
def check_wait(self, child_task_list=None):
"""Determine if all subtasks have finished."""
tasks = self.subtasks()
if child_task_list is not None:
tasks = tasks.filter(id__in=child_task_list)
finished = []
unfinished = []
for task in tasks:
if task.is_finished():
finished.append(task.id)
task.awaited = False
task.save()
else:
unfinished.append(task.id)
return [finished, unfinished]
def set_weight(self, weight):
self.weight = weight
self.save()
def __lock(self, worker_id, new_state=TASK_STATES["ASSIGNED"], initial_states=None):
"""Critical section. Ensures that only one worker takes the task."""
if type(initial_states) in (list, tuple):
# filter out invalid state codes
initial_states = [ i for i, j in TASK_STATES.get_mapping() if i in initial_states ]
if not initial_states:
# initial_states is empty
initial_states = (TASK_STATES["FREE"], )
else:
initial_states = (TASK_STATES["FREE"], )
# it is safe to pass initial_states directly to query,
# because these values are checked in the code above
query = """
UPDATE
hub_task
SET
state=%%s,
worker_id=%%s,
dt_started=%%s,
dt_finished=%%s,
waiting=%%s
WHERE
id=%%s
and state in (%(initial_states)s)
and (worker_id is null or worker_id=%%s)
""" % { "initial_states": ",".join(( "'%s'" % i for i in initial_states )), }
dt_started = self.dt_started
if new_state == TASK_STATES["OPEN"]:
dt_started = datetime.datetime.now()
dt_finished = self.dt_finished
if new_state in FINISHED_STATES:
dt_finished = datetime.datetime.now()
new_worker_id = worker_id
if new_state == TASK_STATES["FREE"]:
new_worker_id = None
waiting = False
with transaction.atomic():
cursor = connection.cursor()
cursor.execute(query, (new_state, new_worker_id, dt_started, dt_finished, waiting, self.id, worker_id))
if cursor.rowcount == 0:
if self.state in FINISHED_STATES:
logger.debug("Trying to interrupt closed task %s, ignoring.", self.id)
return
else:
raise ObjectDoesNotExist()
if cursor.rowcount > 1:
raise MultipleObjectsReturned()
self.dt_started = dt_started
self.dt_finished = dt_finished
if new_worker_id is not None:
self.worker = Worker.objects.get(id=new_worker_id)
self.state = new_state
self.waiting = waiting