class Arch(models.Model):
"""Model for hub_arch table."""
name = models.CharField(max_length=16,
unique=True,
help_text=_("i386, ia64, ..."))
pretty_name = models.CharField(max_length=64,
unique=True,
help_text=_("i386, Itanium, ..."))
class Meta:
ordering = ("name", )
verbose_name_plural = "arches"
def __str__(self):
return u"%s" % self.name
def export(self):
"""Export data for xml-rpc."""
return {
"id": self.id,
"name": self.name,
"pretty_name": self.pretty_name,
}
@property
def worker_count(self):
if self.id is None:
return 0
return Worker.objects.filter(arches__id=self.id).count()
def to_python(self, value):
try:
result = json.loads(value)
except ValueError:
raise ValidationError(_("Cannot deserialize JSON data."))
else:
if not isinstance(result, (dict, list)):
raise ValidationError(_("Data types are restricted to JSON serialized dict or list only."))
return result
def from_db_value(self, value, expression, connection):
if not isinstance(value, six.string_types):
return value
try:
return json.loads(value)
except ValueError:
raise exceptions.ValidationError(
_("Cannot deserialize JSON data. '%s'") % value)
class UserDetailView(ExtraDetailView):
model = get_user_model()
title = _("User detail")
template_name = "user/detail.html"
context_object_name = "usr"
def get_context_data(self, **kwargs):
context = super(UserDetailView, self).get_context_data(**kwargs)
context['tasks'] = kwargs['object'].task_set.count()
return context
def get_db_prep_value(self, value, *args, **kwargs):
if value is None or value == "null":
return None
try:
if self.human_readable:
return json.dumps(value, indent=2, sort_keys=True)
else:
return json.dumps(value)
except Exception:
raise exceptions.ValidationError(_("Cannot serialize JSON data."))
class DetailViewWithWorkers(ExtraDetailView):
model = Channel
template_name = "channel/detail.html"
context_object_name = "channel"
title = _("Architecture detail")
def get_context_data(self, **kwargs):
context = super(DetailViewWithWorkers, self).get_context_data(**kwargs)
context["worker_list"] = kwargs["object"].worker_set.order_by("name")
return context
class ArchDetailView(ExtraDetailView):
model = Arch
template_name = "arch/detail.html"
context_object_name = "arch"
title = _("Architecture detail")
def get_context_data(self, **kwargs):
context = super(ArchDetailView, self).get_context_data(**kwargs)
context["worker_list"] = kwargs["object"].worker_set.order_by("name")
return context
def get_time_display(self):
"""display time in human readable form"""
if self.time is None:
return ""
time = "%02d:%02d:%02d" % (
(self.time.seconds / 60.0 / 60.0),
(self.time.seconds / 60.0 % 60), self.time.seconds % 60)
if self.time.days:
time = _("%s days, %s") % (self.time.days, time)
return time
def to_python(self, value):
if value is None:
return None
if not isinstance(value, six.string_types):
return value
try:
return json.loads(value)
except ValueError:
raise exceptions.ValidationError(
_("Cannot deserialize JSON data. '%s'") % value)
class TaskListView(SearchView):
# TODO: missing kwargs custom queries for backward compatibility
title = _("All tasks")
model = Task
form_class = TaskSearchForm
template_name = "task/list.html"
context_object_name = "task_list"
state = None
order_by = ['-id']
def get_form_kwargs(self):
kwargs = super(TaskListView, self).get_form_kwargs()
kwargs['state'] = self.state
kwargs['order_by'] = self.order_by
return kwargs
def from_db_value(self, value, expression, connection, context=None):
if value is None:
return None
if isinstance(value, (str, six.text_type)) and not value.isdigit():
value = self.state_machine.get_num(value)
try:
value = int(value)
if value < 0:
raise ValueError
except (TypeError, ValueError):
raise exceptions.ValidationError(
_("This value must be a positive integer."))
state_machine = copy(self.state_machine)
state_machine.set_state(state_machine.get_value(value))
return state_machine
class Channel(models.Model):
"""Model for hub_channel table."""
name = models.CharField(max_length=128, help_text=_("Channel name"))
def __str__(self):
return u"%s" % self.name
def export(self):
"""Export data for xml-rpc."""
return {
"id": self.id,
"name": self.name,
}
@property
def worker_count(self):
if self.id is None:
return 0
return Worker.objects.filter(channels__id=self.id).count()
class TaskDetail(ExtraDetailView):
queryset = Task.objects.select_related()
context_object_name = "task"
template_name = "task/detail.html"
title = _("Task detail")
def get_context_data(self, **kwargs):
context = super(TaskDetail, self).get_context_data(**kwargs)
logs = []
for i in kwargs['object'].logs.list:
if self.request.user.has_perm('hub.can_see_traceback'):
logs.append(i)
continue
if not os.path.basename(i).startswith("traceback"):
logs.append(i)
logs.sort()
context["logs"] = logs
context['task_list'] = kwargs['object'].subtasks()
return context
# -*- coding: utf-8 -*-
from django.conf.urls import url
from kobo.django.views.generic import ExtraListView, ExtraDetailView
from kobo.hub.models import Worker
from kobo.django.compat import gettext_lazy as _
urlpatterns = [
url(r"^$",
ExtraListView.as_view(
queryset=Worker.objects.order_by("name"),
template_name="worker/list.html",
context_object_name="worker_list",
title=_("Workers"),
),
name="worker/list"),
url(r"^(?P<pk>\d+)/$",
ExtraDetailView.as_view(
queryset=Worker.objects.select_related(),
template_name="worker/detail.html",
context_object_name="worker",
title=_("Worker detail"),
),
name="worker/detail"),
]
class Meta:
ordering = ("-id", )
permissions = (("can_see_traceback", _("Can see traceback")), )
class Task(models.Model):
"""Model for hub_task table."""
archive = models.BooleanField(
default=False,
help_text=
_("When a task is archived, it disappears from admin interface and cannot be accessed by taskd.<br />Make sure that archived tasks are finished and you won't need them anymore."
))
owner = models.ForeignKey(settings.AUTH_USER_MODEL,
on_delete=models.CASCADE)
worker = models.ForeignKey(
Worker,
null=True,
blank=True,
help_text=_("A worker which has this task assigned."),
on_delete=models.CASCADE)
parent = models.ForeignKey("self",
null=True,
blank=True,
help_text=_("Parent task."),
on_delete=models.CASCADE)
state = models.PositiveIntegerField(default=TASK_STATES["FREE"],
choices=TASK_STATES.get_mapping(),
help_text=_("Current task state."))
label = models.CharField(
max_length=255,
blank=True,
help_text=_("Label, description or any reason for this task."))
exclusive = models.BooleanField(
default=False,
help_text=
_("Exclusive tasks have highest priority. They are used e.g. when shutting down a worker."
))
method = models.CharField(
max_length=255,
help_text=_("Method name represents appropriate task handler."))
args = kobo.django.fields.JSONField(
blank=True,
default={},
help_text=_("Method arguments. JSON serialized dictionary."))
result = models.TextField(
blank=True,
help_text=
_("Task result. Do not store a lot of data here (use HubProxy.upload_task_log instead)."
))
comment = models.TextField(null=True, blank=True)
arch = models.ForeignKey(Arch, on_delete=models.CASCADE)
channel = models.ForeignKey(Channel, on_delete=models.CASCADE)
timeout = models.PositiveIntegerField(
null=True,
blank=True,
help_text=_("Task timeout. Leave blank for no timeout."))
waiting = models.BooleanField(
default=False,
help_text=_("Task is waiting until some subtasks finish."))
awaited = models.BooleanField(
default=False, help_text=_("Task is awaited by another task."))
dt_created = models.DateTimeField(auto_now_add=True)
dt_started = models.DateTimeField(null=True, blank=True)
dt_finished = models.DateTimeField(null=True, blank=True)
priority = models.PositiveIntegerField(default=10,
help_text=_("Priority."))
weight = models.PositiveIntegerField(
default=1,
help_text=
_("Weight determines how many resources is used when processing the task."
))
resubmitted_by = models.ForeignKey(settings.AUTH_USER_MODEL,
null=True,
blank=True,
related_name="resubmitted_by1",
on_delete=models.CASCADE)
resubmitted_from = models.ForeignKey("self",
null=True,
blank=True,
related_name="resubmitted_from1",
on_delete=models.CASCADE)
subtask_count = models.PositiveIntegerField(
default=0,
help_text=_("Subtask count.<br />This is a generated field."))
# override default *objects* Manager
objects = TaskManager()
class Meta:
ordering = ("-id", )
permissions = (("can_see_traceback", _("Can see traceback")), )
def __init__(self, *args, **kwargs):
self.logs = TaskLogs(self)
traceback = kwargs.pop("traceback", None)
if traceback:
self.logs["traceback.log"] = traceback
stdout = kwargs.pop("stdout", None)
if stdout:
self.logs["stdout.log"] = stdout
super(Task, self).__init__(*args, **kwargs)
def __str__(self):
if self.parent:
return u"#%s [method: %s, state: %s, worker: %s, parent: #%s]" % (
self.id, self.method, self.get_state_display(), self.worker,
self.parent.id)
return u"#%s [method: %s, state: %s, worker: %s]" % (
self.id, self.method, self.get_state_display(), self.worker)
def save(self, *args, **kwargs):
# save to db to precalculate subtask counts and obtain an ID (on insert) for stdout and traceback
self.subtask_count = self.subtasks().count()
super(self.__class__, self).save()
self.logs.save()
if self.parent:
self.parent.save(*args, **kwargs)
@classmethod
def get_task_dir(cls, task_id, create=False):
'''Task files (logs, etc.) are saved in TASK_DIR in following structure based on task_id:
TASK_DIR/millions/tens_of_thousands/task_id/*
'''
task_id = int(task_id)
third = task_id
second = task_id // 10000 * 10000
first = task_id // 1000000 * 1000000
task_dir = os.path.abspath(settings.TASK_DIR)
path = os.path.join(task_dir, str(first), str(second), str(third))
path = os.path.abspath(path)
if not path.startswith(task_dir):
raise Exception('Possible hack, trying to read path "%s"' % path)
if create and not os.path.isdir(path):
os.makedirs(path, mode=0o755)
return path
def task_dir(self, create=False):
return Task.get_task_dir(self.id, create)
@classmethod
def create_task(cls,
owner_name,
label,
method,
args=None,
comment=None,
parent_id=None,
worker_name=None,
arch_name="noarch",
channel_name="default",
timeout=None,
priority=10,
weight=1,
exclusive=False,
resubmitted_by=None,
resubmitted_from=None,
state=None):
"""Create a new task."""
task = cls()
task.owner = get_user_model().objects.get(username=owner_name)
task.label = label
task.method = method
task.args = args or {}
task.comment = comment
if parent_id is not None:
task.parent = cls.objects.get(id=parent_id)
if state is not None:
task.state = state
if worker_name is not None:
task.worker = Worker.objects.get(name=worker_name)
task.state = TASK_STATES["ASSIGNED"]
task.resubmitted_by = resubmitted_by
task.resubmitted_from = resubmitted_from
task.arch = Arch.objects.get(name=arch_name)
task.channel = Channel.objects.get(name=channel_name)
task.priority = priority
task.timeout = timeout
task.weight = weight
task.exclusive = exclusive
# TODO: unsupported in Django 1.0
#task.validate()
task.save()
return task.id
@classmethod
def create_shutdown_task(cls, owner_name, worker_name, kill=False):
"""Create a new ShutdownWorker task."""
kwargs = {
"owner_name": owner_name,
"label": "Shutdown a worker.",
"method": "ShutdownWorker",
"args": {
"kill": kill,
},
"worker_name": worker_name,
"weight": 0,
"exclusive": True,
}
return cls.create_task(**kwargs)
def set_args(self, **kwargs):
"""Serialize args dictionary."""
print(
"DeprecationWarning: kobo.hub.models.Task.set_args() is deprecated. Use kobo.hub.models.Task.args instead.",
file=sys.stderr)
self.args = kwargs
def get_args(self):
"""Deserialize args dictionary."""
print(
"DeprecationWarning: kobo.hub.models.Task.get_args() is deprecated. Use kobo.hub.models.Task.args instead.",
file=sys.stderr)
return self.args.copy()
def get_args_display(self):
"""Deserialize args dictionary to human readable form"""
from collections import OrderedDict
result = OrderedDict()
for key, value in sorted(self.args.items()):
result[key] = json.dumps(value)
return result
def export(self, flat=True):
"""Export data for xml-rpc."""
result = {
"id":
self.id,
"owner":
self.owner.username,
"worker":
self.worker_id,
"parent":
self.parent_id,
"state":
self.state,
"label":
self.label,
"method":
self.method,
"args":
self.args,
"result":
self.result,
"exclusive":
self.exclusive,
"arch":
self.arch_id,
"channel":
self.channel_id,
"timeout":
self.timeout,
"waiting":
self.waiting,
"awaited":
self.awaited,
"dt_created":
datetime.datetime.strftime(self.dt_created, "%F %R:%S"),
"dt_started":
self.dt_started and datetime.datetime.strftime(
self.dt_started, "%Y-%m-%d %H:%M:%S") or None,
"dt_finished":
self.dt_finished
and datetime.datetime.strftime(self.dt_finished, "%F %R:%S")
or None,
"priority":
self.priority,
"weight":
self.weight,
"resubmitted_by":
getattr(self.resubmitted_by, "username", None),
"resubmitted_from":
getattr(self.resubmitted_from, "id", None),
# used by task watcher
"state_label":
self.get_state_display(),
"is_finished":
self.is_finished(),
"is_failed":
self.is_failed(),
}
if not flat:
result.update({
"worker": self.worker and self.worker.export() or None,
"parent": self.parent and self.parent.export() or None,
"arch": self.arch.export(),
"channel": self.channel.export(),
"subtask_id_list": [i.id for i in self.subtasks()],
})
return result
def subtasks(self):
return Task.objects.filter(parent=self)
@property
def time(self):
"""return time spent in the task"""
if not self.dt_started:
return None
elif not self.dt_finished:
return datetime.datetime.now() - self.dt_started
else:
return self.dt_finished - self.dt_started
def get_time_display(self):
"""display time in human readable form"""
if self.time is None:
return ""
time = "%02d:%02d:%02d" % (
(self.time.seconds / 60.0 / 60.0),
(self.time.seconds / 60.0 % 60), self.time.seconds % 60)
if self.time.days:
time = _("%s days, %s") % (self.time.days, time)
return time
get_time_display.short_description = "Time"
def __lock(self,
worker_id,
new_state=TASK_STATES["ASSIGNED"],
initial_states=None):
"""Critical section. Ensures that only one worker takes the task."""
if type(initial_states) in (list, tuple):
# filter out invalid state codes
initial_states = [
i for i, j in TASK_STATES.get_mapping() if i in initial_states
]
if not initial_states:
# initial_states is empty
initial_states = (TASK_STATES["FREE"], )
else:
initial_states = (TASK_STATES["FREE"], )
# it is safe to pass initial_states directly to query,
# because these values are checked in the code above
query = dedent( # nosec B608
"""
UPDATE
hub_task
SET
state=%%s,
worker_id=%%s,
dt_started=%%s,
dt_finished=%%s,
waiting=%%s
WHERE
id=%%s
and state in (%(initial_states)s)
and (worker_id is null or worker_id=%%s)
""") % {
"initial_states": ",".join(
("'%s'" % i for i in initial_states))
}
dt_started = self.dt_started
if new_state == TASK_STATES["OPEN"]:
dt_started = datetime.datetime.now()
dt_finished = self.dt_finished
if new_state in FINISHED_STATES:
dt_finished = datetime.datetime.now()
new_worker_id = worker_id
if new_state == TASK_STATES["FREE"]:
new_worker_id = None
waiting = False
with transaction.atomic():
cursor = connection.cursor()
cursor.execute(query, (new_state, new_worker_id, dt_started,
dt_finished, waiting, self.id, worker_id))
if cursor.rowcount == 0:
if self.state in FINISHED_STATES:
logger.debug(
"Trying to interrupt closed task %s, ignoring.",
self.id)
return
else:
raise ObjectDoesNotExist()
if cursor.rowcount > 1:
raise MultipleObjectsReturned()
self.dt_started = dt_started
self.dt_finished = dt_finished
if new_worker_id is not None:
self.worker = Worker.objects.get(id=new_worker_id)
self.state = new_state
self.waiting = waiting
def free_task(self):
"""Free the task."""
try:
self.__lock(self.worker_id,
new_state=TASK_STATES["FREE"],
initial_states=(TASK_STATES["FREE"],
TASK_STATES["ASSIGNED"],
TASK_STATES["CREATED"]))
except (MultipleObjectsReturned, ObjectDoesNotExist):
raise Exception("Cannot free task %d, state is %s" %
(self.id, self.get_state_display()))
def assign_task(self, worker_id=None):
"""Assign the task to a worker identified by worker_id."""
if worker_id is None:
worker_id = self.worker_id
try:
self.__lock(worker_id,
new_state=TASK_STATES["ASSIGNED"],
initial_states=(TASK_STATES["FREE"],
TASK_STATES["CREATED"]))
except (MultipleObjectsReturned, ObjectDoesNotExist):
raise Exception("Cannot assign task %d" % (self.id))
def open_task(self, worker_id=None):
"""Open the task on a worker identified by worker_id."""
if worker_id is None:
worker_id = self.worker_id
try:
self.__lock(worker_id,
new_state=TASK_STATES["OPEN"],
initial_states=(TASK_STATES["FREE"],
TASK_STATES["ASSIGNED"]))
except (MultipleObjectsReturned, ObjectDoesNotExist):
raise Exception("Cannot open task %d, state is %s" %
(self.id, self.get_state_display()))
@transaction.atomic
def close_task(self, task_result=""):
"""Close the task and save result."""
if task_result:
self.result = task_result
self.save()
try:
self.__lock(self.worker_id,
new_state=TASK_STATES["CLOSED"],
initial_states=(TASK_STATES["OPEN"], ))
except (MultipleObjectsReturned, ObjectDoesNotExist):
raise Exception("Cannot close task %d, state is %s" %
(self.id, self.get_state_display()))
self.logs.gzip_logs()
@transaction.atomic
def cancel_task(self, user=None, recursive=True):
"""Cancel the task."""
if user is not None and not user.is_superuser:
if self.owner.username != user.username:
raise Exception("You are not task owner or superuser.")
try:
self.__lock(
self.worker_id,
new_state=TASK_STATES["CANCELED"],
initial_states=(TASK_STATES["FREE"], TASK_STATES["ASSIGNED"],
TASK_STATES["OPEN"], TASK_STATES["CREATED"]))
except (MultipleObjectsReturned, ObjectDoesNotExist):
raise Exception("Cannot cancel task %d, state is %s" %
(self.id, self.get_state_display()))
if recursive:
for task in self.subtasks():
task.cancel_task(recursive=True)
self.logs.gzip_logs()
def cancel_subtasks(self):
"""Cancel all subtasks of the task."""
result = True
for task in self.subtasks():
try:
result &= task.cancel_task()
except (MultipleObjectsReturned, ObjectDoesNotExist):
result = False
return result
@transaction.atomic
def interrupt_task(self, recursive=True):
"""Set the task state to interrupted."""
try:
self.__lock(self.worker_id,
new_state=TASK_STATES["INTERRUPTED"],
initial_states=(TASK_STATES["OPEN"], ))
except (MultipleObjectsReturned, ObjectDoesNotExist):
raise Exception("Cannot interrupt task %d, state is %s" %
(self.id, self.get_state_display()))
if recursive:
for task in self.subtasks():
task.interrupt_task(recursive=True)
self.logs.gzip_logs()
@transaction.atomic
def timeout_task(self, recursive=True):
"""Set the task state to timeout."""
try:
self.__lock(self.worker_id,
new_state=TASK_STATES["TIMEOUT"],
initial_states=(TASK_STATES["OPEN"], ))
except (MultipleObjectsReturned, ObjectDoesNotExist):
raise Exception("Cannot timeout task %d, state is %s" %
(self.id, self.get_state_display()))
if recursive:
for task in self.subtasks():
task.timeout_task(recursive=True)
self.logs.gzip_logs()
@transaction.atomic
def fail_task(self, task_result=""):
"""Fail this task and save result."""
if task_result:
self.result = task_result
self.save()
try:
self.__lock(self.worker_id,
new_state=TASK_STATES["FAILED"],
initial_states=(TASK_STATES["OPEN"], ))
except (MultipleObjectsReturned, ObjectDoesNotExist):
raise Exception("Cannot fail task %i, state is %s" %
(self.id, self.get_state_display()))
self.logs.gzip_logs()
def is_finished(self):
"""Is the task finished? Task state can be one of: closed, interrupted, canceled, failed."""
return self.state in FINISHED_STATES
def is_failed(self):
"""Is the task successfuly finished? Task state must be closed."""
return self.state in FAILED_STATES
def resubmit_task(self, user, force=False, priority=None):
"""Resubmit failed/canceled top-level task."""
if not user.is_superuser:
if self.owner.username != user.username:
raise Exception("You are not task owner or superuser.")
if self.parent:
raise Exception("Task is not top-level: %s" % self.id)
if self.exclusive:
raise Exception("Cannot resubmit exclusive task: %s" % self.id)
if not force and self.state not in FAILED_STATES:
states = [TASK_STATES.get_value(i) for i in FAILED_STATES]
raise Exception("Task '%s' must be in: %s" % (self.id, states))
kwargs = {
"owner_name": self.owner.username,
"label": self.label,
"method": self.method,
"args": self.args,
"comment": self.comment,
"parent_id": None,
"worker_name": None,
"arch_name": self.arch.name,
"channel_name": self.channel.name,
"priority": priority if priority is not None else self.priority,
"weight": self.weight,
"exclusive": self.exclusive,
"resubmitted_by": user,
"resubmitted_from": self,
}
return Task.create_task(**kwargs)
def clone_task(self, user, **kwargs):
"""Clone a task, override field values by kwargs."""
if not user.is_superuser:
raise Exception("You are not superuser.")
if self.parent:
raise Exception("Task is not top-level: %s" % self.id)
kwargs.pop("resubmitted_by", None)
kwargs.pop("resubmitted_from", None)
new_kwargs = {
"owner_name": self.owner.username,
"label": self.label,
"method": self.method,
"args": self.args,
"comment": self.comment,
"parent_id": None,
"worker_name": None,
"arch_name": self.arch.name,
"channel_name": self.channel.name,
"priority": self.priority,
"weight": self.weight,
"exclusive": self.exclusive,
"resubmitted_by": user,
"resubmitted_from": self,
}
new_kwargs.update(kwargs)
return Task.create_task(**new_kwargs)
def wait(self, child_task_list=None):
"""Set this task as waiting and all subtasks in child_task_list as awaited.
If child_task_list is None, process all related subtasks.
"""
tasks = self.subtasks().filter(state__in=(TASK_STATES["FREE"],
TASK_STATES["ASSIGNED"],
TASK_STATES["OPEN"]))
if child_task_list is not None:
tasks = tasks.filter(id__in=child_task_list)
for task in tasks:
task.awaited = True
task.save()
self.waiting = True
self.save()
def check_wait(self, child_task_list=None):
"""Determine if all subtasks have finished."""
tasks = self.subtasks()
if child_task_list is not None:
tasks = tasks.filter(id__in=child_task_list)
finished = []
unfinished = []
for task in tasks:
if task.is_finished():
finished.append(task.id)
task.awaited = False
task.save()
else:
unfinished.append(task.id)
return [finished, unfinished]
def set_weight(self, weight):
self.weight = weight
self.save()
class Worker(models.Model):
"""Model for the hub_worker table."""
worker_key = models.CharField(
max_length=255,
unique=True,
blank=True,
help_text=_(
"Worker authentication key.<br />Leave blank to generate new key.")
)
name = models.CharField(max_length=128,
unique=True,
help_text=_("Worker hostname."))
arches = models.ManyToManyField(Arch,
help_text=_("Supported architectures"))
channels = models.ManyToManyField(Channel)
enabled = models.BooleanField(
default=True,
help_text=_("Enabled workers are allowed to process tasks."))
max_load = models.PositiveIntegerField(
blank=True,
default=1,
help_text=_("Maximum allowed load (sum of task weights)."))
max_tasks = models.PositiveIntegerField(
blank=True,
default=0,
help_text=_("Maximum assigned tasks. (0 = no limit)"))
min_priority = models.PositiveIntegerField(
default=0,
help_text=_("Worker will take only tasks of this or higher priority."))
# redundant fields to improve performance
ready = models.BooleanField(
default=True,
help_text=
_("Is the worker ready to take new tasks?<br />This is a generated field."
))
task_count = models.PositiveIntegerField(
blank=True,
default=0,
help_text=_(
"Count of processed tasks.<br />This is a generated field."))
current_load = models.PositiveIntegerField(
blank=True,
default=0,
help_text=_("Sum of task weights.<br />This is a generated field."))
# override default *objects* Manager
objects = WorkerManager()
def __str__(self):
return u"%s" % self.name
def save(self, *args, **kwargs):
# precompute task count, current load and ready
tasks = Task.objects.opened().filter(worker=self)
self.task_count = tasks.count()
self.current_load = sum(
(task.weight for task in tasks if not task.waiting))
self.ready = self.enabled and (self.current_load < self.max_load
and self.task_count < 3 * self.max_load)
while not self.worker_key:
# if worker_key is empty, generate a new one
key = random_string(64)
if Worker.objects.filter(worker_key=key).count() == 0:
self.worker_key = key
super(self.__class__, self).save(*args, **kwargs)
def export(self):
"""Export data for xml-rpc."""
return {
"id": self.id,
"name": self.name,
"arches": [i.export() for i in self.arches.all()],
"channels": [i.export() for i in self.channels.all()],
"enabled": self.enabled,
"max_load": self.max_load,
"ready": self.ready,
"task_count": self.task_count,
"current_load": self.current_load,
"last_seen": self.last_seen_iso8601,
# Add the hub version.
# This can be used for taskd compatibility checking everytime a worker_info is updated.
"version": self._get_version(),
}
def _get_version(self):
"""Return hub version or None (if settings.VERSION is not set)."""
return getattr(settings, "VERSION", None)
def running_tasks(self):
"""Return list of running tasks on this worker."""
return Task.objects.running().filter(worker=self)
def assigned_tasks(self):
"""Return list of assigned tasks to this worker."""
return Task.objects.assigned().filter(worker=self)
@property
def last_seen(self):
"""Time of this worker's last communication with hub,
or None if unknown.
:rtype: datetime.datetime
"""
try:
stat = os.stat(self._state_path)
except EnvironmentError as error:
if error.errno == errno.ENOENT:
return None
raise
return datetime.datetime.utcfromtimestamp(stat.st_mtime)
@property
def last_seen_iso8601(self):
"""Time of this worker's last communication with hub
as ISO8601-formatted timestamp, or None if unknown.
Example: '2007-04-05T14:30Z'
:rtype: str
"""
when = self.last_seen
if when:
return when.replace(microsecond=0).isoformat() + 'Z'
def update_last_seen(self):
"""Mark worker as having communicated with hub at the current time."""
with open(self._state_path, 'w'):
pass
@property
def _state_path(self):
# Returns path to worker's state file.
# We don't know what characters may have been used in the worker name here,
# so it's base64 encoded first.
safe_name = base64.urlsafe_b64encode(
self.name.encode('utf-8')).decode()
return os.path.join(settings.WORKER_DIR, safe_name)
def update_worker(self, enabled, ready, task_count):
"""Update worker attributes. Return worker_info.
Update only if data provided from a worker differs.
"""
if (self.enabled, self.ready, self.task_count) != (enabled, ready,
task_count):
self.save()
return self.export()
# -*- coding: utf-8 -*-
from django.conf.urls import url
from kobo.django.views.generic import ExtraListView
from kobo.hub.views import ArchDetailView
from kobo.hub.models import Arch
from kobo.django.compat import gettext_lazy as _
urlpatterns = [
url(r"^$", ExtraListView.as_view(
queryset=Arch.objects.order_by("name"),
template_name="arch/list.html",
context_object_name="arch_list",
title=_("Architectures"),
), name="arch/list"),
url(r"^(?P<pk>\d+)/$", ArchDetailView.as_view(), name="arch/detail"),
]
# -*- coding: utf-8 -*-
from django.contrib.auth import get_user_model
from django.conf.urls import url
from kobo.django.views.generic import ExtraListView
from kobo.hub.views import UserDetailView
from kobo.django.compat import gettext_lazy as _
urlpatterns = [
url(r"^$",
ExtraListView.as_view(
queryset=get_user_model().objects.order_by("username"),
template_name="user/list.html",
context_object_name="usr_list",
title=_('Users'),
),
name="user/list"),
url(r"^(?P<pk>\d+)/$", UserDetailView.as_view(), name="user/detail"),
]
# -*- coding: utf-8 -*-
from django.conf.urls import url
from kobo.hub.models import TASK_STATES
from kobo.hub.views import TaskListView, TaskDetail
import kobo.hub.views
from kobo.django.compat import gettext_lazy as _
urlpatterns = [
url(r"^$", TaskListView.as_view(), name="task/index"),
url(r"^(?P<pk>\d+)/$", TaskDetail.as_view(), name="task/detail"),
url(r"^running/$",
TaskListView.as_view(state=(TASK_STATES["FREE"],
TASK_STATES["ASSIGNED"],
TASK_STATES["OPEN"]),
title=_("Running tasks"),
order_by=["id"]),
name="task/running"),
url(r"^finished/$",
TaskListView.as_view(
state=(TASK_STATES["CLOSED"], TASK_STATES["INTERRUPTED"],
TASK_STATES["CANCELED"], TASK_STATES["FAILED"]),
title=_("Finished tasks"),
order_by=["-dt_created", "id"]),
name="task/finished"),
url(r"^(?P<id>\d+)/log/(?P<log_name>.+)$",
kobo.hub.views.task_log,
name="task/log"),
url(r"^(?P<id>\d+)/log-json/(?P<log_name>.+)$",
kobo.hub.views.task_log_json,
name="task/log-json"),
class Meta:
db_table = 'auth_user'
verbose_name = _('user')
verbose_name_plural = _('users')
class User(AbstractBaseUser, PermissionsMixin):
"""
Copy (non-abstract) of AbstractUser with longer username. Removed profile support as it
is deprecated in django 1.5.
Username, password and email are required. Other fields are optional.
"""
username = models.CharField(
_('username'),
max_length=MAX_LENGTH,
unique=True,
help_text=_(
'Required. %s characters or fewer. Letters, digits and @/./+/-/_ only.'
% MAX_LENGTH),
validators=[
validators.RegexValidator(
r'^[\w.@+-]+$',
_('Enter a valid username. '
'This value may contain only letters, numbers '
'and @/./+/-/_ characters.'), 'invalid'),
],
error_messages={
'unique': _("A user with that username already exists."),
})
first_name = models.CharField(_('first name'), max_length=30, blank=True)
last_name = models.CharField(_('last name'), max_length=30, blank=True)
email = models.EmailField(_('email address'), blank=True)
is_staff = models.BooleanField(
_('staff status'),
default=False,
help_text=_('Designates whether the user can log into this admin '
'site.'))
is_active = models.BooleanField(
_('active'),
default=True,
help_text=_('Designates whether this user should be treated as '
'active. Unselect this instead of deleting accounts.'))
date_joined = models.DateTimeField(_('date joined'), default=timezone.now)
objects = UserManager()
USERNAME_FIELD = 'username'
REQUIRED_FIELDS = ['email']
class Meta:
db_table = 'auth_user'
verbose_name = _('user')
verbose_name_plural = _('users')
def get_full_name(self):
"""
Returns the first_name plus the last_name, with a space in between.
"""
full_name = '%s %s' % (self.first_name, self.last_name)
return full_name.strip()
def get_short_name(self):
"Returns the short name for the user."
return self.first_name
def email_user(self, subject, message, from_email=None, **kwargs):
"""
Sends an email to this User.
"""
send_mail(subject, message, from_email, [self.email], **kwargs)
