def update_worker(self):
"""
Look for executors where the connection has broken and tasks need to be re-submitted.
"""
if not self.worker:
self.worker = Worker.objects.create(
name=self.name,
start_timestamp=functions.Now(),
last_heartbeat=functions.Now())
else:
self.worker.refresh_from_db()
if self.worker.status == Worker.LOST:
# someone else marked us as lost, terminate all tasks and exit
logger.warning('Marked as lost, committing harakiri')
self.state = State.terminate
self.executor.mark_terminated(self.executor.get_running_ids())
return
# update our timestamp so no one marks us as lost
self.worker.last_heartbeat = functions.Now()
self.worker.save()
# look for lost workers and re-queue their tasks
Worker.find_lost(self.timeout)
def now():
"""
The current date time.
Usage:
Events.objects.filter(F.start_time > F.now())
"""
return functions.Now()
def ranking_algorithm():
# Based on Hacker News' post rank formula
time_diff = functions.Cast(functions.Now() - F('created_time'),
fields.DurationField())
hours = dtime.duration_to_hours(duration=time_diff)
rank = ExpressionWrapper(
(F('upvote') - F('devote') + 1) / ((hours + 2)**1.8),
output_field=fields.FloatField())
return hours, rank
def form_valid(self, form):
self.object = form.save(commit=False)
self.object.submitter = self.request.user
self.object.updated_at = functions.Now()
self.object.save()
return HttpResponseRedirect(
reverse(
'site_geoservice_detail',
kwargs={'pk': self.object.id},
))
def find_lost(timeout):
from yawn.task.models import Execution
# Make a sparse index so looking up active workers is fast:
# CREATE INDEX yawn_worker_active ON yawn_worker (status) WHERE status = 'active'
lost = Worker.objects.filter(status=Worker.ACTIVE,
last_heartbeat__lt=functions.Now() -
timedelta(seconds=timeout))
for worker in lost:
logger.warning('Marking %r as lost', worker)
worker.status = Worker.LOST
worker.save()
executions = worker.execution_set.filter(status=Execution.RUNNING)
for execution in executions:
logger.warning('Marking %r as lost', execution)
execution.mark_finished(lost=True)
def mark_finished(self, exit_code=None, lost=False):
"""
Update the execution status after it has finished:
successfully, in error, or because it was lost.
Also update the task and workflow; re-queue the task if
it should be retried.
"""
if lost:
self.status = Execution.LOST
self.task.enqueue()
elif exit_code == 0:
self.task.status = Task.SUCCEEDED
self.status = Execution.SUCCEEDED
else:
# queue another run if there are remaining retries
# (current execution is not in count b/c it hasn't been saved yet)
failed_count = self.task.execution_set.filter(
status=Task.FAILED).count()
if failed_count < self.task.template.max_retries:
self.task.enqueue()
else:
self.task.status = Task.FAILED
self.status = Execution.FAILED
if self.task.status != Task.RUNNING:
self.task.save()
with transaction.atomic():
self.task.update_downstream()
if self.task.run:
self.task.run.update_status()
self.stop_timestamp = functions.Now()
self.exit_code = exit_code
# need to be careful not to overwrite stdout/stderr
self.save(update_fields=['status', 'stop_timestamp', 'exit_code'])
def submit_run(self, parameters=None, scheduled_time=None):
"""Create a run of this template"""
from yawn.task.models import Task
run_parameters = self.parameters.copy()
run_parameters.update(parameters or {})
run = Run.objects.create(
workflow=self,
submitted_time=functions.Now(),
scheduled_time=scheduled_time,
parameters=run_parameters,
)
for template in self.template_set.all():
task = Task.objects.create(
run=run,
template=template,
)
if not template.upstream.exists():
task.enqueue()
# refresh to get the actual DB submitted time
run.refresh_from_db()
return run
def form_valid(self, form):
self.object = form.save()
self.object.updated_at = functions.Now()
self.object.save()
return HttpResponseRedirect(self.get_success_url())