def _auto_cancel_jobs(self, ev, recipes): # if a recipe has auto_cancel_on_push then we need to cancel any jobs # on the same branch that are currently running cancel_job_states = [ models.JobStatus.NOT_STARTED, models.JobStatus.RUNNING ] ev_url = reverse('ci:view_event', args=[ev.pk]) msg = "Canceled due to new push <a href='%s'>event</a>" % ev_url for r in recipes: if r.auto_cancel_on_push: js = models.Job.objects.filter( status__in=cancel_job_states, recipe__branch=r.branch, recipe__cause=r.cause, recipe__build_user=r.build_user, recipe__filename=r.filename, ) for j in js.all(): logger.info( 'Job {}: {} canceled by new push event {}: {}'.format( j.pk, j, ev.pk, ev)) # We don't need to update remote Git server status since # we will have new jobs views.set_job_canceled(j, msg) j.event.set_status() j.event.set_complete_if_done() ev.save() # update the timestamp so the js updater works
def handle(self, *args, **options): dryrun = options["dryrun"] days = options["days"] hours = options["hours"] allowed_fail = options["allowed_fail"] client_runner_user = options["client_runner_user"] if days: d = TimeUtils.get_local_time() - timedelta(days=days) elif hours: d = TimeUtils.get_local_time() - timedelta(hours=hours) jobs = models.Job.objects.filter(active=True, ready=True, status=models.JobStatus.NOT_STARTED, created__lt=d) if client_runner_user: if ":" not in client_runner_user: raise CommandError("Invalid format for username: %s" % client_runner_user) host, username = client_runner_user.split(":") git_server = models.GitServer.objects.get(name=host) git_user = models.GitUser.objects.get(name=username, server=git_server) jobs = jobs.filter( (Q(recipe__client_runner_user=None) & Q(recipe__build_user__build_key=git_user.build_key)) | Q(recipe__client_runner_user__build_key=git_user.build_key)) count = jobs.count() prefix = "" if dryrun: prefix = "DRY RUN: " if allowed_fail: err_msg = "Set to allowed to fail due to civet client not running this job in too long a time" status = models.JobStatus.FAILED_OK msg = "Job allowed to fail" else: err_msg = "Canceled due to civet client not running this job in too long a time" status = models.JobStatus.CANCELED msg = "Job canceled" for job in jobs.all(): self.stdout.write("%s%s: %s: %s: %s" % (prefix, msg, job.pk, job, job.created)) if not dryrun: views.set_job_canceled(job, err_msg, status) UpdateRemoteStatus.job_complete(job) job.event.set_complete_if_done() if count == 0: self.stdout.write("No jobs to cancel")
def test_run_job_cancel(self): with test_utils.RecipeDir() as recipe_dir: c, job = self.create_client_and_job(recipe_dir, "JobCancel", sleep=4) self.set_counts() # cancel response, should cancel the job thread = threading.Thread(target=c.run, args=(True,)) thread.start() time.sleep(4) job.refresh_from_db() views.set_job_canceled(job) thread.join() self.compare_counts(num_clients=1, canceled=1, num_events_completed=1, num_jobs_completed=1, active_branches=1, events_canceled=1) self.assertEqual(c.cancel_signal.triggered, False) self.assertEqual(c.graceful_signal.triggered, False) utils.check_canceled_job(self, job)
def ready_jobs(request, build_key, client_name): if request.method != 'GET': return HttpResponseNotAllowed(['GET']) client, created = models.Client.objects.get_or_create( name=client_name, ip=get_client_ip(request)) if created: logger.debug('New client %s : %s seen' % (client_name, get_client_ip(request))) else: # if a client is talking to us here then if they have any running jobs assigned to them they need # to be canceled past_running_jobs = models.Job.objects.filter( client=client, complete=False, status=models.JobStatus.RUNNING) msg = "Canceled due to client %s not finishing job" % client.name for j in past_running_jobs.all(): views.set_job_canceled(j, msg) UpdateRemoteStatus.job_complete(j) client.status_message = 'Looking for work' client.status = models.Client.IDLE client.save() # We need to see if any jobs are part of a push event on a repo # where we want to do custom handling. # Then we need to remove those jobs from the list and do # a separate query with a different sort order and add those # to the list. jobs = (models.Job.objects.filter( (Q(recipe__client_runner_user=None) & Q(recipe__build_user__build_key=build_key)) | Q(recipe__client_runner_user__build_key=build_key), complete=False, active=True, ready=True, status=models.JobStatus.NOT_STARTED, ).select_related('config', 'event__base__branch__repository__user__server').order_by( '-recipe__priority', 'created')) jobs_json = [] current_push_event_branches = set() for job in jobs.all(): if job.event.cause == models.Event.PUSH and job.event.auto_cancel_event_except_current( ): current_push_event_branches.add(job.event.base.branch) else: data = { 'id': job.pk, 'build_key': build_key, 'config': job.config.name, } jobs_json.append(data) if current_push_event_branches: jobs = jobs.filter( event__base__branch__in=current_push_event_branches).order_by( 'created', '-recipe__priority') for job in jobs.all(): data = { 'id': job.pk, 'build_key': build_key, 'config': job.config.name, } jobs_json.append(data) reply = {'jobs': jobs_json} return JsonResponse(reply)