def view_client(request, client_id): """ View details about a client, along with some a list of paginated jobs it has run """ client = get_object_or_404(models.Client, pk=client_id) allowed = Permissions.is_allowed_to_see_clients(request.session) if not allowed: return render(request, 'ci/client.html', { 'client': None, 'allowed': False }) jobs_list = models.Job.objects.filter( client=client).order_by('-last_modified').select_related( 'config', 'event__pull_request', 'event__base__branch__repository__user', 'event__head__branch__repository__user', 'recipe', ) jobs = get_paginated(request, jobs_list) return render(request, 'ci/client.html', { 'client': client, 'jobs': jobs, 'allowed': True })
def get_job_results(request, job_id): """ Just download all the output of the job into a tarball. """ job = get_object_or_404(models.Job.objects.select_related( 'recipe', ).prefetch_related('step_results'), pk=job_id) perms = Permissions.job_permissions(request.session, job) if not perms['can_see_results']: return HttpResponseForbidden('Not allowed to see results') response = HttpResponse(content_type='application/x-gzip') base_name = 'results_{}_{}'.format(job.pk, job.recipe.name) response[ 'Content-Disposition'] = 'attachment; filename="{}.tar.gz"'.format( base_name) tar = tarfile.open(fileobj=response, mode='w:gz') for result in job.step_results.all(): info = tarfile.TarInfo( name='{}/{:02}_{}'.format(base_name, result.position, result.name)) s = BytesIO(result.plain_output().replace('\u2018', "'").replace( "\u2019", "'").encode("utf-8", "replace")) buf = s.getvalue() info.size = len(buf) info.mtime = time.time() tar.addfile(tarinfo=info, fileobj=s) tar.close() return response
def cancel_event(request, event_id): """ Cancel all jobs attached to an event """ if request.method != 'POST': return HttpResponseNotAllowed(['POST']) ev = get_object_or_404(models.Event, pk=event_id) allowed = Permissions.is_collaborator(request.session, ev.build_user, ev.base.repo()) if not allowed: messages.error(request, 'You are not allowed to cancel this event') return redirect('ci:view_event', event_id=ev.pk) signed_in_user = ev.base.server().signed_in_user(request.session) comment = escape(request.POST.get("comment")) post_to_pr = request.POST.get("post_to_pr") == "on" event_url = reverse("ci:view_event", args=[ev.pk]) message = "Parent <a href='%s'>event</a> canceled by %s" % (event_url, signed_in_user) if comment: message += " with comment: %s" % comment if post_to_pr: post_event_change_to_pr(request, ev, "canceled", comment, signed_in_user) event.cancel_event(ev, message, True) logger.info('Event {}: {} canceled by {}'.format(ev.pk, ev, signed_in_user)) messages.info(request, 'Event {} canceled'.format(ev)) return redirect('ci:view_event', event_id=ev.pk)
def cronjobs(request): # TODO: make this check for permission to view cron stuff instead allowed = Permissions.is_allowed_to_see_clients(request.session) if not allowed: return render(request, 'ci/cronjobs.html', { 'recipes': None, 'allowed': False }) recipe_list = models.Recipe.objects.filter( active=True, current=True, scheduler__isnull=False, branch__isnull=False).exclude(scheduler="") local_tz = pytz.timezone('US/Mountain') for r in recipe_list: event_list = (EventsStatus.get_default_events_query().filter( jobs__recipe__filename=r.filename, jobs__recipe__cause=r.cause)) events = get_paginated(request, event_list) evs_info = EventsStatus.multiline_events_info(events) r.most_recent_event = evs_info[0]['id'] if len(evs_info) > 0 else None c = croniter(r.scheduler, start_time=r.last_scheduled.astimezone(local_tz)) r.next_run_time = c.get_next(datetime) # TODO: augment recipes objects with fields that html template will need. data = { 'recipes': recipe_list, 'allowed': True, 'update_interval': settings.HOME_PAGE_UPDATE_INTERVAL, } return render(request, 'ci/cronjobs.html', data)
def view_job(request, job_id): """ View the details of a job, along with any results. """ recipe_q = models.Recipe.objects.prefetch_related("depends_on", "auto_authorized", "viewable_by_teams") q = (models.Job.objects .select_related('recipe__repository__user__server', 'recipe__build_user__server', 'event__pull_request', 'event__base__branch__repository__user__server', 'event__head__branch__repository__user__server', 'config', 'client',) .prefetch_related(Prefetch("recipe", queryset=recipe_q), 'step_results', 'changelog')) job = get_object_or_404(q, pk=job_id) perms = Permissions.job_permissions(request.session, job) clients = None if perms['can_see_client']: clients = sorted_clients(models.Client.objects.exclude(status=models.Client.DOWN)) perms['job'] = job perms['clients'] = clients perms['update_interval'] = settings.JOB_PAGE_UPDATE_INTERVAL return render(request, 'ci/job.html', perms)
def activate_event(request, event_id): """ Endpoint for activating all jobs on an event """ if request.method != 'POST': return HttpResponseNotAllowed(['POST']) ev = get_object_or_404(models.Event, pk=event_id) jobs = ev.jobs.filter(active=False).order_by('-created') if jobs.count() == 0: messages.info(request, 'No jobs to activate') return redirect('ci:view_event', event_id=ev.pk) repo = jobs.first().recipe.repository user = repo.server().signed_in_user(request.session) if not user: raise PermissionDenied('You need to be signed in to activate jobs') collab = Permissions.is_collaborator(request.session, ev.build_user, repo, user=user) if collab: activated_jobs = [] for j in jobs.all(): if set_job_active(request, j, user): activated_jobs.append(j) for j in activated_jobs: j.init_pr_status() ev.make_jobs_ready() else: raise PermissionDenied('Activate event: {} is NOT a collaborator on {}'.format(user, repo)) return redirect('ci:view_event', event_id=ev.pk)
def activate_job(request, job_id): """ Endpoint for activating a job """ if request.method != 'POST': return HttpResponseNotAllowed(['POST']) job = get_object_or_404(models.Job, pk=job_id) server = job.recipe.repository.server() user = server.signed_in_user(request.session) if not user: raise PermissionDenied('You need to be signed in to activate a job') collab = Permissions.is_collaborator(request.session, job.event.build_user, job.recipe.repository, user=user) if collab: if set_job_active(request, job, user): job.init_pr_status() job.event.make_jobs_ready() else: raise PermissionDenied( 'Activate job: {} is NOT a collaborator on {}'.format( user, job.recipe.repository)) return redirect('ci:view_job', job_id=job.pk)
def cancel_job(request, job_id): if request.method != 'POST': return HttpResponseNotAllowed(['POST']) job = get_object_or_404(models.Job, pk=job_id) allowed = Permissions.is_collaborator(request.session, job.event.build_user, job.event.base.repo()) if not allowed: return HttpResponseForbidden('Not allowed to cancel this job') signed_in_user = job.event.base.server().signed_in_user(request.session) message = "Canceled by %s" % signed_in_user comment = escape(request.POST.get('comment')) post_to_pr = request.POST.get('post_to_pr') == 'on' if post_to_pr: post_job_change_to_pr(request, job, "canceled", comment, signed_in_user) if comment: message += "\nwith comment: %s" % comment set_job_canceled(job, message) UpdateRemoteStatus.job_complete(job) logger.info('Job {}: {} on {} canceled by {}'.format( job.pk, job, job.recipe.repository, signed_in_user)) messages.info(request, 'Job {} canceled'.format(job)) return redirect('ci:view_job', job_id=job.pk)
def client_list(request): allowed = Permissions.is_allowed_to_see_clients(request.session) if not allowed: return render(request, 'ci/clients.html', {'clients': None, 'allowed': False}) client_list = clients_info() data = {'clients': client_list, 'allowed': True, 'update_interval': settings.HOME_PAGE_UPDATE_INTERVAL, } return render(request, 'ci/clients.html', data)
def clients_update(request): """ Get the updates for the clients page. """ allowed = Permissions.is_allowed_to_see_clients(request.session) if not allowed: return HttpResponseBadRequest('Not allowed') clients = views.clients_info() return JsonResponse({ 'clients': clients })
def get_result_output(request): if 'result_id' not in request.GET: return HttpResponseBadRequest('Missing parameter') result_id = request.GET['result_id'] result = get_object_or_404(models.StepResult, pk=result_id) if not Permissions.can_see_results(request.session, result.job.recipe): return HttpResponseForbidden("Can't see results") return JsonResponse({'contents': result.clean_output()})
def manual_cron(request, recipe_id): allowed = Permissions.is_allowed_to_see_clients(request.session) if not allowed: return HttpResponseForbidden('Not allowed to start manual cron runs') r = get_object_or_404(models.Recipe, pk=recipe_id) user = r.build_user branch = r.branch latest = user.api().last_sha(branch.repository.user.name, branch.repository.name, branch.name) #likely need to add exception checks for this! if latest: r.last_scheduled = datetime.now(tz=pytz.UTC); r.save(); mev = ManualEvent.ManualEvent(user, branch, latest, "", recipe=r); mev.force = True; mev.save(update_branch_status=True); return redirect('ci:cronjobs')
def view_event(request, event_id): """ Show the details of an Event """ ev = get_object_or_404(EventsStatus.events_with_head(), pk=event_id) evs_info = EventsStatus.multiline_events_info([ev]) allowed = Permissions.is_collaborator(request.session, ev.build_user, ev.base.repo()) has_unactivated = ev.jobs.filter(active=False).count() != 0 context = {'event': ev, 'events': evs_info, 'allowed_to_cancel': allowed, "update_interval": settings.EVENT_PAGE_UPDATE_INTERVAL, "has_unactivated": has_unactivated, } return render(request, 'ci/event.html', context)
def invalidate_event(request, event_id): """ Invalidate all the jobs of an event. The user must be signed in. Input: request: django.http.HttpRequest event_id. models.Event.pk: PK of the event to be invalidated Return: django.http.HttpResponse based object """ if request.method != 'POST': return HttpResponseNotAllowed(['POST']) ev = get_object_or_404(models.Event, pk=event_id) allowed = Permissions.is_collaborator(request.session, ev.build_user, ev.base.repo()) if not allowed: messages.error( request, 'You need to be signed in and be a collaborator to invalidate results.' ) return redirect('ci:view_event', event_id=ev.pk) signed_in_user = ev.base.server().signed_in_user(request.session) comment = escape(request.POST.get("comment")) logger.info('Event {}: {} invalidated by {}'.format( ev.pk, ev, signed_in_user)) event_url = reverse("ci:view_event", args=[ev.pk]) message = "Parent <a href='%s'>event</a> invalidated by %s" % ( event_url, signed_in_user) if comment: message += " with comment: %s" % comment post_to_pr = request.POST.get("post_to_pr") == "on" if post_to_pr: post_event_change_to_pr(request, ev, "invalidated", comment, signed_in_user) same_client = request.POST.get('same_client') == "on" for job in ev.jobs.all(): invalidate_job(request, job, message, same_client, check_ready=False) # Only do this once so that we get the job dependencies setup correctly. ev.make_jobs_ready() return redirect('ci:view_event', event_id=ev.pk)
def invalidate(request, job_id): """ Invalidate the results of a Job. The user must be signed in. Input: request: django.http.HttpRequest job_id: models.Job.pk """ if request.method != 'POST': return HttpResponseNotAllowed(['POST']) job = get_object_or_404(models.Job, pk=job_id) allowed = Permissions.is_collaborator(request.session, job.event.build_user, job.event.base.repo()) if not allowed: raise PermissionDenied('You are not allowed to invalidate results.') same_client = request.POST.get('same_client') == 'on' selected_client = request.POST.get('client_list') comment = escape(request.POST.get('comment')) post_to_pr = request.POST.get('post_to_pr') == 'on' client = None if selected_client: try: client = models.Client.objects.get(pk=int(selected_client)) same_client = True except: pass signed_in_user = job.event.base.server().signed_in_user(request.session) message = "Invalidated by %s" % signed_in_user if comment: message += "\nwith comment: %s" % comment if post_to_pr: post_job_change_to_pr(request, job, "invalidated", comment, signed_in_user) logger.info('Job {}: {} on {} invalidated by {}'.format( job.pk, job, job.recipe.repository, signed_in_user)) invalidate_job(request, job, message, same_client, client) return redirect('ci:view_job', job_id=job.pk)
def update_remote_job_status(request, job_id): """ End point for manually update the remote status of a job. This is needed since sometimes the git server doesn't get updated properly due to timeouts, etc. """ job = get_object_or_404(models.Job.objects, pk=job_id) allowed = Permissions.is_collaborator(request.session, job.event.build_user, job.event.base.repo()) if request.method == "GET": return render(request, 'ci/job_update.html', { "job": job, "allowed": allowed }) elif request.method == "POST": if allowed: UpdateRemoteStatus.job_complete_pr_status(job) else: return HttpResponseNotAllowed("Not allowed") return redirect('ci:view_job', job_id=job.pk)
def view_pr(request, pr_id): """ Show the details of a PR Input: request: django.http.HttpRequest pr_id: pk of models.PullRequest Return: django.http.HttpResponse based object """ pr = get_object_or_404(models.PullRequest.objects.select_related('repository__user'), pk=pr_id) ev = pr.events.select_related('build_user', 'base__branch__repository__user__server').latest() allowed = Permissions.is_collaborator(request.session, ev.build_user, ev.base.repo()) current_alt = [] alt_choices = [] default_choices = [] if allowed: alt_recipes = (models.Recipe.objects .filter(repository=pr.repository, build_user=ev.build_user, current=True, active=True, cause=models.Recipe.CAUSE_PULL_REQUEST_ALT,) .order_by("display_name")) default_recipes = (models.Recipe.objects .filter(repository=pr.repository, build_user=ev.build_user, current=True, active=True, cause=models.Recipe.CAUSE_PULL_REQUEST,) .order_by("display_name")) push_recipes = (models.Recipe.objects .filter(repository=pr.repository, build_user=ev.build_user, current=True, active=True, cause=models.Recipe.CAUSE_PUSH,) .order_by("display_name")) default_recipes = [r for r in default_recipes.all()] current_alt = [ r.pk for r in pr.alternate_recipes.all() ] current_default = [j.recipe.filename for j in pr.events.latest("created").jobs.all() ] push_map = {r.filename: r.branch for r in push_recipes.all()} alt_choices = [] for r in alt_recipes: alt_choices.append({"recipe": r, "selected": r.pk in current_alt, "push_branch": push_map.get(r.filename), }) default_choices = [] for r in default_recipes: default_choices.append({"recipe": r, "pk": r.pk, "disabled": r.filename in current_default, "push_branch": push_map.get(r.filename), }) if alt_choices and request.method == "POST": form_choices = [ (r.pk, r.display_name) for r in alt_recipes ] form = forms.AlternateRecipesForm(request.POST) form.fields["recipes"].choices = form_choices form_default_choices = [] for r in default_choices: if not r["disabled"]: form_default_choices.append((r["pk"], r["recipe"].display_name)) form.fields["default_recipes"].choices = form_default_choices if form.is_valid(): pr.alternate_recipes.clear() for pk in form.cleaned_data["recipes"]: alt = models.Recipe.objects.get(pk=pk) pr.alternate_recipes.add(alt) # do some saves to update the timestamp so that the javascript updater gets activated pr.save() pr.events.latest('created').save() messages.info(request, "Success") pr_event = PullRequestEvent.PullRequestEvent() selected_default_recipes = [] if form.cleaned_data["default_recipes"]: q = models.Recipe.objects.filter(pk__in=form.cleaned_data["default_recipes"]) selected_default_recipes = [r for r in q] pr_event.create_pr_alternates(pr, default_recipes=selected_default_recipes) # update the choices so the new form is correct current_alt = [ r.pk for r in pr.alternate_recipes.all() ] alt_choices = [ {"recipe": r, "selected": r.pk in current_alt} for r in alt_recipes ] else: messages.warning(request, "Invalid form") logger.warning("Invalid form") for field, errors in form.errors.items(): logger.warning("Form error in field: %s: %s" % (field, errors)) events = EventsStatus.events_with_head(pr.events) evs_info = EventsStatus.multiline_events_info(events, events_url=True) context = { "pr": pr, "events": evs_info, "allowed": allowed, "update_interval": settings.EVENT_PAGE_UPDATE_INTERVAL, "alt_choices": alt_choices, "default_choices": default_choices, } return render(request, 'ci/pr.html', context)
def job_script(request, job_id): """ Creates a single shell script that would be similar to what the client ends up running. Used for debugging. Input: job_id: models.Job.pk Return: Http404 if the job doesn't exist or the user doesn't have permission, else HttpResponse """ job = get_object_or_404(models.Job, pk=job_id) perms = Permissions.job_permissions(request.session, job) if not perms['is_owner']: logger.warning("Tried to get job script for %s: %s but not the owner" % (job.pk, job)) raise Http404('Not the owner') script = '<pre>#!/bin/bash' script += '\n# Script for job {}'.format(job) script += '\n# Note that BUILD_ROOT and other environment variables set by the client are not set' script += '\n# It is a good idea to redirect stdin, ie "./script.sh < /dev/null"' script += '\n\n' script += '\nmodule purge' script += '\nexport BUILD_ROOT=""' script += '\nexport MOOSE_JOBS="1"' script += '\n\n' recipe = job.recipe for prestep in recipe.prestepsources.all(): script += '\n{}\n'.format(read_recipe_file(prestep.filename)) for env in recipe.environment_vars.all(): script += '\nexport {}="{}"'.format(env.name, env.value) script += '\nexport CIVET_RECIPE_NAME="{}"'.format(job.recipe.name) script += '\nexport CIVET_JOB_ID="{}"'.format(job.pk) script += '\nexport CIVET_RECIPE_ID="{}"'.format(job.recipe.pk) script += '\nexport CIVET_COMMENTS_URL="{}"'.format(job.event.comments_url) script += '\nexport CIVET_BASE_REPO="{}"'.format(job.event.base.repo()) script += '\nexport CIVET_BASE_REF="{}"'.format(job.event.base.branch.name) script += '\nexport CIVET_BASE_SHA="{}"'.format(job.event.base.sha) script += '\nexport CIVET_BASE_SSH_URL="{}"'.format(job.event.base.ssh_url) script += '\nexport CIVET_HEAD_REPO="{}"'.format(job.event.head.repo()) script += '\nexport CIVET_HEAD_REF="{}"'.format(job.event.head.branch.name) script += '\nexport CIVET_HEAD_SHA="{}"'.format(job.event.head.sha) script += '\nexport CIVET_HEAD_SSH_URL="{}"'.format(job.event.head.ssh_url) script += '\nexport CIVET_EVENT_CAUSE="{}"'.format(job.recipe.cause_str()) script += '\nexport CIVET_BUILD_CONFIG="{}"'.format(job.config.name) script += '\n\n' count = 0 step_cmds = '' for step in recipe.steps.order_by('position').all(): script += '\nfunction step_{}\n{{'.format(count) script += '\n\tlocal CIVET_STEP_NUM="{}"'.format(step.position) script += '\n\tlocal CIVET_STEP_POSITION="{}"'.format(step.position) script += '\n\tlocal CIVET_STEP_NAME="{}"'.format(step.name) script += '\n\tlocal CIVET_STEP_ABORT_ON_FAILURE="{}"'.format( step.abort_on_failure) script += '\n\tlocal CIVET_STEP_ALLOED_TO_FAIL="{}"'.format( step.allowed_to_fail) for env in step.step_environment.all(): script += '\n\tlocal {}="{}"'.format(env.name, env.value) for l in read_recipe_file(step.filename).split('\n'): script += '\n\t{}'.format(l.replace('exit 0', 'return 0')) script += '\n}\n' step_cmds += '\nstep_{}'.format(count) count += 1 script += step_cmds script += '</pre>' return HttpResponse(script)
def job_results(request): """ Returns the job results and job info in JSON. GET parameters: job_id: The pk of the job last_request: A timestamp of when client last requested this information. If the job hasn't been updated since that time we don't have to send as much information. """ if 'last_request' not in request.GET or 'job_id' not in request.GET: return HttpResponseBadRequest('Missing parameters') this_request = TimeUtils.get_local_timestamp() job_id = int(request.GET['job_id']) last_request = int(float(request.GET['last_request'])) # in case it has decimals dt = timezone.localtime(timezone.make_aware(datetime.datetime.utcfromtimestamp(last_request))) job = get_object_or_404(models.Job.objects.select_related("recipe", "client").prefetch_related("step_results"), pk=job_id) if not Permissions.can_see_results(request.session, job.recipe): return HttpResponseForbidden("Can't see results") job_info = { 'id': job.pk, 'complete': job.complete, 'status': job.status_slug(), 'runtime': str(job.seconds), 'ready': job.ready, 'invalidated': job.invalidated, 'active': job.active, 'last_modified': TimeUtils.display_time_str(job.last_modified), 'created': TimeUtils.display_time_str(job.created), 'client_name': '', 'client_url': '', 'recipe_repo_sha': job.recipe_repo_sha[:6], 'recipe_sha': job.recipe.filename_sha[:6], } if job.last_modified < dt: # always return the basic info since we need to update the # "natural" time return JsonResponse({'job_info': job_info, 'results': [], 'last_request': this_request}) if job.client: can_see_client = Permissions.is_allowed_to_see_clients(request.session) if can_see_client: job_info['client_name'] = job.client.name job_info['client_url'] = reverse('ci:view_client', args=[job.client.pk,]) result_info = [] for result in job.step_results.all(): if dt > result.last_modified: continue exit_status = '' if result.complete: exit_status = result.exit_status info = {'id': result.id, 'name': result.name, 'runtime': str(result.seconds), 'exit_status': exit_status, 'output': result.clean_output(), 'status': result.status_slug(), 'running': result.status != models.JobStatus.NOT_STARTED, 'complete': result.complete, 'output_size': result.output_size(), } result_info.append(info) return JsonResponse({'job_info': job_info, 'results': result_info, 'last_request': this_request})
def _check_recipe(self, session, git_api, pr, ev, recipe): """ Check if an individual recipe is active for the PR. If it is not then set a comment on the PR saying that they need to activate the recipe. Input: session[dict]: Session to store collaborator information git_api[GitAPI]: Git API for the build_user pr: models.PullRequest that we are processing ev: models.Event that is attached to this pull request recipe: models.Recipe that we need to process """ if not recipe.active: return [] active = False server = pr.repository.user.server if recipe.automatic == models.Recipe.FULL_AUTO: active = True elif recipe.automatic == models.Recipe.MANUAL: active = False elif recipe.automatic == models.Recipe.AUTO_FOR_AUTHORIZED: if ev.trigger_user: pr_user, created = models.GitUser.objects.get_or_create( name=ev.trigger_user, server=server) if pr_user in recipe.auto_authorized.all(): active = True else: active = Permissions.is_collaborator(session, recipe.build_user, recipe.repository, user=pr_user) if active: logger.info( 'User {} is allowed to activate recipe: {}: {}'.format( pr_user, recipe.pk, recipe)) else: logger.info( 'User {} is NOT allowed to activate recipe {}: {}'. format(pr_user, recipe.pk, recipe)) if created: pr_user.delete() else: logger.info( 'Recipe: {}: {}: not activated because trigger_user is blank' .format(recipe.pk, recipe)) jobs = [] for config in recipe.build_configs.order_by("name").all(): job, created = models.Job.objects.get_or_create(recipe=recipe, event=ev, config=config) if created: job.active = active job.ready = False job.complete = False if job.active: job.status = models.JobStatus.NOT_STARTED else: job.status = models.JobStatus.ACTIVATION_REQUIRED job.save() logger.info('Created job {}: {}: on {}'.format( job.pk, job, recipe.repository)) jobs.append(job) else: logger.info('Job {}: {}: on {} already exists'.format( job.pk, job, recipe.repository)) return jobs