def test_webhook(self):
url = reverse('ci:gitlab:webhook', args=[10000])
# only post allowed
response = self.client.get(url)
self.assertEqual(response.status_code, 405) # not allowed
# no user
data = {'key': 'value'}
response = self.client_post_json(url, data)
self.assertEqual(response.status_code, 400)
# not json
user = utils.get_test_user(server=self.server)
url = reverse('ci:gitlab:webhook', args=[user.build_key])
response = self.client.post(url, data)
self.assertEqual(response.status_code, 400)
# user with no recipes
response = self.client_post_json(url, data)
self.assertEqual(response.status_code, 400)
# unknown json
utils.create_recipe(user=user)
response = self.client_post_json(url, data)
self.assertEqual(response.status_code, 400)
def create_event_with_jobs(self,
commit='1234',
user=None,
branch1=None,
branch2=None,
cause=models.Event.PULL_REQUEST):
ev = utils.create_event(commit2=commit,
user=user,
branch1=branch1,
branch2=branch2,
cause=cause)
ev.base.branch.repository.active = True
ev.base.branch.repository.save()
alt_recipe = utils.create_recipe(
name="alt recipe", cause=models.Recipe.CAUSE_PULL_REQUEST_ALT)
utils.create_step(name="step0_alt", recipe=alt_recipe, position=0)
utils.create_step(name="step1_alt", recipe=alt_recipe, position=1)
if cause == models.Event.PULL_REQUEST:
pr = utils.create_pr(title="Foo {a, b} & <bar> …")
pr.alternate_recipes.add(alt_recipe)
ev.pull_request = pr
ev.save()
r0 = utils.create_recipe(name="r0", cause=cause)
utils.create_step(name="step1_r0", recipe=r0, position=1)
utils.create_step(name="step2_r0", recipe=r0, position=2)
r1 = utils.create_recipe(name="r1", cause=cause)
utils.create_step(name="step1_r1", recipe=r1, position=1)
utils.create_step(name="step2_r1", recipe=r1, position=2)
r1.depends_on.add(r1)
utils.create_job(event=ev, recipe=r0)
utils.create_job(event=ev, recipe=r1)
return ev
def test_recipe(self):
manual = self.create_data()
self.set_counts()
manual.save()
self.compare_counts(events=1, jobs=1, ready=1, commits=1, active=1, active_repos=1)
# now try another event on the Manual but with a new recipe that has the same filename
manual_recipe = models.Recipe.objects.filter(cause=models.Recipe.CAUSE_MANUAL).latest()
new_recipe = utils.create_recipe(name="New recipe",
user=self.build_user,
repo=self.repo,
branch=self.branch,
cause=models.Recipe.CAUSE_MANUAL)
new_recipe.filename = manual_recipe.filename
new_recipe.current = True
new_recipe.active = True
new_recipe.save()
manual_recipe.current = False
manual_recipe.save()
self.set_counts()
manual_recipe.save()
self.compare_counts()
# We have a new latest SHA, everything should be created
manual = self.create_data(latest="10")
self.set_counts()
manual.save()
self.compare_counts(events=1, jobs=1, ready=1, commits=1, active=1)
self.assertEqual(manual_recipe.jobs.count(), 1)
self.assertEqual(new_recipe.jobs.count(), 1)
# save the same Manual and make sure the jobs haven't changed
# and no new events were created.
self.set_counts()
manual.save()
self.compare_counts()
# now a new recipe is added that has a different filename
new_recipe = utils.create_recipe(name="Another New recipe",
user=self.build_user,
repo=self.repo,
branch=self.branch,
cause=models.Recipe.CAUSE_MANUAL)
new_recipe.filename = "Some other filename"
new_recipe.current = True
new_recipe.active = True
new_recipe.save()
self.set_counts()
manual.save()
self.compare_counts(jobs=1, ready=1, active=1)
def setUp(self):
self.client = Client()
self.factory = RequestFactory()
for i in range(5):
repo = utils.create_repo(name="repo%s" % i)
repo.active = True
repo.status = models.JobStatus.SUCCESS
repo.save()
for j in range(2):
b = utils.create_branch(name="branch%s" % j, repo=repo)
b.status = models.JobStatus.SUCCESS
b.save()
for j in range(3):
b = repo.branches.first()
pr = utils.create_pr(title="pr%s" % j, number=j+1, repo=repo)
pr.closed = False
pr.status = models.JobStatus.SUCCESS
pr.save()
ev = utils.create_event(user=repo.user, branch1=b, branch2=b, commit1="%s" % j)
ev.pull_request = pr
ev.save()
for k in range(3):
r = utils.create_recipe(name="%s%s" % (repo.name, k), repo=repo, branch=b)
r.private = False
r.save()
job = utils.create_job(recipe=r, event=ev)
job.status = models.JobStatus.SUCCESS
job.client = utils.create_client(name="client%s/%s" % (repo.name, k))
job.save()
utils.create_step_result(job=job)
utils.create_osversion()
utils.create_loadedmodule()
def test_cancel_event(self):
ev = utils.create_event()
jobs = []
for i in range(3):
r = utils.create_recipe(name="recipe %s" % i, user=ev.build_user)
j = utils.create_job(recipe=r, event=ev, user=ev.build_user)
jobs.append(j)
msg = "Test cancel"
self.set_counts()
event.cancel_event(ev, msg)
# The status on the branch should get updated
self.compare_counts(canceled=3,
events_canceled=1,
num_changelog=3,
num_jobs_completed=3,
num_events_completed=1,
active_branches=1)
ev.refresh_from_db()
self.assertEqual(ev.status, models.JobStatus.CANCELED)
self.assertEqual(ev.complete, True)
for j in jobs:
j.refresh_from_db()
self.assertEqual(j.status, models.JobStatus.CANCELED)
self.assertTrue(j.complete)
def create_job_with_nested_bash(recipe_dir, name="TestJob", sleep=10):
user = utils.get_test_user()
recipe = utils.create_recipe(user=user, name=name)
test_job = utils.create_job(user=user, recipe=recipe)
test_job.ready = True
test_job.client = None
test_job.status = models.JobStatus.NOT_STARTED
test_job.save()
step = utils.create_step(name="step0", recipe=recipe, position=0)
step.filename = "step0.sh"
step.save()
script_filename = os.path.join(recipe_dir, step.filename)
sub_script_filename = os.path.join(recipe_dir, "step0_sub.sh")
sub_sub_script_filename = os.path.join(recipe_dir, "step0_sub_sub.sh")
with open(script_filename, "w") as f:
f.write("#!/bin/bash\necho 'Launching {0}'\n{0}\necho '{0} returned '".
format(sub_script_filename))
with open(sub_script_filename, "w") as f:
f.write("#!/bin/bash\necho 'Launching {0}'\n{0}\necho '{0} returned'".
format(sub_sub_script_filename))
import stat
st = os.stat(sub_script_filename)
os.chmod(sub_script_filename, st.st_mode | stat.S_IEXEC)
with open(sub_sub_script_filename, "w") as f:
f.write(
"#!/bin/bash\necho 'Sleeping {0}...'\nsleep {0}\necho 'Finished sleeping'"
.format(sleep))
st = os.stat(sub_sub_script_filename)
os.chmod(sub_sub_script_filename, st.st_mode | stat.S_IEXEC)
return test_job
def test_recipe(self):
c1_data, c2_data, push = self.create_data()
self.set_counts()
push.save()
self.compare_counts(events=1,
jobs=2,
ready=1,
active=2,
active_repos=1)
# now try another event on the Push but with a new recipe.
push_recipe = models.Recipe.objects.filter(
cause=models.Recipe.CAUSE_PUSH).latest()
new_recipe = utils.create_recipe(name="New recipe",
user=self.build_user,
repo=self.repo,
branch=self.branch,
cause=models.Recipe.CAUSE_PUSH)
new_recipe.filename = push_recipe.filename
new_recipe.save()
push_recipe.current = False
push_recipe.save()
c2_data.sha = '10'
push.head_commit = c2_data
self.set_counts()
push.save()
self.compare_counts(events=1, jobs=2, ready=2, commits=1, active=2)
# save the same push and make sure the jobs haven't changed
# and no new events were created.
self.set_counts()
push.save()
self.compare_counts()
def test_change_recipe(self):
c1_data, c2_data, push = self.create_data()
self.set_counts()
push.save()
self.compare_counts(events=1,
jobs=2,
ready=1,
active=2,
active_repos=1)
# This scenario is one where the event already exists but
# for some reason the same push event gets called and the recipes have changed.
# Nothing should have changed
push_recipe = models.Recipe.objects.filter(
cause=models.Recipe.CAUSE_PUSH).latest()
new_recipe = utils.create_recipe(name="New recipe",
user=self.build_user,
repo=self.repo,
branch=self.branch,
cause=models.Recipe.CAUSE_PUSH)
new_recipe.filename = push_recipe.filename
new_recipe.save()
push_recipe.current = False
push_recipe.save()
self.assertEqual(push_recipe.jobs.count(), 1)
self.set_counts()
push.save()
self.compare_counts()
push_recipe.refresh_from_db()
new_recipe.refresh_from_db()
self.assertEqual(push_recipe.jobs.count(), 1)
self.assertEqual(new_recipe.jobs.count(), 0)
def test_get_paginated(self):
recipes = models.Recipe.objects.all().order_by("-id")
self.assertEqual(models.Recipe.objects.count(), 6)
# there are 6 recipes, so only 1 page
# objs.number is the current page number
request = self.factory.get('/foo?page=1')
objs = views.get_paginated(request, recipes)
self.assertEqual(objs.number, 1)
self.assertEqual(objs.paginator.num_pages, 1)
self.assertEqual(objs.paginator.count, 6)
# Invalid page, so just returns the end page
request = self.factory.get('/foo?page=2')
objs = views.get_paginated(request, recipes)
self.assertEqual(objs.number, 1)
self.assertEqual(objs.paginator.num_pages, 1)
self.assertEqual(objs.paginator.count, 6)
for i in range(10):
utils.create_recipe(name='recipe %s' % i)
# now there are 16 recipes, so page=2 should be
# valid
request = self.factory.get('/foo?page=2')
objs = views.get_paginated(request, recipes, 2)
self.assertEqual(objs.number, 2)
self.assertEqual(objs.paginator.num_pages, 8)
self.assertEqual(objs.paginator.count, 16)
# page=20 doesn't exist so it should return
# the last page
request = self.factory.get('/foo?page=20')
objs = views.get_paginated(request, recipes, 2)
self.assertEqual(objs.number, 8)
self.assertEqual(objs.paginator.num_pages, 8)
self.assertEqual(objs.paginator.count, 16)
# Completely invalid page number so returns
# the first page
request = self.factory.get('/foo?page=foo')
objs = views.get_paginated(request, recipes, 2)
self.assertEqual(objs.number, 1)
self.assertEqual(objs.paginator.num_pages, 8)
self.assertEqual(objs.paginator.count, 16)
def test_recipe_events(self):
response = self.client.get(reverse('ci:recipe_events', args=[1000,]))
self.assertEqual(response.status_code, 404)
rc = utils.create_recipe()
job1 = utils.create_job(recipe=rc)
job1.status = models.JobStatus.SUCCESS
job1.save()
response = self.client.get(reverse('ci:recipe_events', args=[rc.pk]))
self.assertEqual(response.status_code, 200)
def create_events(self):
self.set_counts()
self.build_user = utils.create_user_with_token(name="moosebuild")
self.owner = utils.create_user(name="idaholab")
self.repo = utils.create_repo(name="civet", user=self.owner)
self.branch = utils.create_branch(name="devel", repo=self.repo)
pre = utils.create_recipe(name="Precheck", user=self.build_user, repo=self.repo)
test = utils.create_recipe(name="Test", user=self.build_user, repo=self.repo)
test1 = utils.create_recipe(name="Test1", user=self.build_user, repo=self.repo)
test.depends_on.add(pre)
test1.depends_on.add(pre)
merge = utils.create_recipe(name="Merge", user=self.build_user, repo=self.repo)
merge.depends_on.add(test)
merge.depends_on.add(test1)
pr = utils.create_pr(title="{a, b} & <c> … somereallylongwordthatshouldgettruncated", repo=self.repo)
pr.username = '******'
pr.save()
for commit in ['1234', '2345', '3456']:
e = utils.create_event(user=self.owner, commit1=commit, branch1=self.branch, branch2=self.branch)
e.pull_request = pr
e.description = "some description"
e.save()
j = utils.create_job(recipe=pre, event=e, user=self.build_user)
j.seconds = datetime.timedelta(seconds=10)
j.failed_step = 'failed step'
j.running_step = '3/5'
j.save()
utils.create_job(recipe=test, event=e, user=self.build_user)
utils.create_job(recipe=test1, event=e, user=self.build_user)
utils.create_job(recipe=merge, event=e, user=self.build_user)
self.compare_counts(recipes=4,
deps=4,
current=4,
jobs=12,
active=12,
num_pr_recipes=4,
events=3,
users=2,
repos=1,
branches=1,
commits=3,
prs=1)
def test_job_finished_unrunnable(self, mock_status):
user = utils.get_test_user()
r0 = utils.create_recipe(name='recipe0', user=user)
r1 = utils.create_recipe(name='recipe1', user=user)
r2 = utils.create_recipe(name='recipe2', user=user)
r2.depends_on.add(r1)
r1.depends_on.add(r0)
j0 = utils.create_job(user=user, recipe=r0)
utils.create_job(user=user, recipe=r1)
utils.create_job(user=user, recipe=r2)
post_data = {'seconds': 0, 'complete': True}
client = utils.create_client()
j0.client = client
j0.save()
step_result = utils.create_step_result(job=j0)
step_result.status = models.JobStatus.FAILED
step_result.save()
# should be ok
url = reverse('ci:client:job_finished',
args=[user.build_key, client.name, j0.pk])
self.set_counts()
response = self.client_post_json(url, post_data)
self.compare_counts(num_events_completed=1,
num_jobs_completed=1,
active_branches=1)
self.assertEqual(response.status_code, 200)
self.assertEqual(
mock_status.call_count,
3) # 1 for the job complete update and 2 for the won't run update
step_result.status = models.JobStatus.SUCCESS
step_result.save()
self.set_counts()
response = self.client_post_json(url, post_data)
self.compare_counts(ready=1)
self.assertEqual(response.status_code, 200)
self.assertEqual(mock_status.call_count,
4) # 1 for the job complete update
def test_event_new_job(self):
self.create_event_with_jobs()
self.get()
self.check_repos()
self.check_events()
ev = models.Event.objects.first()
r2 = utils.create_recipe(name="r2")
ev.save() # to trigger the update
utils.create_job(event=ev, recipe=r2)
self.wait_for_js()
self.check_js_error()
self.check_repos()
self.check_events()
def create_client_job(recipe_dir,
name="TestJob",
sleep=1,
n_steps=3,
extra_script=''):
user = utils.get_test_user()
recipe = utils.create_recipe(user=user, name=name)
test_job = utils.create_job(user=user, recipe=recipe)
test_job.ready = True
test_job.client = None
test_job.status = models.JobStatus.NOT_STARTED
test_job.save()
# create a prestep to make sure sourcing functions work
prestep0 = utils.create_prestepsource(
filename="prestep0_{}.sh".format(name), recipe=recipe)
with open(os.path.join(recipe_dir, prestep0.filename), "w") as f:
f.write('function start_message()\n{\n echo start "$*"\n}')
# create a prestep to make sure sourcing functions work
prestep1 = utils.create_prestepsource(
filename="prestep1_{}.sh".format(name), recipe=recipe)
with open(os.path.join(recipe_dir, prestep1.filename), "w") as f:
f.write('function end_message()\n{\n echo end "$*"\n}')
# create a global environment variable to test env works
# as well as BUILD_ROOT replacement
utils.create_recipe_environment(name="GLOBAL_NAME",
value="BUILD_ROOT/global",
recipe=recipe)
count = 0
for s in [f"step{i}".format(i) for i in range(n_steps)]:
step = utils.create_step(name=s, recipe=recipe, position=count)
# create a step environment variable to test env works
# as well as BUILD_ROOT replacement
utils.create_step_environment(name="STEP_NAME",
value="BUILD_ROOT/%s" % s,
step=step)
step.filename = "{}_{}.sh".format(s, name)
step.save()
count += 1
script_filename = os.path.join(recipe_dir, step.filename)
job_script = "echo $GLOBAL_NAME $CIVET_RECIPE_NAME $STEP_NAME\n"
job_script += "start_message {0}:{1}\n".format(recipe.name, s)
job_script += "sleep {0}\n".format(sleep)
job_script += "end_message {0}:{1}\n".format(recipe.name, s)
job_script += extra_script
with open(script_filename, "w") as f:
f.write(job_script)
return test_job
def test_event_new_job(self):
ev = self.create_event_with_jobs()
url = reverse('ci:view_repo', args=[ev.base.branch.repository.pk])
self.get(url)
self.check_repos()
self.check_events()
ev = models.Event.objects.first()
r2 = utils.create_recipe(name="r2")
ev.save() # to trigger the update
utils.create_job(event=ev, recipe=r2)
self.wait_for_js()
self.check_js_error()
self.check_repos()
self.check_events()
def create_default_recipes(self, server_type=settings.GITSERVER_GITHUB):
self.set_counts()
self.server = utils.create_git_server(host_type=server_type)
self.build_user = utils.create_user_with_token(name="moosebuild",
server=self.server)
self.owner = utils.create_user(name="owner", server=self.server)
self.repo = utils.create_repo(name="repo", user=self.owner)
self.branch = utils.create_branch(name="devel", repo=self.repo)
pr = utils.create_recipe(name="PR Base",
user=self.build_user,
repo=self.repo)
pr1 = utils.create_recipe(name="PR With Dep",
user=self.build_user,
repo=self.repo)
pr1.depends_on.add(pr)
push = utils.create_recipe(name="Push Base",
user=self.build_user,
repo=self.repo,
branch=self.branch,
cause=models.Recipe.CAUSE_PUSH)
push1 = utils.create_recipe(name="Push With Dep",
user=self.build_user,
repo=self.repo,
branch=self.branch,
cause=models.Recipe.CAUSE_PUSH)
push1.depends_on.add(push)
alt_pr = utils.create_recipe(
name="Alt PR with dep",
user=self.build_user,
repo=self.repo,
cause=models.Recipe.CAUSE_PULL_REQUEST_ALT)
alt_pr.depends_on.add(pr)
utils.create_recipe(name="Manual",
user=self.build_user,
repo=self.repo,
branch=self.branch,
cause=models.Recipe.CAUSE_MANUAL)
self.compare_counts(
recipes=6,
deps=3,
current=6,
num_push_recipes=2,
num_pr_recipes=2,
num_manual_recipes=1,
num_pr_alt_recipes=1,
users=2,
repos=1,
branches=1,
)
def create_client_job(recipe_dir, name="TestJob", sleep=1):
user = utils.get_test_user()
recipe = utils.create_recipe(user=user, name=name)
test_job = utils.create_job(user=user, recipe=recipe)
test_job.ready = True
test_job.client = None
test_job.status = models.JobStatus.NOT_STARTED
test_job.save()
# create a prestep to make sure sourcing functions work
prestep0 = utils.create_prestepsource(filename="prestep0.sh",
recipe=recipe)
with open(os.path.join(recipe_dir, prestep0.filename), "w") as f:
f.write('function start_message()\n{\n echo start "$*"\n}')
# create a prestep to make sure sourcing functions work
prestep1 = utils.create_prestepsource(filename="prestep1.sh",
recipe=recipe)
with open(os.path.join(recipe_dir, prestep1.filename), "w") as f:
f.write('function end_message()\n{\n echo end "$*"\n}')
# create a global environment variable to test env works
# as well as BUILD_ROOT replacement
utils.create_recipe_environment(name="GLOBAL_NAME",
value="BUILD_ROOT/global",
recipe=recipe)
count = 0
for s in ["step0", "step1", "step2"]:
step = utils.create_step(name=s, recipe=recipe, position=count)
# create a step environment variable to test env works
# as well as BUILD_ROOT replacement
utils.create_step_environment(name="STEP_NAME",
value="BUILD_ROOT/%s" % s,
step=step)
step.filename = "%s.sh" % s
step.save()
count += 1
script_filename = os.path.join(recipe_dir, step.filename)
with open(script_filename, "w") as f:
f.write(
"echo $GLOBAL_NAME $recipe_name $STEP_NAME\nstart_message {0}:{1}\nsleep {2}\nend_message {0}:{1}\n"
.format(recipe.name, s, sleep))
return test_job
def test_create_event_summary(self, mock_get, mock_post):
mock_get.return_value = utils.Response()
mock_post.return_value = utils.Response()
ev = utils.create_event()
ev.comments_url = 'url'
ev.save()
j0 = utils.create_job(event=ev)
config = utils.create_build_config("config1")
j0.recipe.build_configs.add(config)
utils.create_job(event=ev, recipe=j0.recipe, config=config)
r1 = utils.create_recipe(name="r1")
j1 = utils.create_job(recipe=r1, event=ev)
j0.recipe.depends_on.add(r1)
with self.settings(INSTALLED_GITSERVERS=[
utils.github_config(post_event_summary=False,
remote_update=True)
]):
# Not posting the summary so we should not do anything
UpdateRemoteStatus.create_event_summary(ev)
self.assertEqual(mock_post.call_count, 0)
self.assertEqual(mock_get.call_count, 0)
with self.settings(INSTALLED_GITSERVERS=[
utils.github_config(post_event_summary=True,
remote_update=True)
]):
# Configured to post the summary
UpdateRemoteStatus.create_event_summary(ev)
self.assertEqual(mock_post.call_count, 1) # 1 for adding comment
self.assertEqual(mock_get.call_count,
1) # 1 for getting current comments
utils.update_job(j1,
status=models.JobStatus.FAILED,
complete=True,
invalidated=True)
utils.create_step_result(job=j1, status=models.JobStatus.FAILED)
self.assertEqual(len(ev.get_unrunnable_jobs()), 2)
UpdateRemoteStatus.create_event_summary(ev)
self.assertEqual(mock_post.call_count, 2)
self.assertEqual(mock_get.call_count, 2)
def test_update_pull_requests(self):
with test_utils.RecipeDir() as recipes_dir:
creator = self.create_valid_with_check(recipes_dir)
self.set_counts()
creator._update_pull_requests()
self.compare_counts()
# update_pull_requests doesn't depend on recipes in the filesystem
build_user = models.GitUser.objects.get(name="moosebuild")
pr = test_utils.create_pr()
self.assertEqual(
build_user.recipes.filter(
cause=models.Recipe.CAUSE_PULL_REQUEST_ALT).count(), 2)
r1_orig = build_user.recipes.filter(
cause=models.Recipe.CAUSE_PULL_REQUEST_ALT).first()
r1 = test_utils.create_recipe(name="alt_pr",
user=r1_orig.build_user,
repo=r1_orig.repository,
cause=r1_orig.cause)
r1.filename = r1_orig.filename
r1.save()
r1_orig.current = False
r1_orig.save()
self.assertEqual(pr.alternate_recipes.count(), 0)
pr.alternate_recipes.add(r1_orig)
# There is an old alt recipe on the PR, it should be removed and replaced with the new one
self.set_counts()
creator._update_pull_requests()
self.compare_counts()
self.assertEqual(pr.alternate_recipes.count(), 1)
self.assertEqual(pr.alternate_recipes.first().pk, r1.pk)
r1.current = False
r1.save()
self.set_counts()
# Now there are not current alt PR recipes
creator._update_pull_requests()
self.compare_counts(num_pr_alts=-1)
self.assertEqual(pr.alternate_recipes.count(), 0)
def test_change_recipe(self):
"""
Try saving the same pull request but the recipe repo has changed.
This scenario is one where the event already exists but the
user might have just changed something cosmetic about the PR.
So we don't change the current recipes on the event or the jobs either.
But a recipe does get created
"""
c1_data, c2_data, pr = self.create_pr_data()
c1_data, c2_data, pr = self.create_pr_data()
self.set_counts()
pr.save()
self.compare_counts(events=1,
jobs=2,
ready=1,
prs=1,
active=2,
active_repos=1)
new_recipe = utils.create_recipe(
name="New recipe",
user=self.build_user,
repo=self.repo,
branch=self.branch,
cause=models.Recipe.CAUSE_PULL_REQUEST)
pr_recipe = models.Recipe.objects.filter(
cause=models.Recipe.CAUSE_PULL_REQUEST).latest()
new_recipe.filename = pr_recipe.filename
new_recipe.save()
for dep in pr_recipe.depends_on.all():
new_recipe.depends_on.add(dep)
pr_recipe.current = False
pr_recipe.save()
self.set_counts()
pr.save()
self.compare_counts()
def test_change_recipe(self):
manual = self.create_data()
self.set_counts()
manual.save()
self.compare_counts(events=1, jobs=1, ready=1, commits=1, active=1, active_repos=1)
# This scenario is one where the event already exists but the
# for some reason the same event gets called and the recipes have changed.
# Nothing should change
manual_recipe = models.Recipe.objects.filter(cause=models.Recipe.CAUSE_MANUAL).latest()
new_recipe = utils.create_recipe(name="New recipe",
user=self.build_user,
repo=self.repo,
branch=self.branch,
cause=models.Recipe.CAUSE_MANUAL)
new_recipe.filename = manual_recipe.filename
new_recipe.save()
manual_recipe.current = False
manual_recipe.save()
self.set_counts()
manual.save()
self.compare_counts()
self.assertEqual(manual_recipe.jobs.count(), 1)
self.assertEqual(new_recipe.jobs.count(), 0)
def test_view_profile(self):
# invalid git server
response = self.client.get(reverse('ci:view_profile', args=[1000, "no_exist"]))
self.assertEqual(response.status_code, 404)
# not signed in should redirect to sign in
server = utils.create_git_server()
response = self.client.get(reverse('ci:view_profile', args=[server.host_type, server.name]))
self.assertEqual(response.status_code, 302) # redirect
user = utils.get_test_user()
repo1 = utils.create_repo(name='repo1', user=user)
repo2 = utils.create_repo(name='repo2', user=user)
repo3 = utils.create_repo(name='repo3', user=user)
utils.create_recipe(name='r1', user=user, repo=repo1)
utils.create_recipe(name='r2', user=user, repo=repo2)
utils.create_recipe(name='r3', user=user, repo=repo3)
# signed in
utils.simulate_login(self.client.session, user)
response = self.client.get(reverse('ci:view_profile', args=[user.server.host_type, user.server.name]))
self.assertEqual(response.status_code, 200)
def test_view_pr(self, mock_collab):
"""
testing ci:view_pr
"""
# bad pr
url = reverse('ci:view_pr', args=[1000,])
response = self.client.get(url)
self.assertEqual(response.status_code, 404)
pr = utils.create_pr()
ev = utils.create_event()
ev.pull_request = pr
ev.save()
utils.create_job(event=ev)
user = utils.get_test_user()
utils.simulate_login(self.client.session, user)
# user not a collaborator, no alternate recipe form
mock_collab.return_value = False
url = reverse('ci:view_pr', args=[pr.pk,])
self.set_counts()
response = self.client.get(url)
self.compare_counts()
self.assertEqual(response.status_code, 200)
# user a collaborator, they get alternate recipe form
mock_collab.return_value = True
r0 = utils.create_recipe(name="Recipe 0", repo=ev.base.branch.repository, cause=models.Recipe.CAUSE_PULL_REQUEST_ALT)
r1 = utils.create_recipe(name="Recipe 1", repo=ev.base.branch.repository, cause=models.Recipe.CAUSE_PULL_REQUEST_ALT)
self.set_counts()
response = self.client.get(url)
self.compare_counts()
self.assertEqual(response.status_code, 200)
self.set_counts()
# post an invalid alternate recipe form
response = self.client.post(url, {})
self.assertEqual(response.status_code, 200)
self.assertEqual(pr.alternate_recipes.count(), 0)
self.compare_counts()
utils.simulate_login(self.client.session, user)
# post a valid alternate recipe form
self.set_counts()
response = self.client.post(url, {"recipes": [r0.pk, r1.pk]})
self.assertEqual(response.status_code, 200)
self.assertEqual(pr.alternate_recipes.count(), 2)
# The original job plus the two alternate jobs are ready
self.compare_counts(jobs=2, ready=3, active=2, num_pr_alts=2)
# post again with the same recipes
self.set_counts()
response = self.client.post(url, {"recipes": [r0.pk, r1.pk]})
self.assertEqual(response.status_code, 200)
self.assertEqual(pr.alternate_recipes.count(), 2)
self.compare_counts()
# post again different recipes. We don't auto cancel jobs.
self.set_counts()
response = self.client.post(url, {"recipes": [r0.pk]})
self.assertEqual(response.status_code, 200)
self.assertEqual(pr.alternate_recipes.count(), 1)
self.compare_counts(num_pr_alts=-1)
# clear alt recipes
self.set_counts()
response = self.client.post(url, {"recipes": []})
self.assertEqual(response.status_code, 200)
self.assertEqual(pr.alternate_recipes.count(), 0)
self.compare_counts(num_pr_alts=-1)
def test_start_canceled_on_fail(self):
user = utils.get_test_user()
r0 = utils.create_recipe(name='recipe0', user=user)
r1 = utils.create_recipe(name='recipe1', user=user)
r2 = utils.create_recipe(name='recipe2', user=user)
e0 = utils.create_event(user=user, cause=models.Event.PUSH)
j0 = utils.create_job(recipe=r0, event=e0, user=user)
utils.update_job(j0, status=models.JobStatus.SUCCESS, complete=True)
j1 = utils.create_job(recipe=r1, event=e0, user=user)
utils.update_job(j1, status=models.JobStatus.CANCELED, complete=True)
j2 = utils.create_job(recipe=r2, event=e0, user=user)
utils.update_job(j2, status=models.JobStatus.RUNNING, complete=True)
e1 = utils.create_event(user=user,
cause=models.Event.PUSH,
commit1='12345')
j3 = utils.create_job(recipe=r0, event=e1, user=user)
utils.update_job(j3, status=models.JobStatus.SUCCESS, complete=True)
j4 = utils.create_job(recipe=r1, event=e1, user=user)
utils.update_job(j4, status=models.JobStatus.CANCELED, complete=True)
j5 = utils.create_job(recipe=r2, event=e1, user=user)
utils.update_job(j5, status=models.JobStatus.RUNNING, complete=True)
e2 = utils.create_event(user=user,
cause=models.Event.PUSH,
commit1='123456')
j6 = utils.create_job(recipe=r0, event=e2, user=user)
utils.update_job(j6, status=models.JobStatus.SUCCESS, complete=True)
j7 = utils.create_job(recipe=r1, event=e2, user=user)
utils.update_job(j7, status=models.JobStatus.FAILED, complete=True)
j8 = utils.create_job(recipe=r2, event=e2, user=user)
utils.update_job(j8, status=models.JobStatus.FAILED_OK, complete=True)
# If the job isn't a fail then it shouldn't do anything
self.set_counts()
UpdateRemoteStatus.start_canceled_on_fail(j6)
self.compare_counts()
# Normal behavior, a job fails and doesn't do anything to the previous event
self.set_counts()
UpdateRemoteStatus.start_canceled_on_fail(j7)
self.compare_counts()
repo_name = "%s/%s" % (e0.base.branch.repository.user.name,
e0.base.branch.repository.name)
branch_name = e0.base.branch.name
branch_settings = {
"auto_cancel_push_events_except_current": True,
"auto_uncancel_previous_event": True
}
repo_settings = {
repo_name: {
"branch_settings": {
branch_name: branch_settings
}
}
}
with self.settings(INSTALLED_GITSERVERS=[
utils.github_config(repo_settings=repo_settings)
]):
# If the job isn't a fail then it shouldn't do anything
self.set_counts()
UpdateRemoteStatus.start_canceled_on_fail(j3)
self.compare_counts()
# A job fails and should go to the previous event and uncancel any jobs
self.set_counts()
UpdateRemoteStatus.start_canceled_on_fail(j7)
self.compare_counts(ready=1,
active_branches=1,
canceled=-1,
invalidated=1,
num_changelog=1,
num_jobs_completed=-1)
j0.refresh_from_db()
self.assertEqual(j0.status, models.JobStatus.SUCCESS)
j1.refresh_from_db()
self.assertEqual(j1.status, models.JobStatus.CANCELED)
self.assertTrue(j1.complete)
j2.refresh_from_db()
self.assertEqual(j2.status, models.JobStatus.RUNNING)
j3.refresh_from_db()
self.assertEqual(j3.status, models.JobStatus.SUCCESS)
j4.refresh_from_db()
self.assertEqual(j4.status, models.JobStatus.NOT_STARTED)
self.assertFalse(j4.complete)
j5.refresh_from_db()
self.assertEqual(j5.status, models.JobStatus.RUNNING)
# If another job on the same event fails, it shouldn't try to uncancel previous events
utils.update_job(j8, status=models.JobStatus.FAILED)
utils.update_job(j4,
status=models.JobStatus.CANCELED,
complete=True)
self.set_counts()
UpdateRemoteStatus.start_canceled_on_fail(j7)
self.compare_counts()
def test_ready_jobs_client(self):
user = utils.get_test_user()
client = utils.create_client()
request = self.factory.get('/')
client_ip = views.get_client_ip(request)
client.ip = client_ip
client.save()
url = reverse('ci:client:ready_jobs',
args=[user.build_key, client.name])
r0 = utils.create_recipe(name='recipe0', user=user)
r1 = utils.create_recipe(name='recipe1', user=user)
j0 = utils.create_job(user=user, recipe=r0)
j1 = utils.create_job(user=user, recipe=r1)
utils.update_job(j0,
ready=True,
complete=False,
status=models.JobStatus.RUNNING,
client=client)
utils.update_job(j1,
ready=True,
active=True,
status=models.JobStatus.NOT_STARTED)
# we have a client trying to get ready jobs but
# there is a job that is in the RUNNING state
# associated with that client. That must mean
# that the client previously stopped without letting the
# server know, so the previous job should get
# canceled
self.set_counts()
response = self.client.get(url)
self.compare_counts(canceled=1,
num_jobs_completed=1,
num_changelog=1,
active_branches=1)
self.assertEqual(response.status_code, 200)
j0.refresh_from_db()
self.assertEqual(j0.status, models.JobStatus.CANCELED)
# if all jobs are in running, the event should be canceled
utils.update_job(j0, complete=False, status=models.JobStatus.RUNNING)
utils.update_job(j1,
complete=False,
status=models.JobStatus.RUNNING,
client=client)
self.set_counts()
response = self.client.get(url)
self.compare_counts(canceled=2,
num_jobs_completed=2,
num_changelog=2,
events_canceled=1,
num_events_completed=1)
self.assertEqual(response.status_code, 200)
j0.refresh_from_db()
self.assertEqual(j0.status, models.JobStatus.CANCELED)
j1.refresh_from_db()
self.assertEqual(j1.status, models.JobStatus.CANCELED)
# Try again, nothing should change
self.set_counts()
response = self.client.get(url)
self.compare_counts()
self.assertEqual(response.status_code, 200)
def test_job_finished_status(self):
user = utils.get_test_user()
recipe = utils.create_recipe(user=user)
job = utils.create_job(recipe=recipe, user=user)
step0 = utils.create_step(name='step0', recipe=recipe)
step1 = utils.create_step(name='step1', recipe=recipe, position=1)
step0_result = utils.create_step_result(step=step0, job=job)
step1_result = utils.create_step_result(step=step1, job=job)
step0_result.status = models.JobStatus.FAILED_OK
step0_result.save()
step1_result.status = models.JobStatus.SUCCESS
step1_result.save()
client = utils.create_client()
job.client = client
job.save()
job.event.comments_url = 'http://localhost'
job.event.pull_request = utils.create_pr()
job.event.save()
url = reverse('ci:client:job_finished',
args=[user.build_key, client.name, job.pk])
# A step has FAILED_OK
# So final status is FAILED_OK and we update the PR
post_data = {'seconds': 0, 'complete': True}
with patch('ci.github.api.GitHubAPI') as mock_api:
self.set_counts()
response = self.client_post_json(url, post_data)
self.compare_counts(num_events_completed=1, num_jobs_completed=1)
self.assertEqual(response.status_code, 200)
self.assertTrue(mock_api.called)
self.assertTrue(mock_api.return_value.update_pr_status.called)
job.refresh_from_db()
self.assertEqual(job.status, models.JobStatus.FAILED_OK)
os_obj = models.OSVersion.objects.get(name="Other")
self.assertEqual(job.operating_system.pk, os_obj.pk)
self.assertEqual(job.loaded_modules.count(), 1)
self.assertEqual(job.loaded_modules.first().name, "None")
# A step FAILED
# So final status is FAILED and we update the PR
step0_result.status = models.JobStatus.FAILED
step0_result.save()
with patch('ci.github.api.GitHubAPI') as mock_api:
self.set_counts()
response = self.client_post_json(url, post_data)
self.compare_counts()
self.assertEqual(response.status_code, 200)
self.assertTrue(mock_api.called)
self.assertTrue(mock_api.return_value.update_pr_status.called)
job.refresh_from_db()
self.assertEqual(job.status, models.JobStatus.FAILED)
step0_result.status = models.JobStatus.SUCCESS
step0_result.save()
# All steps passed
# So final status is SUCCESS and we update the PR
with patch('ci.github.api.GitHubAPI') as mock_api:
self.set_counts()
response = self.client_post_json(url, post_data)
self.compare_counts()
self.assertEqual(response.status_code, 200)
self.assertTrue(mock_api.called)
self.assertTrue(mock_api.return_value.update_pr_status.called)
job.refresh_from_db()
self.assertEqual(job.status, models.JobStatus.SUCCESS)
step0_result.status = models.JobStatus.FAILED
step0_result.save()
# A step FAILED
# So final status is FAILED and we update the PR
with patch('ci.github.api.GitHubAPI') as mock_api:
self.set_counts()
response = self.client_post_json(url, post_data)
self.compare_counts()
self.assertEqual(response.status_code, 200)
self.assertTrue(mock_api.called)
self.assertTrue(mock_api.return_value.update_pr_status.called)
job.refresh_from_db()
self.assertEqual(job.status, models.JobStatus.FAILED)
def test_release(self, mock_del, mock_get, mock_post):
jdata = [{
"name": "1.0",
"commit": {
"sha": "1234"
},
}]
mock_get.return_value = utils.Response(jdata)
url = reverse('ci:github:webhook', args=[self.build_user.build_key])
data = self.get_data('release.json')
py_data = json.loads(data)
py_data['repository']['owner']['login'] = self.owner.name
py_data['repository']['name'] = self.repo.name
py_data['release']['target_commitish'] = self.branch.name
self.set_counts()
response = self.client_post_json(url, py_data)
self.assertEqual(response.status_code, 200)
self.compare_counts()
self.assertEqual(mock_del.call_count, 0)
self.assertEqual(mock_get.call_count, 1) # getting SHA
self.assertEqual(mock_post.call_count, 0)
# The commit could be a hash, then we assume the branch is master
py_data['release']['target_commitish'] = "1" * 40
mock_get.call_count = 0
self.set_counts()
response = self.client_post_json(url, py_data)
self.assertEqual(response.status_code, 200)
self.compare_counts()
self.assertEqual(mock_del.call_count, 0)
self.assertEqual(mock_get.call_count, 0)
self.assertEqual(mock_post.call_count, 0)
rel = utils.create_recipe(
name="Release1",
user=self.build_user,
repo=self.repo,
branch=self.branch,
cause=models.Recipe.CAUSE_RELEASE,
)
rel1 = utils.create_recipe(
name="Release with dep",
user=self.build_user,
repo=self.repo,
branch=self.branch,
cause=models.Recipe.CAUSE_RELEASE,
)
rel1.depends_on.add(rel)
py_data['release']['target_commitish'] = self.branch.name
self.set_counts()
response = self.client_post_json(url, py_data)
self.assertEqual(response.status_code, 200)
self.compare_counts(events=1, commits=1, jobs=2, ready=1, active=2)
self.assertEqual(mock_del.call_count, 0)
self.assertEqual(mock_get.call_count, 1) # getting SHA
self.assertEqual(mock_post.call_count, 0)
mock_get.call_count = 0
mock_get.side_effect = Exception("Bam!")
self.set_counts()
response = self.client_post_json(url, py_data)
self.assertEqual(response.status_code, 400)
self.compare_counts()
self.assertEqual(mock_del.call_count, 0)
self.assertEqual(mock_get.call_count, 1) # getting SHA
self.assertEqual(mock_post.call_count, 0)
def test_activate_job(self, mock_collab):
# only posts are allowed
response = self.client.get(reverse('ci:activate_job', args=[1000]))
self.assertEqual(response.status_code, 405)
response = self.client.post(reverse('ci:activate_job', args=[1000]))
self.assertEqual(response.status_code, 404)
job = utils.create_job()
job.active = False
job.save()
self.set_counts()
url = reverse('ci:activate_job', args=[job.pk])
self.assertEqual(job.event.base.branch.status, models.JobStatus.NOT_STARTED)
response = self.client.post(url)
self.compare_counts()
# not signed in
self.assertEqual(response.status_code, 403)
user = utils.get_test_user()
utils.simulate_login(self.client.session, user)
mock_collab.return_value = False
self.set_counts()
response = self.client.post(url)
self.compare_counts()
# not a collaborator
job = models.Job.objects.get(pk=job.pk)
self.assertEqual(response.status_code, 403)
self.assertFalse(job.active)
mock_collab.return_value = True
# A collaborator
self.set_counts()
response = self.client.post(url)
self.compare_counts(ready=1, active=1, num_changelog=1)
self.assertEqual(response.status_code, 302) # redirect
job.refresh_from_db()
self.assertTrue(job.active)
# make sure activating a job doesn't mark it as ready
r1 = utils.create_recipe(name='r1')
job.recipe.depends_on.add(r1)
j1 = utils.create_job(recipe=r1)
job.active = False
job.ready = False
job.save()
j1.active = False
j1.ready = False
j1.save()
self.set_counts()
response = self.client.post(url)
self.compare_counts(active=1, num_changelog=1)
self.assertEqual(response.status_code, 302) # redirect
job.refresh_from_db()
self.assertTrue(job.active)
self.assertFalse(job.ready)
# now it should be marked as ready
j1.ready = True
j1.complete = True
j1.status = models.JobStatus.SUCCESS
j1.save()
job.ready = False
job.active = False
job.save()
self.set_counts()
response = self.client.post(url)
# The branch is now set to RUNNING since there is an already finished job
self.compare_counts(ready=1, active=1, num_changelog=1, active_branches=1)
self.assertEqual(response.status_code, 302) # redirect
job.refresh_from_db()
job.event.base.branch.refresh_from_db()
self.assertEqual(job.event.base.branch.status, models.JobStatus.RUNNING)
self.assertTrue(job.active)
self.assertTrue(job.ready)
# Calling activate on an already active job shouldn't do anything
self.set_counts()
response = self.client.post(url)
self.compare_counts()
self.assertEqual(response.status_code, 302) # redirect
def test_ready_jobs_with_current_event(self):
"""
If a branch is specified with "auto_cancel_push_events_except_current" then
jobs on the "current" event get priority over subsequent events. Basically
the normal sort of (-priority, created) gets changed to (created, -priority)
for those jobs.
"""
user = utils.get_test_user()
r0 = utils.create_recipe(name='recipe0', user=user)
r0.priority = 30
r0.save()
r1 = utils.create_recipe(name='recipe1', user=user)
r1.priority = 20
r1.save()
r2 = utils.create_recipe(name='recipe2', user=user)
r2.priority = 10
r2.save()
e0 = utils.create_event(user=user, cause=models.Event.PUSH)
j0 = utils.create_job(recipe=r0, event=e0, user=user)
j1 = utils.create_job(recipe=r1, event=e0, user=user)
j2 = utils.create_job(recipe=r2, event=e0, user=user)
e1 = utils.create_event(user=user,
cause=models.Event.PUSH,
commit1='12345')
j3 = utils.create_job(recipe=r0, event=e1, user=user)
j4 = utils.create_job(recipe=r1, event=e1, user=user)
j5 = utils.create_job(recipe=r2, event=e1, user=user)
for j in models.Job.objects.all():
j.active = True
j.ready = True
j.save()
url = reverse('ci:client:ready_jobs', args=[user.build_key, 'client'])
self.set_counts()
response = self.client.get(url)
self.compare_counts(num_clients=1)
self.assertEqual(response.status_code, 200)
data = response.json()
self.assertIn('jobs', data)
jobs = data["jobs"]
self.assertEqual(len(jobs), 6)
# Normally jobs are sorted by (-priority, created)
self.assertEqual(jobs[0]["id"], j0.pk)
self.assertEqual(jobs[1]["id"], j3.pk)
self.assertEqual(jobs[2]["id"], j1.pk)
self.assertEqual(jobs[3]["id"], j4.pk)
self.assertEqual(jobs[4]["id"], j2.pk)
self.assertEqual(jobs[5]["id"], j5.pk)
repo_name = "%s/%s" % (e0.base.branch.repository.user.name,
e0.base.branch.repository.name)
branch_name = e0.base.branch.name
repo_settings = {
repo_name: {
"branch_settings": {
branch_name: {
"auto_cancel_push_events_except_current": True
}
}
}
}
with self.settings(INSTALLED_GITSERVERS=[
utils.github_config(repo_settings=repo_settings)
]):
response = self.client.get(url)
self.compare_counts(num_clients=1)
self.assertEqual(response.status_code, 200)
data = response.json()
self.assertIn('jobs', data)
jobs = data["jobs"]
self.assertEqual(len(jobs), 6)
# Jobs now are (created, -priority)
self.assertEqual(jobs[0]["id"], j0.pk)
self.assertEqual(jobs[1]["id"], j1.pk)
self.assertEqual(jobs[2]["id"], j2.pk)
self.assertEqual(jobs[3]["id"], j3.pk)
self.assertEqual(jobs[4]["id"], j4.pk)
self.assertEqual(jobs[5]["id"], j5.pk)
def test_event_odd_deps(self):
"""
Had the scenario where we have:
Precheck -> Test:linux, Test:clang -> Merge
where Test had 2 build configs.
But the merge recipe had a depends_on with an outdated
recipe
make_jobs_ready started the merge without waiting for the
two Test jobs to finish
"""
e = utils.create_event()
e.cause = models.Event.PUSH
e.save()
r0 = utils.create_recipe(name='precheck')
r1 = utils.create_recipe(name='test')
r2 = utils.create_recipe(name='merge')
r3 = utils.create_recipe(name='test')
# These two need to have the same filename
r1.filename = "my filename"
r1.save()
r3.filename = r1.filename
r3.save()
r1.build_configs.add(utils.create_build_config("Otherconfig"))
utils.create_recipe_dependency(recipe=r1, depends_on=r0)
utils.create_recipe_dependency(recipe=r2, depends_on=r3)
j0 = utils.create_job(recipe=r0, event=e)
j1a = utils.create_job(recipe=r1,
event=e,
config=r1.build_configs.first())
j1b = utils.create_job(recipe=r1,
event=e,
config=r1.build_configs.last())
j2 = utils.create_job(recipe=r2, event=e)
self.set_counts()
e.make_jobs_ready()
self.compare_counts(ready=1)
j0.refresh_from_db()
j1a.refresh_from_db()
j1b.refresh_from_db()
j2.refresh_from_db()
self.assertEqual(j0.ready, True)
self.assertEqual(j1a.ready, False)
self.assertEqual(j1b.ready, False)
self.assertEqual(j2.ready, False)
j0.complete = True
j0.status = models.JobStatus.SUCCESS
j0.save()
self.set_counts()
e.make_jobs_ready()
self.compare_counts(ready=2)
j0.refresh_from_db()
j1a.refresh_from_db()
j1b.refresh_from_db()
j2.refresh_from_db()
self.assertEqual(j0.ready, True)
self.assertEqual(j1a.ready, True)
self.assertEqual(j1b.ready, True)
self.assertEqual(j2.ready, False)
j1a.complete = True
j1a.status = models.JobStatus.SUCCESS
j1a.save()
self.set_counts()
e.make_jobs_ready()
self.compare_counts()
j1b.complete = True
j1b.status = models.JobStatus.SUCCESS
j1b.save()
self.set_counts()
e.make_jobs_ready()
self.compare_counts(ready=1)
j2.refresh_from_db()
self.assertEqual(j2.ready, True)
