def test_removed(self):
test_utils.create_git_server()
with test_utils.RecipeDir() as recipes_dir:
self.create_valid_with_check(recipes_dir)
self.remove_recipe_from_repo(recipes_dir, "alt.cfg")
self.set_counts()
self.check_load_recipes(recipes_dir, removed=1)
self.compare_counts(
sha_changed=True,
recipes=-1,
deps=-1,
current=-1,
num_pr_alt_recipes=-1,
num_steps=-1,
num_step_envs=-4,
num_recipe_envs=-1,
num_prestep=-2,
)
# change a recipe but now the old ones have jobs attached.
# so 1 new recipe should be added
build_user = models.GitUser.objects.get(name="moosebuild")
for r in models.Recipe.objects.all():
test_utils.create_job(recipe=r, user=build_user)
self.remove_recipe_from_repo(recipes_dir, "all.cfg")
self.set_counts()
self.check_load_recipes(recipes_dir, removed=1)
self.compare_counts(sha_changed=True, current=-4)
def test_job_script(self, mock_collab):
# bad pk
mock_collab.return_value = False
response = self.client.get(reverse('ci:job_script', args=[1000]))
self.assertEqual(response.status_code, 404)
with utils.RecipeDir():
user = utils.get_test_user()
job = utils.create_job(user=user)
job.recipe.build_user = user
job.recipe.save()
utils.create_prestepsource(recipe=job.recipe)
utils.create_recipe_environment(recipe=job.recipe)
step = utils.create_step(recipe=job.recipe,
filename='scripts/1.sh')
utils.create_step_environment(step=step)
url = reverse('ci:job_script', args=[job.pk])
response = self.client.get(url)
# owner doesn't have permission
self.assertEqual(response.status_code, 404)
mock_collab.return_value = True
utils.simulate_login(self.client.session, user)
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertContains(response, job.recipe.name)
def test_repo_changed(self):
with test_utils.RecipeDir() as recipes_dir:
# OK
self.create_recipe_in_repo(recipes_dir, "push_dep.cfg",
"push_dep.cfg")
self.set_counts()
self.check_load_recipes(recipes_dir, new=1)
self.compare_counts(recipes=2,
current=2,
sha_changed=True,
users=2,
repos=1,
branches=1,
num_push_recipes=1,
num_pr_alt_recipes=1,
num_steps=4,
num_step_envs=16,
num_recipe_envs=4,
num_prestep=4)
# Now artificially change the repo SHA
recipe_repo = models.RecipeRepository.load()
recipe_repo.sha = "1234"
recipe_repo.save()
self.set_counts()
self.check_load_recipes(recipes_dir)
self.compare_counts(sha_changed=True)
def test_run_success(self):
with test_utils.RecipeDir() as recipe_dir:
c, job = self.create_client_and_job(recipe_dir, "RunSuccess", sleep=2)
self.set_counts()
c.run(single=True)
self.compare_counts(num_clients=1, num_events_completed=1, num_jobs_completed=1, active_branches=1)
utils.check_complete_job(self, job)
def test_deprecated_environment(self):
with test_utils.RecipeDir() as recipe_dir:
env_before = settings.ENVIRONMENT
settings.ENVIRONMENT = {'FOO': 'bar'}
c = self.create_client("/foo/bar")
self.assertNotIn('FOO', c.get_environment())
extra_script = 'if [ "$FOO" == "bar" ]; then\n'
extra_script += ' echo "FOO=bar"\n'
extra_script += 'fi\n'
job = self.create_job(c,
recipe_dir,
"DeprecatedEnvironment",
n_steps=1,
sleep=2,
extra_script=extra_script)
self.set_counts()
c.run(exit_if=lambda client: True)
self.assertEqual('bar', c.get_environment('FOO'))
self.compare_counts(num_clients=1,
num_events_completed=1,
num_jobs_completed=1,
active_branches=1)
utils.check_complete_job(self,
job,
c,
n_steps=1,
extra_step_msg='FOO=bar\n')
settings.ENVIRONMENT = env_before
def test_get_repo_sha(self):
with utils.RecipeDir() as recipes_dir:
sha = file_utils.get_repo_sha(recipes_dir)
self.assertEqual(len(sha), 40)
sha = file_utils.get_repo_sha("/tmp")
self.assertEqual(sha, "")
def test_run_cancel(self):
with test_utils.RecipeDir() as recipe_dir:
c, job = self.create_client_and_job(recipe_dir, "Cancel", sleep=4)
self.set_counts()
c.client_info["single_shot"] = False
c.client_info["poll"] = 1
# cancel signal, should stop
script = "sleep 3 && kill -USR1 %s" % os.getpid()
proc = subprocess.Popen(script,
shell=True,
executable="/bin/bash",
stdout=subprocess.PIPE)
c.run()
proc.wait()
self.compare_counts(
canceled=1,
num_clients=1,
num_events_completed=1,
num_jobs_completed=1,
active_branches=1,
events_canceled=1,
)
self.assertEqual(c.cancel_signal.triggered, True)
self.assertEqual(c.graceful_signal.triggered, False)
utils.check_canceled_job(self, job, c)
def test_no_modules(self):
with test_utils.RecipeDir() as recipe_dir:
c, job = self.create_client_and_job(recipe_dir,
"NoModules",
sleep=2)
c.client_info["config_modules"] = {}
c.run(exit_if=lambda client: True)
def test_no_recipes(self):
# no recipes, nothing to do
test_utils.create_git_server()
with test_utils.RecipeDir() as recipes_dir:
self.set_counts()
self.check_load_recipes(recipes_dir)
self.compare_counts(sha_changed=True)
def test_bad_thread_join(self):
with test_utils.RecipeDir() as recipe_dir:
c, job = self.create_client_and_job(recipe_dir, "BadThreadJoin", sleep=2)
c.thread_join_wait = 0
self.set_counts()
c.run()
self.compare_counts(num_clients=1, num_events_completed=1, num_jobs_completed=1, active_branches=1)
utils.check_complete_job(self, job)
def test_exception(self, mock_getter):
with test_utils.RecipeDir() as recipe_dir:
# check exception handler
mock_getter.side_effect = Exception("oh no!")
c, job = self.create_client_and_job(recipe_dir, "JobStop", sleep=4)
self.set_counts()
c.run(single=True)
self.compare_counts()
def test_load_deps_ok(self):
with test_utils.RecipeDir() as recipes_dir:
# OK. New idaholab/moose/devel
self.create_valid_with_check(recipes_dir)
r = models.Recipe.objects.filter(auto_cancel_on_push=True)
self.assertEqual(r.count(), 1)
r = models.Recipe.objects.filter(create_issue_on_fail=True)
self.assertEqual(r.count(), 3) # push, release, manual
def test_runner_error(self, mock_getter):
with test_utils.RecipeDir() as recipe_dir:
mock_getter.return_value = None
c, job = self.create_client_and_job(recipe_dir, "JobError")
self.set_counts()
c.runner_error = True
c.run()
self.compare_counts()
def test_no_signals(self):
with test_utils.RecipeDir() as recipe_dir:
# This is just for coverage. We can't really
# test this because if we send a signal it will just quit
import signal
old_signal = signal.SIGUSR2
del signal.SIGUSR2
c, job = self.create_client_and_job(recipe_dir, "No signal", sleep=2)
signal.SIGUSR2 = old_signal
def test_check_server_no_job(self):
with test_utils.RecipeDir() as recipe_dir:
# check no jobs
c, job = self.create_client_and_job(recipe_dir, "JobStop", sleep=4)
job.complete = True
job.save()
self.set_counts()
c.check_server(settings.SERVERS[0])
self.compare_counts(num_clients=1)
def test_no_change(self):
with test_utils.RecipeDir() as recipes_dir:
# Start off with valid recipes
self.create_valid_with_check(recipes_dir)
# OK, repo sha hasn't changed so no changes.
self.set_counts()
self.check_load_recipes(recipes_dir)
self.compare_counts()
def test_no_dep(self):
with test_utils.RecipeDir() as recipes_dir:
# dependency file doesn't exist
self.set_counts()
self.create_recipe_in_repo(recipes_dir, "recipe_all.cfg",
"all.cfg")
with self.assertRaises(
RecipeTester.RecipeRepoReader.InvalidRecipe):
self.check_load_recipes(recipes_dir)
self.compare_counts()
def test_global_abort_on_failure(self):
with utils.RecipeDir() as recipes_dir:
fname = self.create_recipe_in_repo(recipes_dir, "pr.cfg",
"recipe.cfg")
reader = RecipeReader(recipes_dir, fname)
reader.config.remove_option('Step 1', 'abort_on_failure')
reader.config['Global Environment']['abort_on_failure'] = 'true'
reader.read()
self.assertEqual(reader.recipe['steps'][0]['abort_on_failure'],
True)
def test_read_private(self):
with utils.RecipeDir() as recipes_dir:
fname = self.create_recipe_in_repo(recipes_dir,
"recipe_private.cfg",
"private.cfg")
reader = RecipeReader(recipes_dir, fname)
r = reader.read()
self.assertEqual(reader.check(), True)
self.assertEqual(r["private"], True)
self.assertEqual(r["viewable_by_teams"],
["idaholab/MOOSE TEAM", "idaholab/OTHER TEAM"])
def test_get_contents(self):
with utils.RecipeDir() as recipes_dir:
fname = "scripts/1.sh"
self.write_script_to_repo(recipes_dir, "contents", "1.sh")
ret = file_utils.get_contents(recipes_dir, fname)
self.assertEqual(ret, 'contents')
ret = file_utils.get_contents(recipes_dir, 'no_exist')
self.assertEqual(ret, None)
p = os.path.relpath('/etc/passwd', recipes_dir)
ret = file_utils.get_contents(recipes_dir, p)
self.assertEqual(ret, None)
def test_manage_build_root(self):
with test_utils.RecipeDir() as recipe_dir:
temp_dir = tempfile.TemporaryDirectory()
build_root = temp_dir.name + "/build_root"
self.assertEqual(os.path.isdir(build_root), False)
os.mkdir(build_root)
self.assertEqual(os.path.isdir(build_root), True)
manage_build_root_before = settings.MANAGE_BUILD_ROOT
settings.MANAGE_BUILD_ROOT = True
c = self.create_client(build_root)
settings.MANAGE_BUILD_ROOT = manage_build_root_before
self.assertEqual(c.get_build_root(), build_root)
self.assertEqual(c.get_client_info('manage_build_root'), True)
self.assertEqual(c.build_root_exists(), True)
c.check_build_root()
self.assertEqual(c.build_root_exists(), False)
c.create_build_root()
self.assertEqual(c.build_root_exists(), True)
extra_script = 'if [ -d "$BUILD_ROOT" ]; then\n'
extra_script += ' if [ ! -n "$(ls -A "$BUILD_ROOT")" ]; then\n'
extra_script += ' echo BUILD_ROOT_EXISTS_EMPTY\n'
extra_script += ' touch $BUILD_ROOT/build_root_test || exit 1\n'
extra_script += ' fi\n'
extra_script += 'fi\n'
jobs = []
jobs.append(self.create_job(c, recipe_dir, "ManageBuildRoot1", n_steps=1, sleep=2, extra_script=extra_script))
jobs.append(self.create_job(c, recipe_dir, "ManageBuildRoot2", n_steps=1, sleep=2, extra_script=extra_script))
jobs.append(self.create_job(c, recipe_dir, "ManageBuildRoot3", n_steps=1, sleep=2, extra_script=extra_script))
self.set_counts()
c.client_info["poll"] = 1
def exit_create_build_root(client):
self.assertEqual(client.build_root_exists(), False)
client.create_build_root()
self.assertEqual(client.build_root_exists(), True)
return client.get_client_info('jobs_ran') == 3
c.run(exit_if=exit_create_build_root)
self.assertEqual(c.build_root_exists(), False)
self.compare_counts(num_clients=1, num_events_completed=1, num_jobs_completed=3, active_branches=1)
for job in jobs:
utils.check_complete_job(self, job, c, n_steps=1, extra_step_msg='BUILD_ROOT_EXISTS_EMPTY\n')
temp_dir.cleanup()
def test_install_webhooks(self, mock_install):
mock_install.side_effect = Exception("Bam!")
with test_utils.RecipeDir() as recipes_dir:
self.create_valid_recipes(recipes_dir)
creator = RecipeCreator.RecipeCreator(recipes_dir)
creator.install_webhooks()
self.assertEqual(mock_install.call_count, 0)
with self.settings(INSTALLED_GITSERVERS=[
test_utils.github_config(install_webhook=True)
]):
creator.install_webhooks()
self.assertEqual(mock_install.call_count, 1)
def test_dep_invalid(self):
with test_utils.RecipeDir() as recipes_dir:
# dependency file isn't valid
self.set_counts()
self.create_recipe_in_repo(recipes_dir, "recipe_all.cfg",
"all.cfg")
self.create_recipe_in_repo(recipes_dir, "push_dep.cfg",
"pr_dep.cfg")
self.create_recipe_in_repo(recipes_dir, "pr_dep.cfg",
"push_dep.cfg")
with self.assertRaises(
RecipeTester.RecipeRepoReader.InvalidDependency):
self.check_load_recipes(recipes_dir)
self.compare_counts()
def test_client_runner_user(self):
with test_utils.RecipeDir() as recipes_dir:
self.create_valid_with_check(recipes_dir)
build_user = models.GitUser.objects.get(name="moosebuild")
r = models.Recipe.objects.get(filename="recipes/pr_dep.cfg")
self.assertEqual(r.client_runner_user, None)
pr_recipe = self.find_recipe_dict("recipes/pr_dep.cfg")
pr_recipe["client_runner_user"] = build_user.name
self.write_recipe_to_repo(recipes_dir, pr_recipe, "pr_dep.cfg")
self.set_counts()
self.check_load_recipes(recipes_dir, changed=1)
self.compare_counts(sha_changed=True)
r = models.Recipe.objects.get(filename="recipes/pr_dep.cfg")
self.assertEqual(r.client_runner_user, build_user)
def test_is_valid_file(self):
with utils.RecipeDir() as recipes_dir:
self.write_script_to_repo(recipes_dir, "contents", "1.sh")
self.write_script_to_repo(recipes_dir, "contents", "2.sh")
self.assertFalse(file_utils.is_valid_file(recipes_dir, "no_exist"))
self.assertTrue(
file_utils.is_valid_file(recipes_dir, "scripts/1.sh"))
self.assertTrue(
file_utils.is_valid_file(recipes_dir, "scripts/2.sh"))
fname = os.path.join('..', '1.sh')
self.assertFalse(file_utils.is_valid_file(recipes_dir, fname))
if os.path.exists("/etc/passwd"):
fname = os.path.join('..', '..', '..', 'etc', 'passwd')
self.assertFalse(file_utils.is_valid_file(recipes_dir, fname))
def test_run_job_cancel(self):
with test_utils.RecipeDir() as recipe_dir:
c, job = self.create_client_and_job(recipe_dir, "JobCancel", sleep=4)
self.set_counts()
# cancel response, should cancel the job
thread = threading.Thread(target=c.run, args=(True,))
thread.start()
time.sleep(4)
job.refresh_from_db()
views.set_job_canceled(job)
thread.join()
self.compare_counts(num_clients=1, canceled=1, num_events_completed=1, num_jobs_completed=1, active_branches=1, events_canceled=1)
self.assertEqual(c.cancel_signal.triggered, False)
self.assertEqual(c.graceful_signal.triggered, False)
utils.check_canceled_job(self, job)
def test_get_file_sha(self):
with utils.RecipeDir() as recipes_dir:
self.write_script_to_repo(recipes_dir, "contents", "1.sh")
sha = file_utils.get_file_sha(recipes_dir, "scripts/1.sh")
self.assertEqual(len(sha), 40)
sha = file_utils.get_file_sha(recipes_dir, "noexist")
self.assertEqual(sha, "")
fname = os.path.join(recipes_dir, "noexist")
with open(fname, "w") as f:
f.write("noexist")
sha = file_utils.get_file_sha("/tmp/noexist", "noexist")
self.assertEqual(sha, "")
def test_write(self):
with utils.RecipeDir() as recipe_dir:
fname = self.create_recipe_in_repo(recipe_dir, "recipe_all.cfg",
"recipe.cfg")
self.create_recipe_in_repo(recipe_dir, "recipe_push.cfg",
"push_dep.cfg")
self.create_recipe_in_repo(recipe_dir, "recipe_pr.cfg",
"pr_dep.cfg")
self.write_script_to_repo(recipe_dir, "contents", "1.sh")
self.write_script_to_repo(recipe_dir, "contents", "2.sh")
reader = RecipeReader(recipe_dir, fname)
r = reader.read()
self.assertEqual(r.get("repository"),
"git@dummy_git_server:idaholab/civet.git")
r["repository"] = "new_repo"
global_env = r.get("global_env")
self.assertEqual(len(global_env), 2)
r["global_env"]["APPLICATION_NAME"] = "new_app"
global_sources = r.get("global_sources")
self.assertEqual(len(global_sources), 2)
r["global_sources"][0] = "new_source"
deps = r.get("pullrequest_dependencies")
self.assertEqual(len(deps), 1)
r["pullrequest_dependencies"][0] = "new_dep"
steps = r.get("steps")
self.assertEqual(len(steps), 2)
r["steps"][0]["name"] = "new_step"
self.assertTrue(
RecipeWriter.write_recipe_to_repo(recipe_dir, r,
"new_file.cfg"))
reader = RecipeReader(recipe_dir, "new_file.cfg")
r = reader.read()
# we changed the source and the dep so now the recipe doesn't pass the check.
self.assertEqual(r, {})
r = reader.read(do_check=False)
self.assertEqual(r["repository"], "new_repo")
self.assertEqual(r["global_env"]["APPLICATION_NAME"], "new_app")
self.assertEqual(r["global_sources"][0], "new_source")
self.assertEqual(r["pullrequest_dependencies"][0], "new_dep")
self.assertEqual(r["steps"][0]["name"], "new_step")
self.assertFalse(
RecipeWriter.write_recipe_to_repo('/foo', r, '../bar'))
def test_private(self):
test_utils.create_git_server()
with test_utils.RecipeDir() as recipes_dir:
self.create_recipe_in_repo(recipes_dir, "recipe_private.cfg",
"private.cfg")
self.set_counts()
self.check_load_recipes(recipes_dir, new=1)
self.compare_counts(sha_changed=True,
recipes=1,
num_pr_recipes=1,
num_steps=1,
num_step_envs=1,
current=1,
users=2,
repos=1,
viewable_teams=2)
def test_run_job_invalidated_basic(self):
with test_utils.RecipeDir() as recipe_dir:
c, job = self.create_client_and_job(recipe_dir, "JobInvalidated", sleep=40)
# stop response, should stop the job
self.set_counts()
thread = threading.Thread(target=c.run, args=(True,))
thread.start()
start_time = time.time()
time.sleep(4)
job.refresh_from_db()
job.set_invalidated("Test invalidation", check_ready=True)
thread.join()
end_time = time.time()
self.assertGreater(15, end_time-start_time)
self.compare_counts(num_clients=1, invalidated=1, num_changelog=1)
utils.check_stopped_job(self, job)