def test_promote_post(self):
b = Build.create(self.project)
db.session.add(Run(b, 'run0'))
db.session.add(Run(b, 'run1'))
url = 'http://localhost/projects/proj-1/builds/%d/promote' % b.build_id
headers = {
'Content-type': 'application/json',
}
data = {
'name': 'release-x',
'annotation': 'foo bar',
}
# you can't promote an in-progress build
_sign(url, headers, 'POST')
self._post(url, json.dumps(data), headers, 400)
for r in b.runs:
r.set_status(BuildStatus.PASSED)
self._post(url, json.dumps(data), headers, 201)
db.session.refresh(b)
self.assertEqual(BuildStatus.PROMOTED, b.status)
self.assertEqual(data['name'], b.name)
self.assertEqual(data['annotation'], b.annotation)
def test_promote_post(self):
b = Build.create(self.project)
db.session.add(Run(b, "run0"))
db.session.add(Run(b, "run1"))
url = "http://localhost/projects/proj-1/builds/%d/promote" % b.build_id
headers = {
"Content-type": "application/json",
}
data = {
"name": "release-x",
"annotation": "foo bar",
}
# you can't promote an in-progress build
_sign(url, headers, "POST")
self._post(url, json.dumps(data), headers, 400)
for r in b.runs:
r.set_status(BuildStatus.PASSED)
self._post(url, json.dumps(data), headers, 201)
db.session.refresh(b)
self.assertEqual(BuildStatus.PROMOTED, b.status)
self.assertEqual(data["name"], b.name)
self.assertEqual(data["annotation"], b.annotation)
def test_build_status_queued(self):
db.session.add(Run(self.build, 'name1'))
db.session.add(Run(self.build, 'name2'))
db.session.commit()
db.session.refresh(self.build)
self.build.refresh_status()
self.assertEqual(BuildStatus.QUEUED, self.build.status)
def test_run_list(self):
db.session.add(Run(self.build, "run0"))
db.session.add(Run(self.build, "run1"))
runs = self.get_json(self.urlbase)["runs"]
self.assertEqual(2, len(runs))
for i, r in enumerate(runs):
self.assertEqual("run%d" % i, r["name"])
self.assertEqual("QUEUED", r["status"])
def test_promote_list_empty(self, storage):
b = Build.create(self.project)
db.session.add(Run(b, 'run0'))
db.session.add(Run(b, 'run1'))
for r in b.runs:
r.set_status(BuildStatus.PASSED)
url = '/projects/%s/promoted-builds/' % self.project.name
builds = self.get_json(url)['builds']
self.assertEqual(0, len(builds))
def test_run_list(self):
db.session.add(Run(self.build, 'run0'))
db.session.add(Run(self.build, 'run1'))
runs = self.get_json(self.urlbase)['runs']
self.assertEqual(2, len(runs))
for i, r in enumerate(runs):
self.assertEqual('run%d' % i, r['name'])
self.assertEqual('QUEUED', r['status'])
def test_build_status_running(self):
db.session.add(Run(self.build, 'name1'))
r = Run(self.build, 'name2')
r.status = BuildStatus.RUNNING
db.session.add(r)
db.session.commit()
db.session.refresh(self.build)
self.build.refresh_status()
self.assertEqual(BuildStatus.RUNNING, self.build.status)
def test_build_status_running_failed(self):
db.session.add(Run(self.build, 'name1'))
r = Run(self.build, 'name2')
r.status = BuildStatus.FAILED
db.session.add(r)
db.session.commit()
db.session.refresh(self.build)
self.build.refresh_status()
self.assertEqual(BuildStatus.RUNNING_WITH_FAILURES, self.build.status)
def test_run_complete_triggers_name_error(self, storage):
m = Mock()
m.get_project_definition.return_value = json.dumps({
"timeout":
5,
"triggers": [
{
"name":
"github",
"type":
"github_pr",
"runs": [{
"name": "run0",
"host-tag": "foo*",
"triggers": [{
"name": "triggered"
}],
}],
},
{
"name":
"triggered",
"type":
"simple",
"runs": [{
"name": "collision-name",
"host-tag": "bar",
"container": "container-foo",
"script": "test",
}],
},
],
"scripts": {
"test": "#test#",
},
})
m.console_logfd.return_value = open("/dev/null", "w")
m.get_run_definition.return_value = json.dumps({})
m.get_artifact_content.return_value = "#mocked line 1\n"
storage.return_value = m
r = Run(self.build, "run0")
r.trigger = "github"
r.status = BuildStatus.RUNNING
db.session.add(r)
# cause a duplicate name collision
db.session.add(Run(self.build, "collision-name"))
db.session.commit()
headers = [
("Authorization", "Token %s" % r.api_key),
("X-RUN-STATUS", "PASSED"),
]
self._post(self.urlbase + "run0/", None, headers, 200)
self.assertEqual("RUNNING_WITH_FAILURES", r.build.status.name)
def test_build_status_failed(self):
r = Run(self.build, 'name1')
r.status = BuildStatus.PASSED
db.session.add(r)
r = Run(self.build, 'name2')
r.status = BuildStatus.FAILED
db.session.add(r)
db.session.commit()
db.session.refresh(self.build)
self.build.refresh_status()
self.assertEqual(BuildStatus.FAILED, self.build.status)
self.assertEqual(['QUEUED', 'FAILED'],
[x.status.name for x in self.build.status_events])
def test_promote_get(self, storage):
b = Build.create(self.project)
db.session.add(Run(b, "run0"))
db.session.add(Run(b, "run1"))
for r in b.runs:
r.set_status(BuildStatus.PASSED)
t = Test(r, "t1", None, BuildStatus.PASSED)
db.session.add(t)
b.status = BuildStatus.PROMOTED
b.name = "release-X"
b.annotation = "foo bar"
url = "/projects/%s/promoted-builds/release-X/" % self.project.name
build = self.get_json(url)["build"]
self.assertEqual("foo bar", build["annotation"])
def test_promote_get(self, storage):
b = Build.create(self.project)
db.session.add(Run(b, 'run0'))
db.session.add(Run(b, 'run1'))
for r in b.runs:
r.set_status(BuildStatus.PASSED)
t = Test(r, 't1', None, BuildStatus.PASSED)
db.session.add(t)
b.status = BuildStatus.PROMOTED
b.name = 'release-X'
b.annotation = 'foo bar'
url = '/projects/%s/promoted-builds/release-X/' % self.project.name
build = self.get_json(url)['build']
self.assertEqual('foo bar', build['annotation'])
def test_worker_sync_builds_uploading(self, storage):
"""Make sure scheduler takes into account runs that are UPLOADING.
1. Create a "synchronous" Project
2. Add an UPLOADING build and and QUEUED build
Make sure the QUEUED build is not assigned
"""
if db.engine.dialect.name == "sqlite":
self.skipTest("Test requires MySQL")
rundef = {"run_url": "foo", "runner_url": "foo", "env": {}}
storage().get_run_definition.return_value = json.dumps(rundef)
w = Worker("w1", "ubuntu", 12, 2, "aarch64", "key", 2, ["aarch96"])
w.enlisted = True
w.online = True
db.session.add(w)
self.create_projects("job-1")
(p1, ) = Project.query.all()
p1.synchronous_builds = True
db.session.commit()
# add active build
b = Build.create(p1)
r = Run(b, "p1b1r1")
r.status = BuildStatus.UPLOADING
r.host_tag = "aarch96"
db.session.add(r)
b = Build.create(p1)
r = Run(b, "p1b2r1")
r.host_tag = "aarch96"
db.session.add(r)
db.session.commit()
headers = [
("Content-type", "application/json"),
("Authorization", "Token key"),
]
qs = "available_runners=1&foo=2"
# There should be no work available
resp = self.client.get("/workers/w1/",
headers=headers,
query_string=qs)
self.assertEqual(200, resp.status_code, resp.data)
data = json.loads(resp.data.decode())
self.assertNotIn("run-defs", data["data"]["worker"],
data["data"]["worker"])
def test_run_complete_triggers_name_error(self, storage):
m = Mock()
m.get_project_definition.return_value = json.dumps({
'timeout': 5,
'triggers': [
{
'name': 'github',
'type': 'github_pr',
'runs': [{
'name': 'run0',
'host-tag': 'foo*',
'triggers': [
{'name': 'triggered'}
]
}],
},
{
'name': 'triggered',
'type': 'simple',
'runs': [{
'name': 'collision-name',
'host-tag': 'bar',
'container': 'container-foo',
'script': 'test',
}],
},
],
'scripts': {
'test': '#test#',
}
})
m.console_logfd.return_value = open('/dev/null', 'w')
m.get_run_definition.return_value = json.dumps({})
m.get_artifact_content.return_value = '#mocked line 1\n'
storage.return_value = m
r = Run(self.build, 'run0')
r.trigger = 'github'
r.status = BuildStatus.RUNNING
db.session.add(r)
# cause a duplicate name collision
db.session.add(Run(self.build, 'collision-name'))
db.session.commit()
headers = [
('Authorization', 'Token %s' % r.api_key),
('X-RUN-STATUS', 'PASSED'),
]
self._post(self.urlbase + 'run0/', None, headers, 200)
self.assertEqual('RUNNING_WITH_FAILURES', r.build.status.name)
def test_stuck(self, update_run, notify):
"""Ensure stuck runs are failed."""
self.create_projects('proj1')
b = Build.create(Project.query.all()[0])
r = Run(b, 'bla')
r.status = BuildStatus.RUNNING
db.session.add(r)
db.session.commit()
e = RunEvents(r, BuildStatus.RUNNING)
e.time = datetime.datetime.utcnow() - datetime.timedelta(hours=13)
db.session.add(e)
db.session.commit()
_check_stuck()
self.assertEqual('bla', notify.call_args[0][0].name)
notify.rest_mock()
r.status = BuildStatus.CANCELLING
e = RunEvents(r, BuildStatus.RUNNING)
e.time = datetime.datetime.utcnow() - datetime.timedelta(hours=13)
db.session.add(e)
db.session.commit()
_check_stuck()
self.assertEqual('bla', notify.call_args[0][0].name)
self.assertEqual('bla', update_run.call_args[0][0].name)
def test_build_complete_lava_tests(self, storage):
m = Mock()
m.get_project_definition.return_value = json.dumps({
"timeout":
5,
"triggers": [
{
"name": "github",
"type": "github_pr",
"runs": [{
"name": "run0",
}],
},
],
})
m.console_logfd.return_value = open("/dev/null", "w")
m.get_run_definition.return_value = json.dumps({})
storage.return_value = m
r = Run(self.build, "run0")
r.trigger = "github"
r.status = BuildStatus.RUNNING
db.session.add(r)
db.session.commit()
db.session.add(Test(r, "test-1", "ctx", BuildStatus.QUEUED))
db.session.commit()
headers = [
("Authorization", "Token %s" % r.api_key),
("X-RUN-STATUS", "PASSED"),
]
self._post(self.urlbase + "run0/", None, headers, 200)
db.session.refresh(r)
self.assertEqual(BuildStatus.RUNNING, r.status)
def _fail_unexpected(build, exception):
r = Run(build, "build-failure")
db.session.add(r)
r.set_status(BuildStatus.FAILED)
db.session.commit()
storage = Storage()
with storage.console_logfd(r, "a") as f:
f.write("Unexpected error prevented build from running:\n")
f.write(str(exception))
storage.copy_log(r)
if BUILD_URL_FMT:
url = BUILD_URL_FMT.format(project=build.project.name,
build=build.build_id)
else:
url = url_for(
"api_run.run_get_artifact",
proj=build.project.name,
build_id=build.build_id,
run=r.name,
path="console.log",
)
exception = ApiError(500, str(exception))
exception.resp.headers.extend({"Location": url})
return exception
def test_run_stream_not_authenticated(self):
r = Run(self.build, "run0")
db.session.add(r)
db.session.commit()
headers = [("Authorization", "Token badtoken")]
self._post(self.urlbase + "run0/", "message", headers, 401)
def test_build_complete_email(self, build_complete, storage):
m = Mock()
m.get_project_definition.return_value = json.dumps({
'timeout': 5,
'triggers': [
{
'name': 'github',
'type': 'github_pr',
'runs': [{
'name': 'run0',
}],
'email': {
'users': '*****@*****.**',
}
},
],
})
m.console_logfd.return_value = open('/dev/null', 'w')
m.get_run_definition.return_value = json.dumps({})
storage.return_value = m
r = Run(self.build, 'run0')
r.trigger = 'github'
r.status = BuildStatus.RUNNING
db.session.add(r)
db.session.commit()
headers = [
('Authorization', 'Token %s' % r.api_key),
('X-RUN-STATUS', 'PASSED'),
]
self._post(self.urlbase + 'run0/', None, headers, 200)
db.session.refresh(r)
self.assertEqual(self.build, build_complete.call_args_list[0][0][0])
self.assertEqual('*****@*****.**', build_complete.call_args_list[0][0][1])
def test_build_complete_lava_tests(self, storage):
m = Mock()
m.get_project_definition.return_value = json.dumps({
'timeout': 5,
'triggers': [
{
'name': 'github',
'type': 'github_pr',
'runs': [{
'name': 'run0',
}],
},
],
})
m.console_logfd.return_value = open('/dev/null', 'w')
m.get_run_definition.return_value = json.dumps({})
storage.return_value = m
r = Run(self.build, 'run0')
r.trigger = 'github'
r.status = BuildStatus.RUNNING
db.session.add(r)
db.session.commit()
db.session.add(Test(r, 'test-1', 'ctx', BuildStatus.QUEUED))
db.session.commit()
headers = [
('Authorization', 'Token %s' % r.api_key),
('X-RUN-STATUS', 'PASSED'),
]
self._post(self.urlbase + 'run0/', None, headers, 200)
db.session.refresh(r)
self.assertEqual(BuildStatus.RUNNING, r.status)
def test_run_stream_not_authenticated(self):
r = Run(self.build, 'run0')
db.session.add(r)
db.session.commit()
headers = [('Authorization', 'Token badtoken')]
self._post(self.urlbase + 'run0/', 'message', headers, 401)
def test_build_complete(self, webhook, build_complete_email, storage):
m = Mock()
m.get_project_definition.return_value = json.dumps({
"timeout":
5,
"triggers": [
{
"name":
"github",
"type":
"github_pr",
"runs": [{
"name": "run0",
}],
"email": {
"users": "*****@*****.**",
},
"webhooks": [
{
"url": "https://example.com",
"secret_name": "example_secret",
},
],
},
],
})
m.console_logfd.return_value = open("/dev/null", "w")
m.get_run_definition.return_value = json.dumps(
{"secrets": {
"example_secret": "secret_value"
}})
storage.return_value = m
r = Run(self.build, "run0")
r.trigger = "github"
r.status = BuildStatus.RUNNING
db.session.add(r)
db.session.commit()
headers = [
("Authorization", "Token %s" % r.api_key),
("X-RUN-STATUS", "PASSED"),
]
self._post(self.urlbase + "run0/", None, headers, 200)
db.session.refresh(r)
build_complete_email.assert_called_with(self.build, "*****@*****.**")
# dumb but its notified from a thread
sleep(0.1)
webhook.post.assert_called_once()
call = webhook.post.call_args_list[0]
self.assertEqual("https://example.com", call[0][0])
sig = call[1]["headers"]["X-JobServ-Sig"]
data = call[1]["data"]
self.assertEqual("sha256:", sig[:7])
computed = hmac.new(b"secret_value", data.encode(),
"sha256").hexdigest()
self.assertTrue(hmac.compare_digest(computed, sig[7:]))
self.assertIn("reason", data)
def test_run_complete_tests(self, storage):
# Same as test_run_complete_tests_default but with a test name pattern
m = Mock()
m.get_project_definition.return_value = json.dumps({
"timeout":
5,
"triggers": [
{
"name": "github",
"type": "github_pr",
"runs": [{
"name": "run0",
}],
},
],
})
@contextlib.contextmanager
def _logfd(run, mode="r"):
path = os.path.join(jobserv.storage.base.JOBS_DIR, run.name)
with open(path, mode) as f:
yield f
m.console_logfd = _logfd
data = """
t1: PASSED
t2: FAILED
Starting Test: TNAME...
t3: foo
"""
rundef = {
"test-grepping": {
"test-pattern": r".*Starting Test: (?P<name>\S+)...",
"result-pattern": r"\s*(?P<name>\S+): "
"(?P<result>(PASSED|FAILED|foo))",
"fixupdict": {
"foo": "PASSED"
},
}
}
m.get_run_definition.return_value = json.dumps(rundef)
storage.return_value = m
r = Run(self.build, "run0")
r.trigger = "github"
r.status = BuildStatus.RUNNING
db.session.add(r)
db.session.commit()
headers = [
("Authorization", "Token %s" % r.api_key),
("X-RUN-STATUS", "PASSED"),
]
self._post(self.urlbase + "run0/", data, headers, 200)
tests = [(x.name, x.status.name) for x in Test.query.all()]
self.assertEqual([("default", "FAILED"), ("TNAME", "PASSED")], tests)
results = [(x.name, x.status.name)
for x in Test.query.all()[0].results]
expected = [("t1", "PASSED"), ("t2", "FAILED")]
self.assertEqual(expected, results)
def test_surge_complex(self):
# we'll have two amd64 workers and one armhf
worker = Worker('w2', 'd', 1, 1, 'amd64', 'k', 1, 'amd64')
worker.enlisted = True
worker.online = True
db.session.add(worker)
worker = Worker('w3', 'd', 1, 1, 'armhf', 'k', 1, 'armhf')
worker.enlisted = True
worker.online = True
db.session.add(worker)
db.session.commit()
self.create_projects('proj1')
b = Build.create(Project.query.all()[0])
for x in range(SURGE_SUPPORT_RATIO + 1):
r = Run(b, 'amd%d' % x)
r.host_tag = 'amd64'
db.session.add(r)
r = Run(b, 'armhf%d' % x)
r.host_tag = 'armhf'
db.session.add(r)
db.session.commit()
_check_queue()
self.assertFalse(os.path.exists(jobserv.worker.SURGE_FILE + '-amd64'))
self.assertTrue(os.path.exists(jobserv.worker.SURGE_FILE + '-armhf'))
# get us under surge for armhf
db.session.delete(Run.query.filter(Run.host_tag == 'armhf').first())
# and over surge for amd64
for x in range(SURGE_SUPPORT_RATIO + 1):
r = Run(b, 'run%d' % x)
r.host_tag = 'amd64'
db.session.add(r)
db.session.commit()
worker_module.DETECT_FLAPPING = False
_check_queue()
self.assertTrue(os.path.exists(jobserv.worker.SURGE_FILE + '-amd64'))
self.assertFalse(os.path.exists(jobserv.worker.SURGE_FILE + '-armhf'))
# make sure we know about deleted workers
worker.deleted = True
db.session.commit()
_check_queue()
self.assertTrue(os.path.exists(jobserv.worker.SURGE_FILE + '-armhf'))
def test_surge_complex(self):
# we'll have two amd64 workers and one armhf
worker = Worker("w2", "d", 1, 1, "amd64", "k", 1, "amd64")
worker.enlisted = True
worker.online = True
db.session.add(worker)
worker = Worker("w3", "d", 1, 1, "armhf", "k", 1, "armhf")
worker.enlisted = True
worker.online = True
db.session.add(worker)
db.session.commit()
self.create_projects("proj1")
b = Build.create(Project.query.all()[0])
for x in range(SURGE_SUPPORT_RATIO + 1):
r = Run(b, "amd%d" % x)
r.host_tag = "amd64"
db.session.add(r)
r = Run(b, "armhf%d" % x)
r.host_tag = "armhf"
db.session.add(r)
db.session.commit()
_check_queue()
self.assertFalse(os.path.exists(jobserv.worker.SURGE_FILE + "-amd64"))
self.assertTrue(os.path.exists(jobserv.worker.SURGE_FILE + "-armhf"))
# get us under surge for armhf
db.session.delete(Run.query.filter(Run.host_tag == "armhf").first())
# and over surge for amd64
for x in range(SURGE_SUPPORT_RATIO + 1):
r = Run(b, "run%d" % x)
r.host_tag = "amd64"
db.session.add(r)
db.session.commit()
worker_module.DETECT_FLAPPING = False
_check_queue()
self.assertTrue(os.path.exists(jobserv.worker.SURGE_FILE + "-amd64"))
self.assertFalse(os.path.exists(jobserv.worker.SURGE_FILE + "-armhf"))
# make sure we know about deleted workers
worker.deleted = True
db.session.commit()
_check_queue()
self.assertTrue(os.path.exists(jobserv.worker.SURGE_FILE + "-armhf"))
def test_worker_sync_builds_regression(self, storage):
"""Make sure scheduler takes into account other active projects for
sync builds.
"""
if db.engine.dialect.name == "sqlite":
self.skipTest("Test requires MySQL")
rundef = {"run_url": "foo", "runner_url": "foo", "env": {}}
storage().get_run_definition.return_value = json.dumps(rundef)
w = Worker("w1", "ubuntu", 12, 2, "aarch64", "key", 2, ["aarch96"])
w.enlisted = True
w.online = True
db.session.add(w)
self.create_projects("job-1")
self.create_projects("job-2")
p1, p2 = Project.query.all()
p1.synchronous_builds = True
db.session.commit()
# add active build
b = Build.create(p2)
r = Run(b, "p2b1r1")
r.status = BuildStatus.RUNNING
r.host_tag = "aarch96"
db.session.add(r)
b = Build.create(p1)
r = Run(b, "p1b1r1")
r.host_tag = "aarch96"
db.session.add(r)
db.session.commit()
headers = [
("Content-type", "application/json"),
("Authorization", "Token key"),
]
qs = "available_runners=1&foo=2"
# This should make the p1b1r2 run running
resp = self.client.get("/workers/w1/",
headers=headers,
query_string=qs)
self.assertEqual(200, resp.status_code, resp.data)
data = json.loads(resp.data.decode())
self.assertEqual(1, len(data["data"]["worker"]["run-defs"]))
def test_run_complete_triggers_type_upgrade(self, storage):
""" We have a build that's triggered by either a github_pr or a
gitlab_mr. They might have runs that trigger something of type
"simple". This could be the case where a git_poller and github_mr
both trigger a similar set of tests *after* a build. In the
case of the github_pr, we should "upgrade" the type of each run
from simple to github_pr so that it can update the status of the
PR.
"""
m = Mock()
m.get_project_definition.return_value = json.dumps({
'timeout': 5,
'triggers': [
{
'name': 'github',
'type': 'github_pr',
'runs': [{
'name': 'run0',
'host-tag': 'foo*',
'triggers': [
{'name': 'triggered', 'run-names': '{name}-run0'}
]
}],
},
{
'name': 'triggered',
'type': 'simple',
'runs': [{
'name': 'test',
'host-tag': 'bar',
'container': 'container-foo',
'script': 'test',
}],
},
],
'scripts': {
'test': '#test#',
}
})
m.console_logfd.return_value = open('/dev/null', 'w')
m.get_run_definition.return_value = json.dumps({})
storage.return_value = m
r = Run(self.build, 'run0')
r.trigger = 'github'
r.status = BuildStatus.RUNNING
db.session.add(r)
db.session.commit()
headers = [
('Authorization', 'Token %s' % r.api_key),
('X-RUN-STATUS', 'PASSED'),
]
self._post(self.urlbase + 'run0/', None, headers, 200)
run = Run.query.all()[1]
self.assertEqual('test-run0', run.name)
rundef = json.loads(m.set_run_definition.call_args_list[0][0][1])
self.assertEqual('github_pr', rundef['trigger_type'])
def test_promote_list(self, storage):
b = Build.create(self.project)
db.session.add(Run(b, 'run0'))
db.session.add(Run(b, 'run1'))
for r in b.runs:
r.set_status(BuildStatus.PASSED)
t = Test(r, 't1', None, BuildStatus.PASSED)
db.session.add(t)
b.status = BuildStatus.PROMOTED
b.name = 'release-X'
b.annotation = 'foo bar'
url = '/projects/%s/promoted-builds/' % self.project.name
builds = self.get_json(url)['builds']
self.assertEqual(1, len(builds))
self.assertEqual('release-X', builds[0]['name'])
self.assertEqual('foo bar', builds[0]['annotation'])
self.assertEqual(['run0-t1', 'run1-t1'],
[x['name'] for x in builds[0]['tests']])
def test_build_status_passed(self):
r = Run(self.build, 'name')
r.status = BuildStatus.PASSED
db.session.add(r)
db.session.commit()
db.session.refresh(self.build)
self.build.refresh_status()
self.assertEqual(BuildStatus.PASSED, self.build.status)
def setUp(self):
super().setUp()
self.create_projects('job-1')
self.proj = Project.query.filter_by(name='job-1').first_or_404()
self.build = Build.create(self.proj)
self.run = Run(self.build, 'name1')
self.run.status = BuildStatus.RUNNING
db.session.add(self.run)
db.session.commit()