def test_examples(self):
for f in os.listdir(self.examples):
if f[0] == ".": # a vim swap file :)
continue
with open(os.path.join(self.examples, f)) as f:
data = yaml.safe_load(f)
ProjectDefinition.validate_data(data)
def test_host_tag_rundef_loopon_bad(self):
with open(os.path.join(self.examples, 'host-tag.yml')) as f:
data = yaml.safe_load(f)
data['triggers'][0]['runs'][1]['loop-on'][0]['param'] = 'host-tagz'
exp = '"host-tag" or loop-on host-tag parameter required'
with self.assertRaisesRegex(Exception, exp):
ProjectDefinition.validate_data(data)
def test_host_tag_rundef_loopon_bad(self):
with open(os.path.join(self.examples, "host-tag.yml")) as f:
data = yaml.safe_load(f)
data["triggers"][0]["runs"][1]["loop-on"][0]["param"] = "host-tagz"
exp = '"host-tag" or loop-on host-tag parameter required'
with self.assertRaisesRegex(Exception, exp):
ProjectDefinition.validate_data(data)
def test_bad_trigger(self):
with open(os.path.join(self.examples, "python-github.yml")) as f:
data = yaml.safe_load(f)
data["triggers"][0]["type"] = "doesnotexist"
exp = "No such runner"
with self.assertRaisesRegex(Exception, exp):
ProjectDefinition.validate_data(data)
def test_bad_trigger(self):
with open(os.path.join(self.examples, 'python-github.yml')) as f:
data = yaml.safe_load(f)
data['triggers'][0]['type'] = 'doesnotexist'
exp = 'No such runner'
with self.assertRaisesRegex(Exception, exp):
ProjectDefinition.validate_data(data)
def test_run_name_too_long(self):
with open(os.path.join(self.examples, "python-github.yml")) as f:
data = yaml.safe_load(f)
# cause an infinite loop for triggers
data["triggers"][0]["runs"][0]["name"] = "1" * 80
exp = "Name of run must be less than 80 characters"
with self.assertRaisesRegex(Exception, exp):
ProjectDefinition.validate_data(data)
def test_simple_bad(self):
# just make a schema with no "timeout" and ensure it fails
with open(os.path.join(self.examples, "python-github.yml")) as f:
data = yaml.safe_load(f)
del data["timeout"]
exp = "Cannot find required key 'timeout'"
with self.assertRaisesRegex(Exception, exp):
ProjectDefinition.validate_data(data)
def test_run_name_too_long(self):
with open(os.path.join(self.examples, 'python-github.yml')) as f:
data = yaml.safe_load(f)
# cause an infinite loop for triggers
data['triggers'][0]['runs'][0]['name'] = '1' * 80
exp = 'Name of run must be less than 80 characters'
with self.assertRaisesRegex(Exception, exp):
ProjectDefinition.validate_data(data)
def _get_projdef(entry: PollerEntry) -> Optional[ProjectDefinition]:
repo = entry.trigger.definition_repo or ''
defile = entry.trigger.definition_file
if not defile:
defile = entry.trigger.project + '.yml'
gitlab = entry.trigger.secrets.get('gitlabtok')
gheader = entry.trigger.secrets.get('git.http.extraheader')
headers = entry.projdef_headers
token = entry.trigger.secrets.get('githubtok')
if gitlab:
headers['PRIVATE-TOKEN'] = gitlab
p = urlparse(repo)
proj_enc = quote_plus(p.path[1:].replace('.git', ''))
url = p.scheme + '://' + p.netloc + '/api/v4/projects/' + proj_enc + \
'/repository/files/' + quote_plus(defile) + '/raw?ref=master'
elif 'github' in repo:
if token:
headers['Authorization'] = 'token ' + token
url = repo.replace('github.com', 'raw.githubusercontent.com')
if url[-1] != '/':
url += '/'
url += 'master/' + defile
else:
url = repo
if not url.endswith('.git'):
url += '.git'
url = repo + '/plain/' + defile
log.info('Assuming CGit style URL to file: %s', url)
r = requests.get(url, headers=headers)
if r.status_code == 401 and gheader:
log.info('Authorization required using git header in secrets')
key, val = gheader.split(':', 1)
headers[key.strip()] = val.strip()
r = requests.get(url, headers=headers)
if r.status_code == 200:
try:
log.info('New version of project definition found for %s', url)
data = yaml.safe_load(r.text)
ProjectDefinition.validate_data(data)
entry.definition = ProjectDefinition(data)
# allows us to cache the resp
headers['If-None-Match'] = r.headers['ETAG']
except Exception:
log.exception('Validation failed for %s ...skipping', url)
return None
elif r.status_code == 304:
# it hasn't changed
log.debug('Cache hit for %s', url)
else:
log.error('Unable to read definition(%s): %d: %s', url, r.status_code,
r.text)
return None
return entry.definition
def _get_projdef(entry: PollerEntry) -> Optional[ProjectDefinition]:
repo = entry.trigger.definition_repo or ""
defile = entry.trigger.definition_file
if not defile:
defile = entry.trigger.project + ".yml"
gitlab = entry.trigger.secrets.get("gitlabtok")
gheader = entry.trigger.secrets.get("git.http.extraheader")
headers = entry.projdef_headers
token = entry.trigger.secrets.get("githubtok")
if gitlab:
headers["PRIVATE-TOKEN"] = gitlab
p = urlparse(repo)
proj_enc = quote_plus(p.path[1:].replace(".git", ""))
url = (p.scheme + "://" + p.netloc + "/api/v4/projects/" + proj_enc +
"/repository/files/" + quote_plus(defile) + "/raw?ref=master")
elif "github" in repo:
if token:
headers["Authorization"] = "token " + token
url = repo.replace("github.com", "raw.githubusercontent.com")
if url[-1] != "/":
url += "/"
url += "master/" + defile
else:
url = repo
if not url.endswith(".git"):
url += ".git"
url = repo + "/plain/" + defile
log.info("Assuming CGit style URL to file: %s", url)
r = requests.get(url, headers=headers)
if r.status_code == 401 and gheader:
log.info("Authorization required using git header in secrets")
key, val = gheader.split(":", 1)
headers[key.strip()] = val.strip()
r = requests.get(url, headers=headers)
if r.status_code == 200:
try:
log.info("New version of project definition found for %s", url)
data = yaml.safe_load(r.text)
ProjectDefinition.validate_data(data)
entry.definition = ProjectDefinition(data)
# allows us to cache the resp
headers["If-None-Match"] = r.headers["ETAG"]
except Exception:
log.exception("Validation failed for %s ...skipping", url)
return None
elif r.status_code == 304:
# it hasn't changed
log.debug("Cache hit for %s", url)
else:
log.error("Unable to read definition(%s): %d: %s", url, r.status_code,
r.text)
return None
return entry.definition
def simulator_validate():
data = request.get_json()
if not data:
raise ApiError(400, "run-definition must be posted as json data")
try:
ProjectDefinition.validate_data(data)
except Exception as e:
raise ApiError(400, str(e))
return jsendify({})
def test_bad_script_mutual_exclusion(self):
with open(os.path.join(self.examples, "python-github.yml")) as f:
data = yaml.safe_load(f)
data["triggers"][0]["runs"][0]["script-repo"] = {
"name": "doesnotexsit",
"path": "path",
}
exp = '"script" and "script-repo" are mutually exclusive'
with self.assertRaisesRegex(Exception, exp):
ProjectDefinition.validate_data(data)
def test_bad_script_mutual_exclusion(self):
with open(os.path.join(self.examples, 'python-github.yml')) as f:
data = yaml.safe_load(f)
data['triggers'][0]['runs'][0]['script-repo'] = {
'name': 'doesnotexsit',
'path': 'path',
}
exp = '"script" and "script-repo" are mutually exclusive'
with self.assertRaisesRegex(Exception, exp):
ProjectDefinition.validate_data(data)
def test_bad_script_repo(self):
with open(os.path.join(self.examples, "python-github.yml")) as f:
data = yaml.safe_load(f)
del data["triggers"][0]["runs"][0]["script"]
data["triggers"][0]["runs"][0]["script-repo"] = {
"name": "doesnotexsit",
"path": "path",
}
exp = "Script repo does not exist"
with self.assertRaisesRegex(Exception, exp):
ProjectDefinition.validate_data(data)
def test_bad_script_repo(self):
with open(os.path.join(self.examples, 'python-github.yml')) as f:
data = yaml.safe_load(f)
del data['triggers'][0]['runs'][0]['script']
data['triggers'][0]['runs'][0]['script-repo'] = {
'name': 'doesnotexsit',
'path': 'path',
}
exp = 'Script repo does not exist'
with self.assertRaisesRegex(Exception, exp):
ProjectDefinition.validate_data(data)
def test_recursive_build_trigger(self):
with open(os.path.join(self.examples, 'python-github.yml')) as f:
data = yaml.safe_load(f)
# cause an infinite loop for triggers
data['triggers'][0]['triggers'] = [
{
'name': 'unit-test'
},
]
exp = 'Trigger recursion depth exceeded'
with self.assertRaisesRegex(Exception, exp):
ProjectDefinition.validate_data(data)
def test_recursive_build_trigger(self):
with open(os.path.join(self.examples, "python-github.yml")) as f:
data = yaml.safe_load(f)
# cause an infinite loop for triggers
data["triggers"][0]["triggers"] = [
{
"name": "unit-test"
},
]
exp = "Trigger recursion depth exceeded"
with self.assertRaisesRegex(Exception, exp):
ProjectDefinition.validate_data(data)
def test_host_tag_rundef(self):
with open(os.path.join(self.examples, "host-tag.yml")) as f:
data = yaml.safe_load(f)
ProjectDefinition.validate_data(data)
proj = ProjectDefinition(data)
dbrun = Mock()
dbrun.build.project.name = "jobserv"
dbrun.name = "flake8"
dbrun.build.build_id = 1
dbrun.api_key = "123"
trigger = proj._data["triggers"][0]
run = trigger["runs"][0]
rundef = proj.get_run_definition(dbrun, run, trigger, {}, {})
data = json.loads(rundef)
self.assertEqual("aarch6*", data["host-tag"])
self.assertEqual("aarch6*", dbrun.host_tag)
def trigger_build(project,
reason,
trigger_name,
params,
secrets,
proj_def,
queue_priority=0):
proj_def = ProjectDefinition.validate_data(proj_def)
b = Build.create(project)
try:
if reason:
b.reason = reason
if trigger_name:
b.trigger_name = trigger_name
storage = Storage()
storage.create_project_definition(
b, yaml.dump(proj_def._data, default_flow_style=False))
trigger = proj_def.get_trigger(trigger_name)
if not trigger:
raise KeyError("Project(%s) does not have a trigger: %s" %
(project, trigger_name))
if trigger.get("triggers"):
# there's a trigger to run after all the runs for this trigger
# completed. it will need to know the parameters for this job
storage.create_build_params(b, params)
except Exception as e:
raise _fail_unexpected(b, e)
trigger_runs(storage, proj_def, b, trigger, params, secrets, None,
queue_priority)
db.session.commit()
return b
def test_host_tag_rundef(self):
with open(os.path.join(self.examples, 'host-tag.yml')) as f:
data = yaml.safe_load(f)
ProjectDefinition.validate_data(data)
proj = ProjectDefinition(data)
dbrun = Mock()
dbrun.build.project.name = 'jobserv'
dbrun.name = 'flake8'
dbrun.build.build_id = 1
dbrun.api_key = '123'
trigger = proj._data['triggers'][0]
run = trigger['runs'][0]
rundef = proj.get_run_definition(dbrun, run, trigger, {}, {})
data = json.loads(rundef)
self.assertEqual('aarch6*', data['host-tag'])
self.assertEqual('aarch6*', dbrun.host_tag)
def test_script_repo_rundef(self):
with open(os.path.join(self.examples, 'python-github.yml')) as f:
data = yaml.safe_load(f)
del data['triggers'][0]['runs'][0]['script']
data['triggers'][0]['runs'][0]['script-repo'] = {
'name': 'foo',
'path': 'path/foo.sh',
}
data['script-repos'] = {'foo': {'clone-url': 'url'}}
ProjectDefinition.validate_data(data)
proj = ProjectDefinition(data)
dbrun = Mock()
dbrun.build.project.name = 'jobserv'
dbrun.name = 'flake8'
dbrun.build.build_id = 1
dbrun.api_key = '123'
trigger = proj._data['triggers'][0]
run = trigger['runs'][0]
rundef = proj.get_run_definition(dbrun, run, trigger, {}, {})
repo = json.loads(rundef).get('script-repo')
self.assertEqual({'clone-url': 'url', 'path': 'path/foo.sh'}, repo)
def test_script_repo_rundef(self):
with open(os.path.join(self.examples, "python-github.yml")) as f:
data = yaml.safe_load(f)
del data["triggers"][0]["runs"][0]["script"]
data["triggers"][0]["runs"][0]["script-repo"] = {
"name": "foo",
"path": "path/foo.sh",
}
data["script-repos"] = {"foo": {"clone-url": "url"}}
ProjectDefinition.validate_data(data)
proj = ProjectDefinition(data)
dbrun = Mock()
dbrun.build.project.name = "jobserv"
dbrun.name = "flake8"
dbrun.build.build_id = 1
dbrun.api_key = "123"
trigger = proj._data["triggers"][0]
run = trigger["runs"][0]
rundef = proj.get_run_definition(dbrun, run, trigger, {}, {})
repo = json.loads(rundef).get("script-repo")
self.assertEqual({"clone-url": "url", "path": "path/foo.sh"}, repo)
def test_script_repo_token(self, url_for):
url_for.return_value = "blah"
with open(os.path.join(self.examples, "python-github.yml")) as f:
data = yaml.safe_load(f)
del data["triggers"][0]["runs"][0]["script"]
data["triggers"][0]["runs"][0]["script-repo"] = {
"name": "foo",
"path": "path/foo.sh",
}
data["script-repos"] = {"foo": {"clone-url": "url", "token": "f"}}
ProjectDefinition.validate_data(data)
proj = ProjectDefinition(data)
dbrun = Mock()
dbrun.build.project.name = "jobserv"
dbrun.build.build_id = 1
dbrun.name = "flake8"
dbrun.api_key = "secret"
trigger = proj._data["triggers"][0]
run = trigger["runs"][0]
with self.assertRaises(ApiError):
proj.get_run_definition(dbrun, run, trigger, {}, {})
def test_script_repo_token(self, url_for):
url_for.return_value = 'blah'
with open(os.path.join(self.examples, 'python-github.yml')) as f:
data = yaml.safe_load(f)
del data['triggers'][0]['runs'][0]['script']
data['triggers'][0]['runs'][0]['script-repo'] = {
'name': 'foo',
'path': 'path/foo.sh',
}
data['script-repos'] = {'foo': {'clone-url': 'url', 'token': 'f'}}
ProjectDefinition.validate_data(data)
proj = ProjectDefinition(data)
dbrun = Mock()
dbrun.build.project.name = 'jobserv'
dbrun.build.build_id = 1
dbrun.name = 'flake8'
dbrun.api_key = 'secret'
trigger = proj._data['triggers'][0]
run = trigger['runs'][0]
with self.assertRaises(ApiError):
proj.get_run_definition(dbrun, run, trigger, {}, {})
def test_bad_container_auth(self):
with open(os.path.join(self.examples, 'private-container.yml')) as f:
data = yaml.safe_load(f)
proj = ProjectDefinition.validate_data(data)
dbrun = Mock()
dbrun.build.project.name = 'jobserv'
dbrun.build.build_id = 1
dbrun.name = 'flake8'
dbrun.api_key = 'secret'
trigger = proj._data['triggers'][0]
run = trigger['runs'][0]
exp = 'not defined in the run\'s secrets'
with self.assertRaisesRegex(ApiError, exp):
proj.get_run_definition(dbrun, run, trigger, {}, {})
def test_bad_container_auth(self):
with open(os.path.join(self.examples, "private-container.yml")) as f:
data = yaml.safe_load(f)
proj = ProjectDefinition.validate_data(data)
dbrun = Mock()
dbrun.build.project.name = "jobserv"
dbrun.build.build_id = 1
dbrun.name = "flake8"
dbrun.api_key = "secret"
trigger = proj._data["triggers"][0]
run = trigger["runs"][0]
exp = "not defined in the run's secrets"
with self.assertRaisesRegex(ApiError, exp):
proj.get_run_definition(dbrun, run, trigger, {}, {})
def test_params(self):
"""Make sure we get project, trigger, and run params"""
with open(os.path.join(self.examples, "parameters.yml")) as f:
data = yaml.safe_load(f)
proj = ProjectDefinition.validate_data(data)
dbrun = Mock()
dbrun.build.project.name = "jobserv"
dbrun.name = "basic"
dbrun.build.build_id = 1
dbrun.api_key = "123"
trigger = proj._data["triggers"][0]
run = trigger["runs"][0]
rundef = proj.get_run_definition(dbrun, run, trigger, {}, {})
data = json.loads(rundef)
self.assertEqual("GLOBAL", data["env"]["GLOBAL_PARAM"])
self.assertEqual("RUN", data["env"]["RUN_PARAM"])
self.assertEqual("TRIGGER", data["env"]["TRIGGER_PARAM"])
def test_params(self):
"""Make sure we get project, trigger, and run params"""
with open(os.path.join(self.examples, 'parameters.yml')) as f:
data = yaml.safe_load(f)
proj = ProjectDefinition.validate_data(data)
dbrun = Mock()
dbrun.build.project.name = 'jobserv'
dbrun.name = 'basic'
dbrun.build.build_id = 1
dbrun.api_key = '123'
trigger = proj._data['triggers'][0]
run = trigger['runs'][0]
rundef = proj.get_run_definition(dbrun, run, trigger, {}, {})
data = json.loads(rundef)
self.assertEqual('GLOBAL', data['env']['GLOBAL_PARAM'])
self.assertEqual('RUN', data['env']['RUN_PARAM'])
self.assertEqual('TRIGGER', data['env']['TRIGGER_PARAM'])
def test_loop_on(self):
with open(os.path.join(self.examples, 'python-github.yml')) as f:
data = yaml.safe_load(f)
run = {
'name':
'compile-{loop}',
'container':
'foo',
'host-tag':
'amd64',
'script':
'unit-test',
'loop-on': [
{
'param': 'BOARD',
'values': ['carbon', 'nitrogen']
},
{
'param': 'ZEPHYR',
'values': ['upstream', 'dev', 'test']
},
{
'param': 'COMPILER',
'values': ['gcc', 'llvm']
},
],
'triggers': [
{
'name': 'trigger',
'run-names': '{name}-{loop}'
},
]
}
data['triggers'][0]['runs'].insert(1, run)
data['triggers'].append({
'type':
'simple',
'name':
'trigger',
'runs': [{
'name': 'trigger',
'container': 'foo',
'host-tag': 'amd64',
'script': 'flake8',
}]
})
ProjectDefinition.validate_data(data)
runs = ProjectDefinition(data)._data['triggers'][0]['runs']
# we should have 2 + (len(BOARD) * len(ZEPHYR) * len(COMPILER))
self.assertEqual(14, len(runs))
# they should be inserted in between the original runs in a
# predictable order
self.assertEqual('unit-test', runs[0]['name'])
self.assertEqual('compile-carbon-upstream-gcc', runs[1]['name'])
self.assertEqual('compile-carbon-upstream-llvm', runs[2]['name'])
self.assertEqual('compile-carbon-dev-gcc', runs[3]['name'])
self.assertEqual('compile-carbon-dev-llvm', runs[4]['name'])
self.assertEqual('compile-carbon-test-gcc', runs[5]['name'])
self.assertEqual('compile-carbon-test-llvm', runs[6]['name'])
self.assertEqual('compile-nitrogen-upstream-gcc', runs[7]['name'])
self.assertEqual('compile-nitrogen-upstream-llvm', runs[8]['name'])
self.assertEqual('compile-nitrogen-dev-gcc', runs[9]['name'])
self.assertEqual('compile-nitrogen-dev-llvm', runs[10]['name'])
self.assertEqual('compile-nitrogen-test-gcc', runs[11]['name'])
self.assertEqual('compile-nitrogen-test-llvm', runs[12]['name'])
self.assertEqual('flake8', runs[13]['name'])
def test_loop_on(self):
with open(os.path.join(self.examples, "python-github.yml")) as f:
data = yaml.safe_load(f)
run = {
"name":
"compile-{loop}",
"container":
"foo",
"host-tag":
"amd64",
"script":
"unit-test",
"loop-on": [
{
"param": "BOARD",
"values": ["carbon", "nitrogen"]
},
{
"param": "ZEPHYR",
"values": ["upstream", "dev", "test"]
},
{
"param": "COMPILER",
"values": ["gcc", "llvm"]
},
],
"triggers": [
{
"name": "trigger",
"run-names": "{name}-{loop}"
},
],
}
data["triggers"][0]["runs"].insert(1, run)
data["triggers"].append({
"type":
"simple",
"name":
"trigger",
"runs": [{
"name": "trigger",
"container": "foo",
"host-tag": "amd64",
"script": "flake8",
}],
})
ProjectDefinition.validate_data(data)
runs = ProjectDefinition(data)._data["triggers"][0]["runs"]
# we should have 2 + (len(BOARD) * len(ZEPHYR) * len(COMPILER))
self.assertEqual(14, len(runs))
# they should be inserted in between the original runs in a
# predictable order
self.assertEqual("unit-test", runs[0]["name"])
self.assertEqual("compile-carbon-upstream-gcc", runs[1]["name"])
self.assertEqual("compile-carbon-upstream-llvm", runs[2]["name"])
self.assertEqual("compile-carbon-dev-gcc", runs[3]["name"])
self.assertEqual("compile-carbon-dev-llvm", runs[4]["name"])
self.assertEqual("compile-carbon-test-gcc", runs[5]["name"])
self.assertEqual("compile-carbon-test-llvm", runs[6]["name"])
self.assertEqual("compile-nitrogen-upstream-gcc", runs[7]["name"])
self.assertEqual("compile-nitrogen-upstream-llvm", runs[8]["name"])
self.assertEqual("compile-nitrogen-dev-gcc", runs[9]["name"])
self.assertEqual("compile-nitrogen-dev-llvm", runs[10]["name"])
self.assertEqual("compile-nitrogen-test-gcc", runs[11]["name"])
self.assertEqual("compile-nitrogen-test-llvm", runs[12]["name"])
self.assertEqual("flake8", runs[13]["name"])
