def is_repo_allowed(repo):
"""
Check if repo is allowed.
A repository name is checked against a list of denied and allowed repos.
The 'denied_repo' check takes precendence over 'allowed_repo' check.
The list of denied/allowed repos is defined with settings 'denied_repo'
and 'allowed_repo'.
If the settings are not defined,
the repo is not checked against the denied/allowed lists.
Both 'denied_repo' and 'allowed_repo' can have multiple values,
if any of them matches a substring of the repo, the repo is denied/allowed.
Parameters:
-repo : repository name
"""
if repo is None:
logger.warning("Repo is not defined")
return False
denied_message = "Project '%s' is not allowed."
denied_repo = Settings().get_setting("denied_repo")
allowed_repo = Settings().get_setting("allowed_repo")
if denied_repo is not None and \
any(x in repo for x in denied_repo) or \
allowed_repo is not None and \
not any(x in repo for x in allowed_repo):
logger.warning(denied_message, repo)
return False
return True
def is_worker_enabled():
"""Check if a task queue is configured."""
task_queue = Settings().get_setting("task_queue")
# use double not to force boolean evaluation
return check_dict(task_queue, key_list=["broker_url", "backend"]) and \
not not task_queue["broker_url"] and \
not not task_queue["backend"]
def create_worker_app():
"""Create worker app."""
# load settings
settings = Settings()
settings.load_settings(config_file=constants.CONFIG_FILE)
settings.set_client(constants.CLIENT_NAME, constants.CLIENT_VERSION)
if is_worker_enabled():
task_queue = settings.get_setting("task_queue")
worker_app = Celery(
'tasks',
backend=task_queue["backend"],
broker=task_queue["broker_url"]
)
# configure worker
worker_app.conf.update(
CELERY_TASK_SERIALIZER='json',
CELERY_ACCEPT_CONTENT=['json']
)
if worker_app is None:
logger.error("Error connection to task queue")
else:
logger.info(
"Connected to task queue : %s", task_queue["broker_url"]
)
else:
worker_app = Celery()
logger.warning(
"Task queue is not defined,"
" check README.md to configure task queue"
)
return worker_app
def send_build_data(buildjob, detail=None):
"""
Send build data generated by client to keen.io.
Parameters:
- buildjob : BuildJob instance
- detail : Data storage detail level :
'minimal', 'basic', 'full', 'extended'
"""
if not isinstance(buildjob, BuildJob):
raise TypeError("param buildjob should be a BuildJob instance")
data_detail = Settings().get_value_or_setting("data_detail", detail)
if is_writable():
logger.info(
"Sending client build job data to Keen.io (data detail: %s)",
data_detail
)
# store build job data
add_event("build_jobs", {"job": buildjob.to_dict()})
# store build stages
if data_detail in ("full", "extended"):
add_events("build_stages", buildjob.stages_to_list())
def add_project_info_dict(payload):
"""
Add project info to a dictonary.
Param payload: dictonary payload
"""
# check if payload is a dictionary, throws an exception if it isn't
check_dict(payload, "payload")
payload_as_dict = copy.deepcopy(payload)
payload_as_dict[KEEN_PROJECT_INFO_NAME] = Settings().get_project_info()
if "job" in payload:
# override project_name, set to build_job repo
if "repo" in payload["job"]:
payload_as_dict[KEEN_PROJECT_INFO_NAME]["project_name"] = \
payload["job"]["repo"]
# override timestamp, set to finished_at timestamp
if "finished_at" in payload["job"]:
payload_as_dict["keen"] = {
"timestamp": payload["job"]["finished_at"]["isotimestamp"]
}
return payload_as_dict
def __init__(self):
"""Constructor."""
self.settings = Settings()
self.logger = logger
self.file_stats = os.path.join(DASHBOARD_DIR, "stats.html")
self.file_stats_service = os.path.join(DASHBOARD_DIR,
"stats_service.html")
def test_generate_config_file_ioerror(self):
"""
Test dashboard.generate_config_file()
if creation fails because of unexisting path.
"""
# set config file path
Settings().add_setting("dashboard_configfile",
"build/unexisting_path/config_test.js")
# generation should return false
self.assertFalse(dashboard.generate_config_file("test/repo4"))
def load_properties_from_settings(self):
"""Load build properties from settings."""
self.load_property_from_settings("build")
self.load_property_from_settings("job")
self.load_property_from_settings("branch")
self.load_property_from_settings("ci_platform")
self.load_property_from_settings("build_trigger")
self.load_property_from_settings("pull_request")
self.load_property_from_settings("result")
self.load_property_from_settings("build_matrix")
self.add_property("repo", Settings().get_project_name())
def process_notification_payload(payload):
"""
Extract repo slug and build number from Travis notification payload.
Returns a dictionary with "repo" and "build" information,
or an empty dictionary if the payload could not be processed.
Deprecated behaviour : Currently the repo and build information are
also stored in the "settings" object,
but this will be removed in the near future.
Parameters:
- payload : Travis CI notification payload
"""
settings = Settings()
parameters = {}
if payload is None:
logger.warning("Travis notification payload is not set")
return parameters
if not is_string(payload):
logger.warning(
"Travis notification payload is incorrect :"
" string expected, got %s", type(payload))
return parameters
json_payload = json.loads(payload)
logger.info("Travis Payload : %r.", json_payload)
# get repo name from payload
if ("repository" in json_payload
and "owner_name" in json_payload["repository"]
and "name" in json_payload["repository"]):
repo = get_repo_slug(json_payload["repository"]["owner_name"],
json_payload["repository"]["name"])
logger.info("Build repo : %s", repo)
settings.set_project_name(repo)
parameters["repo"] = repo
# get build number from payload
if "number" in json_payload:
logger.info("Build number : %s", str(json_payload["number"]))
settings.add_setting('build', json_payload['number'])
parameters["build"] = json_payload['number']
return parameters
def multi_build(self, repo, first_build, last_build):
"""
Schedule processing multiple consecutive builds.
All builds from first_build until last_build
will be retrieved and processed.
The total number of builds to be scheduled is limited by the
`multi_import.max_builds` config parameter.
Every next scheduled build will be delayed by the
`multi_import.delay` config parameter.
Parameters:
- repo : repo name (fe. buildtimetrend/service)
- first_build : first build number to process (int)
- last_build : last build number to process (int)
"""
first_build = int(first_build)
last_build = int(last_build)
message = ""
multi_import = Settings().get_setting("multi_import")
max_multi_builds = int(multi_import["max_builds"])
if last_build < first_build:
tmp_msg = "Warning : last_build should be equal" \
" or larger than first_build"
self.logger.warning(tmp_msg)
message += tmp_msg + "\n"
last_build = first_build
if (last_build - first_build) > max_multi_builds:
tmp_msg = "Warning : number of multiple builds is limited to {:d}"
self.logger.warning(tmp_msg.format(max_multi_builds))
message += tmp_msg.format(max_multi_builds) + "\n"
last_build = first_build + max_multi_builds
message += "Request to process build(s) #{first_build:d} to" \
" #{last_build:d} of repo {repo}:\n".format(**locals())
build = first_build
delay = 0
while build <= last_build:
message += self.schedule_task(repo, build, delay) + "\n"
delay += multi_import["delay"]
build += 1
return message
def load_property_from_settings(self, property_name, setting_name=None):
"""
Load the value of a setting and set it as a build property.
Parameters
- property_name : name of the build property
- setting_name : name of the setting (takes property_name if not set)
"""
if setting_name is None:
setting_name = property_name
value = Settings().get_setting(setting_name)
if value is not None:
self.add_property(property_name, value)
def setUpClass(cls):
"""Set up test fixture."""
# show full diff in case of assert mismatch
cls.maxDiff = None
cls.settings = Settings()
cls.project_name = buildtimetrend.NAME
cls.project_info = {
"lib_version": buildtimetrend.VERSION,
"schema_version": buildtimetrend.SCHEMA_VERSION,
"client": 'None',
"client_version": 'None',
"project_name": cls.project_name
}
def setUpClass(cls):
"""Set up test fixture."""
cls.settings = Settings()
cls.project_info = cls.settings.get_project_info()
cls.maxDiff = None
cls.test_api_error = {"message": "test message", "error_code": "123"}
# copy Keen.io environment variables
if "KEEN_PROJECT_ID" in os.environ:
cls.copy_keen_project_id = os.environ["KEEN_PROJECT_ID"]
if "KEEN_WRITE_KEY" in os.environ:
cls.copy_keen_write_key = os.environ["KEEN_WRITE_KEY"]
if "KEEN_READ_KEY" in os.environ:
cls.copy_keen_read_key = os.environ["KEEN_READ_KEY"]
if "KEEN_MASTER_KEY" in os.environ:
cls.copy_keen_master_key = os.environ["KEEN_MASTER_KEY"]
def load_all(settings=None):
"""
Load all Travis CI environment variables.
Load Travis CI environment variables and assign their values to
the corresponding setting value :
- general
- build matrix
- pull request
"""
if not isinstance(settings, Settings):
settings = Settings()
load_general_env_vars(settings)
load_build_matrix_env_vars(settings)
load_travis_pr_env_vars(settings)
def __init__(self):
"""
Initialise class.
Load config file and set loglevel, define error page handler
"""
self.settings = Settings()
self.settings.load_settings(config_file=constants.CONFIG_FILE)
self.settings.set_client(constants.CLIENT_NAME,
constants.CLIENT_VERSION)
self.file_index = os.path.join(STATIC_DIR, "index.html")
cherrypy.config.update({'error_page.404': self.error_page_404})
# get logger
self.logger = logger
def check_authorization(repo, auth_header):
"""
Check if Travis CI notification has a correct Authorization header.
This check is enabled if travis_account_token is defined in settings.
More information on the Authorization header :
http://docs.travis-ci.com/user/notifications/#Authorization-for-Webhooks
Returns true if Authorization header is valid, but also if
travis_account_token is not defined.
Parameters:
- repo : git repo name
- auth_header : Travis CI notification Authorization header
"""
# get Travis account token from Settings
token = Settings().get_setting("travis_account_token")
# return True if token is not set
if token is None:
logger.info("Setting travis_account_token is not defined,"
" Travis CI notification Authorization header"
" is not checked.")
return True
# check if parameters are strings
if is_string(repo) and is_string(auth_header) and is_string(token):
# generate hash (encode string to bytes first)
auth_hash = sha256((repo + token).encode('utf-8')).hexdigest()
# compare hash with Authorization header
if auth_hash == auth_header:
logger.info("Travis CI notification Authorization header"
" is correct.")
return True
else:
logger.error("Travis CI notification Authorization header"
" is incorrect.")
return False
else:
logger.debug("repo, auth_header and travis_auth_token"
" should be strings.")
return False
def test_load_properties(self):
"""Test loading properties"""
self.build.load_properties_from_settings()
self.assertDictEqual({
'duration': 0,
"repo": buildtimetrend.NAME
}, self.build.get_properties())
settings = Settings()
settings.add_setting("ci_platform", "travis")
settings.add_setting("build", "123")
settings.add_setting("job", "123.1")
settings.add_setting("branch", "branch1")
settings.add_setting("result", "passed")
settings.add_setting("build_trigger", "push")
settings.add_setting("pull_request", {
"is_pull_request": False,
"title": None,
"number": None
})
settings.set_project_name("test/project")
self.build.load_properties_from_settings()
self.assertDictEqual(
{
'duration': 0,
'ci_platform': "travis",
'build': "123",
'job': "123.1",
'branch': "branch1",
'result': "passed",
'build_trigger': "push",
'pull_request': {
"is_pull_request": False,
"title": None,
"number": None
},
'repo': "test/project"
}, self.build.get_properties())
def test_generate_config_file(self, get_cfg_str_func):
"""Test dashboard.generate_config_file()"""
# set config file path
Settings().add_setting("dashboard_configfile",
constants.DASHBOARD_TEST_CONFIG_FILE)
# check if configfile exists
self.assertFalse(check_file(constants.DASHBOARD_TEST_CONFIG_FILE))
# generate config file with empty repo name
self.assertRaises(TypeError, dashboard.generate_config_file)
# generate config file with empty repo name
self.assertTrue(dashboard.generate_config_file(None))
self.assertTrue(check_file(constants.DASHBOARD_TEST_CONFIG_FILE))
# check if mock was called with correct parameters
args, kwargs = get_cfg_str_func.call_args
self.assertEqual(args, (None, ))
self.assertDictEqual(kwargs, {})
# generate config file
self.assertTrue(dashboard.generate_config_file("test/repo3"))
self.assertTrue(check_file(constants.DASHBOARD_TEST_CONFIG_FILE))
# check if mock was called with correct parameters
args, kwargs = get_cfg_str_func.call_args
self.assertEqual(args, ("test/repo3", ))
self.assertDictEqual(kwargs, {})
# test generated config file contents
with open(constants.DASHBOARD_TEST_CONFIG_FILE, 'r') as config_file:
self.assertEqual("var config = {'projectName': 'test/repo3'};\n",
next(config_file))
self.assertEqual("var keenConfig = {'projectId': '1234abcd'};",
next(config_file))
def test_generate_config_file_fails(self):
"""Test dashboard.generate_config_file() if creation fails"""
# set config file path
Settings().add_setting("dashboard_configfile",
constants.DASHBOARD_TEST_CONFIG_FILE)
# check if configfile exists
self.assertFalse(check_file(constants.DASHBOARD_TEST_CONFIG_FILE))
# init mock
patcher = mock.patch('buildtimetrend.tools.check_file',
return_value=False)
check_file_func = patcher.start()
# generation should return false
self.assertFalse(dashboard.generate_config_file("test/repo4"))
# check if mock was called with correct parameters
args, kwargs = check_file_func.call_args
self.assertEqual(args, (constants.DASHBOARD_TEST_CONFIG_FILE, ))
self.assertDictEqual(kwargs, {})
patcher.stop()
def get_repo_data_detail(repo):
"""
Get level of data detail storage of a repo.
A repository name is checked against a list of repository names.
If a match is found, the corresponding data detail level is used.
Else, the default global setting is returned.
Parameters:
-repo : repository name
"""
settings = Settings()
if repo is None:
logger.warning("Repo is not defined")
else:
repo_settings = settings.get_setting("repo_data_detail")
for repo_substring, setting in repo_settings.items():
if repo_substring in repo:
return setting
# return default global data_detail setting
return settings.get_setting("data_detail")
def setUp():
"""Initialise test environment before each test."""
# reinit settings singleton
Settings().__init__()
def test_load_build_matrix_env_vars_parameters(self):
"""Test load_travis_env_vars, optional parameters"""
# setup Travis env vars
if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
reset_travis_vars = False
copy_os = os.environ["TRAVIS_OS_NAME"]
else:
reset_travis_vars = True
os.environ["TRAVIS"] = "true"
# temporarily remove OS VERSION
if "TRAVIS_OS_NAME" in os.environ:
reset_os = True
copy_os = os.environ["TRAVIS_OS_NAME"]
del os.environ["TRAVIS_OS_NAME"]
else:
reset_os = False
# temporarily remove PYTHON VERSION
if "TRAVIS_PYTHON_VERSION" in os.environ:
reset_python_version = True
copy_python = os.environ["TRAVIS_PYTHON_VERSION"]
del os.environ["TRAVIS_PYTHON_VERSION"]
else:
reset_python_version = False
# test optional build matrix parameters
test_parameters = [{
'env_var': 'TRAVIS_XCODE_SDK',
'parameter': 'xcode_sdk',
'test_value': "test_x_sdk"
}, {
'env_var': 'TRAVIS_XCODE_SCHEME',
'parameter': 'xcode_scheme',
'test_value': "test_x_scheme"
}, {
'env_var': 'TRAVIS_XCODE_PROJECT',
'parameter': 'xcode_project',
'test_value': "test_x_project"
}, {
'env_var': 'TRAVIS_XCODE_WORKSPACE',
'parameter': 'xcode_workspace',
'test_value': "test_x_workspace"
}, {
'env_var': 'CC',
'parameter': 'compiler',
'test_value': "test_gcc"
}, {
'env_var': 'ENV',
'parameter': 'parameters',
'test_value': "test_env"
}]
# test parameters
for parameter in test_parameters:
Settings().__init__()
settings = Settings()
if parameter['env_var'] in os.environ:
reset_travis_parameter = False
expected_param_value = os.environ[parameter['env_var']]
else:
reset_travis_parameter = True
expected_param_value = os.environ[parameter['env_var']] = \
parameter['test_value']
env_var.load_build_matrix_env_vars(settings)
self.assertDictEqual(
{
parameter["parameter"]: expected_param_value,
'summary': expected_param_value
}, settings.get_setting("build_matrix"))
# reset Travis parameters
if reset_travis_parameter:
del os.environ[parameter['env_var']]
# reset test Travis vars
if reset_travis_vars:
del os.environ["TRAVIS"]
# reset removed os name
if reset_os:
os.environ["TRAVIS_OS_NAME"] = copy_os
# reset removed python version
if reset_python_version:
os.environ["TRAVIS_PYTHON_VERSION"] = copy_python
def setUpClass(cls):
"""Set up test fixture."""
cls.settings = Settings()
def setUp(self):
"""Initialise test environment before each test."""
self.build = BuildJob()
# reinitialise settings
Settings().__init__()
def setUpClass(cls):
"""Set up test fixture."""
cls.project_info = Settings().get_project_info()
cls.maxDiff = None
def __init__(self):
"""Initialise class."""
self.settings = Settings()
# get logger
self.logger = logger
def test_load_travis_env_vars(self):
"""Test load_travis_env_vars"""
settings = Settings()
self.assertEqual(None, settings.get_setting("ci_platform"))
self.assertEqual(None, settings.get_setting("build"))
self.assertEqual(None, settings.get_setting("job"))
self.assertEqual(None, settings.get_setting("branch"))
self.assertEqual(None, settings.get_setting("result"))
self.assertEqual(None, settings.get_setting("build_trigger"))
self.assertEqual(None, settings.get_setting("pull_request"))
self.assertEqual(buildtimetrend.NAME, settings.get_project_name())
# setup Travis env vars
if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
reset_travis_vars = False
expected_build = os.environ["TRAVIS_BUILD_NUMBER"]
expected_job = os.environ["TRAVIS_JOB_NUMBER"]
expected_branch = os.environ["TRAVIS_BRANCH"]
expected_project_name = os.environ["TRAVIS_REPO_SLUG"]
copy_pull_request = os.environ["TRAVIS_PULL_REQUEST"]
else:
reset_travis_vars = True
os.environ["TRAVIS"] = "true"
expected_build = os.environ["TRAVIS_BUILD_NUMBER"] = "123"
expected_job = os.environ["TRAVIS_JOB_NUMBER"] = "123.1"
expected_branch = os.environ["TRAVIS_BRANCH"] = "branch1"
expected_project_name = \
os.environ["TRAVIS_REPO_SLUG"] = "test/project"
# setup Travis test result
if "TRAVIS_TEST_RESULT" in os.environ:
reset_travis_result = False
copy_result = os.environ["TRAVIS_TEST_RESULT"]
else:
reset_travis_result = True
os.environ["TRAVIS_TEST_RESULT"] = "0"
os.environ["TRAVIS_PULL_REQUEST"] = "false"
env_var.load_all(settings)
self.assertEqual("travis", settings.get_setting("ci_platform"))
self.assertEqual(expected_build, settings.get_setting("build"))
self.assertEqual(expected_job, settings.get_setting("job"))
self.assertEqual(expected_branch, settings.get_setting("branch"))
self.assertEqual(expected_project_name, settings.get_project_name())
self.assertEqual("passed", settings.get_setting("result"))
self.assertEqual("push", settings.get_setting("build_trigger"))
self.assertDictEqual(
{
'is_pull_request': False,
'title': None,
'number': None
}, settings.get_setting("pull_request"))
os.environ["TRAVIS_TEST_RESULT"] = "1"
# build is a pull request
expected_pull_request = os.environ["TRAVIS_PULL_REQUEST"] = "123"
env_var.load_all()
self.assertEqual("failed", settings.get_setting("result"))
self.assertEqual("pull_request", settings.get_setting("build_trigger"))
self.assertDictEqual(
{
'is_pull_request': True,
'title': "unknown",
'number': expected_pull_request
}, settings.get_setting("pull_request"))
# build is not a pull request
os.environ["TRAVIS_PULL_REQUEST"] = "false"
env_var.load_all(settings)
self.assertEqual("push", settings.get_setting("build_trigger"))
self.assertDictEqual(
{
'is_pull_request': False,
'title': None,
'number': None
}, settings.get_setting("pull_request"))
# reset test Travis vars
if reset_travis_vars:
del os.environ["TRAVIS"]
del os.environ["TRAVIS_BUILD_NUMBER"]
del os.environ["TRAVIS_JOB_NUMBER"]
del os.environ["TRAVIS_BRANCH"]
del os.environ["TRAVIS_REPO_SLUG"]
del os.environ["TRAVIS_PULL_REQUEST"]
else:
os.environ["TRAVIS_PULL_REQUEST"] = copy_pull_request
# reset Travis test result
if reset_travis_result:
del os.environ["TRAVIS_TEST_RESULT"]
else:
os.environ["TRAVIS_TEST_RESULT"] = copy_result
def test_load_build_matrix_env_vars(self):
"""Test load_build_matrix_env_vars"""
settings = Settings()
self.assertEqual(None, settings.get_setting("build_matrix"))
# setup Travis env vars
if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
reset_travis_vars = False
expected_os = os.environ["TRAVIS_OS_NAME"]
copy_python = os.environ["TRAVIS_PYTHON_VERSION"]
del os.environ["TRAVIS_PYTHON_VERSION"]
else:
reset_travis_vars = True
os.environ["TRAVIS"] = "true"
expected_os = os.environ["TRAVIS_OS_NAME"] = "test_os"
# temporarily remove PYTHON VERSION
if "TRAVIS_PYTHON_VERSION" in os.environ:
reset_python_version = True
copy_python = os.environ["TRAVIS_PYTHON_VERSION"]
del os.environ["TRAVIS_PYTHON_VERSION"]
else:
reset_python_version = False
# test language and language versions
test_languages = [{
'env_var': 'TRAVIS_DART_VERSION',
'language': 'dart',
'test_value': "1.1"
}, {
'env_var': 'TRAVIS_GO_VERSION',
'language': 'go',
'test_value': "1.2"
}, {
'env_var': 'TRAVIS_HAXE_VERSION',
'language': 'haxe',
'test_value': "1.3"
}, {
'env_var': 'TRAVIS_JDK_VERSION',
'language': 'java',
'test_value': "1.4"
}, {
'env_var': 'TRAVIS_JULIA_VERSION',
'language': 'julia',
'test_value': "1.5"
}, {
'env_var': 'TRAVIS_NODE_VERSION',
'language': 'javascript',
'test_value': "1.6"
}, {
'env_var': 'TRAVIS_OTP_RELEASE',
'language': 'erlang',
'test_value': "1.7"
}, {
'env_var': 'TRAVIS_PERL_VERSION',
'language': 'perl',
'test_value': "1.8"
}, {
'env_var': 'TRAVIS_PHP_VERSION',
'language': 'php',
'test_value': "1.9"
}, {
'env_var': 'TRAVIS_PYTHON_VERSION',
'language': 'python',
'test_value': "1.10"
}, {
'env_var': 'TRAVIS_R_VERSION',
'language': 'r',
'test_value': "1.11"
}, {
'env_var': 'TRAVIS_RUBY_VERSION',
'language': 'ruby',
'test_value': "1.12"
}, {
'env_var': 'TRAVIS_RUST_VERSION',
'language': 'rust',
'test_value': "1.13"
}, {
'env_var': 'TRAVIS_SCALA_VERSION',
'language': 'scala',
'test_value': "1.14"
}]
# test languages
for language in test_languages:
if language['env_var'] in os.environ:
reset_travis_lang_version = False
expected_lang_version = os.environ[language['env_var']]
else:
reset_travis_lang_version = True
expected_lang_version = \
os.environ[language['env_var']] = language['test_value']
env_var.load_build_matrix_env_vars(settings)
self.assertDictEqual(
{
'os':
expected_os,
'language':
language['language'],
'language_version':
expected_lang_version,
'summary':
"{0!s} {1!s} {2!s}".format(language['language'],
expected_lang_version,
expected_os)
}, settings.get_setting("build_matrix"))
# reset Travis test result
if reset_travis_lang_version:
del os.environ[language['env_var']]
# reset test Travis vars
if reset_travis_vars:
del os.environ["TRAVIS"]
del os.environ["TRAVIS_OS_NAME"]
# reset removed python version
if reset_python_version:
os.environ["TRAVIS_PYTHON_VERSION"] = copy_python
