def is_repo_allowed(repo):
"""
Check if repo is allowed.
A repository name is checked against a list of denied and allowed repos.
The 'denied_repo' check takes precendence over 'allowed_repo' check.
The list of denied/allowed repos is defined with settings 'denied_repo'
and 'allowed_repo'.
If the settings are not defined,
the repo is not checked against the denied/allowed lists.
Both 'denied_repo' and 'allowed_repo' can have multiple values,
if any of them matches a substring of the repo, the repo is denied/allowed.
Parameters:
-repo : repository name
"""
if repo is None:
logger.warning("Repo is not defined")
return False
denied_message = "Project '%s' is not allowed."
denied_repo = Settings().get_setting("denied_repo")
allowed_repo = Settings().get_setting("allowed_repo")
if denied_repo is not None and \
any(x in repo for x in denied_repo) or \
allowed_repo is not None and \
not any(x in repo for x in allowed_repo):
logger.warning(denied_message, repo)
return False
return True
def analyse(argv):
'''
Analyse timestamp file
'''
settings = Settings()
# load Travis environment variables and save them in settings
load_travis_env_vars()
# process command line arguments
process_argv(argv)
# read build data from timestamp CSV file
build = Build(TIMESTAMP_FILE)
# load build properties from settings
build.load_properties_from_settings()
# retrieve data from Travis CI API
if build.get_property("ci_platform") == "travis":
travis_data = TravisData(
build.get_property("repo"),
build.get_property("build"),
)
travis_data.get_build_data()
build.set_started_at(travis_data.get_started_at())
# log data
if settings.get_setting("mode_native") is True:
log_build_native(build)
if settings.get_setting("mode_keen") is True:
log_build_keen(build)
class Root(object):
"""Root handler."""
def __init__(self):
"""
Initialise class.
Load config file and set loglevel, define error page handler
"""
self.settings = Settings()
self.settings.load_settings(config_file=constants.CONFIG_FILE)
self.settings.set_client(constants.CLIENT_NAME,
constants.CLIENT_VERSION)
self.file_index = os.path.join(STATIC_DIR, "index.html")
cherrypy.config.update({'error_page.404': self.error_page_404})
# get logger
self.logger = logger
@cherrypy.expose
def index(self):
"""Index page."""
if check_file(self.file_index):
return open(self.file_index)
else:
raise cherrypy.HTTPError(404, "File not found")
def error_page_404(self, status, message, traceback, version):
"""Error Page (404)."""
self.logger.error(
"Cherrypy %s : Error loading page (%s) : %s\nTraceback : %s",
version, status, message, traceback)
return "This page doesn't exist, please check usage on " \
"the {} website.".format(SERVICE_WEBSITE_LINK)
def send_build_data(buildjob, detail=None):
"""
Send build data generated by client to keen.io.
Parameters:
- buildjob : BuildJob instance
- detail : Data storage detail level :
'minimal', 'basic', 'full', 'extended'
"""
if not isinstance(buildjob, BuildJob):
raise TypeError("param buildjob should be a BuildJob instance")
data_detail = Settings().get_value_or_setting("data_detail", detail)
if is_writable():
logger.info(
"Sending client build job data to Keen.io (data detail: %s)",
data_detail
)
# store build job data
add_event("build_jobs", {"job": buildjob.to_dict()})
# store build stages
if data_detail in ("full", "extended"):
add_events("build_stages", buildjob.stages_to_list())
def add_project_info_dict(payload):
"""
Add project info to a dictonary.
Param payload: dictonary payload
"""
# check if payload is a dictionary, throws an exception if it isn't
check_dict(payload, "payload")
payload_as_dict = copy.deepcopy(payload)
payload_as_dict[KEEN_PROJECT_INFO_NAME] = Settings().get_project_info()
if "job" in payload:
# override project_name, set to build_job repo
if "repo" in payload["job"]:
payload_as_dict[KEEN_PROJECT_INFO_NAME]["project_name"] = \
payload["job"]["repo"]
# override timestamp, set to finished_at timestamp
if "finished_at" in payload["job"]:
payload_as_dict["keen"] = {
"timestamp": payload["job"]["finished_at"]["isotimestamp"]
}
return payload_as_dict
def is_worker_enabled():
"""Check if a task queue is configured."""
task_queue = Settings().get_setting("task_queue")
# use double not to force boolean evaluation
return check_dict(task_queue, key_list=["broker_url", "backend"]) and \
not not task_queue["broker_url"] and \
not not task_queue["backend"]
def generate_trend(argv):
"""Generate trends from analysed buildtime data."""
settings = Settings()
# load settings from config file, env_var and cli parameters
if settings.load_settings(argv) is None:
return
# load Travis CI environment variables
env_var.load_all(settings)
# run trend_keen() always,
# if $KEEN_PROJECT_ID variable is set (checked later), it will be executed
if settings.get_setting("mode_native") is True:
trend_native()
if settings.get_setting("mode_keen") is True:
trend_keen()
def __init__(self):
"""
Initialise class.
Load config file and set loglevel, define error page handler
"""
self.settings = Settings()
self.settings.load_settings(config_file=constants.CONFIG_FILE)
self.settings.set_client(constants.CLIENT_NAME,
constants.CLIENT_VERSION)
self.file_index = os.path.join(STATIC_DIR, "index.html")
cherrypy.config.update({'error_page.404': self.error_page_404})
# get logger
self.logger = logger
def __init__(self):
"""Constructor."""
self.settings = Settings()
self.logger = logger
self.file_stats = os.path.join(DASHBOARD_DIR, "stats.html")
self.file_stats_service = os.path.join(DASHBOARD_DIR,
"stats_service.html")
def create_worker_app():
"""Create worker app."""
# load settings
settings = Settings()
settings.load_settings(config_file=constants.CONFIG_FILE)
settings.set_client(constants.CLIENT_NAME, constants.CLIENT_VERSION)
if is_worker_enabled():
task_queue = settings.get_setting("task_queue")
worker_app = Celery(
'tasks',
backend=task_queue["backend"],
broker=task_queue["broker_url"]
)
# configure worker
worker_app.conf.update(
CELERY_TASK_SERIALIZER='json',
CELERY_ACCEPT_CONTENT=['json']
)
if worker_app is None:
logger.error("Error connection to task queue")
else:
logger.info(
"Connected to task queue : %s", task_queue["broker_url"]
)
else:
worker_app = Celery()
logger.warning(
"Task queue is not defined,"
" check README.md to configure task queue"
)
return worker_app
def generate_trend(argv):
'''
Generate trends from analised buildtime data
'''
settings = Settings()
# load Travis environment variables and save them in settings
load_travis_env_vars()
# process command line arguments
process_argv(argv)
# run trend_keen() always,
# if $KEEN_PROJECT_ID variable is set (checked later), it will be executed
if settings.get_setting("mode_native") is True:
trend_native()
if settings.get_setting("mode_keen") is True:
trend_keen()
def get_read_key(argv):
"""Generate a read key for the project and print that key."""
settings = Settings()
# load settings from config file, env_var and cli parameters
args = settings.load_settings(argv)
# exit script if processing cli parameters failed (result = None)
if args is None:
return
# get project name from argument
# check if collection is empty
if args:
settings.set_project_name(args[0])
# generate a read key
print(keenio.generate_read_key(settings.get_project_name()))
def setUpClass(self):
self.settings = Settings()
self.project_name = buildtimetrend.NAME
self.project_info = {
"version": buildtimetrend.VERSION,
"schema_version": buildtimetrend.SCHEMA_VERSION,
"project_name": self.project_name}
def test_generate_config_file_ioerror(self):
"""
Test dashboard.generate_config_file()
if creation fails because of unexisting path.
"""
# set config file path
Settings().add_setting("dashboard_configfile",
"build/unexisting_path/config_test.js")
# generation should return false
self.assertFalse(dashboard.generate_config_file("test/repo4"))
def load_properties_from_settings(self):
"""Load build properties from settings."""
self.load_property_from_settings("build")
self.load_property_from_settings("job")
self.load_property_from_settings("branch")
self.load_property_from_settings("ci_platform")
self.load_property_from_settings("build_trigger")
self.load_property_from_settings("pull_request")
self.load_property_from_settings("result")
self.load_property_from_settings("build_matrix")
self.add_property("repo", Settings().get_project_name())
def get_repo_data_detail(repo):
"""
Get level of data detail storage of a repo.
A repository name is checked against a list of repository names.
If a match is found, the corresponding data detail level is used.
Else, the default global setting is returned.
Parameters:
-repo : repository name
"""
settings = Settings()
if repo is None:
logger.warning("Repo is not defined")
else:
repo_settings = settings.get_setting("repo_data_detail")
for repo_substring, setting in repo_settings.items():
if repo_substring in repo:
return setting
# return default global data_detail setting
return settings.get_setting("data_detail")
def multi_build(self, repo, first_build, last_build):
"""
Schedule processing multiple consecutive builds.
All builds from first_build until last_build
will be retrieved and processed.
The total number of builds to be scheduled is limited by the
`multi_import.max_builds` config parameter.
Every next scheduled build will be delayed by the
`multi_import.delay` config parameter.
Parameters:
- repo : repo name (fe. buildtimetrend/service)
- first_build : first build number to process (int)
- last_build : last build number to process (int)
"""
first_build = int(first_build)
last_build = int(last_build)
message = ""
multi_import = Settings().get_setting("multi_import")
max_multi_builds = int(multi_import["max_builds"])
if last_build < first_build:
tmp_msg = "Warning : last_build should be equal" \
" or larger than first_build"
self.logger.warning(tmp_msg)
message += tmp_msg + "\n"
last_build = first_build
if (last_build - first_build) > max_multi_builds:
tmp_msg = "Warning : number of multiple builds is limited to {:d}"
self.logger.warning(tmp_msg.format(max_multi_builds))
message += tmp_msg.format(max_multi_builds) + "\n"
last_build = first_build + max_multi_builds
message += "Request to process build(s) #{first_build:d} to" \
" #{last_build:d} of repo {repo}:\n".format(**locals())
build = first_build
delay = 0
while build <= last_build:
message += self.schedule_task(repo, build, delay) + "\n"
delay += multi_import["delay"]
build += 1
return message
def load_travis_env_vars():
'''
Loads Travis CI environment variables and assigns them to
the corresponding settings item.
'''
if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
settings = Settings()
# set ci_platform setting to "travis"
settings.add_setting("ci_platform", "travis")
# set settings with TRAVIS values
env_var_to_settings("TRAVIS_BUILD_NUMBER", "build")
env_var_to_settings("TRAVIS_JOB_NUMBER", "job")
env_var_to_settings("TRAVIS_BRANCH", "branch")
env_var_to_settings("TRAVIS_REPO_SLUG", "project_name")
# convert and set Travis build result
if "TRAVIS_TEST_RESULT" in os.environ:
# map $TRAVIS_TEST_RESULT to a more readable value
settings.add_setting(
"result",
convert_build_result(os.environ["TRAVIS_TEST_RESULT"])
)
def load_property_from_settings(self, property_name, setting_name=None):
"""
Load the value of a setting and set it as a build property.
Parameters
- property_name : name of the build property
- setting_name : name of the setting (takes property_name if not set)
"""
if setting_name is None:
setting_name = property_name
value = Settings().get_setting(setting_name)
if value is not None:
self.add_property(property_name, value)
def setUpClass(cls):
"""Set up test fixture."""
# show full diff in case of assert mismatch
cls.maxDiff = None
cls.settings = Settings()
cls.project_name = buildtimetrend.NAME
cls.project_info = {
"lib_version": buildtimetrend.VERSION,
"schema_version": buildtimetrend.SCHEMA_VERSION,
"client": 'None',
"client_version": 'None',
"project_name": cls.project_name
}
def load_all(settings=None):
"""
Load all Travis CI environment variables.
Load Travis CI environment variables and assign their values to
the corresponding setting value :
- general
- build matrix
- pull request
"""
if not isinstance(settings, Settings):
settings = Settings()
load_general_env_vars(settings)
load_build_matrix_env_vars(settings)
load_travis_pr_env_vars(settings)
def setUpClass(cls):
"""Set up test fixture."""
cls.settings = Settings()
cls.project_info = cls.settings.get_project_info()
cls.maxDiff = None
cls.test_api_error = {"message": "test message", "error_code": "123"}
# copy Keen.io environment variables
if "KEEN_PROJECT_ID" in os.environ:
cls.copy_keen_project_id = os.environ["KEEN_PROJECT_ID"]
if "KEEN_WRITE_KEY" in os.environ:
cls.copy_keen_write_key = os.environ["KEEN_WRITE_KEY"]
if "KEEN_READ_KEY" in os.environ:
cls.copy_keen_read_key = os.environ["KEEN_READ_KEY"]
if "KEEN_MASTER_KEY" in os.environ:
cls.copy_keen_master_key = os.environ["KEEN_MASTER_KEY"]
def get_read_key():
'''
Generate a read key for the project and print that key
'''
settings = Settings()
settings.load_config_file("config.yml")
# get project name from argument
if len(sys.argv) > 1:
settings.set_project_name(sys.argv[1])
# generate a read key
print keen_io_generate_read_key(settings.get_project_name())
def check_authorization(repo, auth_header):
"""
Check if Travis CI notification has a correct Authorization header.
This check is enabled if travis_account_token is defined in settings.
More information on the Authorization header :
http://docs.travis-ci.com/user/notifications/#Authorization-for-Webhooks
Returns true if Authorization header is valid, but also if
travis_account_token is not defined.
Parameters:
- repo : git repo name
- auth_header : Travis CI notification Authorization header
"""
# get Travis account token from Settings
token = Settings().get_setting("travis_account_token")
# return True if token is not set
if token is None:
logger.info("Setting travis_account_token is not defined,"
" Travis CI notification Authorization header"
" is not checked.")
return True
# check if parameters are strings
if is_string(repo) and is_string(auth_header) and is_string(token):
# generate hash (encode string to bytes first)
auth_hash = sha256((repo + token).encode('utf-8')).hexdigest()
# compare hash with Authorization header
if auth_hash == auth_header:
logger.info("Travis CI notification Authorization header"
" is correct.")
return True
else:
logger.error("Travis CI notification Authorization header"
" is incorrect.")
return False
else:
logger.debug("repo, auth_header and travis_auth_token"
" should be strings.")
return False
def process_notification_payload(payload):
"""
Extract repo slug and build number from Travis notification payload.
Returns a dictionary with "repo" and "build" information,
or an empty dictionary if the payload could not be processed.
Deprecated behaviour : Currently the repo and build information are
also stored in the "settings" object,
but this will be removed in the near future.
Parameters:
- payload : Travis CI notification payload
"""
settings = Settings()
parameters = {}
if payload is None:
logger.warning("Travis notification payload is not set")
return parameters
if not is_string(payload):
logger.warning(
"Travis notification payload is incorrect :"
" string expected, got %s", type(payload))
return parameters
json_payload = json.loads(payload)
logger.info("Travis Payload : %r.", json_payload)
# get repo name from payload
if ("repository" in json_payload
and "owner_name" in json_payload["repository"]
and "name" in json_payload["repository"]):
repo = get_repo_slug(json_payload["repository"]["owner_name"],
json_payload["repository"]["name"])
logger.info("Build repo : %s", repo)
settings.set_project_name(repo)
parameters["repo"] = repo
# get build number from payload
if "number" in json_payload:
logger.info("Build number : %s", str(json_payload["number"]))
settings.add_setting('build', json_payload['number'])
parameters["build"] = json_payload['number']
return parameters
def retrieve_and_store_data(argv):
"""
Load timing and build data, process and store it.
Retrieve timing and build data from Travis CI log, parse it
and store the result in Keen.io.
Parameters:
- argv : command line parameters
"""
settings = Settings()
settings.set_client(CLIENT_NAME, CLIENT_VERSION)
# load settings from config file, env_var and cli parameters
if settings.load_settings(argv, "config_service.yml") is None:
return
build = settings.get_setting('build')
if build is None:
print("Build number is not set, use --build=build_id")
return
travis_data = TravisData(settings.get_project_name(), build)
# retrieve build data using Travis CI API
print(
"Retrieve build #{:s} data of {:s} from Travis CI".format(
build, settings.get_project_name()
)
)
travis_data.get_build_data()
# process all build jobs
travis_data.process_build_jobs()
if not keenio.is_writable():
print("Keen IO write key not set, no data was sent")
return
# send build job data to Keen.io
for build_job in travis_data.build_jobs:
print("Send build job #{:s} data to Keen.io".format(build_job))
keenio.send_build_data_service(travis_data.build_jobs[build_job])
def test_generate_config_file(self, get_cfg_str_func):
"""Test dashboard.generate_config_file()"""
# set config file path
Settings().add_setting("dashboard_configfile",
constants.DASHBOARD_TEST_CONFIG_FILE)
# check if configfile exists
self.assertFalse(check_file(constants.DASHBOARD_TEST_CONFIG_FILE))
# generate config file with empty repo name
self.assertRaises(TypeError, dashboard.generate_config_file)
# generate config file with empty repo name
self.assertTrue(dashboard.generate_config_file(None))
self.assertTrue(check_file(constants.DASHBOARD_TEST_CONFIG_FILE))
# check if mock was called with correct parameters
args, kwargs = get_cfg_str_func.call_args
self.assertEqual(args, (None, ))
self.assertDictEqual(kwargs, {})
# generate config file
self.assertTrue(dashboard.generate_config_file("test/repo3"))
self.assertTrue(check_file(constants.DASHBOARD_TEST_CONFIG_FILE))
# check if mock was called with correct parameters
args, kwargs = get_cfg_str_func.call_args
self.assertEqual(args, ("test/repo3", ))
self.assertDictEqual(kwargs, {})
# test generated config file contents
with open(constants.DASHBOARD_TEST_CONFIG_FILE, 'r') as config_file:
self.assertEqual("var config = {'projectName': 'test/repo3'};\n",
next(config_file))
self.assertEqual("var keenConfig = {'projectId': '1234abcd'};",
next(config_file))
def test_generate_config_file_fails(self):
"""Test dashboard.generate_config_file() if creation fails"""
# set config file path
Settings().add_setting("dashboard_configfile",
constants.DASHBOARD_TEST_CONFIG_FILE)
# check if configfile exists
self.assertFalse(check_file(constants.DASHBOARD_TEST_CONFIG_FILE))
# init mock
patcher = mock.patch('buildtimetrend.tools.check_file',
return_value=False)
check_file_func = patcher.start()
# generation should return false
self.assertFalse(dashboard.generate_config_file("test/repo4"))
# check if mock was called with correct parameters
args, kwargs = check_file_func.call_args
self.assertEqual(args, (constants.DASHBOARD_TEST_CONFIG_FILE, ))
self.assertDictEqual(kwargs, {})
patcher.stop()
def setUpClass(cls):
"""Set up test fixture."""
cls.project_info = Settings().get_project_info()
cls.maxDiff = None
def test_load_properties(self):
"""Test loading properties"""
self.build.load_properties_from_settings()
self.assertDictEqual(
{'duration': 0, "repo": buildtimetrend.NAME},
self.build.get_properties())
settings = Settings()
settings.add_setting("ci_platform", "travis")
settings.add_setting("build", "123")
settings.add_setting("job", "123.1")
settings.add_setting("branch", "branch1")
settings.add_setting("result", "passed")
settings.add_setting("build_trigger", "push")
settings.add_setting(
"pull_request",
{
"is_pull_request": False,
"title": None,
"number": None
}
)
settings.set_project_name("test/project")
self.build.load_properties_from_settings()
self.assertDictEqual(
{
'duration': 0,
'ci_platform': "travis",
'build': "123",
'job': "123.1",
'branch': "branch1",
'result': "passed",
'build_trigger': "push",
'pull_request': {
"is_pull_request": False,
"title": None,
"number": None},
'repo': "test/project"
},
self.build.get_properties())
def test_load_properties(self):
"""Test loading properties"""
self.build.load_properties_from_settings()
self.assertDictEqual({
'duration': 0,
"repo": buildtimetrend.NAME
}, self.build.get_properties())
settings = Settings()
settings.add_setting("ci_platform", "travis")
settings.add_setting("build", "123")
settings.add_setting("job", "123.1")
settings.add_setting("branch", "branch1")
settings.add_setting("result", "passed")
settings.add_setting("build_trigger", "push")
settings.add_setting("pull_request", {
"is_pull_request": False,
"title": None,
"number": None
})
settings.set_project_name("test/project")
self.build.load_properties_from_settings()
self.assertDictEqual(
{
'duration': 0,
'ci_platform': "travis",
'build': "123",
'job': "123.1",
'branch': "branch1",
'result': "passed",
'build_trigger': "push",
'pull_request': {
"is_pull_request": False,
"title": None,
"number": None
},
'repo': "test/project"
}, self.build.get_properties())
def setUp(self):
"""Initialise test environment before each test."""
self.build = BuildJob()
# reinitialise settings
Settings().__init__()
def test_load_build_matrix_env_vars(self):
"""Test load_build_matrix_env_vars"""
settings = Settings()
self.assertEqual(None, settings.get_setting("build_matrix"))
# setup Travis env vars
if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
reset_travis_vars = False
expected_os = os.environ["TRAVIS_OS_NAME"]
copy_python = os.environ["TRAVIS_PYTHON_VERSION"]
del os.environ["TRAVIS_PYTHON_VERSION"]
else:
reset_travis_vars = True
os.environ["TRAVIS"] = "true"
expected_os = os.environ["TRAVIS_OS_NAME"] = "test_os"
# temporarily remove PYTHON VERSION
if "TRAVIS_PYTHON_VERSION" in os.environ:
reset_python_version = True
copy_python = os.environ["TRAVIS_PYTHON_VERSION"]
del os.environ["TRAVIS_PYTHON_VERSION"]
else:
reset_python_version = False
# test language and language versions
test_languages = [{
'env_var': 'TRAVIS_DART_VERSION',
'language': 'dart',
'test_value': "1.1"
}, {
'env_var': 'TRAVIS_GO_VERSION',
'language': 'go',
'test_value': "1.2"
}, {
'env_var': 'TRAVIS_HAXE_VERSION',
'language': 'haxe',
'test_value': "1.3"
}, {
'env_var': 'TRAVIS_JDK_VERSION',
'language': 'java',
'test_value': "1.4"
}, {
'env_var': 'TRAVIS_JULIA_VERSION',
'language': 'julia',
'test_value': "1.5"
}, {
'env_var': 'TRAVIS_NODE_VERSION',
'language': 'javascript',
'test_value': "1.6"
}, {
'env_var': 'TRAVIS_OTP_RELEASE',
'language': 'erlang',
'test_value': "1.7"
}, {
'env_var': 'TRAVIS_PERL_VERSION',
'language': 'perl',
'test_value': "1.8"
}, {
'env_var': 'TRAVIS_PHP_VERSION',
'language': 'php',
'test_value': "1.9"
}, {
'env_var': 'TRAVIS_PYTHON_VERSION',
'language': 'python',
'test_value': "1.10"
}, {
'env_var': 'TRAVIS_R_VERSION',
'language': 'r',
'test_value': "1.11"
}, {
'env_var': 'TRAVIS_RUBY_VERSION',
'language': 'ruby',
'test_value': "1.12"
}, {
'env_var': 'TRAVIS_RUST_VERSION',
'language': 'rust',
'test_value': "1.13"
}, {
'env_var': 'TRAVIS_SCALA_VERSION',
'language': 'scala',
'test_value': "1.14"
}]
# test languages
for language in test_languages:
if language['env_var'] in os.environ:
reset_travis_lang_version = False
expected_lang_version = os.environ[language['env_var']]
else:
reset_travis_lang_version = True
expected_lang_version = \
os.environ[language['env_var']] = language['test_value']
env_var.load_build_matrix_env_vars(settings)
self.assertDictEqual(
{
'os':
expected_os,
'language':
language['language'],
'language_version':
expected_lang_version,
'summary':
"{0!s} {1!s} {2!s}".format(language['language'],
expected_lang_version,
expected_os)
}, settings.get_setting("build_matrix"))
# reset Travis test result
if reset_travis_lang_version:
del os.environ[language['env_var']]
# reset test Travis vars
if reset_travis_vars:
del os.environ["TRAVIS"]
del os.environ["TRAVIS_OS_NAME"]
# reset removed python version
if reset_python_version:
os.environ["TRAVIS_PYTHON_VERSION"] = copy_python
def setUpClass(cls):
"""Set up test fixture."""
cls.settings = Settings()
def test_load_build_matrix_env_vars_parameters(self):
"""Test load_travis_env_vars, optional parameters"""
# setup Travis env vars
if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
reset_travis_vars = False
copy_os = os.environ["TRAVIS_OS_NAME"]
else:
reset_travis_vars = True
os.environ["TRAVIS"] = "true"
# temporarily remove OS VERSION
if "TRAVIS_OS_NAME" in os.environ:
reset_os = True
copy_os = os.environ["TRAVIS_OS_NAME"]
del os.environ["TRAVIS_OS_NAME"]
else:
reset_os = False
# temporarily remove PYTHON VERSION
if "TRAVIS_PYTHON_VERSION" in os.environ:
reset_python_version = True
copy_python = os.environ["TRAVIS_PYTHON_VERSION"]
del os.environ["TRAVIS_PYTHON_VERSION"]
else:
reset_python_version = False
# test optional build matrix parameters
test_parameters = [{
'env_var': 'TRAVIS_XCODE_SDK',
'parameter': 'xcode_sdk',
'test_value': "test_x_sdk"
}, {
'env_var': 'TRAVIS_XCODE_SCHEME',
'parameter': 'xcode_scheme',
'test_value': "test_x_scheme"
}, {
'env_var': 'TRAVIS_XCODE_PROJECT',
'parameter': 'xcode_project',
'test_value': "test_x_project"
}, {
'env_var': 'TRAVIS_XCODE_WORKSPACE',
'parameter': 'xcode_workspace',
'test_value': "test_x_workspace"
}, {
'env_var': 'CC',
'parameter': 'compiler',
'test_value': "test_gcc"
}, {
'env_var': 'ENV',
'parameter': 'parameters',
'test_value': "test_env"
}]
# test parameters
for parameter in test_parameters:
Settings().__init__()
settings = Settings()
if parameter['env_var'] in os.environ:
reset_travis_parameter = False
expected_param_value = os.environ[parameter['env_var']]
else:
reset_travis_parameter = True
expected_param_value = os.environ[parameter['env_var']] = \
parameter['test_value']
env_var.load_build_matrix_env_vars(settings)
self.assertDictEqual(
{
parameter["parameter"]: expected_param_value,
'summary': expected_param_value
}, settings.get_setting("build_matrix"))
# reset Travis parameters
if reset_travis_parameter:
del os.environ[parameter['env_var']]
# reset test Travis vars
if reset_travis_vars:
del os.environ["TRAVIS"]
# reset removed os name
if reset_os:
os.environ["TRAVIS_OS_NAME"] = copy_os
# reset removed python version
if reset_python_version:
os.environ["TRAVIS_PYTHON_VERSION"] = copy_python
def setUp():
"""Initialise test environment before each test."""
# reinit settings singleton
Settings().__init__()
def test_load_travis_env_vars(self):
"""Test load_travis_env_vars"""
settings = Settings()
self.assertEqual(None, settings.get_setting("ci_platform"))
self.assertEqual(None, settings.get_setting("build"))
self.assertEqual(None, settings.get_setting("job"))
self.assertEqual(None, settings.get_setting("branch"))
self.assertEqual(None, settings.get_setting("result"))
self.assertEqual(None, settings.get_setting("build_trigger"))
self.assertEqual(None, settings.get_setting("pull_request"))
self.assertEqual(buildtimetrend.NAME, settings.get_project_name())
# setup Travis env vars
if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
reset_travis_vars = False
expected_build = os.environ["TRAVIS_BUILD_NUMBER"]
expected_job = os.environ["TRAVIS_JOB_NUMBER"]
expected_branch = os.environ["TRAVIS_BRANCH"]
expected_project_name = os.environ["TRAVIS_REPO_SLUG"]
copy_pull_request = os.environ["TRAVIS_PULL_REQUEST"]
else:
reset_travis_vars = True
os.environ["TRAVIS"] = "true"
expected_build = os.environ["TRAVIS_BUILD_NUMBER"] = "123"
expected_job = os.environ["TRAVIS_JOB_NUMBER"] = "123.1"
expected_branch = os.environ["TRAVIS_BRANCH"] = "branch1"
expected_project_name = \
os.environ["TRAVIS_REPO_SLUG"] = "test/project"
# setup Travis test result
if "TRAVIS_TEST_RESULT" in os.environ:
reset_travis_result = False
copy_result = os.environ["TRAVIS_TEST_RESULT"]
else:
reset_travis_result = True
os.environ["TRAVIS_TEST_RESULT"] = "0"
os.environ["TRAVIS_PULL_REQUEST"] = "false"
env_var.load_all(settings)
self.assertEqual("travis", settings.get_setting("ci_platform"))
self.assertEqual(expected_build, settings.get_setting("build"))
self.assertEqual(expected_job, settings.get_setting("job"))
self.assertEqual(expected_branch, settings.get_setting("branch"))
self.assertEqual(expected_project_name, settings.get_project_name())
self.assertEqual("passed", settings.get_setting("result"))
self.assertEqual("push", settings.get_setting("build_trigger"))
self.assertDictEqual(
{
'is_pull_request': False,
'title': None,
'number': None
}, settings.get_setting("pull_request"))
os.environ["TRAVIS_TEST_RESULT"] = "1"
# build is a pull request
expected_pull_request = os.environ["TRAVIS_PULL_REQUEST"] = "123"
env_var.load_all()
self.assertEqual("failed", settings.get_setting("result"))
self.assertEqual("pull_request", settings.get_setting("build_trigger"))
self.assertDictEqual(
{
'is_pull_request': True,
'title': "unknown",
'number': expected_pull_request
}, settings.get_setting("pull_request"))
# build is not a pull request
os.environ["TRAVIS_PULL_REQUEST"] = "false"
env_var.load_all(settings)
self.assertEqual("push", settings.get_setting("build_trigger"))
self.assertDictEqual(
{
'is_pull_request': False,
'title': None,
'number': None
}, settings.get_setting("pull_request"))
# reset test Travis vars
if reset_travis_vars:
del os.environ["TRAVIS"]
del os.environ["TRAVIS_BUILD_NUMBER"]
del os.environ["TRAVIS_JOB_NUMBER"]
del os.environ["TRAVIS_BRANCH"]
del os.environ["TRAVIS_REPO_SLUG"]
del os.environ["TRAVIS_PULL_REQUEST"]
else:
os.environ["TRAVIS_PULL_REQUEST"] = copy_pull_request
# reset Travis test result
if reset_travis_result:
del os.environ["TRAVIS_TEST_RESULT"]
else:
os.environ["TRAVIS_TEST_RESULT"] = copy_result
def __init__(self):
"""Initialise class."""
self.settings = Settings()
# get logger
self.logger = logger
class TravisParser(object):
"""
Travis CI build timing and build data parser.
Retrieve timing data from Travis CI, parse it and store it in Keen.io.
"""
def __init__(self):
"""Initialise class."""
self.settings = Settings()
# get logger
self.logger = logger
@cherrypy.expose
def default(self,
repo_owner=None,
repo_name=None,
first_build=None,
last_build=None,
payload=None):
"""
Default handler.
Visiting this page triggers loading and processing the build log
and data of a travis CI build process.
If last_build is defined, all builds from first_build until last_build
will be retrieved and processed.
If only payload is defined, repo and build data
will be extracted from the payload.
Parameters:
- repo_owner : name of the Github repo owner, fe. `buildtimetrend`
- repo_name : name of the Github repo, fe. `service`
- first_build : first build number to process (int)
- last_build : last build number to process (int)
- payload : Travis CI notification payload (json)
"""
cherrypy.response.headers['Content-Type'] = 'text/plain'
# reset settings
self.settings.set_project_name(None)
self.settings.add_setting('build', None)
self.logger.debug("Check Travis headers : %r",
cherrypy.request.headers)
repo = get_repo_slug(repo_owner, repo_name)
# load parameters from the Travis notification payload
if self.check_travis_notification():
payload_params = process_notification_payload(payload)
# assign payload parameters
if repo is None and "repo" in payload_params:
repo = payload_params["repo"]
if first_build is None and last_build is None and \
"build" in payload_params:
first_build = payload_params["build"]
# check parameter validity, check returns error message
# or None if parameters are valid
params_valid = validate_travis_request(repo, first_build)
if params_valid is not None:
self.logger.warning(params_valid)
return params_valid
if last_build is None:
self.logger.warning("Request to process build #%s of repo %s",
first_build, repo)
# schedule task with 10 second delay to give Travis CI time
# to add the finished_at property. (issue #96)
return self.schedule_task(repo, first_build, 10)
return self.multi_build(repo, first_build, last_build)
def multi_build(self, repo, first_build, last_build):
"""
Schedule processing multiple consecutive builds.
All builds from first_build until last_build
will be retrieved and processed.
The total number of builds to be scheduled is limited by the
`multi_import.max_builds` config parameter.
Every next scheduled build will be delayed by the
`multi_import.delay` config parameter.
Parameters:
- repo : repo name (fe. buildtimetrend/service)
- first_build : first build number to process (int)
- last_build : last build number to process (int)
"""
first_build = int(first_build)
last_build = int(last_build)
message = ""
multi_import = Settings().get_setting("multi_import")
max_multi_builds = int(multi_import["max_builds"])
if last_build < first_build:
tmp_msg = "Warning : last_build should be equal" \
" or larger than first_build"
self.logger.warning(tmp_msg)
message += tmp_msg + "\n"
last_build = first_build
if (last_build - first_build) > max_multi_builds:
tmp_msg = "Warning : number of multiple builds is limited to {:d}"
self.logger.warning(tmp_msg.format(max_multi_builds))
message += tmp_msg.format(max_multi_builds) + "\n"
last_build = first_build + max_multi_builds
message += "Request to process build(s) #{first_build:d} to" \
" #{last_build:d} of repo {repo}:\n".format(**locals())
build = first_build
delay = 0
while build <= last_build:
message += self.schedule_task(repo, build, delay) + "\n"
delay += multi_import["delay"]
build += 1
return message
def schedule_task(self, repo, build, delay=0):
"""
Schedule task.
Parameters:
- repo : repo name (fe. buildtimetrend/service)
- build : build number to process (int)
- delay : delay before task should be started, in seconds
"""
# process travis build
if is_worker_enabled():
task = tasks.process_travis_buildlog.apply_async(
(repo, build), countdown=int(delay))
temp_msg = "Task scheduled to process build #{build}" \
" of repo {repo} : {task_id}"
self.logger.warning(
temp_msg.format(build=build, repo=repo, task_id=task.id))
return temp_msg.format(build=cgi.escape(str(build)),
repo=cgi.escape(str(repo)),
task_id=cgi.escape(str(task.id)))
else:
return tasks.process_travis_buildlog(repo, build)
def check_travis_notification(self):
"""
Check Travis CI notification request.
Load Authorization and Travis-Repo-Slug headers and check if
the Authorization header is correct
"""
if "Authorization" not in cherrypy.request.headers:
self.logger.debug("Authorization header is not set")
return True
if "Travis-Repo-Slug" not in cherrypy.request.headers:
self.logger.debug("Travis-Repo-Slug header is not set")
return False
return check_authorization(
cherrypy.request.headers["Travis-Repo-Slug"],
cherrypy.request.headers["Authorization"])
class TestSettings(unittest.TestCase):
@classmethod
def setUpClass(self):
self.settings = Settings()
self.project_name = buildtimetrend.NAME
self.project_info = {
"version": buildtimetrend.VERSION,
"schema_version": buildtimetrend.SCHEMA_VERSION,
"project_name": self.project_name}
def setUp(self):
# reinit settings singleton
if self.settings is not None:
self.settings.__init__()
def test_get_project_info(self):
self.assertDictEqual(self.project_info, self.settings.get_project_info())
def test_get_set_project_name(self):
self.assertEquals(self.project_name, self.settings.get_project_name())
self.settings.set_project_name("test_name")
self.assertEquals("test_name", self.settings.get_project_name())
self.settings.set_project_name(None)
self.assertEquals(None, self.settings.get_project_name())
self.settings.set_project_name("")
self.assertEquals("", self.settings.get_project_name())
def test_get_add_setting(self):
# setting is not set yet
self.assertEquals(None, self.settings.get_setting("test_name"))
self.settings.add_setting("test_name", "test_value")
self.assertEquals("test_value", self.settings.get_setting("test_name"))
self.settings.add_setting("test_name", None)
self.assertEquals(None, self.settings.get_setting("test_name"))
self.settings.add_setting("test_name", "")
self.assertEquals("", self.settings.get_setting("test_name"))
self.settings.add_setting("test_name", 6)
self.assertEquals(6, self.settings.get_setting("test_name"))
def test_get_setting(self):
self.assertEquals(None, self.settings.get_setting("test_name"))
self.assertEquals(
self.project_name,
self.settings.get_setting("project_name"))
self.assertDictEqual(
{
"project_name": self.project_name,
"mode_native": False,
"mode_keen": True
},
self.settings.settings.get_items())
def test_no_config_file(self):
# function should return false when file doesn't exist
self.assertFalse(self.settings.load_config_file('no_file.yml'))
self.assertDictEqual(
{
"project_name": self.project_name,
"mode_native": False,
"mode_keen": True
},
self.settings.settings.get_items())
self.assertFalse(self.settings.load_config_file(''))
self.assertDictEqual(
{
"project_name": self.project_name,
"mode_native": False,
"mode_keen": True
},
self.settings.settings.get_items())
# function should throw an error when no filename is set
self.assertRaises(TypeError, self.settings.load_config_file)
def test_load_config_file(self):
# checking if Keen.io configuration is not set (yet)
self.assertEquals(None, keen.project_id)
self.assertEquals(None, keen.write_key)
self.assertEquals(None, keen.read_key)
# load sample config file
self.assertTrue(self.settings.load_config_file(constants.TEST_SAMPLE_CONFIG_FILE))
self.assertDictEqual(
{"project_name": "test_project",
"mode_native": True,
"mode_keen": False,
"setting1": "test_value1"},
self.settings.settings.get_items())
# checking if Keen.io configuration is set
self.assertEquals("1234", keen.project_id)
self.assertEquals("12345678", keen.write_key)
self.assertEquals("abcdefg", keen.read_key)
self.assertTrue(keen_is_readable())
self.assertTrue(keen_is_writable())
def test_load_properties(self):
self.build.load_properties_from_settings()
self.assertDictEqual(
{'duration': 0, "repo": buildtimetrend.NAME},
self.build.get_properties())
settings = Settings()
settings.add_setting("ci_platform", "travis")
settings.add_setting("build", "123")
settings.add_setting("job", "123.1")
settings.add_setting("branch", "branch1")
settings.add_setting("result", "passed")
settings.set_project_name("test/project")
self.build.load_properties_from_settings()
self.assertDictEqual(
{ 'duration': 0,
'ci_platform': "travis",
'build': "123",
'job': "123.1",
'branch': "branch1",
'result': "passed",
'repo': "test/project"},
self.build.get_properties())
