def analyse(argv):
'''
Analyse timestamp file
'''
settings = Settings()
# load Travis environment variables and save them in settings
load_travis_env_vars()
# process command line arguments
process_argv(argv)
# read build data from timestamp CSV file
build = Build(TIMESTAMP_FILE)
# load build properties from settings
build.load_properties_from_settings()
# retrieve data from Travis CI API
if build.get_property("ci_platform") == "travis":
travis_data = TravisData(
build.get_property("repo"),
build.get_property("build"),
)
travis_data.get_build_data()
build.set_started_at(travis_data.get_started_at())
# log data
if settings.get_setting("mode_native") is True:
log_build_native(build)
if settings.get_setting("mode_keen") is True:
log_build_keen(build)
def create_worker_app():
"""Create worker app."""
# load settings
settings = Settings()
settings.load_settings(config_file=constants.CONFIG_FILE)
settings.set_client(constants.CLIENT_NAME, constants.CLIENT_VERSION)
if is_worker_enabled():
task_queue = settings.get_setting("task_queue")
worker_app = Celery(
'tasks',
backend=task_queue["backend"],
broker=task_queue["broker_url"]
)
# configure worker
worker_app.conf.update(
CELERY_TASK_SERIALIZER='json',
CELERY_ACCEPT_CONTENT=['json']
)
if worker_app is None:
logger.error("Error connection to task queue")
else:
logger.info(
"Connected to task queue : %s", task_queue["broker_url"]
)
else:
worker_app = Celery()
logger.warning(
"Task queue is not defined,"
" check README.md to configure task queue"
)
return worker_app
def generate_trend(argv):
"""Generate trends from analysed buildtime data."""
settings = Settings()
# load settings from config file, env_var and cli parameters
if settings.load_settings(argv) is None:
return
# load Travis CI environment variables
env_var.load_all(settings)
# run trend_keen() always,
# if $KEEN_PROJECT_ID variable is set (checked later), it will be executed
if settings.get_setting("mode_native") is True:
trend_native()
if settings.get_setting("mode_keen") is True:
trend_keen()
def generate_trend(argv):
'''
Generate trends from analised buildtime data
'''
settings = Settings()
# load Travis environment variables and save them in settings
load_travis_env_vars()
# process command line arguments
process_argv(argv)
# run trend_keen() always,
# if $KEEN_PROJECT_ID variable is set (checked later), it will be executed
if settings.get_setting("mode_native") is True:
trend_native()
if settings.get_setting("mode_keen") is True:
trend_keen()
def get_repo_data_detail(repo):
"""
Get level of data detail storage of a repo.
A repository name is checked against a list of repository names.
If a match is found, the corresponding data detail level is used.
Else, the default global setting is returned.
Parameters:
-repo : repository name
"""
settings = Settings()
if repo is None:
logger.warning("Repo is not defined")
else:
repo_settings = settings.get_setting("repo_data_detail")
for repo_substring, setting in repo_settings.items():
if repo_substring in repo:
return setting
# return default global data_detail setting
return settings.get_setting("data_detail")
def retrieve_and_store_data(argv):
"""
Load timing and build data, process and store it.
Retrieve timing and build data from Travis CI log, parse it
and store the result in Keen.io.
Parameters:
- argv : command line parameters
"""
settings = Settings()
settings.set_client(CLIENT_NAME, CLIENT_VERSION)
# load settings from config file, env_var and cli parameters
if settings.load_settings(argv, "config_service.yml") is None:
return
build = settings.get_setting('build')
if build is None:
print("Build number is not set, use --build=build_id")
return
travis_data = TravisData(settings.get_project_name(), build)
# retrieve build data using Travis CI API
print(
"Retrieve build #{:s} data of {:s} from Travis CI".format(
build, settings.get_project_name()
)
)
travis_data.get_build_data()
# process all build jobs
travis_data.process_build_jobs()
if not keenio.is_writable():
print("Keen IO write key not set, no data was sent")
return
# send build job data to Keen.io
for build_job in travis_data.build_jobs:
print("Send build job #{:s} data to Keen.io".format(build_job))
keenio.send_build_data_service(travis_data.build_jobs[build_job])
class TestSettings(unittest.TestCase):
@classmethod
def setUpClass(self):
self.settings = Settings()
self.project_name = buildtimetrend.NAME
self.project_info = {
"version": buildtimetrend.VERSION,
"schema_version": buildtimetrend.SCHEMA_VERSION,
"project_name": self.project_name}
def setUp(self):
# reinit settings singleton
if self.settings is not None:
self.settings.__init__()
def test_get_project_info(self):
self.assertDictEqual(self.project_info, self.settings.get_project_info())
def test_get_set_project_name(self):
self.assertEquals(self.project_name, self.settings.get_project_name())
self.settings.set_project_name("test_name")
self.assertEquals("test_name", self.settings.get_project_name())
self.settings.set_project_name(None)
self.assertEquals(None, self.settings.get_project_name())
self.settings.set_project_name("")
self.assertEquals("", self.settings.get_project_name())
def test_get_add_setting(self):
# setting is not set yet
self.assertEquals(None, self.settings.get_setting("test_name"))
self.settings.add_setting("test_name", "test_value")
self.assertEquals("test_value", self.settings.get_setting("test_name"))
self.settings.add_setting("test_name", None)
self.assertEquals(None, self.settings.get_setting("test_name"))
self.settings.add_setting("test_name", "")
self.assertEquals("", self.settings.get_setting("test_name"))
self.settings.add_setting("test_name", 6)
self.assertEquals(6, self.settings.get_setting("test_name"))
def test_get_setting(self):
self.assertEquals(None, self.settings.get_setting("test_name"))
self.assertEquals(
self.project_name,
self.settings.get_setting("project_name"))
self.assertDictEqual(
{
"project_name": self.project_name,
"mode_native": False,
"mode_keen": True
},
self.settings.settings.get_items())
def test_no_config_file(self):
# function should return false when file doesn't exist
self.assertFalse(self.settings.load_config_file('no_file.yml'))
self.assertDictEqual(
{
"project_name": self.project_name,
"mode_native": False,
"mode_keen": True
},
self.settings.settings.get_items())
self.assertFalse(self.settings.load_config_file(''))
self.assertDictEqual(
{
"project_name": self.project_name,
"mode_native": False,
"mode_keen": True
},
self.settings.settings.get_items())
# function should throw an error when no filename is set
self.assertRaises(TypeError, self.settings.load_config_file)
def test_load_config_file(self):
# checking if Keen.io configuration is not set (yet)
self.assertEquals(None, keen.project_id)
self.assertEquals(None, keen.write_key)
self.assertEquals(None, keen.read_key)
# load sample config file
self.assertTrue(self.settings.load_config_file(constants.TEST_SAMPLE_CONFIG_FILE))
self.assertDictEqual(
{"project_name": "test_project",
"mode_native": True,
"mode_keen": False,
"setting1": "test_value1"},
self.settings.settings.get_items())
# checking if Keen.io configuration is set
self.assertEquals("1234", keen.project_id)
self.assertEquals("12345678", keen.write_key)
self.assertEquals("abcdefg", keen.read_key)
self.assertTrue(keen_is_readable())
self.assertTrue(keen_is_writable())
def test_load_travis_env_vars(self):
"""Test load_travis_env_vars"""
settings = Settings()
self.assertEqual(None, settings.get_setting("ci_platform"))
self.assertEqual(None, settings.get_setting("build"))
self.assertEqual(None, settings.get_setting("job"))
self.assertEqual(None, settings.get_setting("branch"))
self.assertEqual(None, settings.get_setting("result"))
self.assertEqual(None, settings.get_setting("build_trigger"))
self.assertEqual(None, settings.get_setting("pull_request"))
self.assertEqual(buildtimetrend.NAME, settings.get_project_name())
# setup Travis env vars
if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
reset_travis_vars = False
expected_build = os.environ["TRAVIS_BUILD_NUMBER"]
expected_job = os.environ["TRAVIS_JOB_NUMBER"]
expected_branch = os.environ["TRAVIS_BRANCH"]
expected_project_name = os.environ["TRAVIS_REPO_SLUG"]
copy_pull_request = os.environ["TRAVIS_PULL_REQUEST"]
else:
reset_travis_vars = True
os.environ["TRAVIS"] = "true"
expected_build = os.environ["TRAVIS_BUILD_NUMBER"] = "123"
expected_job = os.environ["TRAVIS_JOB_NUMBER"] = "123.1"
expected_branch = os.environ["TRAVIS_BRANCH"] = "branch1"
expected_project_name = \
os.environ["TRAVIS_REPO_SLUG"] = "test/project"
# setup Travis test result
if "TRAVIS_TEST_RESULT" in os.environ:
reset_travis_result = False
copy_result = os.environ["TRAVIS_TEST_RESULT"]
else:
reset_travis_result = True
os.environ["TRAVIS_TEST_RESULT"] = "0"
os.environ["TRAVIS_PULL_REQUEST"] = "false"
env_var.load_all(settings)
self.assertEqual("travis", settings.get_setting("ci_platform"))
self.assertEqual(expected_build, settings.get_setting("build"))
self.assertEqual(expected_job, settings.get_setting("job"))
self.assertEqual(expected_branch, settings.get_setting("branch"))
self.assertEqual(expected_project_name, settings.get_project_name())
self.assertEqual("passed", settings.get_setting("result"))
self.assertEqual("push", settings.get_setting("build_trigger"))
self.assertDictEqual(
{
'is_pull_request': False,
'title': None,
'number': None
}, settings.get_setting("pull_request"))
os.environ["TRAVIS_TEST_RESULT"] = "1"
# build is a pull request
expected_pull_request = os.environ["TRAVIS_PULL_REQUEST"] = "123"
env_var.load_all()
self.assertEqual("failed", settings.get_setting("result"))
self.assertEqual("pull_request", settings.get_setting("build_trigger"))
self.assertDictEqual(
{
'is_pull_request': True,
'title': "unknown",
'number': expected_pull_request
}, settings.get_setting("pull_request"))
# build is not a pull request
os.environ["TRAVIS_PULL_REQUEST"] = "false"
env_var.load_all(settings)
self.assertEqual("push", settings.get_setting("build_trigger"))
self.assertDictEqual(
{
'is_pull_request': False,
'title': None,
'number': None
}, settings.get_setting("pull_request"))
# reset test Travis vars
if reset_travis_vars:
del os.environ["TRAVIS"]
del os.environ["TRAVIS_BUILD_NUMBER"]
del os.environ["TRAVIS_JOB_NUMBER"]
del os.environ["TRAVIS_BRANCH"]
del os.environ["TRAVIS_REPO_SLUG"]
del os.environ["TRAVIS_PULL_REQUEST"]
else:
os.environ["TRAVIS_PULL_REQUEST"] = copy_pull_request
# reset Travis test result
if reset_travis_result:
del os.environ["TRAVIS_TEST_RESULT"]
else:
os.environ["TRAVIS_TEST_RESULT"] = copy_result
def test_load_build_matrix_env_vars_parameters(self):
"""Test load_travis_env_vars, optional parameters"""
# setup Travis env vars
if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
reset_travis_vars = False
copy_os = os.environ["TRAVIS_OS_NAME"]
else:
reset_travis_vars = True
os.environ["TRAVIS"] = "true"
# temporarily remove OS VERSION
if "TRAVIS_OS_NAME" in os.environ:
reset_os = True
copy_os = os.environ["TRAVIS_OS_NAME"]
del os.environ["TRAVIS_OS_NAME"]
else:
reset_os = False
# temporarily remove PYTHON VERSION
if "TRAVIS_PYTHON_VERSION" in os.environ:
reset_python_version = True
copy_python = os.environ["TRAVIS_PYTHON_VERSION"]
del os.environ["TRAVIS_PYTHON_VERSION"]
else:
reset_python_version = False
# test optional build matrix parameters
test_parameters = [{
'env_var': 'TRAVIS_XCODE_SDK',
'parameter': 'xcode_sdk',
'test_value': "test_x_sdk"
}, {
'env_var': 'TRAVIS_XCODE_SCHEME',
'parameter': 'xcode_scheme',
'test_value': "test_x_scheme"
}, {
'env_var': 'TRAVIS_XCODE_PROJECT',
'parameter': 'xcode_project',
'test_value': "test_x_project"
}, {
'env_var': 'TRAVIS_XCODE_WORKSPACE',
'parameter': 'xcode_workspace',
'test_value': "test_x_workspace"
}, {
'env_var': 'CC',
'parameter': 'compiler',
'test_value': "test_gcc"
}, {
'env_var': 'ENV',
'parameter': 'parameters',
'test_value': "test_env"
}]
# test parameters
for parameter in test_parameters:
Settings().__init__()
settings = Settings()
if parameter['env_var'] in os.environ:
reset_travis_parameter = False
expected_param_value = os.environ[parameter['env_var']]
else:
reset_travis_parameter = True
expected_param_value = os.environ[parameter['env_var']] = \
parameter['test_value']
env_var.load_build_matrix_env_vars(settings)
self.assertDictEqual(
{
parameter["parameter"]: expected_param_value,
'summary': expected_param_value
}, settings.get_setting("build_matrix"))
# reset Travis parameters
if reset_travis_parameter:
del os.environ[parameter['env_var']]
# reset test Travis vars
if reset_travis_vars:
del os.environ["TRAVIS"]
# reset removed os name
if reset_os:
os.environ["TRAVIS_OS_NAME"] = copy_os
# reset removed python version
if reset_python_version:
os.environ["TRAVIS_PYTHON_VERSION"] = copy_python
def test_load_build_matrix_env_vars(self):
"""Test load_build_matrix_env_vars"""
settings = Settings()
self.assertEqual(None, settings.get_setting("build_matrix"))
# setup Travis env vars
if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
reset_travis_vars = False
expected_os = os.environ["TRAVIS_OS_NAME"]
copy_python = os.environ["TRAVIS_PYTHON_VERSION"]
del os.environ["TRAVIS_PYTHON_VERSION"]
else:
reset_travis_vars = True
os.environ["TRAVIS"] = "true"
expected_os = os.environ["TRAVIS_OS_NAME"] = "test_os"
# temporarily remove PYTHON VERSION
if "TRAVIS_PYTHON_VERSION" in os.environ:
reset_python_version = True
copy_python = os.environ["TRAVIS_PYTHON_VERSION"]
del os.environ["TRAVIS_PYTHON_VERSION"]
else:
reset_python_version = False
# test language and language versions
test_languages = [{
'env_var': 'TRAVIS_DART_VERSION',
'language': 'dart',
'test_value': "1.1"
}, {
'env_var': 'TRAVIS_GO_VERSION',
'language': 'go',
'test_value': "1.2"
}, {
'env_var': 'TRAVIS_HAXE_VERSION',
'language': 'haxe',
'test_value': "1.3"
}, {
'env_var': 'TRAVIS_JDK_VERSION',
'language': 'java',
'test_value': "1.4"
}, {
'env_var': 'TRAVIS_JULIA_VERSION',
'language': 'julia',
'test_value': "1.5"
}, {
'env_var': 'TRAVIS_NODE_VERSION',
'language': 'javascript',
'test_value': "1.6"
}, {
'env_var': 'TRAVIS_OTP_RELEASE',
'language': 'erlang',
'test_value': "1.7"
}, {
'env_var': 'TRAVIS_PERL_VERSION',
'language': 'perl',
'test_value': "1.8"
}, {
'env_var': 'TRAVIS_PHP_VERSION',
'language': 'php',
'test_value': "1.9"
}, {
'env_var': 'TRAVIS_PYTHON_VERSION',
'language': 'python',
'test_value': "1.10"
}, {
'env_var': 'TRAVIS_R_VERSION',
'language': 'r',
'test_value': "1.11"
}, {
'env_var': 'TRAVIS_RUBY_VERSION',
'language': 'ruby',
'test_value': "1.12"
}, {
'env_var': 'TRAVIS_RUST_VERSION',
'language': 'rust',
'test_value': "1.13"
}, {
'env_var': 'TRAVIS_SCALA_VERSION',
'language': 'scala',
'test_value': "1.14"
}]
# test languages
for language in test_languages:
if language['env_var'] in os.environ:
reset_travis_lang_version = False
expected_lang_version = os.environ[language['env_var']]
else:
reset_travis_lang_version = True
expected_lang_version = \
os.environ[language['env_var']] = language['test_value']
env_var.load_build_matrix_env_vars(settings)
self.assertDictEqual(
{
'os':
expected_os,
'language':
language['language'],
'language_version':
expected_lang_version,
'summary':
"{0!s} {1!s} {2!s}".format(language['language'],
expected_lang_version,
expected_os)
}, settings.get_setting("build_matrix"))
# reset Travis test result
if reset_travis_lang_version:
del os.environ[language['env_var']]
# reset test Travis vars
if reset_travis_vars:
del os.environ["TRAVIS"]
del os.environ["TRAVIS_OS_NAME"]
# reset removed python version
if reset_python_version:
os.environ["TRAVIS_PYTHON_VERSION"] = copy_python
