def test_load_properties(self):
self.build.load_properties_from_settings()
self.assertDictEqual(
{'duration': 0, "repo": buildtimetrend.NAME},
self.build.get_properties())
settings = Settings()
settings.add_setting("ci_platform", "travis")
settings.add_setting("build", "123")
settings.add_setting("job", "123.1")
settings.add_setting("branch", "branch1")
settings.add_setting("result", "passed")
settings.set_project_name("test/project")
self.build.load_properties_from_settings()
self.assertDictEqual(
{ 'duration': 0,
'ci_platform': "travis",
'build': "123",
'job': "123.1",
'branch': "branch1",
'result': "passed",
'repo': "test/project"},
self.build.get_properties())
def get_read_key():
'''
Generate a read key for the project and print that key
'''
settings = Settings()
settings.load_config_file("config.yml")
# get project name from argument
if len(sys.argv) > 1:
settings.set_project_name(sys.argv[1])
# generate a read key
print keen_io_generate_read_key(settings.get_project_name())
def process_notification_payload(payload):
"""
Extract repo slug and build number from Travis notification payload.
Returns a dictionary with "repo" and "build" information,
or an empty dictionary if the payload could not be processed.
Deprecated behaviour : Currently the repo and build information are
also stored in the "settings" object,
but this will be removed in the near future.
Parameters:
- payload : Travis CI notification payload
"""
settings = Settings()
parameters = {}
if payload is None:
logger.warning("Travis notification payload is not set")
return parameters
if not is_string(payload):
logger.warning(
"Travis notification payload is incorrect :"
" string expected, got %s", type(payload))
return parameters
json_payload = json.loads(payload)
logger.info("Travis Payload : %r.", json_payload)
# get repo name from payload
if ("repository" in json_payload
and "owner_name" in json_payload["repository"]
and "name" in json_payload["repository"]):
repo = get_repo_slug(json_payload["repository"]["owner_name"],
json_payload["repository"]["name"])
logger.info("Build repo : %s", repo)
settings.set_project_name(repo)
parameters["repo"] = repo
# get build number from payload
if "number" in json_payload:
logger.info("Build number : %s", str(json_payload["number"]))
settings.add_setting('build', json_payload['number'])
parameters["build"] = json_payload['number']
return parameters
def get_read_key(argv):
"""Generate a read key for the project and print that key."""
settings = Settings()
# load settings from config file, env_var and cli parameters
args = settings.load_settings(argv)
# exit script if processing cli parameters failed (result = None)
if args is None:
return
# get project name from argument
# check if collection is empty
if args:
settings.set_project_name(args[0])
# generate a read key
print(keenio.generate_read_key(settings.get_project_name()))
def test_load_properties(self):
"""Test loading properties"""
self.build.load_properties_from_settings()
self.assertDictEqual(
{'duration': 0, "repo": buildtimetrend.NAME},
self.build.get_properties())
settings = Settings()
settings.add_setting("ci_platform", "travis")
settings.add_setting("build", "123")
settings.add_setting("job", "123.1")
settings.add_setting("branch", "branch1")
settings.add_setting("result", "passed")
settings.add_setting("build_trigger", "push")
settings.add_setting(
"pull_request",
{
"is_pull_request": False,
"title": None,
"number": None
}
)
settings.set_project_name("test/project")
self.build.load_properties_from_settings()
self.assertDictEqual(
{
'duration': 0,
'ci_platform': "travis",
'build': "123",
'job': "123.1",
'branch': "branch1",
'result': "passed",
'build_trigger': "push",
'pull_request': {
"is_pull_request": False,
"title": None,
"number": None},
'repo': "test/project"
},
self.build.get_properties())
def test_load_properties(self):
"""Test loading properties"""
self.build.load_properties_from_settings()
self.assertDictEqual({
'duration': 0,
"repo": buildtimetrend.NAME
}, self.build.get_properties())
settings = Settings()
settings.add_setting("ci_platform", "travis")
settings.add_setting("build", "123")
settings.add_setting("job", "123.1")
settings.add_setting("branch", "branch1")
settings.add_setting("result", "passed")
settings.add_setting("build_trigger", "push")
settings.add_setting("pull_request", {
"is_pull_request": False,
"title": None,
"number": None
})
settings.set_project_name("test/project")
self.build.load_properties_from_settings()
self.assertDictEqual(
{
'duration': 0,
'ci_platform': "travis",
'build': "123",
'job': "123.1",
'branch': "branch1",
'result': "passed",
'build_trigger': "push",
'pull_request': {
"is_pull_request": False,
"title": None,
"number": None
},
'repo': "test/project"
}, self.build.get_properties())
class TravisParser(object):
"""
Travis CI build timing and build data parser.
Retrieve timing data from Travis CI, parse it and store it in Keen.io.
"""
def __init__(self):
"""Initialise class."""
self.settings = Settings()
# get logger
self.logger = logger
@cherrypy.expose
def default(self,
repo_owner=None,
repo_name=None,
first_build=None,
last_build=None,
payload=None):
"""
Default handler.
Visiting this page triggers loading and processing the build log
and data of a travis CI build process.
If last_build is defined, all builds from first_build until last_build
will be retrieved and processed.
If only payload is defined, repo and build data
will be extracted from the payload.
Parameters:
- repo_owner : name of the Github repo owner, fe. `buildtimetrend`
- repo_name : name of the Github repo, fe. `service`
- first_build : first build number to process (int)
- last_build : last build number to process (int)
- payload : Travis CI notification payload (json)
"""
cherrypy.response.headers['Content-Type'] = 'text/plain'
# reset settings
self.settings.set_project_name(None)
self.settings.add_setting('build', None)
self.logger.debug("Check Travis headers : %r",
cherrypy.request.headers)
repo = get_repo_slug(repo_owner, repo_name)
# load parameters from the Travis notification payload
if self.check_travis_notification():
payload_params = process_notification_payload(payload)
# assign payload parameters
if repo is None and "repo" in payload_params:
repo = payload_params["repo"]
if first_build is None and last_build is None and \
"build" in payload_params:
first_build = payload_params["build"]
# check parameter validity, check returns error message
# or None if parameters are valid
params_valid = validate_travis_request(repo, first_build)
if params_valid is not None:
self.logger.warning(params_valid)
return params_valid
if last_build is None:
self.logger.warning("Request to process build #%s of repo %s",
first_build, repo)
# schedule task with 10 second delay to give Travis CI time
# to add the finished_at property. (issue #96)
return self.schedule_task(repo, first_build, 10)
return self.multi_build(repo, first_build, last_build)
def multi_build(self, repo, first_build, last_build):
"""
Schedule processing multiple consecutive builds.
All builds from first_build until last_build
will be retrieved and processed.
The total number of builds to be scheduled is limited by the
`multi_import.max_builds` config parameter.
Every next scheduled build will be delayed by the
`multi_import.delay` config parameter.
Parameters:
- repo : repo name (fe. buildtimetrend/service)
- first_build : first build number to process (int)
- last_build : last build number to process (int)
"""
first_build = int(first_build)
last_build = int(last_build)
message = ""
multi_import = Settings().get_setting("multi_import")
max_multi_builds = int(multi_import["max_builds"])
if last_build < first_build:
tmp_msg = "Warning : last_build should be equal" \
" or larger than first_build"
self.logger.warning(tmp_msg)
message += tmp_msg + "\n"
last_build = first_build
if (last_build - first_build) > max_multi_builds:
tmp_msg = "Warning : number of multiple builds is limited to {:d}"
self.logger.warning(tmp_msg.format(max_multi_builds))
message += tmp_msg.format(max_multi_builds) + "\n"
last_build = first_build + max_multi_builds
message += "Request to process build(s) #{first_build:d} to" \
" #{last_build:d} of repo {repo}:\n".format(**locals())
build = first_build
delay = 0
while build <= last_build:
message += self.schedule_task(repo, build, delay) + "\n"
delay += multi_import["delay"]
build += 1
return message
def schedule_task(self, repo, build, delay=0):
"""
Schedule task.
Parameters:
- repo : repo name (fe. buildtimetrend/service)
- build : build number to process (int)
- delay : delay before task should be started, in seconds
"""
# process travis build
if is_worker_enabled():
task = tasks.process_travis_buildlog.apply_async(
(repo, build), countdown=int(delay))
temp_msg = "Task scheduled to process build #{build}" \
" of repo {repo} : {task_id}"
self.logger.warning(
temp_msg.format(build=build, repo=repo, task_id=task.id))
return temp_msg.format(build=cgi.escape(str(build)),
repo=cgi.escape(str(repo)),
task_id=cgi.escape(str(task.id)))
else:
return tasks.process_travis_buildlog(repo, build)
def check_travis_notification(self):
"""
Check Travis CI notification request.
Load Authorization and Travis-Repo-Slug headers and check if
the Authorization header is correct
"""
if "Authorization" not in cherrypy.request.headers:
self.logger.debug("Authorization header is not set")
return True
if "Travis-Repo-Slug" not in cherrypy.request.headers:
self.logger.debug("Travis-Repo-Slug header is not set")
return False
return check_authorization(
cherrypy.request.headers["Travis-Repo-Slug"],
cherrypy.request.headers["Authorization"])
class TestSettings(unittest.TestCase):
@classmethod
def setUpClass(self):
self.settings = Settings()
self.project_name = buildtimetrend.NAME
self.project_info = {
"version": buildtimetrend.VERSION,
"schema_version": buildtimetrend.SCHEMA_VERSION,
"project_name": self.project_name}
def setUp(self):
# reinit settings singleton
if self.settings is not None:
self.settings.__init__()
def test_get_project_info(self):
self.assertDictEqual(self.project_info, self.settings.get_project_info())
def test_get_set_project_name(self):
self.assertEquals(self.project_name, self.settings.get_project_name())
self.settings.set_project_name("test_name")
self.assertEquals("test_name", self.settings.get_project_name())
self.settings.set_project_name(None)
self.assertEquals(None, self.settings.get_project_name())
self.settings.set_project_name("")
self.assertEquals("", self.settings.get_project_name())
def test_get_add_setting(self):
# setting is not set yet
self.assertEquals(None, self.settings.get_setting("test_name"))
self.settings.add_setting("test_name", "test_value")
self.assertEquals("test_value", self.settings.get_setting("test_name"))
self.settings.add_setting("test_name", None)
self.assertEquals(None, self.settings.get_setting("test_name"))
self.settings.add_setting("test_name", "")
self.assertEquals("", self.settings.get_setting("test_name"))
self.settings.add_setting("test_name", 6)
self.assertEquals(6, self.settings.get_setting("test_name"))
def test_get_setting(self):
self.assertEquals(None, self.settings.get_setting("test_name"))
self.assertEquals(
self.project_name,
self.settings.get_setting("project_name"))
self.assertDictEqual(
{
"project_name": self.project_name,
"mode_native": False,
"mode_keen": True
},
self.settings.settings.get_items())
def test_no_config_file(self):
# function should return false when file doesn't exist
self.assertFalse(self.settings.load_config_file('no_file.yml'))
self.assertDictEqual(
{
"project_name": self.project_name,
"mode_native": False,
"mode_keen": True
},
self.settings.settings.get_items())
self.assertFalse(self.settings.load_config_file(''))
self.assertDictEqual(
{
"project_name": self.project_name,
"mode_native": False,
"mode_keen": True
},
self.settings.settings.get_items())
# function should throw an error when no filename is set
self.assertRaises(TypeError, self.settings.load_config_file)
def test_load_config_file(self):
# checking if Keen.io configuration is not set (yet)
self.assertEquals(None, keen.project_id)
self.assertEquals(None, keen.write_key)
self.assertEquals(None, keen.read_key)
# load sample config file
self.assertTrue(self.settings.load_config_file(constants.TEST_SAMPLE_CONFIG_FILE))
self.assertDictEqual(
{"project_name": "test_project",
"mode_native": True,
"mode_keen": False,
"setting1": "test_value1"},
self.settings.settings.get_items())
# checking if Keen.io configuration is set
self.assertEquals("1234", keen.project_id)
self.assertEquals("12345678", keen.write_key)
self.assertEquals("abcdefg", keen.read_key)
self.assertTrue(keen_is_readable())
self.assertTrue(keen_is_writable())
