def test_check_dict(self):
"""Test check_dict()"""
# error is thrown when called without parameters
self.assertRaises(TypeError, check_dict)
# error is thrown when called with an invalid parameter
with self.assertRaises(TypeError) as exc_msg:
check_dict(None, "name")
self.assertEqual("param name should be a dictionary",
str(exc_msg.exception))
with self.assertRaises(TypeError) as exc_msg:
check_dict("string", "string_name")
self.assertEqual("param string_name should be a dictionary",
str(exc_msg.exception))
# should return true if parameter is a dictionary
self.assertTrue(check_dict({"string": "test"}, "name"))
# should return false if parameter is not a list (and name is not set)
self.assertFalse(check_dict(None))
self.assertFalse(check_dict("string"))
# should return true if key is found in dictionary
self.assertTrue(check_dict({"string": "test"}, "name", "string"))
self.assertTrue(
check_dict({
"string": "test",
7: "test"
}, "name", list({7, "string"})))
# should return false if key is not found in dictionary
self.assertFalse(check_dict({"string": "test"}, "name", list({7})))
def add_project_info_dict(payload):
"""
Add project info to a dictonary.
Param payload: dictonary payload
"""
# check if payload is a dictionary, throws an exception if it isn't
check_dict(payload, "payload")
payload_as_dict = copy.deepcopy(payload)
payload_as_dict[KEEN_PROJECT_INFO_NAME] = Settings().get_project_info()
if "job" in payload:
# override project_name, set to build_job repo
if "repo" in payload["job"]:
payload_as_dict[KEEN_PROJECT_INFO_NAME]["project_name"] = \
payload["job"]["repo"]
# override timestamp, set to finished_at timestamp
if "finished_at" in payload["job"]:
payload_as_dict["keen"] = {
"timestamp": payload["job"]["finished_at"]["isotimestamp"]
}
return payload_as_dict
def get_latest_buildtime(repo=None):
"""
Query Keen.io database and retrieve buildtime duration of last build.
Parameters :
- repo : repo name (fe. buildtimetrend/python-lib)
"""
if repo is None or not is_readable():
return -1
try:
result = keen.extraction(
"build_jobs",
property_names="job.duration",
latest=1,
filters=[get_repo_filter(repo)]
)
except requests.ConnectionError:
logger.error("Connection to Keen.io API failed")
return -1
except keen.exceptions.KeenApiError as msg:
logger.error("Error in keenio.get_latest_buildtime() : " + str(msg))
return -1
if is_list(result) and len(result) > 0 and \
check_dict(result[0], None, ['job']) and \
check_dict(result[0]['job'], None, ['duration']):
return result[0]['job']['duration']
return -1
def add_item(self, name, value):
"""
Add an item to the collection.
Parameters :
- name : Item name
- value : Item value
"""
if check_dict(value) and name in self.items and \
check_dict(self.items[name]):
self.items[name].update(value)
else:
self.items[name] = value
def is_worker_enabled():
"""Check if a task queue is configured."""
task_queue = Settings().get_setting("task_queue")
# use double not to force boolean evaluation
return check_dict(task_queue, key_list=["broker_url", "backend"]) and \
not not task_queue["broker_url"] and \
not not task_queue["backend"]
def process_start_stage(self, tags_dict):
"""
Process parsed start_stage tags.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
tag_list = list({'start_stage', 'start_substage'})
if not check_dict(tags_dict, "tags_dict", tag_list):
return False
logger.debug("Start stage : %s", tags_dict)
result = False
if self.has_started():
logger.info("Substage already started")
else:
name = "{stage:s}.{substage:s}".format(
stage=tags_dict['start_stage'],
substage=tags_dict['start_substage'])
result = self.set_name(name)
return result
def process_end_stage(self, tags_dict):
"""
Process parsed end_stage tags.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
tag_list = list({'end_stage', 'end_substage'})
if not check_dict(tags_dict, "tags_dict", tag_list):
return False
logger.debug("End stage : %s", tags_dict)
# construct substage name
end_stagename = "{stage}.{substage}".format(
stage=tags_dict['end_stage'], substage=tags_dict['end_substage'])
# check if stage was started
# and if substage name matches
if not self.has_name() or self.stage.data["name"] != end_stagename:
logger.info("Substage was not started or name doesn't match")
self.finished_incomplete = True
return False
# stage finished successfully
self.finished = True
logger.info("Stage %s finished successfully", self.get_name())
return True
def process_end_stage(self, tags_dict):
"""
Process parsed end_stage tags.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
tag_list = list({'end_stage', 'end_substage'})
if not check_dict(tags_dict, "tags_dict", tag_list):
return False
logger.debug("End stage : %s", tags_dict)
# construct substage name
end_stagename = "{stage}.{substage}".format(
stage=tags_dict['end_stage'],
substage=tags_dict['end_substage']
)
# check if stage was started
# and if substage name matches
if not self.has_name() or self.stage.data["name"] != end_stagename:
logger.info("Substage was not started or name doesn't match")
self.finished_incomplete = True
return False
# stage finished successfully
self.finished = True
logger.info("Stage %s finished successfully", self.get_name())
return True
def process_start_stage(self, tags_dict):
"""
Process parsed start_stage tags.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
tag_list = list({'start_stage', 'start_substage'})
if not check_dict(tags_dict, "tags_dict", tag_list):
return False
logger.debug("Start stage : %s", tags_dict)
result = False
if self.has_started():
logger.info("Substage already started")
else:
name = "{stage:s}.{substage:s}".format(
stage=tags_dict['start_stage'],
substage=tags_dict['start_substage']
)
result = self.set_name(name)
return result
def process_start_stage(self, tags_dict):
'''
Processes parsed start_stage tags
Parameters:
- tags_dict : dictionary with parsed tags
'''
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
tag_list = list({'start_stage', 'start_substage'})
if not check_dict(tags_dict, "tags_dict", tag_list):
return False
logger = get_logger()
logger.debug("Start stage : %s", tags_dict)
result = False
if self.has_started():
logger.warning("Substage already started")
else:
name = "%s.%s" % (
tags_dict['start_stage'], tags_dict['start_substage']
)
result = self.stage.set_name(name)
return result
def add_items(self, items_dict):
"""
Add items as a dictionary to the collection.
Parameters:
- items_dict : dictionary with items
"""
if check_dict(items_dict, "items_dict"):
# append dictionary with items to the existing collection
self.items.update(items_dict)
def process_end_time(self, tags_dict):
"""
Process parsed end_time tags.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
tag_list = list({
'end_hash',
'start_timestamp',
'finish_timestamp',
'duration'
})
if not check_dict(tags_dict, "tags_dict", tag_list):
return False
logger.debug("End time : %s", tags_dict)
result = False
# check if timing was started
# and if hash matches
if (not self.has_timing_hash() or
self.timing_hash != tags_dict['end_hash']):
logger.info("Substage timing was not started or"
" hash doesn't match")
self.finished_incomplete = True
else:
set_started = set_finished = set_duration = False
# Set started timestamp
if self.stage.set_started_at_nano(tags_dict['start_timestamp']):
logger.info("Stage started at %s",
self.stage.data["started_at"]["isotimestamp"])
set_started = True
# Set finished timestamp
if self.stage.set_finished_at_nano(tags_dict['finish_timestamp']):
logger.info("Stage finished at %s",
self.stage.data["finished_at"]["isotimestamp"])
set_finished = True
# Set duration
if self.stage.set_duration_nano(tags_dict['duration']):
logger.info("Stage duration : %ss",
self.stage.data['duration'])
set_duration = True
result = set_started and set_finished and set_duration
return result
def test_check_dict(self):
"""Test check_dict()"""
# error is thrown when called without parameters
self.assertRaises(TypeError, check_dict)
# error is thrown when called with an invalid parameter
with self.assertRaises(TypeError) as exc_msg:
check_dict(None, "name")
self.assertEqual(
"param name should be a dictionary", str(exc_msg.exception)
)
with self.assertRaises(TypeError) as exc_msg:
check_dict("string", "string_name")
self.assertEqual(
"param string_name should be a dictionary", str(exc_msg.exception)
)
# should return true if parameter is a dictionary
self.assertTrue(check_dict({"string": "test"}, "name"))
# should return false if parameter is not a list (and name is not set)
self.assertFalse(check_dict(None))
self.assertFalse(check_dict("string"))
# should return true if key is found in dictionary
self.assertTrue(check_dict({"string": "test"}, "name", "string"))
self.assertTrue(check_dict(
{"string": "test", 7: "test"},
"name",
list({7, "string"})
))
# should return false if key is not found in dictionary
self.assertFalse(check_dict(
{"string": "test"},
"name",
list({7})
))
def process_end_time(self, tags_dict):
"""
Process parsed end_time tags.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
tag_list = list(
{'end_hash', 'start_timestamp', 'finish_timestamp', 'duration'})
if not check_dict(tags_dict, "tags_dict", tag_list):
return False
logger.debug("End time : %s", tags_dict)
result = False
# check if timing was started
# and if hash matches
if (not self.has_timing_hash()
or self.timing_hash != tags_dict['end_hash']):
logger.info("Substage timing was not started or"
" hash doesn't match")
self.finished_incomplete = True
else:
set_started = set_finished = set_duration = False
# Set started timestamp
if self.stage.set_started_at_nano(tags_dict['start_timestamp']):
logger.info("Stage started at %s",
self.stage.data["started_at"]["isotimestamp"])
set_started = True
# Set finished timestamp
if self.stage.set_finished_at_nano(tags_dict['finish_timestamp']):
logger.info("Stage finished at %s",
self.stage.data["finished_at"]["isotimestamp"])
set_finished = True
# Set duration
if self.stage.set_duration_nano(tags_dict['duration']):
logger.info("Stage duration : %ss",
self.stage.data['duration'])
set_duration = True
result = set_started and set_finished and set_duration
return result
def _handle_request(self, request, params=None):
"""
Retrieve Travis CI data using API.
Parameters:
- request : request to be sent to API
- params : HTTP request parameters
"""
request_url = self.api_url + request
request_params = self.request_params.copy()
if params is not None and check_dict(params, "params"):
request_params.update(params)
req = Request(request_url, None, request_params)
opener = build_opener()
logger.info("Request from Travis CI API : %s", request_url)
return opener.open(req)
def add_project_info_dict(payload):
'''
Adds project info to a dictonary
Param payload: dictonary payload
'''
if not check_dict(payload, "payload"):
return None
payload_as_dict = copy.deepcopy(payload)
payload_as_dict["buildtime_trend"] = Settings().get_project_info()
# override timestamp, set to finished_at timestamp
if "build" in payload and "finished_at" in payload["build"]:
payload_as_dict["keen"] = {
"timestamp": payload["build"]["finished_at"]["isotimestamp"]
}
return payload_as_dict
def parse_travis_worker_tag(self, line):
'''
Parse and process Travis CI worker tag
Param line : line from logfile containing Travis CI tags
'''
get_logger().debug('line : %s', line)
# parse Travis CI worker tags
result = re.search(TRAVIS_LOG_PARSE_WORKER_STRING, line)
if not result:
return
worker_tags = result.groupdict()
# check if parameter worker_tags is a dictionary and
# if it contains all required tags
tag_list = list({'hostname', 'os'})
if check_dict(worker_tags, "worker_tags", tag_list):
get_logger().debug("Worker tags : %s", worker_tags)
self.current_job.add_property("worker", worker_tags)
def parse_travis_worker_tag(self, line):
"""
Parse and process Travis CI worker tag.
Parameters:
- line : line from logfile containing Travis CI tags
"""
logger.debug('line : %s', line)
# parse Travis CI worker tags
result = re.search(TRAVIS_LOG_PARSE_WORKER_STRING, line)
if not result:
return
worker_tags = result.groupdict()
# check if parameter worker_tags is a dictionary and
# if it contains all required tags
tag_list = list({'hostname', 'os'})
if tools.check_dict(worker_tags, "worker_tags", tag_list):
logger.debug("Worker tags : %s", worker_tags)
self.current_job.add_property("worker", worker_tags)
def process_start_time(self, tags_dict):
"""
Process parsed start_time tags.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
if not check_dict(tags_dict, "tags_dict", 'start_hash'):
return False
logger.debug("Start time : %s", tags_dict)
if self.has_timing_hash():
logger.info("Substage timing already set")
return False
self.timing_hash = tags_dict['start_hash']
logger.info("Set timing hash : %s", self.timing_hash)
return True
def process_start_time(self, tags_dict):
"""
Process parsed start_time tags.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
if not check_dict(tags_dict, "tags_dict", 'start_hash'):
return False
logger.debug("Start time : %s", tags_dict)
if self.has_timing_hash():
logger.info("Substage timing already set")
return False
self.timing_hash = tags_dict['start_hash']
logger.info("Set timing hash : %s", self.timing_hash)
return True
def process_parsed_tags(self, tags_dict):
'''
Processes parsed tags and calls the corresponding handler method
Parameters:
- tags_dict : dictionary with parsed tags
'''
result = False
# check if parameter tags_dict is a dictionary
if check_dict(tags_dict, "tags_dict"):
if 'start_stage' in tags_dict:
result = self.process_start_stage(tags_dict)
elif 'start_hash' in tags_dict:
result = self.process_start_time(tags_dict)
elif 'command' in tags_dict:
result = self.process_command(tags_dict)
elif 'end_hash' in tags_dict:
result = self.process_end_time(tags_dict)
elif 'end_stage' in tags_dict:
result = self.process_end_stage(tags_dict)
return result
def _handle_request(self, request, params=None):
"""
Retrieve Travis CI data using API.
Parameters:
- request : request to be sent to API
- params : HTTP request parameters
"""
request_url = self.api_url + request
request_params = self.request_params.copy()
if params is not None and check_dict(params, "params"):
request_params.update(params)
req = Request(
request_url,
None,
request_params
)
opener = build_opener()
logger.info("Request from Travis CI API : %s", request_url)
return opener.open(req)
def process_command(self, tags_dict):
"""
Process parsed command tag.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
if not check_dict(tags_dict, "tags_dict", 'command'):
return False
logger.debug("Command : %s", tags_dict)
result = False
if self.has_command():
logger.info("Command is already set")
elif self.stage.set_command(tags_dict['command']):
logger.info("Set command : %s", tags_dict['command'])
result = True
return result
def process_command(self, tags_dict):
"""
Process parsed command tag.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
if not check_dict(tags_dict, "tags_dict", 'command'):
return False
logger.debug("Command : %s", tags_dict)
result = False
if self.has_command():
logger.info("Command is already set")
elif self.stage.set_command(tags_dict['command']):
logger.info("Set command : %s", tags_dict['command'])
result = True
return result
def process_parsed_tags(self, tags_dict):
"""
Process parsed tags and calls the corresponding handler method.
Parameters:
- tags_dict : dictionary with parsed tags
"""
result = False
# check if parameter tags_dict is a dictionary
if check_dict(tags_dict, "tags_dict"):
if 'start_stage' in tags_dict:
result = self.process_start_stage(tags_dict)
elif 'start_hash' in tags_dict:
result = self.process_start_time(tags_dict)
elif 'command' in tags_dict:
result = self.process_command(tags_dict)
elif 'end_hash' in tags_dict:
result = self.process_end_time(tags_dict)
elif 'end_stage' in tags_dict:
result = self.process_end_stage(tags_dict)
return result
def get_finished_at(self):
"""Retrieve timestamp when build finished."""
if tools.check_dict(self.current_build_data, key_list=["finished_at"]):
return self.current_build_data['finished_at']
else:
return None