def process_end_stage(self, tags_dict):
"""
Process parsed end_stage tags.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
tag_list = list({'end_stage', 'end_substage'})
if not check_dict(tags_dict, "tags_dict", tag_list):
return False
logger.debug("End stage : %s", tags_dict)
# construct substage name
end_stagename = "{stage}.{substage}".format(
stage=tags_dict['end_stage'],
substage=tags_dict['end_substage']
)
# check if stage was started
# and if substage name matches
if not self.has_name() or self.stage.data["name"] != end_stagename:
logger.info("Substage was not started or name doesn't match")
self.finished_incomplete = True
return False
# stage finished successfully
self.finished = True
logger.info("Stage %s finished successfully", self.get_name())
return True
def process_start_stage(self, tags_dict):
"""
Process parsed start_stage tags.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
tag_list = list({'start_stage', 'start_substage'})
if not check_dict(tags_dict, "tags_dict", tag_list):
return False
logger.debug("Start stage : %s", tags_dict)
result = False
if self.has_started():
logger.info("Substage already started")
else:
name = "{stage:s}.{substage:s}".format(
stage=tags_dict['start_stage'],
substage=tags_dict['start_substage'])
result = self.set_name(name)
return result
def get_substage_name(self, command):
"""
Resolve Travis CI substage name that corresponds to a cli command.
Parameters:
- command : cli command
"""
if not tools.is_string(command):
return ""
if len(self.current_build_data) > 0 and \
"config" in self.current_build_data:
build_config = self.current_build_data["config"]
else:
logger.warning("Travis CI build config is not set")
return ""
# check if build_config collection is empty
if build_config:
for stage_name, commands in build_config.items():
if tools.is_list(commands) and command in commands:
substage_number = commands.index(command) + 1
substage_name = "{stage}.{substage:d}".format(
stage=stage_name, substage=substage_number)
logger.debug("Substage %s corresponds to '%s'",
substage_name, command)
return substage_name
return ""
def parse_travis_time_tag(self, line):
"""
Parse and process Travis CI timing tags.
Parameters:
- line : line from logfile containing Travis CI tags
"""
if self.travis_substage is None:
self.travis_substage = TravisSubstage()
escaped_line = line.replace('\x0d', '*').replace('\x1b', 'ESC')
logger.debug('line : %s', escaped_line)
# parse Travis CI timing tags
for parse_string in TRAVIS_LOG_PARSE_TIMING_STRINGS:
result = re.search(parse_string, line)
if result:
self.travis_substage.process_parsed_tags(result.groupdict())
# when finished : log stage and create a new instance
if self.travis_substage.has_finished():
# set substage name, if it is not set
if not self.travis_substage.has_name() and \
self.travis_substage.has_command():
self.travis_substage.set_name(
self.get_substage_name(
self.travis_substage.get_command()))
# only log complete substages
if not self.travis_substage.finished_incomplete:
self.current_job.add_stage(self.travis_substage.stage)
self.travis_substage = TravisSubstage()
def process_start_stage(self, tags_dict):
"""
Process parsed start_stage tags.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
tag_list = list({'start_stage', 'start_substage'})
if not check_dict(tags_dict, "tags_dict", tag_list):
return False
logger.debug("Start stage : %s", tags_dict)
result = False
if self.has_started():
logger.info("Substage already started")
else:
name = "{stage:s}.{substage:s}".format(
stage=tags_dict['start_stage'],
substage=tags_dict['start_substage']
)
result = self.set_name(name)
return result
def process_end_stage(self, tags_dict):
"""
Process parsed end_stage tags.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
tag_list = list({'end_stage', 'end_substage'})
if not check_dict(tags_dict, "tags_dict", tag_list):
return False
logger.debug("End stage : %s", tags_dict)
# construct substage name
end_stagename = "{stage}.{substage}".format(
stage=tags_dict['end_stage'], substage=tags_dict['end_substage'])
# check if stage was started
# and if substage name matches
if not self.has_name() or self.stage.data["name"] != end_stagename:
logger.info("Substage was not started or name doesn't match")
self.finished_incomplete = True
return False
# stage finished successfully
self.finished = True
logger.info("Stage %s finished successfully", self.get_name())
return True
def process_end_time(self, tags_dict):
"""
Process parsed end_time tags.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
tag_list = list({
'end_hash',
'start_timestamp',
'finish_timestamp',
'duration'
})
if not check_dict(tags_dict, "tags_dict", tag_list):
return False
logger.debug("End time : %s", tags_dict)
result = False
# check if timing was started
# and if hash matches
if (not self.has_timing_hash() or
self.timing_hash != tags_dict['end_hash']):
logger.info("Substage timing was not started or"
" hash doesn't match")
self.finished_incomplete = True
else:
set_started = set_finished = set_duration = False
# Set started timestamp
if self.stage.set_started_at_nano(tags_dict['start_timestamp']):
logger.info("Stage started at %s",
self.stage.data["started_at"]["isotimestamp"])
set_started = True
# Set finished timestamp
if self.stage.set_finished_at_nano(tags_dict['finish_timestamp']):
logger.info("Stage finished at %s",
self.stage.data["finished_at"]["isotimestamp"])
set_finished = True
# Set duration
if self.stage.set_duration_nano(tags_dict['duration']):
logger.info("Stage duration : %ss",
self.stage.data['duration'])
set_duration = True
result = set_started and set_finished and set_duration
return result
def trend_native():
"""Generate native trend with matplotlib : chart in PNG format."""
from buildtimetrend.trend import Trend
# use parameter for timestamps file and check if file exists
result_file = os.getenv('BUILD_TREND_OUTPUTFILE',
'dashboard/buildtimes.xml')
chart_file = os.getenv('BUILD_TREND_TRENDFILE', 'dashboard/trend.png')
trend = Trend()
if trend.gather_data(result_file):
# log number of builds and list of buildnames
logger.debug('Builds (%d) : %s', len(trend.builds), trend.builds)
logger.debug('Stages (%d) : %s', len(trend.stages), trend.stages)
trend.generate(chart_file)
def process_end_time(self, tags_dict):
"""
Process parsed end_time tags.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
tag_list = list(
{'end_hash', 'start_timestamp', 'finish_timestamp', 'duration'})
if not check_dict(tags_dict, "tags_dict", tag_list):
return False
logger.debug("End time : %s", tags_dict)
result = False
# check if timing was started
# and if hash matches
if (not self.has_timing_hash()
or self.timing_hash != tags_dict['end_hash']):
logger.info("Substage timing was not started or"
" hash doesn't match")
self.finished_incomplete = True
else:
set_started = set_finished = set_duration = False
# Set started timestamp
if self.stage.set_started_at_nano(tags_dict['start_timestamp']):
logger.info("Stage started at %s",
self.stage.data["started_at"]["isotimestamp"])
set_started = True
# Set finished timestamp
if self.stage.set_finished_at_nano(tags_dict['finish_timestamp']):
logger.info("Stage finished at %s",
self.stage.data["finished_at"]["isotimestamp"])
set_finished = True
# Set duration
if self.stage.set_duration_nano(tags_dict['duration']):
logger.info("Stage duration : %ss",
self.stage.data['duration'])
set_duration = True
result = set_started and set_finished and set_duration
return result
def get_job_data(self, job_id):
"""
Retrieve Travis CI job data.
Parameters:
- job_id : ID of the job to process
"""
request = 'jobs/{:s}'.format(str(job_id))
job_data = self.connector.json_request(request)
# log job_data
logger.debug("Job #%s data : %s", str(job_id),
json.dumps(job_data, sort_keys=True, indent=2))
return job_data
def check_authorization(repo, auth_header):
"""
Check if Travis CI notification has a correct Authorization header.
This check is enabled if travis_account_token is defined in settings.
More information on the Authorization header :
http://docs.travis-ci.com/user/notifications/#Authorization-for-Webhooks
Returns true if Authorization header is valid, but also if
travis_account_token is not defined.
Parameters:
- repo : git repo name
- auth_header : Travis CI notification Authorization header
"""
# get Travis account token from Settings
token = Settings().get_setting("travis_account_token")
# return True if token is not set
if token is None:
logger.info("Setting travis_account_token is not defined,"
" Travis CI notification Authorization header"
" is not checked.")
return True
# check if parameters are strings
if is_string(repo) and is_string(auth_header) and is_string(token):
# generate hash (encode string to bytes first)
auth_hash = sha256((repo + token).encode('utf-8')).hexdigest()
# compare hash with Authorization header
if auth_hash == auth_header:
logger.info("Travis CI notification Authorization header"
" is correct.")
return True
else:
logger.error("Travis CI notification Authorization header"
" is incorrect.")
return False
else:
logger.debug("repo, auth_header and travis_auth_token"
" should be strings.")
return False
def gather_data(self, result_file):
"""
Get buildtime data from an xml file.
Parameters
- result_file : xml file containing the buildtime data
"""
# load buildtimes file
if check_file(result_file):
root_xml = etree.parse(result_file).getroot()
else:
return False
index = 0
# print content of buildtimes file
for build_xml in root_xml:
build_id = build_xml.get('build')
job_id = build_xml.get('job')
if job_id is None and build_id is None:
build_name = "#{0:d}".format(index + 1)
elif job_id is not None:
build_name = job_id
else:
build_name = build_id
self.builds.append(build_name)
# add 0 to each existing stage, to make sure that
# the indexes of each value
# are correct, even if a stage does not exist in a build
# if a stage exists, the zero will be replaced by its duration
for stage in self.stages:
self.stages[stage].append(0)
# add duration of each stage to stages list
for build_child in build_xml:
if build_child.tag == 'stages':
stage_count = len(build_child)
self.parse_xml_stages(build_child, index)
logger.debug("Build ID : %s, Job : %s, stages : %d", build_id,
job_id, stage_count)
index += 1
return True
def gather_data(self, result_file):
"""
Get buildtime data from an xml file.
Parameters
- result_file : xml file containing the buildtime data
"""
# load buildtimes file
if check_file(result_file):
root_xml = etree.parse(result_file).getroot()
else:
return False
index = 0
# print content of buildtimes file
for build_xml in root_xml:
build_id = build_xml.get('build')
job_id = build_xml.get('job')
if job_id is None and build_id is None:
build_name = "#{0:d}".format(index + 1)
elif job_id is not None:
build_name = job_id
else:
build_name = build_id
self.builds.append(build_name)
# add 0 to each existing stage, to make sure that
# the indexes of each value
# are correct, even if a stage does not exist in a build
# if a stage exists, the zero will be replaced by its duration
for stage in self.stages:
self.stages[stage].append(0)
# add duration of each stage to stages list
for build_child in build_xml:
if build_child.tag == 'stages':
stage_count = len(build_child)
self.parse_xml_stages(build_child, index)
logger.debug("Build ID : %s, Job : %s, stages : %d",
build_id, job_id, stage_count)
index += 1
return True
def get_build_data(self):
"""
Retrieve Travis CI build data.
Returns true if retrieving data was succesful, false on error.
"""
request = 'repos/{repo}/builds?number={build_id}'.format(
repo=self.repo, build_id=self.build_id)
try:
self.builds_data = self.connector.json_request(request)
except (HTTPError, URLError) as msg:
logger.error("Error getting build data from Travis CI: %s", msg)
return False
# log builds_data
logger.debug("Build #%s data : %s", str(self.build_id),
json.dumps(self.builds_data, sort_keys=True, indent=2))
return True
def env_var_to_settings(self, env_var_name, settings_name):
"""
Store environment variable value as a setting.
Parameters:
- env_var_name : Name of the environment variable
- settings_name : Name of the corresponding settings value
"""
if env_var_name in os.environ:
self.add_setting(settings_name, os.environ[env_var_name])
logger.debug("Setting %s was set to %s", settings_name,
os.environ[env_var_name])
return True
else:
logger.debug(
"Setting %s was not set,"
" environment variable %s doesn't exist", settings_name,
env_var_name)
return False
def get_pct_passed_build_jobs(repo=None, interval=None):
"""
Calculate percentage of passed build jobs.
Parameters :
- repo : repo name (fe. buildtimetrend/service)
- interval : timeframe, possible values : 'week', 'month', 'year',
anything else defaults to 'week'
"""
total_jobs = get_total_build_jobs(repo, interval)
passed_jobs = get_passed_build_jobs(repo, interval)
logger.debug("passed/total build jobs : %d/%d", passed_jobs, total_jobs)
# calculate percentage if at least one job was executed
# passed is a valid number (not -1)
if total_jobs > 0 and passed_jobs >= 0:
return int(float(passed_jobs) / float(total_jobs) * 100.0)
return -1
def env_var_to_settings(self, env_var_name, settings_name):
"""
Store environment variable value as a setting.
Parameters:
- env_var_name : Name of the environment variable
- settings_name : Name of the corresponding settings value
"""
if env_var_name in os.environ:
self.add_setting(settings_name, os.environ[env_var_name])
logger.debug(
"Setting %s was set to %s",
settings_name, os.environ[env_var_name])
return True
else:
logger.debug(
"Setting %s was not set,"
" environment variable %s doesn't exist",
settings_name, env_var_name)
return False
def process_start_time(self, tags_dict):
"""
Process parsed start_time tags.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
if not check_dict(tags_dict, "tags_dict", 'start_hash'):
return False
logger.debug("Start time : %s", tags_dict)
if self.has_timing_hash():
logger.info("Substage timing already set")
return False
self.timing_hash = tags_dict['start_hash']
logger.info("Set timing hash : %s", self.timing_hash)
return True
def parse_travis_worker_tag(self, line):
"""
Parse and process Travis CI worker tag.
Parameters:
- line : line from logfile containing Travis CI tags
"""
logger.debug('line : %s', line)
# parse Travis CI worker tags
result = re.search(TRAVIS_LOG_PARSE_WORKER_STRING, line)
if not result:
return
worker_tags = result.groupdict()
# check if parameter worker_tags is a dictionary and
# if it contains all required tags
tag_list = list({'hostname', 'os'})
if tools.check_dict(worker_tags, "worker_tags", tag_list):
logger.debug("Worker tags : %s", worker_tags)
self.current_job.add_property("worker", worker_tags)
def process_start_time(self, tags_dict):
"""
Process parsed start_time tags.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
if not check_dict(tags_dict, "tags_dict", 'start_hash'):
return False
logger.debug("Start time : %s", tags_dict)
if self.has_timing_hash():
logger.info("Substage timing already set")
return False
self.timing_hash = tags_dict['start_hash']
logger.info("Set timing hash : %s", self.timing_hash)
return True
def process_command(self, tags_dict):
"""
Process parsed command tag.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
if not check_dict(tags_dict, "tags_dict", 'command'):
return False
logger.debug("Command : %s", tags_dict)
result = False
if self.has_command():
logger.info("Command is already set")
elif self.stage.set_command(tags_dict['command']):
logger.info("Set command : %s", tags_dict['command'])
result = True
return result
def process_command(self, tags_dict):
"""
Process parsed command tag.
Parameters:
- tags_dict : dictionary with parsed tags
"""
# check if parameter tags_dict is a dictionary and
# if it contains all required tags
if not check_dict(tags_dict, "tags_dict", 'command'):
return False
logger.debug("Command : %s", tags_dict)
result = False
if self.has_command():
logger.info("Command is already set")
elif self.stage.set_command(tags_dict['command']):
logger.info("Set command : %s", tags_dict['command'])
result = True
return result
