def process_end_stage(self, tags_dict): """ Process parsed end_stage tags. Parameters: - tags_dict : dictionary with parsed tags """ # check if parameter tags_dict is a dictionary and # if it contains all required tags tag_list = list({'end_stage', 'end_substage'}) if not check_dict(tags_dict, "tags_dict", tag_list): return False logger.debug("End stage : %s", tags_dict) # construct substage name end_stagename = "{stage}.{substage}".format( stage=tags_dict['end_stage'], substage=tags_dict['end_substage'] ) # check if stage was started # and if substage name matches if not self.has_name() or self.stage.data["name"] != end_stagename: logger.info("Substage was not started or name doesn't match") self.finished_incomplete = True return False # stage finished successfully self.finished = True logger.info("Stage %s finished successfully", self.get_name()) return True
def process_start_stage(self, tags_dict): """ Process parsed start_stage tags. Parameters: - tags_dict : dictionary with parsed tags """ # check if parameter tags_dict is a dictionary and # if it contains all required tags tag_list = list({'start_stage', 'start_substage'}) if not check_dict(tags_dict, "tags_dict", tag_list): return False logger.debug("Start stage : %s", tags_dict) result = False if self.has_started(): logger.info("Substage already started") else: name = "{stage:s}.{substage:s}".format( stage=tags_dict['start_stage'], substage=tags_dict['start_substage']) result = self.set_name(name) return result
def get_substage_name(self, command): """ Resolve Travis CI substage name that corresponds to a cli command. Parameters: - command : cli command """ if not tools.is_string(command): return "" if len(self.current_build_data) > 0 and \ "config" in self.current_build_data: build_config = self.current_build_data["config"] else: logger.warning("Travis CI build config is not set") return "" # check if build_config collection is empty if build_config: for stage_name, commands in build_config.items(): if tools.is_list(commands) and command in commands: substage_number = commands.index(command) + 1 substage_name = "{stage}.{substage:d}".format( stage=stage_name, substage=substage_number) logger.debug("Substage %s corresponds to '%s'", substage_name, command) return substage_name return ""
def parse_travis_time_tag(self, line): """ Parse and process Travis CI timing tags. Parameters: - line : line from logfile containing Travis CI tags """ if self.travis_substage is None: self.travis_substage = TravisSubstage() escaped_line = line.replace('\x0d', '*').replace('\x1b', 'ESC') logger.debug('line : %s', escaped_line) # parse Travis CI timing tags for parse_string in TRAVIS_LOG_PARSE_TIMING_STRINGS: result = re.search(parse_string, line) if result: self.travis_substage.process_parsed_tags(result.groupdict()) # when finished : log stage and create a new instance if self.travis_substage.has_finished(): # set substage name, if it is not set if not self.travis_substage.has_name() and \ self.travis_substage.has_command(): self.travis_substage.set_name( self.get_substage_name( self.travis_substage.get_command())) # only log complete substages if not self.travis_substage.finished_incomplete: self.current_job.add_stage(self.travis_substage.stage) self.travis_substage = TravisSubstage()
def process_start_stage(self, tags_dict): """ Process parsed start_stage tags. Parameters: - tags_dict : dictionary with parsed tags """ # check if parameter tags_dict is a dictionary and # if it contains all required tags tag_list = list({'start_stage', 'start_substage'}) if not check_dict(tags_dict, "tags_dict", tag_list): return False logger.debug("Start stage : %s", tags_dict) result = False if self.has_started(): logger.info("Substage already started") else: name = "{stage:s}.{substage:s}".format( stage=tags_dict['start_stage'], substage=tags_dict['start_substage'] ) result = self.set_name(name) return result
def process_end_stage(self, tags_dict): """ Process parsed end_stage tags. Parameters: - tags_dict : dictionary with parsed tags """ # check if parameter tags_dict is a dictionary and # if it contains all required tags tag_list = list({'end_stage', 'end_substage'}) if not check_dict(tags_dict, "tags_dict", tag_list): return False logger.debug("End stage : %s", tags_dict) # construct substage name end_stagename = "{stage}.{substage}".format( stage=tags_dict['end_stage'], substage=tags_dict['end_substage']) # check if stage was started # and if substage name matches if not self.has_name() or self.stage.data["name"] != end_stagename: logger.info("Substage was not started or name doesn't match") self.finished_incomplete = True return False # stage finished successfully self.finished = True logger.info("Stage %s finished successfully", self.get_name()) return True
def process_end_time(self, tags_dict): """ Process parsed end_time tags. Parameters: - tags_dict : dictionary with parsed tags """ # check if parameter tags_dict is a dictionary and # if it contains all required tags tag_list = list({ 'end_hash', 'start_timestamp', 'finish_timestamp', 'duration' }) if not check_dict(tags_dict, "tags_dict", tag_list): return False logger.debug("End time : %s", tags_dict) result = False # check if timing was started # and if hash matches if (not self.has_timing_hash() or self.timing_hash != tags_dict['end_hash']): logger.info("Substage timing was not started or" " hash doesn't match") self.finished_incomplete = True else: set_started = set_finished = set_duration = False # Set started timestamp if self.stage.set_started_at_nano(tags_dict['start_timestamp']): logger.info("Stage started at %s", self.stage.data["started_at"]["isotimestamp"]) set_started = True # Set finished timestamp if self.stage.set_finished_at_nano(tags_dict['finish_timestamp']): logger.info("Stage finished at %s", self.stage.data["finished_at"]["isotimestamp"]) set_finished = True # Set duration if self.stage.set_duration_nano(tags_dict['duration']): logger.info("Stage duration : %ss", self.stage.data['duration']) set_duration = True result = set_started and set_finished and set_duration return result
def trend_native(): """Generate native trend with matplotlib : chart in PNG format.""" from buildtimetrend.trend import Trend # use parameter for timestamps file and check if file exists result_file = os.getenv('BUILD_TREND_OUTPUTFILE', 'dashboard/buildtimes.xml') chart_file = os.getenv('BUILD_TREND_TRENDFILE', 'dashboard/trend.png') trend = Trend() if trend.gather_data(result_file): # log number of builds and list of buildnames logger.debug('Builds (%d) : %s', len(trend.builds), trend.builds) logger.debug('Stages (%d) : %s', len(trend.stages), trend.stages) trend.generate(chart_file)
def process_end_time(self, tags_dict): """ Process parsed end_time tags. Parameters: - tags_dict : dictionary with parsed tags """ # check if parameter tags_dict is a dictionary and # if it contains all required tags tag_list = list( {'end_hash', 'start_timestamp', 'finish_timestamp', 'duration'}) if not check_dict(tags_dict, "tags_dict", tag_list): return False logger.debug("End time : %s", tags_dict) result = False # check if timing was started # and if hash matches if (not self.has_timing_hash() or self.timing_hash != tags_dict['end_hash']): logger.info("Substage timing was not started or" " hash doesn't match") self.finished_incomplete = True else: set_started = set_finished = set_duration = False # Set started timestamp if self.stage.set_started_at_nano(tags_dict['start_timestamp']): logger.info("Stage started at %s", self.stage.data["started_at"]["isotimestamp"]) set_started = True # Set finished timestamp if self.stage.set_finished_at_nano(tags_dict['finish_timestamp']): logger.info("Stage finished at %s", self.stage.data["finished_at"]["isotimestamp"]) set_finished = True # Set duration if self.stage.set_duration_nano(tags_dict['duration']): logger.info("Stage duration : %ss", self.stage.data['duration']) set_duration = True result = set_started and set_finished and set_duration return result
def get_job_data(self, job_id): """ Retrieve Travis CI job data. Parameters: - job_id : ID of the job to process """ request = 'jobs/{:s}'.format(str(job_id)) job_data = self.connector.json_request(request) # log job_data logger.debug("Job #%s data : %s", str(job_id), json.dumps(job_data, sort_keys=True, indent=2)) return job_data
def check_authorization(repo, auth_header): """ Check if Travis CI notification has a correct Authorization header. This check is enabled if travis_account_token is defined in settings. More information on the Authorization header : http://docs.travis-ci.com/user/notifications/#Authorization-for-Webhooks Returns true if Authorization header is valid, but also if travis_account_token is not defined. Parameters: - repo : git repo name - auth_header : Travis CI notification Authorization header """ # get Travis account token from Settings token = Settings().get_setting("travis_account_token") # return True if token is not set if token is None: logger.info("Setting travis_account_token is not defined," " Travis CI notification Authorization header" " is not checked.") return True # check if parameters are strings if is_string(repo) and is_string(auth_header) and is_string(token): # generate hash (encode string to bytes first) auth_hash = sha256((repo + token).encode('utf-8')).hexdigest() # compare hash with Authorization header if auth_hash == auth_header: logger.info("Travis CI notification Authorization header" " is correct.") return True else: logger.error("Travis CI notification Authorization header" " is incorrect.") return False else: logger.debug("repo, auth_header and travis_auth_token" " should be strings.") return False
def gather_data(self, result_file): """ Get buildtime data from an xml file. Parameters - result_file : xml file containing the buildtime data """ # load buildtimes file if check_file(result_file): root_xml = etree.parse(result_file).getroot() else: return False index = 0 # print content of buildtimes file for build_xml in root_xml: build_id = build_xml.get('build') job_id = build_xml.get('job') if job_id is None and build_id is None: build_name = "#{0:d}".format(index + 1) elif job_id is not None: build_name = job_id else: build_name = build_id self.builds.append(build_name) # add 0 to each existing stage, to make sure that # the indexes of each value # are correct, even if a stage does not exist in a build # if a stage exists, the zero will be replaced by its duration for stage in self.stages: self.stages[stage].append(0) # add duration of each stage to stages list for build_child in build_xml: if build_child.tag == 'stages': stage_count = len(build_child) self.parse_xml_stages(build_child, index) logger.debug("Build ID : %s, Job : %s, stages : %d", build_id, job_id, stage_count) index += 1 return True
def get_build_data(self): """ Retrieve Travis CI build data. Returns true if retrieving data was succesful, false on error. """ request = 'repos/{repo}/builds?number={build_id}'.format( repo=self.repo, build_id=self.build_id) try: self.builds_data = self.connector.json_request(request) except (HTTPError, URLError) as msg: logger.error("Error getting build data from Travis CI: %s", msg) return False # log builds_data logger.debug("Build #%s data : %s", str(self.build_id), json.dumps(self.builds_data, sort_keys=True, indent=2)) return True
def env_var_to_settings(self, env_var_name, settings_name): """ Store environment variable value as a setting. Parameters: - env_var_name : Name of the environment variable - settings_name : Name of the corresponding settings value """ if env_var_name in os.environ: self.add_setting(settings_name, os.environ[env_var_name]) logger.debug("Setting %s was set to %s", settings_name, os.environ[env_var_name]) return True else: logger.debug( "Setting %s was not set," " environment variable %s doesn't exist", settings_name, env_var_name) return False
def get_pct_passed_build_jobs(repo=None, interval=None): """ Calculate percentage of passed build jobs. Parameters : - repo : repo name (fe. buildtimetrend/service) - interval : timeframe, possible values : 'week', 'month', 'year', anything else defaults to 'week' """ total_jobs = get_total_build_jobs(repo, interval) passed_jobs = get_passed_build_jobs(repo, interval) logger.debug("passed/total build jobs : %d/%d", passed_jobs, total_jobs) # calculate percentage if at least one job was executed # passed is a valid number (not -1) if total_jobs > 0 and passed_jobs >= 0: return int(float(passed_jobs) / float(total_jobs) * 100.0) return -1
def env_var_to_settings(self, env_var_name, settings_name): """ Store environment variable value as a setting. Parameters: - env_var_name : Name of the environment variable - settings_name : Name of the corresponding settings value """ if env_var_name in os.environ: self.add_setting(settings_name, os.environ[env_var_name]) logger.debug( "Setting %s was set to %s", settings_name, os.environ[env_var_name]) return True else: logger.debug( "Setting %s was not set," " environment variable %s doesn't exist", settings_name, env_var_name) return False
def process_start_time(self, tags_dict): """ Process parsed start_time tags. Parameters: - tags_dict : dictionary with parsed tags """ # check if parameter tags_dict is a dictionary and # if it contains all required tags if not check_dict(tags_dict, "tags_dict", 'start_hash'): return False logger.debug("Start time : %s", tags_dict) if self.has_timing_hash(): logger.info("Substage timing already set") return False self.timing_hash = tags_dict['start_hash'] logger.info("Set timing hash : %s", self.timing_hash) return True
def parse_travis_worker_tag(self, line): """ Parse and process Travis CI worker tag. Parameters: - line : line from logfile containing Travis CI tags """ logger.debug('line : %s', line) # parse Travis CI worker tags result = re.search(TRAVIS_LOG_PARSE_WORKER_STRING, line) if not result: return worker_tags = result.groupdict() # check if parameter worker_tags is a dictionary and # if it contains all required tags tag_list = list({'hostname', 'os'}) if tools.check_dict(worker_tags, "worker_tags", tag_list): logger.debug("Worker tags : %s", worker_tags) self.current_job.add_property("worker", worker_tags)
def process_command(self, tags_dict): """ Process parsed command tag. Parameters: - tags_dict : dictionary with parsed tags """ # check if parameter tags_dict is a dictionary and # if it contains all required tags if not check_dict(tags_dict, "tags_dict", 'command'): return False logger.debug("Command : %s", tags_dict) result = False if self.has_command(): logger.info("Command is already set") elif self.stage.set_command(tags_dict['command']): logger.info("Set command : %s", tags_dict['command']) result = True return result