def test_check_dict(self): """Test check_dict()""" # error is thrown when called without parameters self.assertRaises(TypeError, check_dict) # error is thrown when called with an invalid parameter with self.assertRaises(TypeError) as exc_msg: check_dict(None, "name") self.assertEqual("param name should be a dictionary", str(exc_msg.exception)) with self.assertRaises(TypeError) as exc_msg: check_dict("string", "string_name") self.assertEqual("param string_name should be a dictionary", str(exc_msg.exception)) # should return true if parameter is a dictionary self.assertTrue(check_dict({"string": "test"}, "name")) # should return false if parameter is not a list (and name is not set) self.assertFalse(check_dict(None)) self.assertFalse(check_dict("string")) # should return true if key is found in dictionary self.assertTrue(check_dict({"string": "test"}, "name", "string")) self.assertTrue( check_dict({ "string": "test", 7: "test" }, "name", list({7, "string"}))) # should return false if key is not found in dictionary self.assertFalse(check_dict({"string": "test"}, "name", list({7})))
def add_project_info_dict(payload): """ Add project info to a dictonary. Param payload: dictonary payload """ # check if payload is a dictionary, throws an exception if it isn't check_dict(payload, "payload") payload_as_dict = copy.deepcopy(payload) payload_as_dict[KEEN_PROJECT_INFO_NAME] = Settings().get_project_info() if "job" in payload: # override project_name, set to build_job repo if "repo" in payload["job"]: payload_as_dict[KEEN_PROJECT_INFO_NAME]["project_name"] = \ payload["job"]["repo"] # override timestamp, set to finished_at timestamp if "finished_at" in payload["job"]: payload_as_dict["keen"] = { "timestamp": payload["job"]["finished_at"]["isotimestamp"] } return payload_as_dict
def get_latest_buildtime(repo=None): """ Query Keen.io database and retrieve buildtime duration of last build. Parameters : - repo : repo name (fe. buildtimetrend/python-lib) """ if repo is None or not is_readable(): return -1 try: result = keen.extraction( "build_jobs", property_names="job.duration", latest=1, filters=[get_repo_filter(repo)] ) except requests.ConnectionError: logger.error("Connection to Keen.io API failed") return -1 except keen.exceptions.KeenApiError as msg: logger.error("Error in keenio.get_latest_buildtime() : " + str(msg)) return -1 if is_list(result) and len(result) > 0 and \ check_dict(result[0], None, ['job']) and \ check_dict(result[0]['job'], None, ['duration']): return result[0]['job']['duration'] return -1
def add_item(self, name, value): """ Add an item to the collection. Parameters : - name : Item name - value : Item value """ if check_dict(value) and name in self.items and \ check_dict(self.items[name]): self.items[name].update(value) else: self.items[name] = value
def is_worker_enabled(): """Check if a task queue is configured.""" task_queue = Settings().get_setting("task_queue") # use double not to force boolean evaluation return check_dict(task_queue, key_list=["broker_url", "backend"]) and \ not not task_queue["broker_url"] and \ not not task_queue["backend"]
def process_start_stage(self, tags_dict): """ Process parsed start_stage tags. Parameters: - tags_dict : dictionary with parsed tags """ # check if parameter tags_dict is a dictionary and # if it contains all required tags tag_list = list({'start_stage', 'start_substage'}) if not check_dict(tags_dict, "tags_dict", tag_list): return False logger.debug("Start stage : %s", tags_dict) result = False if self.has_started(): logger.info("Substage already started") else: name = "{stage:s}.{substage:s}".format( stage=tags_dict['start_stage'], substage=tags_dict['start_substage']) result = self.set_name(name) return result
def process_end_stage(self, tags_dict): """ Process parsed end_stage tags. Parameters: - tags_dict : dictionary with parsed tags """ # check if parameter tags_dict is a dictionary and # if it contains all required tags tag_list = list({'end_stage', 'end_substage'}) if not check_dict(tags_dict, "tags_dict", tag_list): return False logger.debug("End stage : %s", tags_dict) # construct substage name end_stagename = "{stage}.{substage}".format( stage=tags_dict['end_stage'], substage=tags_dict['end_substage']) # check if stage was started # and if substage name matches if not self.has_name() or self.stage.data["name"] != end_stagename: logger.info("Substage was not started or name doesn't match") self.finished_incomplete = True return False # stage finished successfully self.finished = True logger.info("Stage %s finished successfully", self.get_name()) return True
def process_end_stage(self, tags_dict): """ Process parsed end_stage tags. Parameters: - tags_dict : dictionary with parsed tags """ # check if parameter tags_dict is a dictionary and # if it contains all required tags tag_list = list({'end_stage', 'end_substage'}) if not check_dict(tags_dict, "tags_dict", tag_list): return False logger.debug("End stage : %s", tags_dict) # construct substage name end_stagename = "{stage}.{substage}".format( stage=tags_dict['end_stage'], substage=tags_dict['end_substage'] ) # check if stage was started # and if substage name matches if not self.has_name() or self.stage.data["name"] != end_stagename: logger.info("Substage was not started or name doesn't match") self.finished_incomplete = True return False # stage finished successfully self.finished = True logger.info("Stage %s finished successfully", self.get_name()) return True
def process_start_stage(self, tags_dict): """ Process parsed start_stage tags. Parameters: - tags_dict : dictionary with parsed tags """ # check if parameter tags_dict is a dictionary and # if it contains all required tags tag_list = list({'start_stage', 'start_substage'}) if not check_dict(tags_dict, "tags_dict", tag_list): return False logger.debug("Start stage : %s", tags_dict) result = False if self.has_started(): logger.info("Substage already started") else: name = "{stage:s}.{substage:s}".format( stage=tags_dict['start_stage'], substage=tags_dict['start_substage'] ) result = self.set_name(name) return result
def process_start_stage(self, tags_dict): ''' Processes parsed start_stage tags Parameters: - tags_dict : dictionary with parsed tags ''' # check if parameter tags_dict is a dictionary and # if it contains all required tags tag_list = list({'start_stage', 'start_substage'}) if not check_dict(tags_dict, "tags_dict", tag_list): return False logger = get_logger() logger.debug("Start stage : %s", tags_dict) result = False if self.has_started(): logger.warning("Substage already started") else: name = "%s.%s" % ( tags_dict['start_stage'], tags_dict['start_substage'] ) result = self.stage.set_name(name) return result
def add_items(self, items_dict): """ Add items as a dictionary to the collection. Parameters: - items_dict : dictionary with items """ if check_dict(items_dict, "items_dict"): # append dictionary with items to the existing collection self.items.update(items_dict)
def process_end_time(self, tags_dict): """ Process parsed end_time tags. Parameters: - tags_dict : dictionary with parsed tags """ # check if parameter tags_dict is a dictionary and # if it contains all required tags tag_list = list({ 'end_hash', 'start_timestamp', 'finish_timestamp', 'duration' }) if not check_dict(tags_dict, "tags_dict", tag_list): return False logger.debug("End time : %s", tags_dict) result = False # check if timing was started # and if hash matches if (not self.has_timing_hash() or self.timing_hash != tags_dict['end_hash']): logger.info("Substage timing was not started or" " hash doesn't match") self.finished_incomplete = True else: set_started = set_finished = set_duration = False # Set started timestamp if self.stage.set_started_at_nano(tags_dict['start_timestamp']): logger.info("Stage started at %s", self.stage.data["started_at"]["isotimestamp"]) set_started = True # Set finished timestamp if self.stage.set_finished_at_nano(tags_dict['finish_timestamp']): logger.info("Stage finished at %s", self.stage.data["finished_at"]["isotimestamp"]) set_finished = True # Set duration if self.stage.set_duration_nano(tags_dict['duration']): logger.info("Stage duration : %ss", self.stage.data['duration']) set_duration = True result = set_started and set_finished and set_duration return result
def test_check_dict(self): """Test check_dict()""" # error is thrown when called without parameters self.assertRaises(TypeError, check_dict) # error is thrown when called with an invalid parameter with self.assertRaises(TypeError) as exc_msg: check_dict(None, "name") self.assertEqual( "param name should be a dictionary", str(exc_msg.exception) ) with self.assertRaises(TypeError) as exc_msg: check_dict("string", "string_name") self.assertEqual( "param string_name should be a dictionary", str(exc_msg.exception) ) # should return true if parameter is a dictionary self.assertTrue(check_dict({"string": "test"}, "name")) # should return false if parameter is not a list (and name is not set) self.assertFalse(check_dict(None)) self.assertFalse(check_dict("string")) # should return true if key is found in dictionary self.assertTrue(check_dict({"string": "test"}, "name", "string")) self.assertTrue(check_dict( {"string": "test", 7: "test"}, "name", list({7, "string"}) )) # should return false if key is not found in dictionary self.assertFalse(check_dict( {"string": "test"}, "name", list({7}) ))
def process_end_time(self, tags_dict): """ Process parsed end_time tags. Parameters: - tags_dict : dictionary with parsed tags """ # check if parameter tags_dict is a dictionary and # if it contains all required tags tag_list = list( {'end_hash', 'start_timestamp', 'finish_timestamp', 'duration'}) if not check_dict(tags_dict, "tags_dict", tag_list): return False logger.debug("End time : %s", tags_dict) result = False # check if timing was started # and if hash matches if (not self.has_timing_hash() or self.timing_hash != tags_dict['end_hash']): logger.info("Substage timing was not started or" " hash doesn't match") self.finished_incomplete = True else: set_started = set_finished = set_duration = False # Set started timestamp if self.stage.set_started_at_nano(tags_dict['start_timestamp']): logger.info("Stage started at %s", self.stage.data["started_at"]["isotimestamp"]) set_started = True # Set finished timestamp if self.stage.set_finished_at_nano(tags_dict['finish_timestamp']): logger.info("Stage finished at %s", self.stage.data["finished_at"]["isotimestamp"]) set_finished = True # Set duration if self.stage.set_duration_nano(tags_dict['duration']): logger.info("Stage duration : %ss", self.stage.data['duration']) set_duration = True result = set_started and set_finished and set_duration return result
def _handle_request(self, request, params=None): """ Retrieve Travis CI data using API. Parameters: - request : request to be sent to API - params : HTTP request parameters """ request_url = self.api_url + request request_params = self.request_params.copy() if params is not None and check_dict(params, "params"): request_params.update(params) req = Request(request_url, None, request_params) opener = build_opener() logger.info("Request from Travis CI API : %s", request_url) return opener.open(req)
def add_project_info_dict(payload): ''' Adds project info to a dictonary Param payload: dictonary payload ''' if not check_dict(payload, "payload"): return None payload_as_dict = copy.deepcopy(payload) payload_as_dict["buildtime_trend"] = Settings().get_project_info() # override timestamp, set to finished_at timestamp if "build" in payload and "finished_at" in payload["build"]: payload_as_dict["keen"] = { "timestamp": payload["build"]["finished_at"]["isotimestamp"] } return payload_as_dict
def parse_travis_worker_tag(self, line): ''' Parse and process Travis CI worker tag Param line : line from logfile containing Travis CI tags ''' get_logger().debug('line : %s', line) # parse Travis CI worker tags result = re.search(TRAVIS_LOG_PARSE_WORKER_STRING, line) if not result: return worker_tags = result.groupdict() # check if parameter worker_tags is a dictionary and # if it contains all required tags tag_list = list({'hostname', 'os'}) if check_dict(worker_tags, "worker_tags", tag_list): get_logger().debug("Worker tags : %s", worker_tags) self.current_job.add_property("worker", worker_tags)
def parse_travis_worker_tag(self, line): """ Parse and process Travis CI worker tag. Parameters: - line : line from logfile containing Travis CI tags """ logger.debug('line : %s', line) # parse Travis CI worker tags result = re.search(TRAVIS_LOG_PARSE_WORKER_STRING, line) if not result: return worker_tags = result.groupdict() # check if parameter worker_tags is a dictionary and # if it contains all required tags tag_list = list({'hostname', 'os'}) if tools.check_dict(worker_tags, "worker_tags", tag_list): logger.debug("Worker tags : %s", worker_tags) self.current_job.add_property("worker", worker_tags)
def process_start_time(self, tags_dict): """ Process parsed start_time tags. Parameters: - tags_dict : dictionary with parsed tags """ # check if parameter tags_dict is a dictionary and # if it contains all required tags if not check_dict(tags_dict, "tags_dict", 'start_hash'): return False logger.debug("Start time : %s", tags_dict) if self.has_timing_hash(): logger.info("Substage timing already set") return False self.timing_hash = tags_dict['start_hash'] logger.info("Set timing hash : %s", self.timing_hash) return True
def process_parsed_tags(self, tags_dict): ''' Processes parsed tags and calls the corresponding handler method Parameters: - tags_dict : dictionary with parsed tags ''' result = False # check if parameter tags_dict is a dictionary if check_dict(tags_dict, "tags_dict"): if 'start_stage' in tags_dict: result = self.process_start_stage(tags_dict) elif 'start_hash' in tags_dict: result = self.process_start_time(tags_dict) elif 'command' in tags_dict: result = self.process_command(tags_dict) elif 'end_hash' in tags_dict: result = self.process_end_time(tags_dict) elif 'end_stage' in tags_dict: result = self.process_end_stage(tags_dict) return result
def _handle_request(self, request, params=None): """ Retrieve Travis CI data using API. Parameters: - request : request to be sent to API - params : HTTP request parameters """ request_url = self.api_url + request request_params = self.request_params.copy() if params is not None and check_dict(params, "params"): request_params.update(params) req = Request( request_url, None, request_params ) opener = build_opener() logger.info("Request from Travis CI API : %s", request_url) return opener.open(req)
def process_command(self, tags_dict): """ Process parsed command tag. Parameters: - tags_dict : dictionary with parsed tags """ # check if parameter tags_dict is a dictionary and # if it contains all required tags if not check_dict(tags_dict, "tags_dict", 'command'): return False logger.debug("Command : %s", tags_dict) result = False if self.has_command(): logger.info("Command is already set") elif self.stage.set_command(tags_dict['command']): logger.info("Set command : %s", tags_dict['command']) result = True return result
def process_parsed_tags(self, tags_dict): """ Process parsed tags and calls the corresponding handler method. Parameters: - tags_dict : dictionary with parsed tags """ result = False # check if parameter tags_dict is a dictionary if check_dict(tags_dict, "tags_dict"): if 'start_stage' in tags_dict: result = self.process_start_stage(tags_dict) elif 'start_hash' in tags_dict: result = self.process_start_time(tags_dict) elif 'command' in tags_dict: result = self.process_command(tags_dict) elif 'end_hash' in tags_dict: result = self.process_end_time(tags_dict) elif 'end_stage' in tags_dict: result = self.process_end_stage(tags_dict) return result
def get_finished_at(self): """Retrieve timestamp when build finished.""" if tools.check_dict(self.current_build_data, key_list=["finished_at"]): return self.current_build_data['finished_at'] else: return None