def test_pod(self): self.assertEqual(digest( 'pod-blah', error_re=regex.wordRE("pod"), strip=False), '<span class="highlight">pod-blah</span>') self.assertEqual(digest('0 1 2 3 4 5 pod 6 7 8 9 10', error_re=regex.wordRE("pod"), filters={"pod": "pod", "UID": "", "Namespace": "", "ContainerID":""}), 's2 2 3 4 5 pod 6 7 8 9 10')
def test_pod(self): self.assertEqual(digest( 'pod-blah', error_re=regex.wordRE("pod"), strip=False), '<span class="hilight">pod-blah</span>') self.assertEqual(digest('0 1 2 3 4 5 pod 6 7 8 9 10', error_re=regex.wordRE("pod"), filters={"pod": "pod", "UID": "", "Namespace": "", "ContainerID":""}), 's2 2 3 4 5 pod 6 7 8 9 10')
def test_pod(self): self.assertEqual(digest( 'pod-blah', error_re=regex.wordRE("pod"), filters={"pod": "pod", "UID": "", "Namespace": ""}, strip=False), '<span class="hilight"><span class="keyword">' 'pod</span>-blah</span>') self.assertEqual(digest('0 1 2 3 4 5 pod 6 7 8 9 10', error_re=regex.wordRE("pod"), filters={"pod": "pod", "UID": "", "Namespace": ""}), 's2 ( 0 1 ) 2 3 4 5 pod 6 7 8 9')
def get_woven_logs(log_files, pod, filters, objref_dict): lines = [] combined_lines = [] first_combined = "" pod_re = regex.wordRE(pod) # Produce a list of lines of all the selected logs for log_file in log_files: log = gcs_async.read(log_file).get_result() log = log.decode('utf8', 'replace') lines.extend(log.split('\n')) # Combine lines without timestamp into previous line, except if it comes at the # beginning of the file, in which case add it to the line with the first timestamp for line in lines: timestamp_re = regex.timestamp(line) if timestamp_re and timestamp_re.group(0): if not combined_lines: # add beginning of file to first timestamp line line = first_combined + line combined_lines.append(line) else: if not combined_lines: first_combined = first_combined + line else: combined_lines[-1] = combined_lines[-1] + line lines = sorted(combined_lines, key=regex.sub_timestamp) data = '\n'.join(lines) woven_logs = log_parser.digest(data, error_re=pod_re, filters=filters, objref_dict=objref_dict) return woven_logs
def parse(lines, error_re, hilight_words, filters): """ Given filters returns indeces of wanted lines from the kubelet log Args: lines: array of kubelet log lines error_re: regular expression of the failed pod name hilight_words: array of words that need to be bolded filters: dictionary of which filters to apply Returns: matched_lines: ordered array of indeces of lines to display hilight_words: updated hilight_words """ matched_lines = [] uid = "" end = 0 for n, line in enumerate(lines): if error_re.search(line): matched_lines.append(n) if filters["uid"] and uid == "": s = regex.uidobj_re.search(line) if s and s.group(1) != "": end = n uid = s.group(1) hilight_words.append(uid) if uid != "" and matched_lines[-1] != n: uid_re = regex.wordRE(uid) if uid_re.search(line): matched_lines.append(n) matched_lines.sort() return matched_lines, hilight_words
def test_make_dict(self): """Test make_dict works""" objref_dict = kubelet_parser.make_dict(lines, regex.wordRE("abc")) self.assertEqual(objref_dict, { "UID": "uid", "Namespace": "podName", "Name": "abc" })
def test_pod(self): self.assertEqual( digest('pod-blah', error_re=regex.wordRE("pod"), filters={ "pod": "pod", "UID": "", "Namespace": "" }, strip=False), '<span class="hilight"><span class="keyword">' 'pod</span>-blah</span>') self.assertEqual( digest('0 1 2 3 4 5 pod 6 7 8 9 10', error_re=regex.wordRE("pod"), filters={ "pod": "pod", "UID": "", "Namespace": "" }), 's2 ( 0 1 ) 2 3 4 5 pod 6 7 8 9')
def test_wordRE(self): for text, matches in [ ('/abcdef/', True), ('Pod abcdef failed', True), ('abcdef', True), ('cdabcdef', False), ('abc def', False), ('Podname(abcdef)', True), ]: self.assertEqual(bool(regex.wordRE("abcdef").search(text)), matches, 'wordRE(abcdef).search(%r) should be %r' % (text, matches))
def parse_log_file(log_filename, pod, filters=None, make_dict=False, objref_dict=None): """Based on make_dict, either returns the objref_dict or the parsed log file""" log = gcs_async.read(log_filename).get_result() if log is None: return None pod_re = regex.wordRE(pod) if make_dict: return kubelet_parser.make_dict(log.decode('utf8', 'replace'), pod_re) else: return log_parser.digest(log.decode('utf8', 'replace'), error_re=pod_re, filters=filters, objref_dict=objref_dict)
def parse_log_file(log_filename, pod, filters=None, make_dict=False, objref_dict=None): """Based on make_dict, either returns the objref_dict or the parsed log file""" log = gcs_async.read(log_filename).get_result() if log is None: return {}, False if make_dict else None if pod: bold_re = regex.wordRE(pod) else: bold_re = regex.error_re if objref_dict is None: objref_dict = {} if make_dict and pod: return kubelet_parser.make_dict(log.decode('utf8', 'replace'), bold_re, objref_dict) else: return log_parser.digest(log.decode('utf8', 'replace'), error_re=bold_re, filters=filters, objref_dict=objref_dict)
def parse_kubelet(pod, junit, build_dir, filters): junit_file = junit + ".xml" tmps = [f.filename for f in gcs_ls('%s/artifacts' % build_dir) if re.match(r'.*/tmp-node.*', f.filename)] junit_regex = r".*" + junit_file + r".*" kubelet_filename = "" for folder in tmps: tmp_contents = [f.filename for f in gcs_ls(folder)] for f in tmp_contents: if re.match(junit_regex, f): for file in tmp_contents: if re.match(r'.*kubelet\.log', file): kubelet_filename = file if kubelet_filename == "": return False kubelet_log = gcs_async.read(kubelet_filename).get_result() if kubelet_log: pod_re = regex.wordRE(pod) kubelet_log = log_parser.digest(kubelet_log.decode('utf8', 'replace'), error_re=pod_re, filters=filters) return kubelet_log
def test_make_dict_fail(self): """Test when objref line not in file""" objref_dict = kubelet_parser.make_dict(["pod failed"], regex.wordRE("abc")) self.assertEqual(objref_dict, None)
def test_make_dict(self): """Test make_dict works""" objref_dict = kubelet_parser.make_dict(lines, regex.wordRE("abc")) self.assertEqual( objref_dict, {"UID": "uid", "Namespace": "podName", "Name": "abc"})