forked from cms-sw/cms-bot
-
Notifications
You must be signed in to change notification settings - Fork 0
/
parse_jenkins_builds.py
executable file
·100 lines (99 loc) · 2.76 KB
/
parse_jenkins_builds.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
#!/usr/bin/python
from hashlib import sha1
import os , re , sys , json
import xml.etree.ElementTree as ET
from es_utils import send_payload
from es_utils import get_payload
query_url='http://cmses-master01.cern.ch:9200/jenkins/_search'
query_running_builds = """ {
"query": {
"filtered": {
"query": {
"bool": {
"should": [
{
"query_string": {
"query": "job_status:Running",
"lowercase_expanded_terms": false
}
}
]
}
},
"filter": {
"bool": {
"must": [
{
"match_all": {}
}
]
}
}
}
},
"highlight": {
"fields": {},
"fragment_size": 2147483647,
"pre_tags": [
"@start-highlight@"
],
"post_tags": [
"@end-highlight@"
]
},
"size": 500,
"sort": [
{
"_score": {
"order": "desc",
"ignore_unmapped": true
}
}
]
} """
all_local = list()
path = '/build/jobs'
index = "jenkins"
document = "builds-data"
rematch = re.compile(".*/builds/\d+$")
for root, dirs, files in os.walk(path):
if rematch.match(root):
logFile = root + '/build.xml'
flagFile = root + '/check.done'
if os.path.exists(logFile) and not os.path.exists(flagFile):
payload = {}
job_info = root.split('/')
payload['job_name'] = job_info[3]
payload['build_number'] = job_info[-1]
payload['url'] = "https://cmssdt.cern.ch/jenkins/job/" + job_info[3] + "/" + job_info[-1] + "/"
id = sha1(root).hexdigest()
try:
tree = ET.parse(logFile)
root = tree.getroot()
payload['@timestamp'] = root.find('startTime').text
payload['slave_node'] = root.find('builtOn').text
build_result = root.find('result')
if build_result is not None:
payload['build_result'] = build_result.text
payload['build_duration'] = int(int(root.find('duration').text)/1000)
payload['job_status'] = 'Finished'
os.system('touch "' + flagFile + '"')
else:
payload['job_status'] = 'Running'
all_local.append(id)
send_payload(index,document,id,json.dumps(payload))
except Exception as e:
print "Xml parsing error" , e
running_builds_elastic=list()
content = get_payload(query_url,query_running_builds)
if content == "":
running_builds_elastic = []
else:
content_hash = json.loads(content)
last=int(len(content_hash["hits"]["hits"]))
for i in range(0,last):
running_builds_elastic.append(str(content_hash["hits"]["hits"][i]["_id"]))
for build in running_builds_elastic:
if build not in all_local:
send_payload(index,document,build,'{"job_status":"Failed"}')
print "job status marked as Failed"