forked from cms-sw/cms-bot
-
Notifications
You must be signed in to change notification settings - Fork 0
/
parse_jenkins_builds.py
executable file
·90 lines (85 loc) · 3.21 KB
/
parse_jenkins_builds.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
#!/usr/bin/python
from hashlib import sha1
import os , re , sys , json
import xml.etree.ElementTree as ET
from es_utils import send_payload,get_payload,resend_payload
JENKINS_PREFIX="jenkins"
try: JENKINS_PREFIX=os.environ['JENKINS_URL'].strip("/").split("/")[-1]
except: JENKINS_PREFIX="jenkins"
def findParametersAction(root):
if root.tag=='parameters': return root
for x in root:
p=findParametersAction(x)
if p is not None: return p
return None
def getParameters(root, payload):
n=root.find('name')
if n is not None:
if n.text is None: return
v=root.find('value')
vv = "None"
if v is not None: vv = str(v.text)
payload['parameter_'+n.text]=vv
else:
for x in root: getParameters(x, payload)
query_running_builds = """{
"query": {"bool": {"must": {"query_string": {"query": "job_status:running"}}}},
"from": 0,
"size": 10000
}"""
all_local = list()
path = '/build/jobs'
document = "builds-data"
rematch = re.compile(".*/builds/\d+$")
for root, dirs, files in os.walk(path):
if rematch.match(root):
logFile = root + '/build.xml'
flagFile = root + '/check.done'
if os.path.exists(logFile) and not os.path.exists(flagFile):
payload = {}
job_info = root.split('/')
payload['job_name'] = job_info[3]
payload['build_number'] = job_info[-1]
payload['url'] = "https://cmssdt.cern.ch/"+JENKINS_PREFIX+"/job/" + job_info[3] + "/" + job_info[-1] + "/"
id = sha1(root).hexdigest()
try:
tree = ET.parse(logFile)
root = tree.getroot()
pa=findParametersAction(root)
if pa is not None: getParameters(pa, payload)
jstime = root.find('startTime').text
payload['@timestamp'] = jstime
payload['slave_node'] = root.find('builtOn').text
build_result = root.find('result')
if build_result is not None:
payload['build_result'] = build_result.text
payload['build_duration'] = int(int(root.find('duration').text)/1000)
payload['job_status'] = 'Finished'
os.system('touch "' + flagFile + '"')
else:
payload['job_status'] = 'Running'
all_local.append(id)
weekindex="jenkins-jobs-"+str(int((((int(jstime)/1000)/86400)+4)/7))
send_payload(weekindex,document,id,json.dumps(payload), passwd_file="/var/lib/jenkins/secrets/github_hook_secret_cmsbot")
except Exception as e:
print "Xml parsing error",logFile , e
running_builds_elastic={}
content = get_payload('jenkins-*',query_running_builds)
if content == "":
running_builds_elastic = []
else:
content_hash = json.loads(content)
if (not 'hits' in content_hash) or (not 'hits' in content_hash['hits']):
print "ERROR: ",content
sys.exit(1)
for hit in content_hash['hits']['hits']:
if hit["_index"].startswith("jenkins-jobs-") or hit["_index"].startswith("cmssdt-jenkins-jobs-"):
try:print "Running:",hit["_source"]['job_name'],hit["_source"]['build_number'],hit["_index"],hit['_id']
except: pass
running_builds_elastic[hit['_id']]=hit
for build in running_builds_elastic:
if build not in all_local:
hit = running_builds_elastic[build]
hit["_source"]["job_status"]="Failed"
resend_payload(hit)
print "job status marked as Failed"