def process (line, count):
payload = {}
items = line.split(" ")
if len(items)<10: return True
if not (items[3][0]=='[' and items[4][-1]==']'): return True
payload["ip"]=items[0]
payload["ident"]=items[1]
payload["auth"]=items[2]
payload["verb"]=items[5][1:]
payload["request"]=items[6]
payload["httpversion"]=items[7][:-1]
payload["response"]=items[8]
try:
payload["bytes"]=int(items[9])
except:
payload["bytes"]=0
payload["@timestamp"]=int(mktime(datetime.strptime(items[3][1:],'%d/%b/%Y:%H:%M:%S').timetuple())*1000)
if len(items)>10: payload["referrer"]=items[10][1:-1]
if len(items)>11 and re.match('^"[0-9]+(\.[0-9]+)+"$', items[11]):
payload["ip"]=items[11][1:-1]
if len(items)>12:
agent = " ".join(items[12:]).replace('"','')
payload["agent"] = agent
payload["agent_type"]=agent.replace(" ","-").split("/",1)[0].upper()
id = sha1(line).hexdigest()
if (count%1000)==0: print "Processed entries",count
if not send_payload("apache-cmssdt","access_log", id, dumps(payload), passwd_file="/data/es/es_secret"):
return False
if payload["request"].startswith("/SDT/releases.map?release="):
xpayload = dict(item.split("=") for item in payload["request"].split("?",1)[1].split("&"))
for x in ["@timestamp","ip"]: xpayload[x] = payload[x]
return send_payload("scram-access","cmssw-releases", id, dumps(xpayload), passwd_file="/data/es/es_secret")
return True
def process_matrix_log(logFile):
t = os.path.getmtime(logFile)
timestp = int(t*1000)
payload = {}
pathInfo = logFile.split('/')
architecture = pathInfo[4]
release = pathInfo[8]
workflow = pathInfo[10].split('_')[0]
step = pathInfo[11].split('_')[0]
dat = re.findall('\d{4}-\d{2}-\d{2}',release)[0]
week = strftime("%U",strptime(dat,"%Y-%m-%d"))
id = sha1(release + architecture + workflow + str(step)).hexdigest()
dataset = {"type" : "relvals", "name" : "%s/%s" % (workflow, step), "ds_block" : "UNKNOWN", "ds_status" : "UNKNOWN", "ds_owner" : "UNKNOWN", "ds_files" : "0", "at_cern" : "UNKNOWN", "dataset" : "UNKNOWN"}
dataset["release"]=release
dataset["architecture"]=architecture
dataset["@timestamp"]=timestp
datasets=[]
lines = file(logFile).read()
for l in lines.split("\n"):
if not " Initiating request to open file " in l: continue
rootfile = l.split(" Initiating request to open file ")[1].split(" ")[0]
if (not "file:" in rootfile) and (not rootfile in datasets): datasets.append(rootfile)
for ds in datasets:
ds_items = ds.split("?",1)
ds_items.append("")
dataset["protocol"]=ds_items[0].split("/store/",1)[0]
dataset["protocol_opts"]=ds_items[1]
dataset["lfn"]="/store/"+ds_items[0].split("/store/",1)[1]
idx = sha1(id + ds).hexdigest()
send_payload("ib-dataset-"+week,"relvals-dataset",idx,json.dumps(dataset))
return
def send_unittest_dataset(datasets, payload, id, index, doc):
for ds in datasets:
ds_items = ds.split("?",1)
ds_items.append("")
payload["protocol"]=ds_items[0].split("/store/",1)[0]
payload["protocol_opts"]=ds_items[1]
payload["lfn"]="/store/"+ds_items[0].split("/store/",1)[1]
send_payload(index, doc, sha1(id + ds).hexdigest(), json.dumps(payload))
def send_unittest_dataset(datasets, payload, id, index, doc):
for ds in datasets:
ds_items = ds.split("?", 1)
ds_items.append("")
payload["protocol"] = ds_items[0].split("/store/", 1)[0]
payload["protocol_opts"] = ds_items[1]
payload["lfn"] = "/store/" + ds_items[0].split("/store/", 1)[1]
send_payload(index, doc,
sha1(id + ds).hexdigest(), json.dumps(payload))
def es_parse_log(logFile):
t = os.path.getmtime(logFile)
timestp = datetime.datetime.fromtimestamp(int(t)).strftime('%Y-%m-%d %H:%M:%S')
payload = {}
pathInfo = logFile.split('/')
architecture = pathInfo[4]
release = pathInfo[8]
workflow = pathInfo[10].split('_')[0]
step = pathInfo[11].split('_')[0]
dat = re.findall('\d{4}-\d{2}-\d{2}',release)[0]
week = strftime("%U",strptime(dat,"%Y-%m-%d"))
index = "ib-matrix-" + week
document = "runTheMatrix-data"
id = sha1(release + architecture + workflow + str(step)).hexdigest()
payload["workflow"] = workflow
payload["release"] = release
payload["architecture"] = architecture
payload["step"] = step
payload["hostname"] = gethostname()
payload["timestamp"] = timestp
exception = ""
error = ""
errors = []
inException = False
inError = False
if exists(logFile):
lines = file(logFile).read()
payload["url"] = logFile.replace('/data/sdt/' , 'https://cmssdt.cern.ch/SDT/cgi-bin/')
for l in lines.split("\n"):
if l.startswith("----- Begin Fatal Exception"):
inException = True
continue
if l.startswith("----- End Fatal Exception"):
inException = False
continue
if l.startswith("%MSG-e"):
inError = True
error = l
error_kind = re.split(" [0-9a-zA-Z-]* [0-9:]{8} CET", error)[0].replace("%MSG-e ", "")
continue
if inError == True and l.startswith("%MSG"):
inError = False
errors.append({"error": error, "kind": error_kind})
error = ""
error_kind = ""
continue
if inException:
exception += l + "\n"
if inError:
error += l + "\n"
if exception:
payload["exception"] = exception
if errors:
payload["errors"] = errors
send_payload(index,document,id,json.dumps(payload))
def es_parse_log(logFile):
t = os.path.getmtime(logFile)
timestp = datetime.datetime.fromtimestamp(int(t)).strftime('%Y-%m-%d %H:%M:%S')
payload = {}
pathInfo = logFile.split('/')
architecture = pathInfo[4]
release = pathInfo[8]
workflow = pathInfo[10].split('_')[0]
step = pathInfo[11].split('_')[0]
dat = re.findall('\d{4}-\d{2}-\d{2}',release)[0]
week = strftime("%U",strptime(dat,"%Y-%m-%d"))
index = "ib-matrix-" + week
document = "runTheMatrix-data"
id = sha1(release + architecture + workflow + str(step)).hexdigest()
payload["workflow"] = workflow
payload["release"] = release
payload["architecture"] = architecture
payload["step"] = step
payload["hostname"] = gethostname()
payload["timestp"] = timestp
exception = ""
error = ""
errors = []
inException = False
inError = False
if exists(logFile):
lines = file(logFile).read()
payload["url"] = logFile.replace('/data/sdt/' , 'https://cmssdt.cern.ch/SDT/cgi-bin/')
for l in lines.split("\n"):
if l.startswith("----- Begin Fatal Exception"):
inException = True
continue
if l.startswith("----- End Fatal Exception"):
inException = False
continue
if l.startswith("%MSG-e"):
inError = True
error = l
error_kind = re.split(" [0-9a-zA-Z-]* [0-9:]{8} CET", error)[0].replace("%MSG-e ", "")
continue
if inError == True and l.startswith("%MSG"):
inError = False
errors.append({"error": error, "kind": error_kind})
error = ""
error_kind = ""
continue
if inException:
exception += l + "\n"
if inError:
error += l + "\n"
if exception:
payload["exception"] = exception
if errors:
payload["errors"] = errors
send_payload(index,document,id,json.dumps(payload))
def send_unittest_dataset(datasets, payload, id, index, doc):
for ds in datasets:
ds_items = ds.split("?",1)
ds_items.append("")
ibeos = "/store/user/cmsbuild"
if ibeos in ds_items[0]: ds_items[0] = ds_items[0].replace(ibeos,"")
else: ibeos=""
payload["protocol"]=ds_items[0].split("/store/",1)[0]+ibeos
payload["protocol_opts"]=ds_items[1]
payload["lfn"]="/store/"+ds_items[0].split("/store/",1)[1]
send_payload(index, doc, sha1(id + ds).hexdigest(), json.dumps(payload))
def send_unittest_dataset(datasets, payload, id, index, doc):
for ds in datasets:
ds_items = ds.split("?",1)
ds_items.append("")
ibeos = "/store/user/cmsbuild"
if ibeos in ds_items[0]: ds_items[0] = ds_items[0].replace(ibeos,"")
else: ibeos=""
payload["protocol"]=ds_items[0].split("/store/",1)[0]+ibeos
payload["protocol_opts"]=ds_items[1]
payload["lfn"]="/store/"+ds_items[0].split("/store/",1)[1]
send_payload(index, doc, sha1(id + ds).hexdigest(), json.dumps(payload))
def process (line, count):
for agent in LOGWATCH_APACHE_IGNORE_AGENTS:
if agent in line: return True
payload = {}
items = line.split(" ")
if len(items)<12: return True
if not (items[3][0]=='[' and items[4][-1]==']'): return True
payload["ip"]=items[0]
payload["ident"]=items[1]
payload["auth"]=items[2]
payload["verb"]=items[5][1:]
payload["request"]=items[6]
payload["httpversion"]=items[7][:-1]
payload["response"]=items[8]
try:
payload["bytes"]=int(items[9])
except:
payload["bytes"]=0
payload["referrer"]=items[10][1:-1]
agent = " ".join(items[11:]).replace('"','')
if "CMSPKG-v" in agent: agent = agent.replace("-v","/")
payload["agent"]=agent
payload["agent_type"]=agent.replace(" ","-").split("/",1)[0].upper()
tsec = mktime(datetime.strptime(items[3][1:],'%d/%b/%Y:%H:%M:%S').timetuple())
week = str(int(tsec/(86400*7)))
payload["@timestamp"]=int(tsec*1000)
id = sha1(line).hexdigest()
if (count%1000)==0: print "Processed entries",count
if not send_payload("apache-cmsrep-"+week,"access_log", id, dumps(payload), passwd_file="/data/es/es_secret"):
return False
if payload["verb"] != "GET": return True
items = payload["request"].replace("/cms/cpt/Software/download/","/cmssw/",1).split("/")
if len(items)<6: return True
if items[3] == "apt": items[3]="PRMS"
if items[3] != "RPMS": return True
pkg, cmspkg, arch, repo, dev = items[-1], "apt", "" , "", 0
if "?" in pkg:
pkg, pkgopts = pkg.split("?",1)
if "version=" in pkgopts: cmspkg = pkgopts.split("version=",1)[1].split("&",1)[0]
if not pkg.endswith(".rpm"): return True
if (items[1] == "cgi-bin") and items[2].startswith("cmspkg"):
if len(items)<8: return True
if items[2].endswith('-dev'): dev=1
repo, arch = items[4], items[5]
elif items[1] == "cmssw":
repo, arch = items[2], items[4]
else:
return True
from urllib import unquote
xpayload = {'dev' : dev, 'repository' : unquote(repo), 'architecture' : unquote(arch), 'package' : unquote(pkg).split("-1-",1)[0], 'cmspkg' : unquote(cmspkg)}
for x in ["@timestamp","ip"]: xpayload[x] = payload[x]
return send_payload("cmspkg-access-"+week,"rpm-packages", id, dumps(xpayload), passwd_file="/data/es/es_secret")
def process(wfnum, s, sfile, hostname, exit_code, details=False):
global release, arch, rel_msec, week, ex_fields, cmsThreads
try:
stats = json.load(open(sfile))
xdata = {}
for stat in stats:
for item in stat:
try: xdata[item].append(stat[item])
except:
xdata[item]=[]
xdata[item].append(stat[item])
stat["@timestamp"]=rel_msec+(stat["time"]*1000)
stat["release"]=release
stat["step"]=s
stat["workflow"]=wfnum
stat["cmsthreads"]=cmsThreads
stat["architecture"]=arch
idx = sha1(release + arch + wfnum + s + str(stat["time"])).hexdigest()
del stat["time"]
if details:
try:send_payload("relvals_stats_details-"+week,"runtime-stats-details",idx,json.dumps(stat))
except Exception as e: print(e)
print("Working on ",release, arch, wfnum, s, len(stats))
sdata = {"release":release, "architecture":arch, "step":s, "@timestamp":rel_msec, "workflow":wfnum, "hostname":hostname, "exit_code":exit_code, "cmsthreads":cmsThreads}
for x in xdata:
data = sorted(xdata[x])
if x in ["time","num_threads","processes","num_fds"]:
sdata[x]=data[-1]
continue
if not x in ex_fields: continue
dlen = len(data)
for t in ["min", "max", "avg", "median", "25", "75"]: sdata[x+"_"+t]=0
if dlen>0:
sdata[x+"_min"]=data[0]
sdata[x+"_max"]=data[-1]
if dlen>1:
dlen2=int(dlen/2)
if (dlen%2)==0: sdata[x+"_median"]=int((data[dlen2-1]+data[dlen2])/2)
else: sdata[x+"_median"]=data[dlen2]
sdata[x+"_avg"]=int(sum(data)/dlen)
for t in [25, 75]:
sdata[x+"_"+str(t)]=int(percentile(t,data, dlen))
else:
for t in ["25", "75", "avg", "median"]:
sdata[x+"_"+t]=data[0]
idx = sha1(release + arch + wfnum + s + str(rel_sec)).hexdigest()
try:send_payload("relvals_stats_summary-"+week,"runtime-stats-summary",idx,json.dumps(sdata))
except Exception as e: print(e)
except Exception as e: print(e)
return
def process(wfnum, s, sfile, hostname, exit_code, details=False):
global release, arch, rel_msec, week, ex_fields, cmsThreads
try:
stats = json.load(open(sfile))
xdata = {}
for stat in stats:
for item in stat:
try: xdata[item].append(stat[item])
except:
xdata[item]=[]
xdata[item].append(stat[item])
stat["@timestamp"]=rel_msec+(stat["time"]*1000)
stat["release"]=release
stat["step"]=s
stat["workflow"]=wfnum
stat["cmsthreads"]=cmsThreads
stat["architecture"]=arch
idx = sha1(release + arch + wfnum + s + str(stat["time"])).hexdigest()
del stat["time"]
if details:
try:send_payload("relvals_stats_details-"+week,"runtime-stats-details",idx,json.dumps(stat))
except Exception as e: print e
print "Working on ",release, arch, wfnum, s, len(stats)
sdata = {"release":release, "architecture":arch, "step":s, "@timestamp":rel_msec, "workflow":wfnum, "hostname":hostname, "exit_code":exit_code, "cmsthreads":cmsThreads}
for x in xdata:
data = sorted(xdata[x])
if x in ["time","num_threads","processes","num_fds"]:
sdata[x]=data[-1]
continue
if not x in ex_fields: continue
dlen = len(data)
for t in ["min", "max", "avg", "median", "25", "75"]: sdata[x+"_"+t]=0
if dlen>0:
sdata[x+"_min"]=data[0]
sdata[x+"_max"]=data[-1]
if dlen>1:
dlen2=int(dlen/2)
if (dlen%2)==0: sdata[x+"_median"]=int((data[dlen2-1]+data[dlen2])/2)
else: sdata[x+"_median"]=data[dlen2]
sdata[x+"_avg"]=int(sum(data)/dlen)
for t in [25, 75]:
sdata[x+"_"+str(t)]=int(percentile(t,data, dlen))
else:
for t in ["25", "75", "avg", "median"]:
sdata[x+"_"+t]=data[0]
idx = sha1(release + arch + wfnum + s + str(rel_sec)).hexdigest()
try:send_payload("relvals_stats_summary-"+week,"runtime-stats-summary",idx,json.dumps(sdata))
except Exception as e: print e
except Exception as e: print e
return
def process (line, count):
for agent in LOGWATCH_APACHE_IGNORE_AGENTS:
if agent in line: return True
payload = {}
items = line.split(" ")
if len(items)<10: return True
if not (items[3][0]=='[' and items[4][-1]==']'): return True
payload["ip"]=items[0]
payload["ident"]=items[1]
payload["auth"]=items[2]
payload["verb"]=items[5][1:]
payload["request"]=items[6]
payload["httpversion"]=items[7][:-1]
payload["response"]=items[8]
try:
payload["bytes"]=int(items[9])
except:
payload["bytes"]=0
payload["@timestamp"]=int(mktime(datetime.strptime(items[3][1:],'%d/%b/%Y:%H:%M:%S').timetuple())*1000)
if len(items)>10: payload["referrer"]=items[10][1:-1]
if len(items)>11 and re.match('^"[0-9]+(\.[0-9]+)+"$', items[11]):
payload["ip"]=items[11][1:-1]
if len(items)>12:
agent = " ".join(items[12:]).replace('"','')
payload["agent"] = agent
payload["agent_type"]=agent.replace(" ","-").split("/",1)[0].upper()
id = sha1(line).hexdigest()
if (count%1000)==0: print "Processed entries",count
if not send_payload("apache-doxygen","access_log", id, dumps(payload), passwd_file="/data/es/es_secret"):
return False
return True
def process (line, count):
for agent in LOGWATCH_APACHE_IGNORE_AGENTS:
if agent in line: return True
payload = {}
items = line.split(" ")
if len(items)<10: return True
if not (items[3][0]=='[' and items[4][-1]==']'): return True
payload["ip"]=items[0]
payload["ident"]=items[1]
payload["auth"]=items[2]
payload["verb"]=items[5][1:]
payload["request"]=items[6]
payload["httpversion"]=items[7][:-1]
payload["response"]=items[8]
try:
payload["bytes"]=int(items[9])
except:
payload["bytes"]=0
tsec = mktime(datetime.strptime(items[3][1:],'%d/%b/%Y:%H:%M:%S').timetuple())
week = str(int(tsec/(86400*7)))
payload["@timestamp"]=int(tsec*1000)
if len(items)>10: payload["referrer"]=items[10][1:-1]
if len(items)>11 and re.match('^"[0-9]+(\.[0-9]+)+"$', items[11]):
payload["ip"]=items[11][1:-1]
if len(items)>12:
agent = " ".join(items[12:]).replace('"','')
payload["agent"] = agent
payload["agent_type"]=agent.replace(" ","-").split("/",1)[0].upper()
id = sha1(line).hexdigest()
if (count%1000)==0: print "Processed entries",count
if not send_payload("apache-doxygen-"+week,"access_log", id, dumps(payload), passwd_file="/data/es/es_secret"):
return False
return True
def process (line, count):
payload = {}
items = line.split(" ")
if len(items)<12: return True
if not (items[3][0]=='[' and items[4][-1]==']'): return True
payload["ip"]=items[0]
payload["ident"]=items[1]
payload["auth"]=items[2]
payload["verb"]=items[5][1:]
payload["request"]=items[6]
payload["httpversion"]=items[7][:-1]
payload["response"]=items[8]
try:
payload["bytes"]=int(items[9])
except:
payload["bytes"]=0
payload["referrer"]=items[10][1:-1]
agent = " ".join(items[11:]).replace('"','')
if "CMSPKG-v" in agent: agent = agent.replace("-v","/")
payload["agent"]=agent
payload["agent_type"]=agent.replace(" ","-").split("/",1)[0].upper()
payload["@timestamp"]=int(mktime(datetime.strptime(items[3][1:],'%d/%b/%Y:%H:%M:%S').timetuple())*1000)
id = sha1(line).hexdigest()
if (count%1000)==0: print "Processed entries",count
return send_payload("apache-cmsrep","access_log", id, dumps(payload), passwd_file="/data/es/es_secret")
def send_unittest_dataset(datasets, payload, id, index, doc):
for ds in datasets:
print("Processing ", ds)
if not 'root://' in ds: continue
ds_items = ds.split("?", 1)
ds_items.append("")
ibeos = "/store/user/cmsbuild"
if ibeos in ds_items[0]: ds_items[0] = ds_items[0].replace(ibeos, "")
else: ibeos = ""
payload["protocol"] = ds_items[0].split("/store/", 1)[0] + ibeos
payload["protocol_opts"] = ds_items[1]
payload["lfn"] = "/store/" + ds_items[0].split("/store/", 1)[1].strip()
print("Sending", index, doc,
sha1(id + ds).hexdigest(), json.dumps(payload))
send_payload(index, doc,
sha1(id + ds).hexdigest(), json.dumps(payload))
def process_ib_utests(logFile):
t = getmtime(logFile)
timestp = datetime.datetime.fromtimestamp(
int(t)).strftime('%Y-%m-%d %H:%M:%S')
payload = {}
pathInfo = logFile.split('/')
architecture = pathInfo[4]
release = pathInfo[8]
#dat = re.findall('\d{4}-\d{2}-\d{2}',release)[0]
index = "ibs"
document = "unittests"
payload["release"] = release
payload["architecture"] = architecture
payload["@timestamp"] = timestp
if exists(logFile):
with open(logFile) as f:
try:
it = iter(f)
line = it.next()
while '--------' not in line:
line = it.next()
while True:
line = it.next().strip()
if ":" in line:
pkg = line.split(':')[0].strip()
payload[
"url"] = 'https://cmssdt.cern.ch/SDT/cgi-bin/buildlogs/' + architecture + '/' + release + '/unitTestLogs/' + pkg
line = it.next().strip()
while ':' not in line:
if "had ERRORS" in line:
payload["status"] = 1
else:
payload["status"] = 0
utest = line.split(' ')[0]
payload["package"] = pkg
payload["name"] = utest
id = sha1(release + architecture +
utest).hexdigest()
send_payload(index, document, id,
json.dumps(payload))
line = it.next().strip()
except Exception as e:
print "File processed:", e
else:
print "Invalid File Path"
def process_ib_utests(logFile):
t = getmtime(logFile)
timestp = int(t * 1000)
payload = {}
pathInfo = logFile.split('/')
architecture = pathInfo[4]
release = pathInfo[8]
week, rel_sec = cmsswIB2Week(release)
index = "ibs-" + week
document = "unittests"
payload["release"] = release
payload["architecture"] = architecture
payload["@timestamp"] = timestp
if exists(logFile):
with open(logFile) as f:
try:
it = iter(f)
line = next(it)
while '--------' not in line:
line = next(it)
while True:
line = it.next().strip()
if ":" in line:
pkg = line.split(':')[0].strip()
payload[
"url"] = 'https://cmssdt.cern.ch/SDT/cgi-bin/buildlogs/' + architecture + '/' + release + '/unitTestLogs/' + pkg
line = it.next().strip()
while ':' not in line:
if "had ERRORS" in line:
payload["status"] = 1
else:
payload["status"] = 0
utest = line.split(' ')[0]
payload["package"] = pkg
payload["name"] = utest
id = sha1(release + architecture +
utest).hexdigest()
send_payload(index, document, id,
json.dumps(payload))
line = it.next().strip()
except Exception as e:
print("ERROR: File processed: %s" % e)
else:
print("Invalid File Path")
def process_matrix_log(logFile):
t = os.path.getmtime(logFile)
timestp = int(t * 1000)
payload = {}
pathInfo = logFile.split('/')
architecture = pathInfo[4]
release = pathInfo[8]
workflow = pathInfo[10].split('_')[0]
step = pathInfo[11].split('_')[0]
dat = re.findall('\d{4}-\d{2}-\d{2}', release)[0]
week = strftime("%U", strptime(dat, "%Y-%m-%d"))
id = sha1(release + architecture + workflow + str(step)).hexdigest()
dataset = {
"type": "relvals",
"name": "%s/%s" % (workflow, step),
"ds_block": "UNKNOWN",
"ds_status": "UNKNOWN",
"ds_owner": "UNKNOWN",
"ds_files": "0",
"at_cern": "UNKNOWN",
"dataset": "UNKNOWN"
}
dataset["release"] = release
dataset["architecture"] = architecture
dataset["@timestamp"] = timestp
datasets = []
lines = file(logFile).read()
for l in lines.split("\n"):
if not " Initiating request to open file " in l: continue
rootfile = l.split(" Initiating request to open file ")[1].split(
" ")[0]
if (not "file:" in rootfile) and (not rootfile in datasets):
datasets.append(rootfile)
for ds in datasets:
ds_items = ds.split("?", 1)
ds_items.append("")
dataset["protocol"] = ds_items[0].split("/store/", 1)[0]
dataset["protocol_opts"] = ds_items[1]
dataset["lfn"] = "/store/" + ds_items[0].split("/store/", 1)[1]
idx = sha1(id + ds).hexdigest()
send_payload("ib-dataset-" + week, "relvals-dataset", idx,
json.dumps(dataset))
return
def process (line, count):
payload = {}
items = line.split(" ")
if len(items)<12: return True
if not (items[3][0]=='[' and items[4][-1]==']'): return True
payload["ip"]=items[0]
payload["ident"]=items[1]
payload["auth"]=items[2]
payload["verb"]=items[5][1:]
payload["request"]=items[6]
payload["httpversion"]=items[7][:-1]
payload["response"]=items[8]
try:
payload["bytes"]=int(items[9])
except:
payload["bytes"]=0
payload["referrer"]=items[10][1:-1]
agent = " ".join(items[11:]).replace('"','')
if "CMSPKG-v" in agent: agent = agent.replace("-v","/")
payload["agent"]=agent
payload["agent_type"]=agent.replace(" ","-").split("/",1)[0].upper()
payload["@timestamp"]=int(mktime(datetime.strptime(items[3][1:],'%d/%b/%Y:%H:%M:%S').timetuple())*1000)
id = sha1(line).hexdigest()
if (count%1000)==0: print "Processed entries",count
if not send_payload("apache-cmsrep","access_log", id, dumps(payload), passwd_file="/data/es/es_secret"):
return False
if not payload["request"].startswith("/cgi-bin/cmspkg"): return True
items = payload["request"].split("/")
if len(items)<8: return True
if items[3] != "RPMS": return True
dev = 0
if items[2].endswith('-dev'): dev=1
xpayload = {}
try:
from urllib import unquote
pkg, rest = items[7].split("?",1)
cmspkg = rest.split("version=",1)[1].split("&",1)[0]
xpayload = {'dev' : dev, 'repository' : items[4], 'architecture' : items[5], 'package' : unquote(pkg).split("-1-",1)[0], 'cmspkg' : cmspkg}
except:
return True
for x in ["@timestamp","ip"]: xpayload[x] = payload[x]
return send_payload("cmspkg-access","rpm-packages", id, dumps(xpayload), passwd_file="/data/es/es_secret")
def process_ib_utests(logFile):
t = getmtime(logFile)
timestp = int(t*1000)
payload = {}
pathInfo = logFile.split('/')
architecture = pathInfo[4]
release = pathInfo[8]
week, rel_sec = cmsswIB2Week (release)
index = "ibs-"+week
document = "unittests"
payload["release"] = release
payload["architecture"] = architecture
payload["@timestamp"] = timestp
if exists(logFile):
with open(logFile) as f:
try:
it = iter(f)
line = next(it)
while '--------' not in line:
line = next(it)
while True:
line=it.next().strip()
if ":" in line:
pkg = line.split(':')[0].strip()
payload["url"] = 'https://cmssdt.cern.ch/SDT/cgi-bin/buildlogs/'+ architecture +'/'+ release +'/unitTestLogs/' + pkg
line = it.next().strip()
while ':' not in line:
if "had ERRORS" in line:
payload["status"] = 1
else:
payload["status"] = 0
utest= line.split(' ')[0]
payload["package"] = pkg
payload["name"] = utest
id = sha1(release + architecture + utest).hexdigest()
send_payload(index,document,id,json.dumps(payload))
line = it.next().strip()
except Exception as e:
print("File processed:", e)
else:
print("Invalid File Path")
def process_ib_utests(logFile):
t = os.path.getmtime(logFile)
timestp = datetime.datetime.fromtimestamp(int(t)).strftime('%Y-%m-%d %H:%M:%S')
payload = {}
pathInfo = logFile.split('/')
architecture = pathInfo[4]
release = pathInfo[8]
#dat = re.findall('\d{4}-\d{2}-\d{2}',release)[0]
index = "ibs"
document = "unittests"
payload["release"] = release
payload["architecture"] = architecture
payload["@timestamp"] = timestp
if exists(logFile):
with open(logFile) as f:
try:
it = iter(f)
line = it.next()
while '--------' not in line:
line = it.next()
while True:
line=it.next().strip()
if ":" in line:
pkg = line.split(':')[0].strip()
payload["url"] = 'https://cmssdt.cern.ch/SDT/cgi-bin/buildlogs/'+ architecture +'/'+ release +'/unitTestLogs/' + pkg
line = it.next().strip()
while ':' not in line:
if "had ERRORS" in line:
payload["status"] = 1
else:
payload["status"] = 0
utest= line.split(' ')[0]
payload["package"] = pkg
payload["name"] = utest
id = sha1(release + architecture + utest).hexdigest()
send_payload(index,document,id,json.dumps(payload))
line = it.next().strip()
except Exception as e:
print "File processed:", e
else:
print "Invalid File Path"
from os.path import dirname, abspath
import sys
sys.path.append(dirname(dirname(abspath(__file__))))
from hashlib import sha1
import json
from es_utils import send_payload
from _py2with3compatibility import run_cmd
from cmsutils import cmsswIB2Week
err, logs = run_cmd(
"find /data/sdt/SDT/jenkins-artifacts/cmssw-afs-eos-comparison -mindepth 1 -maxdepth 1 -name '*.json' -type f"
)
for jfile in logs.split('\n'):
if not jfile: continue
print("Processing file", jfile)
payload = {}
try:
payload = json.load(open(jfile))
except ValueError as err:
print(err)
run_cmd("rm -f %s" % jfile)
continue
week, rel_sec = cmsswIB2Week(payload["release"])
payload["@timestamp"] = rel_sec * 1000
id = sha1("%s-%s-%s" % (payload["release"], payload["architecture"],
payload["fstype"])).hexdigest()
print(payload)
if send_payload("cmssw-afs-eos-%s" % week, "build", id,
json.dumps(payload)):
run_cmd("rm -f %s" % jfile)
id = sha1(root).hexdigest()
try:
tree = ET.parse(logFile)
root = tree.getroot()
payload['@timestamp'] = root.find('startTime').text
payload['slave_node'] = root.find('builtOn').text
build_result = root.find('result')
if build_result is not None:
payload['build_result'] = build_result.text
payload['build_duration'] = int(int(root.find('duration').text)/1000)
payload['job_status'] = 'Finished'
os.system('touch "' + flagFile + '"')
else:
payload['job_status'] = 'Running'
all_local.append(id)
send_payload(index,document,id,json.dumps(payload))
except Exception as e:
print "Xml parsing error" , e
running_builds_elastic=list()
content = get_payload(query_url,query_running_builds)
if content == "":
running_builds_elastic = []
else:
content_hash = json.loads(content)
last=int(len(content_hash["hits"]["hits"]))
for i in range(0,last):
running_builds_elastic.append(str(content_hash["hits"]["hits"][i]["_id"]))
for build in running_builds_elastic:
if build not in all_local:
send_payload(index,document,build,'{"job_status":"Failed"}')
print "job status marked as Failed"
process_all = False
files_to_process=[]
cmd_to_get_logs = "ls -rt "+os.path.join(apache_log_dir,ssl_error_log)+"*"
if len(sys.argv)==1:
process_all = True
cmd_to_get_logs = cmd_to_get_logs + " | tail -2"
prev_hour = datetime.now()-timedelta(hours=1)
filter_search = " | grep '"+prev_hour.strftime("^\[%a %b %d %H:[0-5][0-9]:[0-5][0-9] %Y\] ")+"'"
err, out = getstatusoutput(cmd_to_get_logs)
if err:
print out
sys.exit(1)
ReTime = re.compile('^\[[A-Za-z]{3} ([A-Za-z]{3} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [0-9]{4})\] \[[^\]]+\] \[client (.+)\]\s(.+)')
for log in out.split("\n"):
find_cmd = "grep '%s' %s %s" % (search_for, log, filter_search)
err, out = getstatusoutput(find_cmd)
for line in out.split("\n"):
m = ReTime.match(line)
if m:
timestamp = int(datetime.strptime(m.group(1), "%b %d %H:%M:%S %Y").strftime('%s'))*1000
payload = {}
payload['@timestamp'] = timestamp
payload['ip'] = m.group(2)
payload['message'] = line
id = sha1(str(timestamp) + m.group(2)).hexdigest()
send_payload("hypernews","hn-timeouts",id, dumps(payload), passwd_file="/data/es/es_secret")
payload = {}
payload['@timestamp'] = int(time()*1000)
send_payload("hypernews","hn-heartbeat",str(payload['@timestamp']), dumps(payload), passwd_file="/data/es/es_secret")
lfns = {}
ti = 0
updated = 0
for item in json_out:
for h in item["hits"]["hits"]:
ti += 1
lfn_info = h["_source"]
lfn_data = cmsweb.search(lfn_info["lfn"], lfn_info)
update = False
for item in lfn_data:
if (not item in lfn_info) or (lfn_data[item] !=
lfn_info[item]):
lfn_info[item] = lfn_data[item]
update = True
print "Processing .... %s/%s (LFNS: %s, NEW LFN: %s, DS: %s, BLK:%s, ERR: %s): %s" % (
ti, total_hits, len(cmsweb.cache['lfns']),
len(cmsweb.cache['new_lfns']),
len(cmsweb.cache['datasets']),
len(cmsweb.cache['blocks']), cmsweb.errors, update)
if update:
send_payload(h['_index'], h['_type'], h['_id'],
json.dumps(lfn_info))
updated += 1
print "Updated Records:", updated
exit(0)
lfns = []
for lfn in datasets:
lfns.append([lfn, datasets[lfn]])
download(lfns, opts.store, opts.job, opts.dryRun)
print "Total Files:", len(datasets)
pa=findParametersAction(root)
if pa is not None: getParameters(pa, payload)
jstime = root.find('startTime').text
payload['@timestamp'] = jstime
payload['slave_node'] = root.find('builtOn').text
build_result = root.find('result')
if build_result is not None:
payload['build_result'] = build_result.text
payload['build_duration'] = int(int(root.find('duration').text)/1000)
payload['job_status'] = 'Finished'
os.system('touch "' + flagFile + '"')
else:
payload['job_status'] = 'Running'
all_local.append(id)
weekindex="jenkins-jobs-"+str(int((((int(jstime)/1000)/86400)+4)/7))
send_payload(weekindex,document,id,json.dumps(payload), passwd_file="/var/lib/jenkins/secrets/github_hook_secret_cmsbot")
except Exception as e:
print "Xml parsing error",logFile , e
running_builds_elastic={}
content = get_payload(query_url,query_running_builds)
if content == "":
running_builds_elastic = []
else:
content_hash = json.loads(content)
for hit in content_hash['hits']['hits']:
if hit["_index"]=="jenkins" or hit["_index"].startswith("jenkins-jobs-"):
try:print "Running:",hit["_source"]['job_name'],hit["_source"]['build_number'],hit["_index"],hit['_id']
except: pass
running_builds_elastic[hit['_id']]=hit
for build in running_builds_elastic:
if build not in all_local:
def process_build_any_ib(logFile):
rel = ""
arch = ""
uploadTime = 0
stime = 0
upload = False
jstart = 0
jend = 0
patch = 0
finished = False
with open(logFile) as f:
for line in f:
line = line.strip()
if not jstart:
m = ReDate.match(line)
if m:
jstart = datetime.strptime(
m.group(1) + m.group(2), "%b %d %H:%M:%S %Y")
continue
if not arch:
m = ReArch.match(line)
if m: arch = m.group(1)
continue
if not rel:
m = ReRel.match(line)
if m: rel = m.group(1)
continue
if ReFinish.match(line):
finished = True
if "ABORTED" in line: return True
break
if ReUpload.match(line):
upload = True
print "Upload: ", stime, line
continue
if ReType.match(line): patch = 1
m = ReDate.match(line)
if not m: continue
xtime = datetime.strptime(
m.group(1) + m.group(2), "%b %d %H:%M:%S %Y")
jend = xtime
if not upload:
stime = xtime
else:
upload = False
dtime = xtime - stime
uploadTime += dtime.seconds
print "FINISHED: ", finished, rel, arch, uploadTime, jstart, upload, patch
if not rel or not arch or not finished: return finished
urlx = logFile.split("/")
url = "https://cmssdt.cern.ch/jenkins/job/build-any-ib/" + logFile.split(
"/")[-2] + "/console"
timestp = getmtime(logFile)
ttime = 0
if jend and jstart:
dtime = jend - jstart
ttime = dtime.seconds
print ttime, uploadTime, rel, arch, patch, url
payload = {}
payload["release"] = rel
payload["architecture"] = arch
payload["total_time"] = ttime
payload["upload_time"] = uploadTime
payload["patch"] = patch
payload["@timestamp"] = int(timestp * 1000)
payload["url"] = url
week, rel_sec = cmsswIB2Week(rel)
print payload
id = sha1(rel + arch).hexdigest()
send_payload("jenkins-ibs-" + week, "timings", id, json.dumps(payload))
return finished
continue
if inException:
exception += l + "\n"
if inError:
error += l + "\n"
if exception:
payload["exception"] = exception
if errors:
payload["errors"] = errors
payload["error_count"] = error_count
try:
payload = es_parse_jobreport(payload,logFile)
except Exception, e:
print e
print "sending data for ",logFile
try:send_payload(index,document,id,json.dumps(payload))
except:pass
if datasets:
dataset = {"type" : "relvals", "name" : "%s/%s" % (payload["workflow"], payload["step"])}
for fld in ["release","architecture","@timestamp"]: dataset[fld] = payload[fld]
for ds in datasets:
ds_items = ds.split("?",1)
ds_items.append("")
ibeos = "/store/user/cmsbuild"
if ibeos in ds_items[0]: ds_items[0] = ds_items[0].replace(ibeos,"")
else: ibeos=""
dataset["protocol"]=ds_items[0].split("/store/",1)[0]+ibeos
dataset["protocol_opts"]=ds_items[1]
dataset["lfn"]="/store/"+ds_items[0].split("/store/",1)[1]
idx = sha1(id + ds).hexdigest()
print dataset
cmd_to_get_logs = "ls -rt "+os.path.join(apache_log_dir,ssl_error_log)+"*"
if len(sys.argv)==1:
process_all = True
cmd_to_get_logs = cmd_to_get_logs + " | tail -2"
prev_hour = datetime.now()-timedelta(hours=1)
filter_search = " | grep '"+prev_hour.strftime("^\[%a %b %d %H:[0-5][0-9]:[0-5][0-9] %Y\] ")+"'"
err, out = run_cmd(cmd_to_get_logs)
if err:
print(out)
sys.exit(1)
ReTime = re.compile('^\[[A-Za-z]{3} ([A-Za-z]{3} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [0-9]{4})\] \[[^\]]+\] \[client (.+)\]\s(.+)')
for log in out.split("\n"):
find_cmd = "grep '%s' %s %s" % (search_for, log, filter_search)
err, out = run_cmd(find_cmd)
for line in out.split("\n"):
m = ReTime.match(line)
if m:
tsec = int(datetime.strptime(m.group(1), "%b %d %H:%M:%S %Y").strftime('%s'))
week = str(int(tsec/(86400*7)))
timestamp = tsec*1000
payload = {}
payload['@timestamp'] = timestamp
payload['ip'] = m.group(2)
payload['message'] = line
id = sha1(str(timestamp) + m.group(2)).hexdigest()
send_payload("hypernews-"+week,"hn-timeouts",id, dumps(payload))
payload = {}
payload['@timestamp'] = int(time()*1000)
send_payload("hypernews","hn-heartbeat",str(payload['@timestamp']), dumps(payload))
root = tree.getroot()
pa = findParametersAction(root)
if pa is not None: getParameters(pa, payload)
payload['@timestamp'] = root.find('startTime').text
payload['slave_node'] = root.find('builtOn').text
build_result = root.find('result')
if build_result is not None:
payload['build_result'] = build_result.text
payload['build_duration'] = int(
int(root.find('duration').text) / 1000)
payload['job_status'] = 'Finished'
os.system('touch "' + flagFile + '"')
else:
payload['job_status'] = 'Running'
all_local.append(id)
send_payload(index, document, id, json.dumps(payload))
except Exception as e:
print "Xml parsing error", e
running_builds_elastic = list()
content = get_payload(query_url, query_running_builds)
if content == "":
running_builds_elastic = []
else:
content_hash = json.loads(content)
last = int(len(content_hash["hits"]["hits"]))
for i in range(0, last):
running_builds_elastic.append(
str(content_hash["hits"]["hits"][i]["_id"]))
for build in running_builds_elastic:
if build not in all_local:
send_payload(index, document, build, '{"job_status":"Failed"}')
#process info
yesterday = datetime.date.today() - datetime.timedelta(1)
timestp = int(yesterday.strftime("%s")) * 1000
temp_fails = dict(
list(
map(
map_int_val,
list((dict([
list(map(cust_strip,
x.split(' ')[-3:-1])) for x in temp_fails
])).items()))))
msgs_num = list(map(str.strip, msgs_num.split(':')))
adrpts = list(map(str.strip, adrpts.split(':')))
byttr = list(map(str.strip, byttr.split(':')))
egrp_emails = dict(list(map(rm_extra, [x.strip() for x in egrps])))
#send info
payload['@timestamp'] = timestp
payload['msgs_to_rcpts'] = msgs_num[1]
payload['addressed_rcpts'] = adrpts[1]
payload['byts_transfd'] = byttr[1]
total = 0
for k in egrp_emails:
payload['hn-' + k] = egrp_emails[k]
total += egrp_emails[k]
payload['posts'] = total
payload['forums'] = len(egrp_emails)
for k in temp_fails:
payload['fail-' + k] = temp_fails[k]
send_payload("hypernews", "mailinfo", timestp, dumps(payload))
def es_parse_log(logFile):
t = os.path.getmtime(logFile)
timestp = int(t * 1000)
payload = {}
pathInfo = logFile.split('/')
architecture = pathInfo[4]
release = pathInfo[8]
workflow = pathInfo[10].split('_')[0]
step = pathInfo[11].split('_')[0]
week, rel_sec = cmsswIB2Week(release)
index = "ib-matrix-" + week
document = "runTheMatrix-data"
id = sha1(release + architecture + workflow + str(step)).hexdigest()
logdir = '/'.join(logFile.split('/')[:-1])
cmdfile = logdir + '/cmdLog'
cmd_step = find_step_cmd(cmdfile, step)
if cmd_step: payload["command"] = cmd_step
wf_log = logdir + '/workflow.log'
exitcode = get_exit_code(wf_log, int(step[-1]))
if exitcode != -1: payload["exitcode"] = exitcode
payload["workflow"] = workflow
payload["release"] = release
payload["architecture"] = architecture
payload["step"] = step
payload["@timestamp"] = timestp
hostFile = "/".join(logFile.split('/')[:-1]) + "/hostname"
if os.path.exists(hostFile):
with open(hostFile, 'r') as hname:
payload["hostname"] = hname.readlines()[0].strip()
exception = ""
error = ""
errors = []
inException = False
inError = False
datasets = []
error_count = 0
if exists(logFile):
with open(logFile) as f:
lines = f.readlines()
payload[
"url"] = 'https://cmssdt.cern.ch/SDT/cgi-bin/buildlogs/' + pathInfo[
4] + '/' + pathInfo[8] + '/pyRelValMatrixLogs/run/' + pathInfo[
-2] + '/' + pathInfo[-1]
total_lines = len(lines)
for i in range(total_lines):
l = lines[i].strip()
if " Initiating request to open file " in l:
try:
rootfile = l.split(
" Initiating request to open file ")[1].split(" ")[0]
if (not "file:" in rootfile) and (not rootfile
in datasets):
#if (i+2)<total_lines:
# if (rootfile in lines[i+1]) and (rootfile in lines[i+2]) and ("Successfully opened file " in lines[i+1]) and ("Closed file " in lines[i+2]):
# print "File read with no valid events: %s" % rootfile
# continue
datasets.append(rootfile)
except:
pass
continue
if l.startswith("----- Begin Fatal Exception"):
inException = True
continue
if l.startswith("----- End Fatal Exception"):
inException = False
continue
if l.startswith("%MSG-e"):
inError = True
error = l
error_kind = re.split(" [0-9a-zA-Z-]* [0-9:]{8} CET",
error)[0].replace("%MSG-e ", "")
continue
if inError == True and l.startswith("%MSG"):
inError = False
if len(errors) < 10:
errors.append({"error": error, "kind": error_kind})
error_count += 1
error = ""
error_kind = ""
continue
if inException:
exception += l + "\n"
if inError:
error += l + "\n"
if exception:
payload["exception"] = exception
if errors:
payload["errors"] = errors
payload["error_count"] = error_count
try:
payload = es_parse_jobreport(payload, logFile)
except Exception as e:
print(e)
print("sending data for ", logFile)
try:
send_payload(index, document, id, json.dumps(payload))
except:
pass
if datasets:
dataset = {
"type": "relvals",
"name": "%s/%s" % (payload["workflow"], payload["step"])
}
for fld in ["release", "architecture", "@timestamp"]:
dataset[fld] = payload[fld]
for ds in datasets:
ds_items = ds.split("?", 1)
ds_items.append("")
ibeos = "/store/user/cmsbuild"
if ibeos in ds_items[0]:
ds_items[0] = ds_items[0].replace(ibeos, "")
else:
ibeos = ""
dataset["protocol"] = ds_items[0].split("/store/", 1)[0] + ibeos
dataset["protocol_opts"] = ds_items[1]
dataset["lfn"] = "/store/" + ds_items[0].split("/store/",
1)[1].strip()
idx = sha1(id + ds).hexdigest()
print(dataset)
send_payload("ib-dataset-" + week, "relvals-dataset", idx,
json.dumps(dataset))
sys.exit(1)
for line in cmd_out.split('\n'):
if re.match(match_tmp,line): temp_fails.append(line)
elif 'Messages To Recipients:' in line: msgs_num = line
elif 'Addressed Recipients:' in line : adrpts = line
elif 'Bytes Transferred:' in line: byttr = line
elif re.match(match_hn,line): egrps.append(line)
#process info
yesterday = datetime.date.today() - datetime.timedelta(1)
timestp = int(yesterday.strftime("%s")) * 1000
temp_fails = dict(list(map(map_int_val,list((dict([list(map(cust_strip,x.split(' ')[-3:-1])) for x in temp_fails])).items()))))
msgs_num= list(map(str.strip,msgs_num.split(':')))
adrpts = list(map(str.strip, adrpts.split(':')))
byttr = list(map(str.strip, byttr.split(':')))
egrp_emails = dict(list(map(rm_extra,[x.strip() for x in egrps])))
#send info
payload['@timestamp'] = timestp
payload['msgs_to_rcpts'] = msgs_num[1]
payload['addressed_rcpts'] = adrpts[1]
payload['byts_transfd'] = byttr[1]
total=0
for k in egrp_emails:
payload['hn-'+k] = egrp_emails[k]
total += egrp_emails[k]
payload['posts'] = total
payload['forums'] = len(egrp_emails)
for k in temp_fails:
payload['fail-'+k] = temp_fails[k]
send_payload("hypernews","mailinfo",timestp, dumps(payload), passwd_file="/data/es/es_secret")
if inException:
exception += l + "\n"
if inError:
error += l + "\n"
if exception:
payload["exception"] = exception
if errors:
payload["errors"] = errors
payload["error_count"] = error_count
try:
payload = es_parse_jobreport(payload, logFile)
except Exception, e:
print e
print "sending data for ", logFile
try:
send_payload(index, document, id, json.dumps(payload))
except:
pass
if datasets:
dataset = {
"type": "relvals",
"name": "%s/%s" % (payload["workflow"], payload["step"])
}
for fld in ["release", "architecture", "@timestamp"]:
dataset[fld] = payload[fld]
for ds in datasets:
ds_items = ds.split("?", 1)
ds_items.append("")
ibeos = "/store/user/cmsbuild"
if ibeos in ds_items[0]:
ds_items[0] = ds_items[0].replace(ibeos, "")
continue
if l.startswith("%MSG-e"):
inError = True
error = l
error_kind = re.split(" [0-9a-zA-Z-]* [0-9:]{8} CET", error)[0].replace("%MSG-e ", "")
continue
if inError == True and l.startswith("%MSG"):
inError = False
errors.append({"error": error, "kind": error_kind})
error = ""
error_kind = ""
continue
if inException:
exception += l + "\n"
if inError:
error += l + "\n"
if exception:
payload["exception"] = exception
if errors:
payload["errors"] = errors
try:
payload = es_parse_jobreport(payload,logFile)
except Exception, e:
print e
print "sending data for ",logFile
send_payload(index,document,id,json.dumps(payload))
if __name__ == "__main__":
print "Processing ",sys.argv[1]
es_parse_log(sys.argv[1])
if opts.show_tests: t = " ("+",".join(sorted(rels[k][b][r].keys()))+")"
x.append("%s%s" % (r,t))
print " Releases: %s " % ", ".join(x)
else:
print json.dumps(json_out, indent=2, sort_keys=True, separators=(',',': '))
else:
cmsweb=CMSWeb()
lfns = {}
ti = 0
updated = 0
for item in json_out:
for h in item["hits"]["hits"]:
ti+=1
lfn_info = h["_source"]
lfn_data = cmsweb.search(lfn_info["lfn"], lfn_info)
update = False
for item in lfn_data:
if (not item in lfn_info) or (lfn_data[item] != lfn_info[item]):
lfn_info[item]=lfn_data[item]
update = True
print "Processing .... %s/%s (LFNS: %s, NEW LFN: %s, DS: %s, BLK:%s, ERR: %s): %s" % (ti, total_hits ,len(cmsweb.cache['lfns']), len(cmsweb.cache['new_lfns']), len(cmsweb.cache['datasets']), len(cmsweb.cache['blocks']), cmsweb.errors, update)
if update:
send_payload(h['_index'], h['_type'], h['_id'], json.dumps(lfn_info))
updated += 1
print "Updated Records:", updated
exit(0)
lfns = []
for lfn in datasets: lfns.append([lfn, datasets[lfn]])
download(lfns, opts.store, opts.job,opts.dryRun)
print "Total Files:",len(datasets)
def process_build_any_ib(logFile):
rel = ""
arch = ""
uploadTime=0
stime=0
upload=False
jstart = 0
jend=0
patch = 0
finished = False
with open(logFile) as f:
for line in f:
line = line.strip()
if not jstart:
m=ReDate.match(line)
if m:
jstart = datetime.strptime(m.group(1)+m.group(2), "%b %d %H:%M:%S %Y")
continue
if not arch:
m=ReArch.match(line)
if m: arch=m.group(1)
continue
if not rel:
m=ReRel.match(line)
if m: rel=m.group(1)
continue
if ReFinish.match(line):
finished = True
if "ABORTED" in line: return True
break
if ReUpload.match(line):
upload=True
continue
if ReType.match(line): patch=1
m=ReDate.match(line)
if not m: continue
xtime = datetime.strptime(m.group(1)+m.group(2), "%b %d %H:%M:%S %Y")
jend = xtime
if not upload:
stime = xtime
else:
upload=False
dtime = xtime - stime
uploadTime += dtime.seconds
print "FINISHED: ",finished,rel, arch,uploadTime
if not rel or not arch or not finished: return finished
urlx = logFile.split("/")
url = "https://cmssdt.cern.ch/jenkins/job/build-any-ib/"+logFile.split("/")[-2]+"/console"
timestp = getmtime(logFile)
ttime = jend - jstart
print ttime.seconds, uploadTime, rel, arch, patch, url
payload = {}
payload["release"] = rel
payload["architecture"] = arch
payload["total_time"] = ttime.seconds
payload["upload_time"] = uploadTime
payload["patch"] = patch
payload["@timestamp"] = int(timestp*1000)
payload["url"]=url
print payload
id = sha1(rel + arch).hexdigest()
send_payload("jenkins-ibs","timings",id,json.dumps(payload))
return finished
def es_parse_log(logFile):
t = os.path.getmtime(logFile)
timestp = int(t*1000)
payload = {}
pathInfo = logFile.split('/')
architecture = pathInfo[4]
release = pathInfo[8]
workflow = pathInfo[10].split('_')[0]
step = pathInfo[11].split('_')[0]
week, rel_sec = cmsswIB2Week(release)
index = "ib-matrix-" + week
document = "runTheMatrix-data"
id = sha1(release + architecture + workflow + str(step)).hexdigest()
logdir = '/'.join(logFile.split('/')[:-1])
cmdfile = logdir + '/cmdLog'
cmd_step = find_step_cmd(cmdfile,step)
if cmd_step: payload["command"] = cmd_step
wf_log = logdir + '/workflow.log'
exitcode = get_exit_code(wf_log,int(step[-1]))
if exitcode != -1 : payload["exitcode"] = exitcode
payload["workflow"] = workflow
payload["release"] = release
payload["architecture"] = architecture
payload["step"] = step
payload["@timestamp"] = timestp
hostFile = "/".join(logFile.split('/')[:-1]) + "/hostname"
if os.path.exists (hostFile):
with open(hostFile,'r') as hname:
payload["hostname"] = hname.readlines()[0].strip()
exception = ""
error = ""
errors = []
inException = False
inError = False
datasets = []
error_count = 0
if exists(logFile):
with open(logFile) as f:
lines = f.readlines()
payload["url"] = 'https://cmssdt.cern.ch/SDT/cgi-bin/buildlogs/'+pathInfo[4]+'/'+pathInfo[8]+'/pyRelValMatrixLogs/run/'+pathInfo[-2]+'/'+pathInfo[-1]
total_lines = len(lines)
for i in range(total_lines):
l = lines[i].strip()
if " Initiating request to open file " in l:
try:
rootfile = l.split(" Initiating request to open file ")[1].split(" ")[0]
if (not "file:" in rootfile) and (not rootfile in datasets):
#if (i+2)<total_lines:
# if (rootfile in lines[i+1]) and (rootfile in lines[i+2]) and ("Successfully opened file " in lines[i+1]) and ("Closed file " in lines[i+2]):
# print "File read with no valid events: %s" % rootfile
# continue
datasets.append(rootfile)
except: pass
continue
if l.startswith("----- Begin Fatal Exception"):
inException = True
continue
if l.startswith("----- End Fatal Exception"):
inException = False
continue
if l.startswith("%MSG-e"):
inError = True
error = l
error_kind = re.split(" [0-9a-zA-Z-]* [0-9:]{8} CET", error)[0].replace("%MSG-e ", "")
continue
if inError == True and l.startswith("%MSG"):
inError = False
if len(errors)<10:
errors.append({"error": error, "kind": error_kind})
error_count += 1
error = ""
error_kind = ""
continue
if inException:
exception += l + "\n"
if inError:
error += l + "\n"
if exception:
payload["exception"] = exception
if errors:
payload["errors"] = errors
payload["error_count"] = error_count
try:
payload = es_parse_jobreport(payload,logFile)
except Exception as e:
print(e)
print("sending data for ",logFile)
try:send_payload(index,document,id,json.dumps(payload))
except:pass
if datasets:
dataset = {"type" : "relvals", "name" : "%s/%s" % (payload["workflow"], payload["step"])}
for fld in ["release","architecture","@timestamp"]: dataset[fld] = payload[fld]
for ds in datasets:
ds_items = ds.split("?",1)
ds_items.append("")
ibeos = "/store/user/cmsbuild"
if ibeos in ds_items[0]: ds_items[0] = ds_items[0].replace(ibeos,"")
else: ibeos=""
dataset["protocol"]=ds_items[0].split("/store/",1)[0]+ibeos
dataset["protocol_opts"]=ds_items[1]
dataset["lfn"]="/store/"+ds_items[0].split("/store/",1)[1].strip()
idx = sha1(id + ds).hexdigest()
print(dataset)
send_payload("ib-dataset-"+week,"relvals-dataset",idx,json.dumps(dataset))
'^\[[A-Za-z]{3} ([A-Za-z]{3} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [0-9]{4})\] \[[^\]]+\] \[client (.+)\]\s(.+)'
)
for log in out.split("\n"):
find_cmd = "grep '%s' %s %s" % (search_for, log, filter_search)
err, out = run_cmd(find_cmd)
for line in out.split("\n"):
m = ReTime.match(line)
if m:
tsec = int(
datetime.strptime(m.group(1),
"%b %d %H:%M:%S %Y").strftime('%s'))
week = str(int(tsec / (86400 * 7)))
timestamp = tsec * 1000
payload = {}
payload['@timestamp'] = timestamp
payload['ip'] = m.group(2)
payload['message'] = line
id = sha1(str(timestamp) + m.group(2)).hexdigest()
send_payload("hypernews-" + week,
"hn-timeouts",
id,
dumps(payload),
passwd_file="/data/es/es_secret")
payload = {}
payload['@timestamp'] = int(time() * 1000)
send_payload("hypernews",
"hn-heartbeat",
str(payload['@timestamp']),
dumps(payload),
passwd_file="/data/es/es_secret")
#process info
yesterday = datetime.date.today() - datetime.timedelta(1)
timestp = int(yesterday.strftime("%s")) * 1000
temp_fails = dict(
map(map_int_val,
(dict([map(cust_strip,
x.split(' ')[-3:-1]) for x in temp_fails])).items()))
msgs_num = map(str.strip, msgs_num.split(':'))
adrpts = map(str.strip, adrpts.split(':'))
byttr = map(str.strip, byttr.split(':'))
egrp_emails = dict(map(rm_extra, [x.strip() for x in egrps]))
#send info
payload['@timestamp'] = timestp
payload['msgs_to_rcpts'] = msgs_num[1]
payload['addressed_rcpts'] = adrpts[1]
payload['byts_transfd'] = byttr[1]
total = 0
for k in egrp_emails:
payload['hn-' + k] = egrp_emails[k]
total += egrp_emails[k]
payload['posts'] = total
payload['forums'] = len(egrp_emails)
for k in temp_fails:
payload['fail-' + k] = temp_fails[k]
send_payload("hypernews",
"mailinfo",
timestp,
dumps(payload),
passwd_file="/data/es/es_secret")
build_result = root.find('result')
if build_result is not None:
payload['build_result'] = build_result.text
payload['build_duration'] = int(
int(root.find('duration').text) / 1000)
payload['job_status'] = 'Finished'
os.system('touch "' + flagFile + '"')
else:
payload['job_status'] = 'Running'
all_local.append(id)
weekindex = "jenkins-jobs-" + str(
int((((int(jstime) / 1000) / 86400) + 4) / 7))
send_payload(
weekindex,
document,
id,
json.dumps(payload),
passwd_file=
"/var/lib/jenkins/secrets/github_hook_secret_cmsbot")
except Exception as e:
print "Xml parsing error", logFile, e
running_builds_elastic = {}
content = get_payload('jenkins-*', query_running_builds)
if content == "":
running_builds_elastic = []
else:
content_hash = json.loads(content)
if (not 'hits' in content_hash) or (not 'hits' in content_hash['hits']):
print "ERROR: ", content
sys.exit(1)
for hit in content_hash['hits']['hits']:
jstime = root.find('startTime').text
payload['@timestamp'] = int(jstime)
payload['slave_node'] = root.find('builtOn').text
payload['jenkins_server'] = JENKINS_PREFIX
build_result = root.find('result')
if build_result is not None:
payload['build_result'] = build_result.text
payload['build_duration'] = int(int(root.find('duration').text)/1000)
payload['job_status'] = 'Finished'
os.system('touch "' + flagFile + '"')
else:
payload['job_status'] = 'Running'
all_local.append(id)
weekindex="jenkins-jobs-"+str(int((((int(jstime)/1000)/86400)+4)/7))
print("==>",id,payload['job_name'],payload['build_number'],payload['job_status'])
send_payload(weekindex,document,id,json.dumps(payload))
except Exception as e:
print("Xml parsing error",logFile , e)
running_builds_elastic={}
content_hash = get_payload_wscroll('jenkins-*',query_running_builds)
if not content_hash:
running_builds_elastic = {}
else:
if (not 'hits' in content_hash) or (not 'hits' in content_hash['hits']):
print("ERROR: ",content)
sys.exit(1)
print("Found:", len(content_hash['hits']['hits']))
for hit in content_hash['hits']['hits']:
if hit["_index"].startswith("cmssdt-jenkins-jobs-"):
if not "jenkins_server" in hit["_source"]: hit["_source"]["jenkins_server"] = JENKINS_PREFIX
if hit["_source"]["jenkins_server"]!=JENKINS_PREFIX: continue
