def process (line, count): payload = {} items = line.split(" ") if len(items)<10: return True if not (items[3][0]=='[' and items[4][-1]==']'): return True payload["ip"]=items[0] payload["ident"]=items[1] payload["auth"]=items[2] payload["verb"]=items[5][1:] payload["request"]=items[6] payload["httpversion"]=items[7][:-1] payload["response"]=items[8] try: payload["bytes"]=int(items[9]) except: payload["bytes"]=0 payload["@timestamp"]=int(mktime(datetime.strptime(items[3][1:],'%d/%b/%Y:%H:%M:%S').timetuple())*1000) if len(items)>10: payload["referrer"]=items[10][1:-1] if len(items)>11 and re.match('^"[0-9]+(\.[0-9]+)+"$', items[11]): payload["ip"]=items[11][1:-1] if len(items)>12: agent = " ".join(items[12:]).replace('"','') payload["agent"] = agent payload["agent_type"]=agent.replace(" ","-").split("/",1)[0].upper() id = sha1(line).hexdigest() if (count%1000)==0: print "Processed entries",count if not send_payload("apache-cmssdt","access_log", id, dumps(payload), passwd_file="/data/es/es_secret"): return False if payload["request"].startswith("/SDT/releases.map?release="): xpayload = dict(item.split("=") for item in payload["request"].split("?",1)[1].split("&")) for x in ["@timestamp","ip"]: xpayload[x] = payload[x] return send_payload("scram-access","cmssw-releases", id, dumps(xpayload), passwd_file="/data/es/es_secret") return True
def process_matrix_log(logFile): t = os.path.getmtime(logFile) timestp = int(t*1000) payload = {} pathInfo = logFile.split('/') architecture = pathInfo[4] release = pathInfo[8] workflow = pathInfo[10].split('_')[0] step = pathInfo[11].split('_')[0] dat = re.findall('\d{4}-\d{2}-\d{2}',release)[0] week = strftime("%U",strptime(dat,"%Y-%m-%d")) id = sha1(release + architecture + workflow + str(step)).hexdigest() dataset = {"type" : "relvals", "name" : "%s/%s" % (workflow, step), "ds_block" : "UNKNOWN", "ds_status" : "UNKNOWN", "ds_owner" : "UNKNOWN", "ds_files" : "0", "at_cern" : "UNKNOWN", "dataset" : "UNKNOWN"} dataset["release"]=release dataset["architecture"]=architecture dataset["@timestamp"]=timestp datasets=[] lines = file(logFile).read() for l in lines.split("\n"): if not " Initiating request to open file " in l: continue rootfile = l.split(" Initiating request to open file ")[1].split(" ")[0] if (not "file:" in rootfile) and (not rootfile in datasets): datasets.append(rootfile) for ds in datasets: ds_items = ds.split("?",1) ds_items.append("") dataset["protocol"]=ds_items[0].split("/store/",1)[0] dataset["protocol_opts"]=ds_items[1] dataset["lfn"]="/store/"+ds_items[0].split("/store/",1)[1] idx = sha1(id + ds).hexdigest() send_payload("ib-dataset-"+week,"relvals-dataset",idx,json.dumps(dataset)) return
def send_unittest_dataset(datasets, payload, id, index, doc): for ds in datasets: ds_items = ds.split("?",1) ds_items.append("") payload["protocol"]=ds_items[0].split("/store/",1)[0] payload["protocol_opts"]=ds_items[1] payload["lfn"]="/store/"+ds_items[0].split("/store/",1)[1] send_payload(index, doc, sha1(id + ds).hexdigest(), json.dumps(payload))
def send_unittest_dataset(datasets, payload, id, index, doc): for ds in datasets: ds_items = ds.split("?", 1) ds_items.append("") payload["protocol"] = ds_items[0].split("/store/", 1)[0] payload["protocol_opts"] = ds_items[1] payload["lfn"] = "/store/" + ds_items[0].split("/store/", 1)[1] send_payload(index, doc, sha1(id + ds).hexdigest(), json.dumps(payload))
def es_parse_log(logFile): t = os.path.getmtime(logFile) timestp = datetime.datetime.fromtimestamp(int(t)).strftime('%Y-%m-%d %H:%M:%S') payload = {} pathInfo = logFile.split('/') architecture = pathInfo[4] release = pathInfo[8] workflow = pathInfo[10].split('_')[0] step = pathInfo[11].split('_')[0] dat = re.findall('\d{4}-\d{2}-\d{2}',release)[0] week = strftime("%U",strptime(dat,"%Y-%m-%d")) index = "ib-matrix-" + week document = "runTheMatrix-data" id = sha1(release + architecture + workflow + str(step)).hexdigest() payload["workflow"] = workflow payload["release"] = release payload["architecture"] = architecture payload["step"] = step payload["hostname"] = gethostname() payload["timestamp"] = timestp exception = "" error = "" errors = [] inException = False inError = False if exists(logFile): lines = file(logFile).read() payload["url"] = logFile.replace('/data/sdt/' , 'https://cmssdt.cern.ch/SDT/cgi-bin/') for l in lines.split("\n"): if l.startswith("----- Begin Fatal Exception"): inException = True continue if l.startswith("----- End Fatal Exception"): inException = False continue if l.startswith("%MSG-e"): inError = True error = l error_kind = re.split(" [0-9a-zA-Z-]* [0-9:]{8} CET", error)[0].replace("%MSG-e ", "") continue if inError == True and l.startswith("%MSG"): inError = False errors.append({"error": error, "kind": error_kind}) error = "" error_kind = "" continue if inException: exception += l + "\n" if inError: error += l + "\n" if exception: payload["exception"] = exception if errors: payload["errors"] = errors send_payload(index,document,id,json.dumps(payload))
def es_parse_log(logFile): t = os.path.getmtime(logFile) timestp = datetime.datetime.fromtimestamp(int(t)).strftime('%Y-%m-%d %H:%M:%S') payload = {} pathInfo = logFile.split('/') architecture = pathInfo[4] release = pathInfo[8] workflow = pathInfo[10].split('_')[0] step = pathInfo[11].split('_')[0] dat = re.findall('\d{4}-\d{2}-\d{2}',release)[0] week = strftime("%U",strptime(dat,"%Y-%m-%d")) index = "ib-matrix-" + week document = "runTheMatrix-data" id = sha1(release + architecture + workflow + str(step)).hexdigest() payload["workflow"] = workflow payload["release"] = release payload["architecture"] = architecture payload["step"] = step payload["hostname"] = gethostname() payload["timestp"] = timestp exception = "" error = "" errors = [] inException = False inError = False if exists(logFile): lines = file(logFile).read() payload["url"] = logFile.replace('/data/sdt/' , 'https://cmssdt.cern.ch/SDT/cgi-bin/') for l in lines.split("\n"): if l.startswith("----- Begin Fatal Exception"): inException = True continue if l.startswith("----- End Fatal Exception"): inException = False continue if l.startswith("%MSG-e"): inError = True error = l error_kind = re.split(" [0-9a-zA-Z-]* [0-9:]{8} CET", error)[0].replace("%MSG-e ", "") continue if inError == True and l.startswith("%MSG"): inError = False errors.append({"error": error, "kind": error_kind}) error = "" error_kind = "" continue if inException: exception += l + "\n" if inError: error += l + "\n" if exception: payload["exception"] = exception if errors: payload["errors"] = errors send_payload(index,document,id,json.dumps(payload))
def send_unittest_dataset(datasets, payload, id, index, doc): for ds in datasets: ds_items = ds.split("?",1) ds_items.append("") ibeos = "/store/user/cmsbuild" if ibeos in ds_items[0]: ds_items[0] = ds_items[0].replace(ibeos,"") else: ibeos="" payload["protocol"]=ds_items[0].split("/store/",1)[0]+ibeos payload["protocol_opts"]=ds_items[1] payload["lfn"]="/store/"+ds_items[0].split("/store/",1)[1] send_payload(index, doc, sha1(id + ds).hexdigest(), json.dumps(payload))
def process (line, count): for agent in LOGWATCH_APACHE_IGNORE_AGENTS: if agent in line: return True payload = {} items = line.split(" ") if len(items)<12: return True if not (items[3][0]=='[' and items[4][-1]==']'): return True payload["ip"]=items[0] payload["ident"]=items[1] payload["auth"]=items[2] payload["verb"]=items[5][1:] payload["request"]=items[6] payload["httpversion"]=items[7][:-1] payload["response"]=items[8] try: payload["bytes"]=int(items[9]) except: payload["bytes"]=0 payload["referrer"]=items[10][1:-1] agent = " ".join(items[11:]).replace('"','') if "CMSPKG-v" in agent: agent = agent.replace("-v","/") payload["agent"]=agent payload["agent_type"]=agent.replace(" ","-").split("/",1)[0].upper() tsec = mktime(datetime.strptime(items[3][1:],'%d/%b/%Y:%H:%M:%S').timetuple()) week = str(int(tsec/(86400*7))) payload["@timestamp"]=int(tsec*1000) id = sha1(line).hexdigest() if (count%1000)==0: print "Processed entries",count if not send_payload("apache-cmsrep-"+week,"access_log", id, dumps(payload), passwd_file="/data/es/es_secret"): return False if payload["verb"] != "GET": return True items = payload["request"].replace("/cms/cpt/Software/download/","/cmssw/",1).split("/") if len(items)<6: return True if items[3] == "apt": items[3]="PRMS" if items[3] != "RPMS": return True pkg, cmspkg, arch, repo, dev = items[-1], "apt", "" , "", 0 if "?" in pkg: pkg, pkgopts = pkg.split("?",1) if "version=" in pkgopts: cmspkg = pkgopts.split("version=",1)[1].split("&",1)[0] if not pkg.endswith(".rpm"): return True if (items[1] == "cgi-bin") and items[2].startswith("cmspkg"): if len(items)<8: return True if items[2].endswith('-dev'): dev=1 repo, arch = items[4], items[5] elif items[1] == "cmssw": repo, arch = items[2], items[4] else: return True from urllib import unquote xpayload = {'dev' : dev, 'repository' : unquote(repo), 'architecture' : unquote(arch), 'package' : unquote(pkg).split("-1-",1)[0], 'cmspkg' : unquote(cmspkg)} for x in ["@timestamp","ip"]: xpayload[x] = payload[x] return send_payload("cmspkg-access-"+week,"rpm-packages", id, dumps(xpayload), passwd_file="/data/es/es_secret")
def process(wfnum, s, sfile, hostname, exit_code, details=False): global release, arch, rel_msec, week, ex_fields, cmsThreads try: stats = json.load(open(sfile)) xdata = {} for stat in stats: for item in stat: try: xdata[item].append(stat[item]) except: xdata[item]=[] xdata[item].append(stat[item]) stat["@timestamp"]=rel_msec+(stat["time"]*1000) stat["release"]=release stat["step"]=s stat["workflow"]=wfnum stat["cmsthreads"]=cmsThreads stat["architecture"]=arch idx = sha1(release + arch + wfnum + s + str(stat["time"])).hexdigest() del stat["time"] if details: try:send_payload("relvals_stats_details-"+week,"runtime-stats-details",idx,json.dumps(stat)) except Exception as e: print(e) print("Working on ",release, arch, wfnum, s, len(stats)) sdata = {"release":release, "architecture":arch, "step":s, "@timestamp":rel_msec, "workflow":wfnum, "hostname":hostname, "exit_code":exit_code, "cmsthreads":cmsThreads} for x in xdata: data = sorted(xdata[x]) if x in ["time","num_threads","processes","num_fds"]: sdata[x]=data[-1] continue if not x in ex_fields: continue dlen = len(data) for t in ["min", "max", "avg", "median", "25", "75"]: sdata[x+"_"+t]=0 if dlen>0: sdata[x+"_min"]=data[0] sdata[x+"_max"]=data[-1] if dlen>1: dlen2=int(dlen/2) if (dlen%2)==0: sdata[x+"_median"]=int((data[dlen2-1]+data[dlen2])/2) else: sdata[x+"_median"]=data[dlen2] sdata[x+"_avg"]=int(sum(data)/dlen) for t in [25, 75]: sdata[x+"_"+str(t)]=int(percentile(t,data, dlen)) else: for t in ["25", "75", "avg", "median"]: sdata[x+"_"+t]=data[0] idx = sha1(release + arch + wfnum + s + str(rel_sec)).hexdigest() try:send_payload("relvals_stats_summary-"+week,"runtime-stats-summary",idx,json.dumps(sdata)) except Exception as e: print(e) except Exception as e: print(e) return
def process(wfnum, s, sfile, hostname, exit_code, details=False): global release, arch, rel_msec, week, ex_fields, cmsThreads try: stats = json.load(open(sfile)) xdata = {} for stat in stats: for item in stat: try: xdata[item].append(stat[item]) except: xdata[item]=[] xdata[item].append(stat[item]) stat["@timestamp"]=rel_msec+(stat["time"]*1000) stat["release"]=release stat["step"]=s stat["workflow"]=wfnum stat["cmsthreads"]=cmsThreads stat["architecture"]=arch idx = sha1(release + arch + wfnum + s + str(stat["time"])).hexdigest() del stat["time"] if details: try:send_payload("relvals_stats_details-"+week,"runtime-stats-details",idx,json.dumps(stat)) except Exception as e: print e print "Working on ",release, arch, wfnum, s, len(stats) sdata = {"release":release, "architecture":arch, "step":s, "@timestamp":rel_msec, "workflow":wfnum, "hostname":hostname, "exit_code":exit_code, "cmsthreads":cmsThreads} for x in xdata: data = sorted(xdata[x]) if x in ["time","num_threads","processes","num_fds"]: sdata[x]=data[-1] continue if not x in ex_fields: continue dlen = len(data) for t in ["min", "max", "avg", "median", "25", "75"]: sdata[x+"_"+t]=0 if dlen>0: sdata[x+"_min"]=data[0] sdata[x+"_max"]=data[-1] if dlen>1: dlen2=int(dlen/2) if (dlen%2)==0: sdata[x+"_median"]=int((data[dlen2-1]+data[dlen2])/2) else: sdata[x+"_median"]=data[dlen2] sdata[x+"_avg"]=int(sum(data)/dlen) for t in [25, 75]: sdata[x+"_"+str(t)]=int(percentile(t,data, dlen)) else: for t in ["25", "75", "avg", "median"]: sdata[x+"_"+t]=data[0] idx = sha1(release + arch + wfnum + s + str(rel_sec)).hexdigest() try:send_payload("relvals_stats_summary-"+week,"runtime-stats-summary",idx,json.dumps(sdata)) except Exception as e: print e except Exception as e: print e return
def process (line, count): for agent in LOGWATCH_APACHE_IGNORE_AGENTS: if agent in line: return True payload = {} items = line.split(" ") if len(items)<10: return True if not (items[3][0]=='[' and items[4][-1]==']'): return True payload["ip"]=items[0] payload["ident"]=items[1] payload["auth"]=items[2] payload["verb"]=items[5][1:] payload["request"]=items[6] payload["httpversion"]=items[7][:-1] payload["response"]=items[8] try: payload["bytes"]=int(items[9]) except: payload["bytes"]=0 payload["@timestamp"]=int(mktime(datetime.strptime(items[3][1:],'%d/%b/%Y:%H:%M:%S').timetuple())*1000) if len(items)>10: payload["referrer"]=items[10][1:-1] if len(items)>11 and re.match('^"[0-9]+(\.[0-9]+)+"$', items[11]): payload["ip"]=items[11][1:-1] if len(items)>12: agent = " ".join(items[12:]).replace('"','') payload["agent"] = agent payload["agent_type"]=agent.replace(" ","-").split("/",1)[0].upper() id = sha1(line).hexdigest() if (count%1000)==0: print "Processed entries",count if not send_payload("apache-doxygen","access_log", id, dumps(payload), passwd_file="/data/es/es_secret"): return False return True
def process (line, count): for agent in LOGWATCH_APACHE_IGNORE_AGENTS: if agent in line: return True payload = {} items = line.split(" ") if len(items)<10: return True if not (items[3][0]=='[' and items[4][-1]==']'): return True payload["ip"]=items[0] payload["ident"]=items[1] payload["auth"]=items[2] payload["verb"]=items[5][1:] payload["request"]=items[6] payload["httpversion"]=items[7][:-1] payload["response"]=items[8] try: payload["bytes"]=int(items[9]) except: payload["bytes"]=0 tsec = mktime(datetime.strptime(items[3][1:],'%d/%b/%Y:%H:%M:%S').timetuple()) week = str(int(tsec/(86400*7))) payload["@timestamp"]=int(tsec*1000) if len(items)>10: payload["referrer"]=items[10][1:-1] if len(items)>11 and re.match('^"[0-9]+(\.[0-9]+)+"$', items[11]): payload["ip"]=items[11][1:-1] if len(items)>12: agent = " ".join(items[12:]).replace('"','') payload["agent"] = agent payload["agent_type"]=agent.replace(" ","-").split("/",1)[0].upper() id = sha1(line).hexdigest() if (count%1000)==0: print "Processed entries",count if not send_payload("apache-doxygen-"+week,"access_log", id, dumps(payload), passwd_file="/data/es/es_secret"): return False return True
def process (line, count): payload = {} items = line.split(" ") if len(items)<12: return True if not (items[3][0]=='[' and items[4][-1]==']'): return True payload["ip"]=items[0] payload["ident"]=items[1] payload["auth"]=items[2] payload["verb"]=items[5][1:] payload["request"]=items[6] payload["httpversion"]=items[7][:-1] payload["response"]=items[8] try: payload["bytes"]=int(items[9]) except: payload["bytes"]=0 payload["referrer"]=items[10][1:-1] agent = " ".join(items[11:]).replace('"','') if "CMSPKG-v" in agent: agent = agent.replace("-v","/") payload["agent"]=agent payload["agent_type"]=agent.replace(" ","-").split("/",1)[0].upper() payload["@timestamp"]=int(mktime(datetime.strptime(items[3][1:],'%d/%b/%Y:%H:%M:%S').timetuple())*1000) id = sha1(line).hexdigest() if (count%1000)==0: print "Processed entries",count return send_payload("apache-cmsrep","access_log", id, dumps(payload), passwd_file="/data/es/es_secret")
def send_unittest_dataset(datasets, payload, id, index, doc): for ds in datasets: print("Processing ", ds) if not 'root://' in ds: continue ds_items = ds.split("?", 1) ds_items.append("") ibeos = "/store/user/cmsbuild" if ibeos in ds_items[0]: ds_items[0] = ds_items[0].replace(ibeos, "") else: ibeos = "" payload["protocol"] = ds_items[0].split("/store/", 1)[0] + ibeos payload["protocol_opts"] = ds_items[1] payload["lfn"] = "/store/" + ds_items[0].split("/store/", 1)[1].strip() print("Sending", index, doc, sha1(id + ds).hexdigest(), json.dumps(payload)) send_payload(index, doc, sha1(id + ds).hexdigest(), json.dumps(payload))
def process_ib_utests(logFile): t = getmtime(logFile) timestp = datetime.datetime.fromtimestamp( int(t)).strftime('%Y-%m-%d %H:%M:%S') payload = {} pathInfo = logFile.split('/') architecture = pathInfo[4] release = pathInfo[8] #dat = re.findall('\d{4}-\d{2}-\d{2}',release)[0] index = "ibs" document = "unittests" payload["release"] = release payload["architecture"] = architecture payload["@timestamp"] = timestp if exists(logFile): with open(logFile) as f: try: it = iter(f) line = it.next() while '--------' not in line: line = it.next() while True: line = it.next().strip() if ":" in line: pkg = line.split(':')[0].strip() payload[ "url"] = 'https://cmssdt.cern.ch/SDT/cgi-bin/buildlogs/' + architecture + '/' + release + '/unitTestLogs/' + pkg line = it.next().strip() while ':' not in line: if "had ERRORS" in line: payload["status"] = 1 else: payload["status"] = 0 utest = line.split(' ')[0] payload["package"] = pkg payload["name"] = utest id = sha1(release + architecture + utest).hexdigest() send_payload(index, document, id, json.dumps(payload)) line = it.next().strip() except Exception as e: print "File processed:", e else: print "Invalid File Path"
def process_ib_utests(logFile): t = getmtime(logFile) timestp = int(t * 1000) payload = {} pathInfo = logFile.split('/') architecture = pathInfo[4] release = pathInfo[8] week, rel_sec = cmsswIB2Week(release) index = "ibs-" + week document = "unittests" payload["release"] = release payload["architecture"] = architecture payload["@timestamp"] = timestp if exists(logFile): with open(logFile) as f: try: it = iter(f) line = next(it) while '--------' not in line: line = next(it) while True: line = it.next().strip() if ":" in line: pkg = line.split(':')[0].strip() payload[ "url"] = 'https://cmssdt.cern.ch/SDT/cgi-bin/buildlogs/' + architecture + '/' + release + '/unitTestLogs/' + pkg line = it.next().strip() while ':' not in line: if "had ERRORS" in line: payload["status"] = 1 else: payload["status"] = 0 utest = line.split(' ')[0] payload["package"] = pkg payload["name"] = utest id = sha1(release + architecture + utest).hexdigest() send_payload(index, document, id, json.dumps(payload)) line = it.next().strip() except Exception as e: print("ERROR: File processed: %s" % e) else: print("Invalid File Path")
def process_matrix_log(logFile): t = os.path.getmtime(logFile) timestp = int(t * 1000) payload = {} pathInfo = logFile.split('/') architecture = pathInfo[4] release = pathInfo[8] workflow = pathInfo[10].split('_')[0] step = pathInfo[11].split('_')[0] dat = re.findall('\d{4}-\d{2}-\d{2}', release)[0] week = strftime("%U", strptime(dat, "%Y-%m-%d")) id = sha1(release + architecture + workflow + str(step)).hexdigest() dataset = { "type": "relvals", "name": "%s/%s" % (workflow, step), "ds_block": "UNKNOWN", "ds_status": "UNKNOWN", "ds_owner": "UNKNOWN", "ds_files": "0", "at_cern": "UNKNOWN", "dataset": "UNKNOWN" } dataset["release"] = release dataset["architecture"] = architecture dataset["@timestamp"] = timestp datasets = [] lines = file(logFile).read() for l in lines.split("\n"): if not " Initiating request to open file " in l: continue rootfile = l.split(" Initiating request to open file ")[1].split( " ")[0] if (not "file:" in rootfile) and (not rootfile in datasets): datasets.append(rootfile) for ds in datasets: ds_items = ds.split("?", 1) ds_items.append("") dataset["protocol"] = ds_items[0].split("/store/", 1)[0] dataset["protocol_opts"] = ds_items[1] dataset["lfn"] = "/store/" + ds_items[0].split("/store/", 1)[1] idx = sha1(id + ds).hexdigest() send_payload("ib-dataset-" + week, "relvals-dataset", idx, json.dumps(dataset)) return
def process (line, count): payload = {} items = line.split(" ") if len(items)<12: return True if not (items[3][0]=='[' and items[4][-1]==']'): return True payload["ip"]=items[0] payload["ident"]=items[1] payload["auth"]=items[2] payload["verb"]=items[5][1:] payload["request"]=items[6] payload["httpversion"]=items[7][:-1] payload["response"]=items[8] try: payload["bytes"]=int(items[9]) except: payload["bytes"]=0 payload["referrer"]=items[10][1:-1] agent = " ".join(items[11:]).replace('"','') if "CMSPKG-v" in agent: agent = agent.replace("-v","/") payload["agent"]=agent payload["agent_type"]=agent.replace(" ","-").split("/",1)[0].upper() payload["@timestamp"]=int(mktime(datetime.strptime(items[3][1:],'%d/%b/%Y:%H:%M:%S').timetuple())*1000) id = sha1(line).hexdigest() if (count%1000)==0: print "Processed entries",count if not send_payload("apache-cmsrep","access_log", id, dumps(payload), passwd_file="/data/es/es_secret"): return False if not payload["request"].startswith("/cgi-bin/cmspkg"): return True items = payload["request"].split("/") if len(items)<8: return True if items[3] != "RPMS": return True dev = 0 if items[2].endswith('-dev'): dev=1 xpayload = {} try: from urllib import unquote pkg, rest = items[7].split("?",1) cmspkg = rest.split("version=",1)[1].split("&",1)[0] xpayload = {'dev' : dev, 'repository' : items[4], 'architecture' : items[5], 'package' : unquote(pkg).split("-1-",1)[0], 'cmspkg' : cmspkg} except: return True for x in ["@timestamp","ip"]: xpayload[x] = payload[x] return send_payload("cmspkg-access","rpm-packages", id, dumps(xpayload), passwd_file="/data/es/es_secret")
def process_ib_utests(logFile): t = getmtime(logFile) timestp = int(t*1000) payload = {} pathInfo = logFile.split('/') architecture = pathInfo[4] release = pathInfo[8] week, rel_sec = cmsswIB2Week (release) index = "ibs-"+week document = "unittests" payload["release"] = release payload["architecture"] = architecture payload["@timestamp"] = timestp if exists(logFile): with open(logFile) as f: try: it = iter(f) line = next(it) while '--------' not in line: line = next(it) while True: line=it.next().strip() if ":" in line: pkg = line.split(':')[0].strip() payload["url"] = 'https://cmssdt.cern.ch/SDT/cgi-bin/buildlogs/'+ architecture +'/'+ release +'/unitTestLogs/' + pkg line = it.next().strip() while ':' not in line: if "had ERRORS" in line: payload["status"] = 1 else: payload["status"] = 0 utest= line.split(' ')[0] payload["package"] = pkg payload["name"] = utest id = sha1(release + architecture + utest).hexdigest() send_payload(index,document,id,json.dumps(payload)) line = it.next().strip() except Exception as e: print("File processed:", e) else: print("Invalid File Path")
def process_ib_utests(logFile): t = os.path.getmtime(logFile) timestp = datetime.datetime.fromtimestamp(int(t)).strftime('%Y-%m-%d %H:%M:%S') payload = {} pathInfo = logFile.split('/') architecture = pathInfo[4] release = pathInfo[8] #dat = re.findall('\d{4}-\d{2}-\d{2}',release)[0] index = "ibs" document = "unittests" payload["release"] = release payload["architecture"] = architecture payload["@timestamp"] = timestp if exists(logFile): with open(logFile) as f: try: it = iter(f) line = it.next() while '--------' not in line: line = it.next() while True: line=it.next().strip() if ":" in line: pkg = line.split(':')[0].strip() payload["url"] = 'https://cmssdt.cern.ch/SDT/cgi-bin/buildlogs/'+ architecture +'/'+ release +'/unitTestLogs/' + pkg line = it.next().strip() while ':' not in line: if "had ERRORS" in line: payload["status"] = 1 else: payload["status"] = 0 utest= line.split(' ')[0] payload["package"] = pkg payload["name"] = utest id = sha1(release + architecture + utest).hexdigest() send_payload(index,document,id,json.dumps(payload)) line = it.next().strip() except Exception as e: print "File processed:", e else: print "Invalid File Path"
from os.path import dirname, abspath import sys sys.path.append(dirname(dirname(abspath(__file__)))) from hashlib import sha1 import json from es_utils import send_payload from _py2with3compatibility import run_cmd from cmsutils import cmsswIB2Week err, logs = run_cmd( "find /data/sdt/SDT/jenkins-artifacts/cmssw-afs-eos-comparison -mindepth 1 -maxdepth 1 -name '*.json' -type f" ) for jfile in logs.split('\n'): if not jfile: continue print("Processing file", jfile) payload = {} try: payload = json.load(open(jfile)) except ValueError as err: print(err) run_cmd("rm -f %s" % jfile) continue week, rel_sec = cmsswIB2Week(payload["release"]) payload["@timestamp"] = rel_sec * 1000 id = sha1("%s-%s-%s" % (payload["release"], payload["architecture"], payload["fstype"])).hexdigest() print(payload) if send_payload("cmssw-afs-eos-%s" % week, "build", id, json.dumps(payload)): run_cmd("rm -f %s" % jfile)
id = sha1(root).hexdigest() try: tree = ET.parse(logFile) root = tree.getroot() payload['@timestamp'] = root.find('startTime').text payload['slave_node'] = root.find('builtOn').text build_result = root.find('result') if build_result is not None: payload['build_result'] = build_result.text payload['build_duration'] = int(int(root.find('duration').text)/1000) payload['job_status'] = 'Finished' os.system('touch "' + flagFile + '"') else: payload['job_status'] = 'Running' all_local.append(id) send_payload(index,document,id,json.dumps(payload)) except Exception as e: print "Xml parsing error" , e running_builds_elastic=list() content = get_payload(query_url,query_running_builds) if content == "": running_builds_elastic = [] else: content_hash = json.loads(content) last=int(len(content_hash["hits"]["hits"])) for i in range(0,last): running_builds_elastic.append(str(content_hash["hits"]["hits"][i]["_id"])) for build in running_builds_elastic: if build not in all_local: send_payload(index,document,build,'{"job_status":"Failed"}') print "job status marked as Failed"
process_all = False files_to_process=[] cmd_to_get_logs = "ls -rt "+os.path.join(apache_log_dir,ssl_error_log)+"*" if len(sys.argv)==1: process_all = True cmd_to_get_logs = cmd_to_get_logs + " | tail -2" prev_hour = datetime.now()-timedelta(hours=1) filter_search = " | grep '"+prev_hour.strftime("^\[%a %b %d %H:[0-5][0-9]:[0-5][0-9] %Y\] ")+"'" err, out = getstatusoutput(cmd_to_get_logs) if err: print out sys.exit(1) ReTime = re.compile('^\[[A-Za-z]{3} ([A-Za-z]{3} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [0-9]{4})\] \[[^\]]+\] \[client (.+)\]\s(.+)') for log in out.split("\n"): find_cmd = "grep '%s' %s %s" % (search_for, log, filter_search) err, out = getstatusoutput(find_cmd) for line in out.split("\n"): m = ReTime.match(line) if m: timestamp = int(datetime.strptime(m.group(1), "%b %d %H:%M:%S %Y").strftime('%s'))*1000 payload = {} payload['@timestamp'] = timestamp payload['ip'] = m.group(2) payload['message'] = line id = sha1(str(timestamp) + m.group(2)).hexdigest() send_payload("hypernews","hn-timeouts",id, dumps(payload), passwd_file="/data/es/es_secret") payload = {} payload['@timestamp'] = int(time()*1000) send_payload("hypernews","hn-heartbeat",str(payload['@timestamp']), dumps(payload), passwd_file="/data/es/es_secret")
lfns = {} ti = 0 updated = 0 for item in json_out: for h in item["hits"]["hits"]: ti += 1 lfn_info = h["_source"] lfn_data = cmsweb.search(lfn_info["lfn"], lfn_info) update = False for item in lfn_data: if (not item in lfn_info) or (lfn_data[item] != lfn_info[item]): lfn_info[item] = lfn_data[item] update = True print "Processing .... %s/%s (LFNS: %s, NEW LFN: %s, DS: %s, BLK:%s, ERR: %s): %s" % ( ti, total_hits, len(cmsweb.cache['lfns']), len(cmsweb.cache['new_lfns']), len(cmsweb.cache['datasets']), len(cmsweb.cache['blocks']), cmsweb.errors, update) if update: send_payload(h['_index'], h['_type'], h['_id'], json.dumps(lfn_info)) updated += 1 print "Updated Records:", updated exit(0) lfns = [] for lfn in datasets: lfns.append([lfn, datasets[lfn]]) download(lfns, opts.store, opts.job, opts.dryRun) print "Total Files:", len(datasets)
pa=findParametersAction(root) if pa is not None: getParameters(pa, payload) jstime = root.find('startTime').text payload['@timestamp'] = jstime payload['slave_node'] = root.find('builtOn').text build_result = root.find('result') if build_result is not None: payload['build_result'] = build_result.text payload['build_duration'] = int(int(root.find('duration').text)/1000) payload['job_status'] = 'Finished' os.system('touch "' + flagFile + '"') else: payload['job_status'] = 'Running' all_local.append(id) weekindex="jenkins-jobs-"+str(int((((int(jstime)/1000)/86400)+4)/7)) send_payload(weekindex,document,id,json.dumps(payload), passwd_file="/var/lib/jenkins/secrets/github_hook_secret_cmsbot") except Exception as e: print "Xml parsing error",logFile , e running_builds_elastic={} content = get_payload(query_url,query_running_builds) if content == "": running_builds_elastic = [] else: content_hash = json.loads(content) for hit in content_hash['hits']['hits']: if hit["_index"]=="jenkins" or hit["_index"].startswith("jenkins-jobs-"): try:print "Running:",hit["_source"]['job_name'],hit["_source"]['build_number'],hit["_index"],hit['_id'] except: pass running_builds_elastic[hit['_id']]=hit for build in running_builds_elastic: if build not in all_local:
def process_build_any_ib(logFile): rel = "" arch = "" uploadTime = 0 stime = 0 upload = False jstart = 0 jend = 0 patch = 0 finished = False with open(logFile) as f: for line in f: line = line.strip() if not jstart: m = ReDate.match(line) if m: jstart = datetime.strptime( m.group(1) + m.group(2), "%b %d %H:%M:%S %Y") continue if not arch: m = ReArch.match(line) if m: arch = m.group(1) continue if not rel: m = ReRel.match(line) if m: rel = m.group(1) continue if ReFinish.match(line): finished = True if "ABORTED" in line: return True break if ReUpload.match(line): upload = True print "Upload: ", stime, line continue if ReType.match(line): patch = 1 m = ReDate.match(line) if not m: continue xtime = datetime.strptime( m.group(1) + m.group(2), "%b %d %H:%M:%S %Y") jend = xtime if not upload: stime = xtime else: upload = False dtime = xtime - stime uploadTime += dtime.seconds print "FINISHED: ", finished, rel, arch, uploadTime, jstart, upload, patch if not rel or not arch or not finished: return finished urlx = logFile.split("/") url = "https://cmssdt.cern.ch/jenkins/job/build-any-ib/" + logFile.split( "/")[-2] + "/console" timestp = getmtime(logFile) ttime = 0 if jend and jstart: dtime = jend - jstart ttime = dtime.seconds print ttime, uploadTime, rel, arch, patch, url payload = {} payload["release"] = rel payload["architecture"] = arch payload["total_time"] = ttime payload["upload_time"] = uploadTime payload["patch"] = patch payload["@timestamp"] = int(timestp * 1000) payload["url"] = url week, rel_sec = cmsswIB2Week(rel) print payload id = sha1(rel + arch).hexdigest() send_payload("jenkins-ibs-" + week, "timings", id, json.dumps(payload)) return finished
continue if inException: exception += l + "\n" if inError: error += l + "\n" if exception: payload["exception"] = exception if errors: payload["errors"] = errors payload["error_count"] = error_count try: payload = es_parse_jobreport(payload,logFile) except Exception, e: print e print "sending data for ",logFile try:send_payload(index,document,id,json.dumps(payload)) except:pass if datasets: dataset = {"type" : "relvals", "name" : "%s/%s" % (payload["workflow"], payload["step"])} for fld in ["release","architecture","@timestamp"]: dataset[fld] = payload[fld] for ds in datasets: ds_items = ds.split("?",1) ds_items.append("") ibeos = "/store/user/cmsbuild" if ibeos in ds_items[0]: ds_items[0] = ds_items[0].replace(ibeos,"") else: ibeos="" dataset["protocol"]=ds_items[0].split("/store/",1)[0]+ibeos dataset["protocol_opts"]=ds_items[1] dataset["lfn"]="/store/"+ds_items[0].split("/store/",1)[1] idx = sha1(id + ds).hexdigest() print dataset
cmd_to_get_logs = "ls -rt "+os.path.join(apache_log_dir,ssl_error_log)+"*" if len(sys.argv)==1: process_all = True cmd_to_get_logs = cmd_to_get_logs + " | tail -2" prev_hour = datetime.now()-timedelta(hours=1) filter_search = " | grep '"+prev_hour.strftime("^\[%a %b %d %H:[0-5][0-9]:[0-5][0-9] %Y\] ")+"'" err, out = run_cmd(cmd_to_get_logs) if err: print(out) sys.exit(1) ReTime = re.compile('^\[[A-Za-z]{3} ([A-Za-z]{3} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [0-9]{4})\] \[[^\]]+\] \[client (.+)\]\s(.+)') for log in out.split("\n"): find_cmd = "grep '%s' %s %s" % (search_for, log, filter_search) err, out = run_cmd(find_cmd) for line in out.split("\n"): m = ReTime.match(line) if m: tsec = int(datetime.strptime(m.group(1), "%b %d %H:%M:%S %Y").strftime('%s')) week = str(int(tsec/(86400*7))) timestamp = tsec*1000 payload = {} payload['@timestamp'] = timestamp payload['ip'] = m.group(2) payload['message'] = line id = sha1(str(timestamp) + m.group(2)).hexdigest() send_payload("hypernews-"+week,"hn-timeouts",id, dumps(payload)) payload = {} payload['@timestamp'] = int(time()*1000) send_payload("hypernews","hn-heartbeat",str(payload['@timestamp']), dumps(payload))
root = tree.getroot() pa = findParametersAction(root) if pa is not None: getParameters(pa, payload) payload['@timestamp'] = root.find('startTime').text payload['slave_node'] = root.find('builtOn').text build_result = root.find('result') if build_result is not None: payload['build_result'] = build_result.text payload['build_duration'] = int( int(root.find('duration').text) / 1000) payload['job_status'] = 'Finished' os.system('touch "' + flagFile + '"') else: payload['job_status'] = 'Running' all_local.append(id) send_payload(index, document, id, json.dumps(payload)) except Exception as e: print "Xml parsing error", e running_builds_elastic = list() content = get_payload(query_url, query_running_builds) if content == "": running_builds_elastic = [] else: content_hash = json.loads(content) last = int(len(content_hash["hits"]["hits"])) for i in range(0, last): running_builds_elastic.append( str(content_hash["hits"]["hits"][i]["_id"])) for build in running_builds_elastic: if build not in all_local: send_payload(index, document, build, '{"job_status":"Failed"}')
#process info yesterday = datetime.date.today() - datetime.timedelta(1) timestp = int(yesterday.strftime("%s")) * 1000 temp_fails = dict( list( map( map_int_val, list((dict([ list(map(cust_strip, x.split(' ')[-3:-1])) for x in temp_fails ])).items())))) msgs_num = list(map(str.strip, msgs_num.split(':'))) adrpts = list(map(str.strip, adrpts.split(':'))) byttr = list(map(str.strip, byttr.split(':'))) egrp_emails = dict(list(map(rm_extra, [x.strip() for x in egrps]))) #send info payload['@timestamp'] = timestp payload['msgs_to_rcpts'] = msgs_num[1] payload['addressed_rcpts'] = adrpts[1] payload['byts_transfd'] = byttr[1] total = 0 for k in egrp_emails: payload['hn-' + k] = egrp_emails[k] total += egrp_emails[k] payload['posts'] = total payload['forums'] = len(egrp_emails) for k in temp_fails: payload['fail-' + k] = temp_fails[k] send_payload("hypernews", "mailinfo", timestp, dumps(payload))
def es_parse_log(logFile): t = os.path.getmtime(logFile) timestp = int(t * 1000) payload = {} pathInfo = logFile.split('/') architecture = pathInfo[4] release = pathInfo[8] workflow = pathInfo[10].split('_')[0] step = pathInfo[11].split('_')[0] week, rel_sec = cmsswIB2Week(release) index = "ib-matrix-" + week document = "runTheMatrix-data" id = sha1(release + architecture + workflow + str(step)).hexdigest() logdir = '/'.join(logFile.split('/')[:-1]) cmdfile = logdir + '/cmdLog' cmd_step = find_step_cmd(cmdfile, step) if cmd_step: payload["command"] = cmd_step wf_log = logdir + '/workflow.log' exitcode = get_exit_code(wf_log, int(step[-1])) if exitcode != -1: payload["exitcode"] = exitcode payload["workflow"] = workflow payload["release"] = release payload["architecture"] = architecture payload["step"] = step payload["@timestamp"] = timestp hostFile = "/".join(logFile.split('/')[:-1]) + "/hostname" if os.path.exists(hostFile): with open(hostFile, 'r') as hname: payload["hostname"] = hname.readlines()[0].strip() exception = "" error = "" errors = [] inException = False inError = False datasets = [] error_count = 0 if exists(logFile): with open(logFile) as f: lines = f.readlines() payload[ "url"] = 'https://cmssdt.cern.ch/SDT/cgi-bin/buildlogs/' + pathInfo[ 4] + '/' + pathInfo[8] + '/pyRelValMatrixLogs/run/' + pathInfo[ -2] + '/' + pathInfo[-1] total_lines = len(lines) for i in range(total_lines): l = lines[i].strip() if " Initiating request to open file " in l: try: rootfile = l.split( " Initiating request to open file ")[1].split(" ")[0] if (not "file:" in rootfile) and (not rootfile in datasets): #if (i+2)<total_lines: # if (rootfile in lines[i+1]) and (rootfile in lines[i+2]) and ("Successfully opened file " in lines[i+1]) and ("Closed file " in lines[i+2]): # print "File read with no valid events: %s" % rootfile # continue datasets.append(rootfile) except: pass continue if l.startswith("----- Begin Fatal Exception"): inException = True continue if l.startswith("----- End Fatal Exception"): inException = False continue if l.startswith("%MSG-e"): inError = True error = l error_kind = re.split(" [0-9a-zA-Z-]* [0-9:]{8} CET", error)[0].replace("%MSG-e ", "") continue if inError == True and l.startswith("%MSG"): inError = False if len(errors) < 10: errors.append({"error": error, "kind": error_kind}) error_count += 1 error = "" error_kind = "" continue if inException: exception += l + "\n" if inError: error += l + "\n" if exception: payload["exception"] = exception if errors: payload["errors"] = errors payload["error_count"] = error_count try: payload = es_parse_jobreport(payload, logFile) except Exception as e: print(e) print("sending data for ", logFile) try: send_payload(index, document, id, json.dumps(payload)) except: pass if datasets: dataset = { "type": "relvals", "name": "%s/%s" % (payload["workflow"], payload["step"]) } for fld in ["release", "architecture", "@timestamp"]: dataset[fld] = payload[fld] for ds in datasets: ds_items = ds.split("?", 1) ds_items.append("") ibeos = "/store/user/cmsbuild" if ibeos in ds_items[0]: ds_items[0] = ds_items[0].replace(ibeos, "") else: ibeos = "" dataset["protocol"] = ds_items[0].split("/store/", 1)[0] + ibeos dataset["protocol_opts"] = ds_items[1] dataset["lfn"] = "/store/" + ds_items[0].split("/store/", 1)[1].strip() idx = sha1(id + ds).hexdigest() print(dataset) send_payload("ib-dataset-" + week, "relvals-dataset", idx, json.dumps(dataset))
sys.exit(1) for line in cmd_out.split('\n'): if re.match(match_tmp,line): temp_fails.append(line) elif 'Messages To Recipients:' in line: msgs_num = line elif 'Addressed Recipients:' in line : adrpts = line elif 'Bytes Transferred:' in line: byttr = line elif re.match(match_hn,line): egrps.append(line) #process info yesterday = datetime.date.today() - datetime.timedelta(1) timestp = int(yesterday.strftime("%s")) * 1000 temp_fails = dict(list(map(map_int_val,list((dict([list(map(cust_strip,x.split(' ')[-3:-1])) for x in temp_fails])).items())))) msgs_num= list(map(str.strip,msgs_num.split(':'))) adrpts = list(map(str.strip, adrpts.split(':'))) byttr = list(map(str.strip, byttr.split(':'))) egrp_emails = dict(list(map(rm_extra,[x.strip() for x in egrps]))) #send info payload['@timestamp'] = timestp payload['msgs_to_rcpts'] = msgs_num[1] payload['addressed_rcpts'] = adrpts[1] payload['byts_transfd'] = byttr[1] total=0 for k in egrp_emails: payload['hn-'+k] = egrp_emails[k] total += egrp_emails[k] payload['posts'] = total payload['forums'] = len(egrp_emails) for k in temp_fails: payload['fail-'+k] = temp_fails[k] send_payload("hypernews","mailinfo",timestp, dumps(payload), passwd_file="/data/es/es_secret")
if inException: exception += l + "\n" if inError: error += l + "\n" if exception: payload["exception"] = exception if errors: payload["errors"] = errors payload["error_count"] = error_count try: payload = es_parse_jobreport(payload, logFile) except Exception, e: print e print "sending data for ", logFile try: send_payload(index, document, id, json.dumps(payload)) except: pass if datasets: dataset = { "type": "relvals", "name": "%s/%s" % (payload["workflow"], payload["step"]) } for fld in ["release", "architecture", "@timestamp"]: dataset[fld] = payload[fld] for ds in datasets: ds_items = ds.split("?", 1) ds_items.append("") ibeos = "/store/user/cmsbuild" if ibeos in ds_items[0]: ds_items[0] = ds_items[0].replace(ibeos, "")
continue if l.startswith("%MSG-e"): inError = True error = l error_kind = re.split(" [0-9a-zA-Z-]* [0-9:]{8} CET", error)[0].replace("%MSG-e ", "") continue if inError == True and l.startswith("%MSG"): inError = False errors.append({"error": error, "kind": error_kind}) error = "" error_kind = "" continue if inException: exception += l + "\n" if inError: error += l + "\n" if exception: payload["exception"] = exception if errors: payload["errors"] = errors try: payload = es_parse_jobreport(payload,logFile) except Exception, e: print e print "sending data for ",logFile send_payload(index,document,id,json.dumps(payload)) if __name__ == "__main__": print "Processing ",sys.argv[1] es_parse_log(sys.argv[1])
if opts.show_tests: t = " ("+",".join(sorted(rels[k][b][r].keys()))+")" x.append("%s%s" % (r,t)) print " Releases: %s " % ", ".join(x) else: print json.dumps(json_out, indent=2, sort_keys=True, separators=(',',': ')) else: cmsweb=CMSWeb() lfns = {} ti = 0 updated = 0 for item in json_out: for h in item["hits"]["hits"]: ti+=1 lfn_info = h["_source"] lfn_data = cmsweb.search(lfn_info["lfn"], lfn_info) update = False for item in lfn_data: if (not item in lfn_info) or (lfn_data[item] != lfn_info[item]): lfn_info[item]=lfn_data[item] update = True print "Processing .... %s/%s (LFNS: %s, NEW LFN: %s, DS: %s, BLK:%s, ERR: %s): %s" % (ti, total_hits ,len(cmsweb.cache['lfns']), len(cmsweb.cache['new_lfns']), len(cmsweb.cache['datasets']), len(cmsweb.cache['blocks']), cmsweb.errors, update) if update: send_payload(h['_index'], h['_type'], h['_id'], json.dumps(lfn_info)) updated += 1 print "Updated Records:", updated exit(0) lfns = [] for lfn in datasets: lfns.append([lfn, datasets[lfn]]) download(lfns, opts.store, opts.job,opts.dryRun) print "Total Files:",len(datasets)
def process_build_any_ib(logFile): rel = "" arch = "" uploadTime=0 stime=0 upload=False jstart = 0 jend=0 patch = 0 finished = False with open(logFile) as f: for line in f: line = line.strip() if not jstart: m=ReDate.match(line) if m: jstart = datetime.strptime(m.group(1)+m.group(2), "%b %d %H:%M:%S %Y") continue if not arch: m=ReArch.match(line) if m: arch=m.group(1) continue if not rel: m=ReRel.match(line) if m: rel=m.group(1) continue if ReFinish.match(line): finished = True if "ABORTED" in line: return True break if ReUpload.match(line): upload=True continue if ReType.match(line): patch=1 m=ReDate.match(line) if not m: continue xtime = datetime.strptime(m.group(1)+m.group(2), "%b %d %H:%M:%S %Y") jend = xtime if not upload: stime = xtime else: upload=False dtime = xtime - stime uploadTime += dtime.seconds print "FINISHED: ",finished,rel, arch,uploadTime if not rel or not arch or not finished: return finished urlx = logFile.split("/") url = "https://cmssdt.cern.ch/jenkins/job/build-any-ib/"+logFile.split("/")[-2]+"/console" timestp = getmtime(logFile) ttime = jend - jstart print ttime.seconds, uploadTime, rel, arch, patch, url payload = {} payload["release"] = rel payload["architecture"] = arch payload["total_time"] = ttime.seconds payload["upload_time"] = uploadTime payload["patch"] = patch payload["@timestamp"] = int(timestp*1000) payload["url"]=url print payload id = sha1(rel + arch).hexdigest() send_payload("jenkins-ibs","timings",id,json.dumps(payload)) return finished
def es_parse_log(logFile): t = os.path.getmtime(logFile) timestp = int(t*1000) payload = {} pathInfo = logFile.split('/') architecture = pathInfo[4] release = pathInfo[8] workflow = pathInfo[10].split('_')[0] step = pathInfo[11].split('_')[0] week, rel_sec = cmsswIB2Week(release) index = "ib-matrix-" + week document = "runTheMatrix-data" id = sha1(release + architecture + workflow + str(step)).hexdigest() logdir = '/'.join(logFile.split('/')[:-1]) cmdfile = logdir + '/cmdLog' cmd_step = find_step_cmd(cmdfile,step) if cmd_step: payload["command"] = cmd_step wf_log = logdir + '/workflow.log' exitcode = get_exit_code(wf_log,int(step[-1])) if exitcode != -1 : payload["exitcode"] = exitcode payload["workflow"] = workflow payload["release"] = release payload["architecture"] = architecture payload["step"] = step payload["@timestamp"] = timestp hostFile = "/".join(logFile.split('/')[:-1]) + "/hostname" if os.path.exists (hostFile): with open(hostFile,'r') as hname: payload["hostname"] = hname.readlines()[0].strip() exception = "" error = "" errors = [] inException = False inError = False datasets = [] error_count = 0 if exists(logFile): with open(logFile) as f: lines = f.readlines() payload["url"] = 'https://cmssdt.cern.ch/SDT/cgi-bin/buildlogs/'+pathInfo[4]+'/'+pathInfo[8]+'/pyRelValMatrixLogs/run/'+pathInfo[-2]+'/'+pathInfo[-1] total_lines = len(lines) for i in range(total_lines): l = lines[i].strip() if " Initiating request to open file " in l: try: rootfile = l.split(" Initiating request to open file ")[1].split(" ")[0] if (not "file:" in rootfile) and (not rootfile in datasets): #if (i+2)<total_lines: # if (rootfile in lines[i+1]) and (rootfile in lines[i+2]) and ("Successfully opened file " in lines[i+1]) and ("Closed file " in lines[i+2]): # print "File read with no valid events: %s" % rootfile # continue datasets.append(rootfile) except: pass continue if l.startswith("----- Begin Fatal Exception"): inException = True continue if l.startswith("----- End Fatal Exception"): inException = False continue if l.startswith("%MSG-e"): inError = True error = l error_kind = re.split(" [0-9a-zA-Z-]* [0-9:]{8} CET", error)[0].replace("%MSG-e ", "") continue if inError == True and l.startswith("%MSG"): inError = False if len(errors)<10: errors.append({"error": error, "kind": error_kind}) error_count += 1 error = "" error_kind = "" continue if inException: exception += l + "\n" if inError: error += l + "\n" if exception: payload["exception"] = exception if errors: payload["errors"] = errors payload["error_count"] = error_count try: payload = es_parse_jobreport(payload,logFile) except Exception as e: print(e) print("sending data for ",logFile) try:send_payload(index,document,id,json.dumps(payload)) except:pass if datasets: dataset = {"type" : "relvals", "name" : "%s/%s" % (payload["workflow"], payload["step"])} for fld in ["release","architecture","@timestamp"]: dataset[fld] = payload[fld] for ds in datasets: ds_items = ds.split("?",1) ds_items.append("") ibeos = "/store/user/cmsbuild" if ibeos in ds_items[0]: ds_items[0] = ds_items[0].replace(ibeos,"") else: ibeos="" dataset["protocol"]=ds_items[0].split("/store/",1)[0]+ibeos dataset["protocol_opts"]=ds_items[1] dataset["lfn"]="/store/"+ds_items[0].split("/store/",1)[1].strip() idx = sha1(id + ds).hexdigest() print(dataset) send_payload("ib-dataset-"+week,"relvals-dataset",idx,json.dumps(dataset))
'^\[[A-Za-z]{3} ([A-Za-z]{3} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [0-9]{4})\] \[[^\]]+\] \[client (.+)\]\s(.+)' ) for log in out.split("\n"): find_cmd = "grep '%s' %s %s" % (search_for, log, filter_search) err, out = run_cmd(find_cmd) for line in out.split("\n"): m = ReTime.match(line) if m: tsec = int( datetime.strptime(m.group(1), "%b %d %H:%M:%S %Y").strftime('%s')) week = str(int(tsec / (86400 * 7))) timestamp = tsec * 1000 payload = {} payload['@timestamp'] = timestamp payload['ip'] = m.group(2) payload['message'] = line id = sha1(str(timestamp) + m.group(2)).hexdigest() send_payload("hypernews-" + week, "hn-timeouts", id, dumps(payload), passwd_file="/data/es/es_secret") payload = {} payload['@timestamp'] = int(time() * 1000) send_payload("hypernews", "hn-heartbeat", str(payload['@timestamp']), dumps(payload), passwd_file="/data/es/es_secret")
#process info yesterday = datetime.date.today() - datetime.timedelta(1) timestp = int(yesterday.strftime("%s")) * 1000 temp_fails = dict( map(map_int_val, (dict([map(cust_strip, x.split(' ')[-3:-1]) for x in temp_fails])).items())) msgs_num = map(str.strip, msgs_num.split(':')) adrpts = map(str.strip, adrpts.split(':')) byttr = map(str.strip, byttr.split(':')) egrp_emails = dict(map(rm_extra, [x.strip() for x in egrps])) #send info payload['@timestamp'] = timestp payload['msgs_to_rcpts'] = msgs_num[1] payload['addressed_rcpts'] = adrpts[1] payload['byts_transfd'] = byttr[1] total = 0 for k in egrp_emails: payload['hn-' + k] = egrp_emails[k] total += egrp_emails[k] payload['posts'] = total payload['forums'] = len(egrp_emails) for k in temp_fails: payload['fail-' + k] = temp_fails[k] send_payload("hypernews", "mailinfo", timestp, dumps(payload), passwd_file="/data/es/es_secret")
build_result = root.find('result') if build_result is not None: payload['build_result'] = build_result.text payload['build_duration'] = int( int(root.find('duration').text) / 1000) payload['job_status'] = 'Finished' os.system('touch "' + flagFile + '"') else: payload['job_status'] = 'Running' all_local.append(id) weekindex = "jenkins-jobs-" + str( int((((int(jstime) / 1000) / 86400) + 4) / 7)) send_payload( weekindex, document, id, json.dumps(payload), passwd_file= "/var/lib/jenkins/secrets/github_hook_secret_cmsbot") except Exception as e: print "Xml parsing error", logFile, e running_builds_elastic = {} content = get_payload('jenkins-*', query_running_builds) if content == "": running_builds_elastic = [] else: content_hash = json.loads(content) if (not 'hits' in content_hash) or (not 'hits' in content_hash['hits']): print "ERROR: ", content sys.exit(1) for hit in content_hash['hits']['hits']:
jstime = root.find('startTime').text payload['@timestamp'] = int(jstime) payload['slave_node'] = root.find('builtOn').text payload['jenkins_server'] = JENKINS_PREFIX build_result = root.find('result') if build_result is not None: payload['build_result'] = build_result.text payload['build_duration'] = int(int(root.find('duration').text)/1000) payload['job_status'] = 'Finished' os.system('touch "' + flagFile + '"') else: payload['job_status'] = 'Running' all_local.append(id) weekindex="jenkins-jobs-"+str(int((((int(jstime)/1000)/86400)+4)/7)) print("==>",id,payload['job_name'],payload['build_number'],payload['job_status']) send_payload(weekindex,document,id,json.dumps(payload)) except Exception as e: print("Xml parsing error",logFile , e) running_builds_elastic={} content_hash = get_payload_wscroll('jenkins-*',query_running_builds) if not content_hash: running_builds_elastic = {} else: if (not 'hits' in content_hash) or (not 'hits' in content_hash['hits']): print("ERROR: ",content) sys.exit(1) print("Found:", len(content_hash['hits']['hits'])) for hit in content_hash['hits']['hits']: if hit["_index"].startswith("cmssdt-jenkins-jobs-"): if not "jenkins_server" in hit["_source"]: hit["_source"]["jenkins_server"] = JENKINS_PREFIX if hit["_source"]["jenkins_server"]!=JENKINS_PREFIX: continue