def process_addon_log(logFile): t = getmtime(logFile) timestp = int(t*1000) pathInfo = logFile.split('/') architecture = pathInfo[4] release = pathInfo[8] week, rel_sec = cmsswIB2Week (release) datasets = [] payload = {"type" : "addon"} payload["release"]=release payload["architecture"]=architecture payload["@timestamp"]=timestp payload["name"] = pathInfo[-1].split("-")[1].split("_cmsRun_")[0].split("_cmsDriver.py_")[0] id = sha1(release + architecture + "addon" + payload["name"]).hexdigest() config_list = [] for index, l in enumerate(file(logFile).read().split("\n")): config_list = add_exception_to_config(l,index, config_list) if " Initiating request to open file " in l: try: rootfile = l.split(" Initiating request to open file ")[1].split(" ")[0] if (not "file:" in rootfile) and (not rootfile in datasets): datasets.append(rootfile) except: pass send_unittest_dataset(datasets, payload, id, "ib-dataset-"+week,"addon-dataset") write_config_file(logFile + "-read_config", config_list) return
def process_addon_log(logFile): t = getmtime(logFile) timestp = int(t*1000) pathInfo = logFile.split('/') architecture = pathInfo[4] release = pathInfo[8] week, rel_sec = cmsswIB2Week (release) datasets = [] payload = {"type" : "addon"} payload["release"]=release payload["architecture"]=architecture payload["@timestamp"]=timestp payload["name"] = pathInfo[-1].split("-")[1].split("_cmsRun_")[0].split("_cmsDriver.py_")[0] id = sha1(release + architecture + "addon" + payload["name"]).hexdigest() config_list = [] with open(logFile) as f: for index, l in enumerate(f): l = l.strip() config_list = add_exception_to_config(l,index, config_list) if " Initiating request to open file " in l: try: rootfile = l.split(" Initiating request to open file ")[1].split(" ")[0] if (not "file:" in rootfile) and (not rootfile in datasets): datasets.append(rootfile) except: pass send_unittest_dataset(datasets, payload, id, "ib-dataset-"+week,"addon-dataset") transform_and_write_config_file(logFile + "-read_config", config_list) return
def test_unittestlogs(self): config_list = [] custom_rule_set = [ {"str_to_match": "test (.*) had ERRORS", "name": "{0}{1}{2} failed", "control_type": ResultTypeEnum.ISSUE}, {"str_to_match": '===== Test "([^\s]+)" ====', "name": "{0}", "control_type": ResultTypeEnum.TEST} ] for index, l in enumerate(unittestlog.split("\n")): config_list = add_exception_to_config(l, index, config_list, custom_rule_set) transform_and_write_config_file("/tmp/unittestlogs.log-read_config", config_list) print("Example config file in %s" % ("/tmp/unittestlogs.log-read_config"))
def process_unittest_log(logFile): t = getmtime(logFile) timestp = int(t * 1000) pathInfo = logFile.split('/') architecture = pathInfo[4] release = pathInfo[8] week, rel_sec = cmsswIB2Week(release) package = pathInfo[-3] + "/" + pathInfo[-2] payload = {"type": "unittest"} payload["release"] = release payload["architecture"] = architecture payload["@timestamp"] = timestp config_list = [] custom_rule_set = [{ "str_to_match": "test (.*) had ERRORS", "name": "{0} failed", 'control_type': ResultTypeEnum.ISSUE }, { "str_to_match": '===== Test "([^\s]+)" ====', "name": "{0}", 'control_type': ResultTypeEnum.TEST }] with open(logFile) as f: utname = None datasets = [] xid = None for index, l in enumerate(f): l = l.strip() config_list = add_exception_to_config(l, index, config_list, custom_rule_set) if l.startswith('===== Test "') and l.endswith('" ===='): if utname: send_unittest_dataset(datasets, payload, xid, "ib-dataset-" + week, "unittest-dataset") datasets = [] utname = l.split('"')[1] payload["name"] = "%s/%s" % (package, utname) xid = sha1(release + architecture + package + str(utname)).hexdigest() elif " Initiating request to open file " in l: try: rootfile = l.split( " Initiating request to open file ")[1].split(" ")[0] if (not "file:" in rootfile) and (not rootfile in datasets): datasets.append(rootfile) except Exception as e: print("ERROR: ", logFile, e) traceback.print_exc(file=sys.stdout) if datasets and xid: send_unittest_dataset(datasets, payload, xid, "ib-dataset-" + week, "unittest-dataset") transform_and_write_config_file(logFile + "-read_config", config_list) return
def test_unittestlogs(self): config_list = [] custom_rule_set = [{ "str_to_match": "test (.*) had ERRORS", "name": "{0}{1}{2} failed" }, { "str_to_match": '===== Test "([^\s]+)" ====', "name": "{0}" }] for index, l in enumerate(unittestlog.split("\n")): config_list = add_exception_to_config(l, index, config_list, custom_rule_set) write_config_file("tmp/unittestlogs.log" + "-read_config", config_list)
def process_unittest_log(logFile): t = getmtime(logFile) timestp = int(t * 1000) pathInfo = logFile.split('/') architecture = pathInfo[4] release = pathInfo[8] week, rel_sec = cmsswIB2Week(release) package = pathInfo[-3] + "/" + pathInfo[-2] utname = None datasets = [] payload = {"type": "unittest"} payload["release"] = release payload["architecture"] = architecture payload["@timestamp"] = timestp id = None config_list = [] custom_rule_set = [{ "str_to_match": "test (.*) had ERRORS", "name": "{0} failed" }, { "str_to_match": '===== Test "([^\s]+)" ====', "name": "{0}" }] for index, l in enumerate(file(logFile).read().split("\n")): config_list = add_exception_to_config(l, index, config_list, custom_rule_set) if l.startswith('===== Test "') and l.endswith('" ===='): if utname: send_unittest_dataset(datasets, payload, id, "ib-dataset-" + week, "unittest-dataset") datasets = [] utname = l.split('"')[1] payload["name"] = "%s/%s" % (package, utname) id = sha1(release + architecture + package + str(utname)).hexdigest() elif " Initiating request to open file " in l: try: rootfile = l.split( " Initiating request to open file ")[1].split(" ")[0] if (not "file:" in rootfile) and (not rootfile in datasets): datasets.append(rootfile) except: pass if datasets: send_unittest_dataset(datasets, payload, id, "ib-dataset-" + week, "unittest-dataset") write_config_file(logFile + "-read_config", config_list) return
def test_unittestlogs(self): config_list = [] custom_rule_set = [{ "str_to_match": "test (.*) had ERRORS", "name": "{0}{1}{2} failed", "control_type": ResultTypeEnum.ISSUE }, { "str_to_match": '===== Test "([^\s]+)" ====', "name": "{0}", "control_type": ResultTypeEnum.TEST }] for index, l in enumerate(unittestlog.split("\n")): config_list = add_exception_to_config(l, index, config_list, custom_rule_set) transform_and_write_config_file( "/tmp/unittestlogs.log" + "-read_config", config_list)
def process_unittest_log(logFile): t = getmtime(logFile) timestp = int(t*1000) pathInfo = logFile.split('/') architecture = pathInfo[4] release = pathInfo[8] week, rel_sec = cmsswIB2Week (release) package = pathInfo[-3]+"/"+ pathInfo[-2] payload = {"type" : "unittest"} payload["release"]=release payload["architecture"]=architecture payload["@timestamp"]=timestp config_list = [] custom_rule_set = [ {"str_to_match": "test (.*) had ERRORS", "name": "{0} failed", 'control_type': ResultTypeEnum.ISSUE }, {"str_to_match": '===== Test "([^\s]+)" ====', "name": "{0}", 'control_type': ResultTypeEnum.TEST } ] with open(logFile) as f: utname = None datasets = [] xid = None for index, l in enumerate(f): l = l.strip() config_list = add_exception_to_config(l,index,config_list,custom_rule_set) if l.startswith('===== Test "') and l.endswith('" ===='): if utname: send_unittest_dataset(datasets, payload, xid, "ib-dataset-"+week, "unittest-dataset") datasets = [] utname = l.split('"')[1] payload["name"] = "%s/%s" % (package, utname) xid = sha1(release + architecture + package + str(utname)).hexdigest() elif " Initiating request to open file " in l: try: rootfile = l.split(" Initiating request to open file ")[1].split(" ")[0] if (not "file:" in rootfile) and (not rootfile in datasets): datasets.append(rootfile) except Exception as e: print("ERROR: ",e) if datasets and xid: send_unittest_dataset(datasets, payload, xid, "ib-dataset-"+week,"unittest-dataset") transform_and_write_config_file(logFile + "-read_config", config_list) return
def parseLog(self): logData = {} logRE = re.compile('^.*/([1-9][0-9]*(\.[0-9]+|))_[^/]+/step([1-9])_.*\.log$') max_steps = 0 for logFile in glob.glob(self.basedir+'/[1-9]*/step[0-9]*.log'): m = logRE.match(logFile) if not m: continue wf = m.group(1) step = int(m.group(3)) if step>max_steps: max_steps=step if wf not in logData: logData[wf] = {'steps': {}, 'events' : [], 'failed' : [], 'warning' : []} if step not in logData[wf]['steps']: logData[wf]['steps'][step]=logFile cache_read=0 log_processed=0 for wf in logData: for k in logData[wf]: if k == 'steps': continue for s in range(0, max_steps): logData[wf][k].append(-1) index =0 for step in sorted(logData[wf]['steps']): data = [0, 0, 0] logFile = logData[wf]['steps'][step] json_cache = os.path.dirname(logFile)+"/logcache_"+str(step)+".json" log_reader_config_path = logFile + "-read_config" config_list = [] cache_ok = False if (os.path.exists(json_cache)) and (os.path.getmtime(logFile)<=os.path.getmtime(json_cache)): try: jfile = open(json_cache,"r") data = json.load(jfile) jfile.close() cache_read+=1 cache_ok = True except: os.remove(json_cache) if not cache_ok: try: es_parse_log(logFile) except Exception as e: print("Sending log information to elasticsearch failed" , str(e)) inFile = open(logFile) for line_nr, line in enumerate(inFile): config_list = add_exception_to_config(line, line_nr, config_list) if '%MSG-w' in line: data[1]=data[1]+1 if '%MSG-e' in line: data[2]=data[2]+1 if 'Begin processing the ' in line: data[0]=data[0]+1 inFile.close() jfile = open(json_cache,"w") json.dump(data,jfile) jfile.close() transform_and_write_config_file(log_reader_config_path, config_list) log_processed+=1 logData[wf]['events'][index] = data[0] logData[wf]['failed'][index] = data[2] logData[wf]['warning'][index] = data[1] index+=1 del logData[wf]['steps'] print("Log processed: ",log_processed) print("Caches read:",cache_read) from pickle import Pickler outFile = open(os.path.join(self.outdir,'runTheMatrixMsgs.pkl'), 'w') pklFile = Pickler(outFile) pklFile.dump(logData) outFile.close() return
def parseLog(self): logData = {} logRE = re.compile('^.*/([1-9][0-9]*(\.[0-9]+|))_[^/]+/step([1-9])_.*\.log$') max_steps = 0 for logFile in glob.glob(self.basedir+'/[1-9]*/step[0-9]*.log'): m = logRE.match(logFile) if not m: continue wf = m.group(1) step = int(m.group(3)) if step>max_steps: max_steps=step if not logData.has_key(wf): logData[wf] = {'steps': {}, 'events' : [], 'failed' : [], 'warning' : []} if not logData[wf]['steps'].has_key(step): logData[wf]['steps'][step]=logFile cache_read=0 log_processed=0 for wf in logData: for k in logData[wf]: if k == 'steps': continue for s in range(0, max_steps): logData[wf][k].append(-1) index =0 for step in sorted(logData[wf]['steps']): data = [0, 0, 0] logFile = logData[wf]['steps'][step] json_cache = os.path.dirname(logFile)+"/logcache_"+str(step)+".json" log_reader_config_path = logFile + "-read_config" config_list = [] cache_ok = False if (os.path.exists(json_cache)) and (os.path.getmtime(logFile)<=os.path.getmtime(json_cache)): try: jfile = open(json_cache,"r") data = json.load(jfile) jfile.close() cache_read+=1 cache_ok = True except: os.remove(json_cache) if not cache_ok: try: es_parse_log(logFile) except Exception as e: print "Sending log information to elasticsearch failed" , str(e) inFile = open(logFile) for line_nr, line in enumerate(inFile): config_list = add_exception_to_config(line, line_nr, config_list) if '%MSG-w' in line: data[1]=data[1]+1 if '%MSG-e' in line: data[2]=data[2]+1 if 'Begin processing the ' in line: data[0]=data[0]+1 inFile.close() jfile = open(json_cache,"w") json.dump(data,jfile) jfile.close() write_config_file(log_reader_config_path,config_list) log_processed+=1 logData[wf]['events'][index] = data[0] logData[wf]['failed'][index] = data[2] logData[wf]['warning'][index] = data[1] index+=1 del logData[wf]['steps'] print "Log processed: ",log_processed print "Caches read:",cache_read from pickle import Pickler outFile = open(os.path.join(self.outdir,'runTheMatrixMsgs.pkl'), 'w') pklFile = Pickler(outFile) pklFile.dump(logData) outFile.close() return