def parseATransLogData(device, conn, cursor, lines): print "trans lines count:", len(lines) okc=0; errorLogs=[] #解析出错、不正确数据的行 errorLines=[] #发生保存错误的记录 cacheLines=[] #本次提交的行 sqls=[] lc=0 for l in lines: if not l: break lc+=1 eMsg=""; alog="" try: log=line_to_log(device, l) except Exception, e: eMsg=u"%s"%e errorLogs.append("%s\t--%s"%(l, eMsg)) log=None if log: sqls.append(log) cacheLines.append(l) #先记住还没有提交数据,commit不成功的话可以知道哪些数据没有提交成功 if len(cacheLines)>=600: #达到600行就提交一次 try: cursor=commitLog(conn, cursor, sqls) okc+=len(cacheLines) print "\tcommit ", len(cacheLines) alog=cacheLines[0] except: errorLines+=cacheLines cacheLines=[] sqls=[]
cursor=commitLog(conn, cursor, sqls) okc+=len(cacheLines) print "\tcommit last:", len(cacheLines) if not alog: alog=cacheLines[0] except: print_exc() errorLines+=cacheLines if errorLines: #重新保存上面提交失败的数据,每条记录提交一次,最小化失败记录数 cacheLines=errorLines errorLines=[] for line in cacheLines: print line if line not in errorLogs: try: log=line_to_log(device, line) cursor=commitLog(conn, cursor, log) okc+=1 print "\tcommit last error:", line except Exception, e: eMsg=u"%s"%e if "Duplicate" not in eMsg: errorLines.append("%s\t--%s"%(line, eMsg)) errorLines+=errorLogs dlogObj="" try: if okc==1: dlogObj=alog elif okc>1: dlogObj=alog + ", ..." except:pass