def checkmakedir(dname,fname): try: d=os.path.dirname(dname); if not os.path.exists(d): os.makedirs(d); except: fl=statslog.logcreate(fname); statslog.logwrite(fl,"Error reported (create dir): "+str(sys.exc_info()[1])); return;
def parserunner(): fl = statslog.logcreate('log/parserunner.log') try: while True: p = Process(target=tabber) p.start() time.sleep(600) except: statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]), 'parserunner')
def splitrunner(): fl = statslog.logcreate('log/splitrunner.log') try: while (1): p = Process(target=splitter) p.start() time.sleep(30) except: statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]), 'splitrunner')
def tabFile(fileid, filepath, filename, filetype, inservserial, datepart, statsfirstrowversion, datafolder): oraconn = oracon.openconnect(constr) fl = statslog.logcreate('log/tabber_' + str(fileid) + '.log') try: outdir = datafolder + '/processed/' + str(inservserial) + '/' checkmakedir(outdir) dtsplit = string.split(datepart, ' ') dt = string.join(dtsplit, '.') outputfilename = filetype + '.' + str(inservserial) + '.' + str( dt) + '.out' outfile = open(outdir + outputfilename, 'w') statslog.logwrite(fl, 'Opening file....' + filename) infile = open(filepath + '/' + filename) lines = infile.readlines() infile.close() datastruct = {} if statsfirstrowversion == 'True': if len(lines) > 1: statslog.logwrite(fl, 'Generating data...', 'tabFile') datarowstruct = {} datarowstruct = generateData(lines, filetype, inservserial, datepart, fl) datarow = datarowstruct[0] structid = datarowstruct[1] outfile.write(datarow) sqlstmt = 'begin dataload.ADDOUTPUTFILE (STATS_SPLITFILE_ID=>' + str( fileid) sqlstmt += ',STATS_OUTPUTFILE_NAME=>\'' + outputfilename + '\',STATS_OUTPUTFILE_PATH =>\'' + outdir + '\',STATS_STRUCTURE_ID=>' + str( structid) + '); end;' oracon.execSql(oraconn, sqlstmt) sqlstmt = 'begin Update stats_split_files set FILE_PROCESS_STATUS=1,file_process_date=sysdate where stats_splitfile_id=' + str( fileid) + '; commit; end;' oracon.execSql(oraconn, sqlstmt) statslog.logwrite(fl, 'Done processing :' + filename, 'tabFile') except: statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]), 'tabFile') sqlstmt = 'begin Update stats_split_files set FILE_PROCESS_STATUS=3 where stats_splitfile_id=' + str( fileid) + '; commit; end;' oracon.execSql(oraconn, sqlstmt)
def splitrunner(): fl = statslog.logcreate('log/splitrunner.log') try: ipadd = hostlib.retHostIP() sqlstmt = 'select number_of_files_per_run from statsprocessingmachine where ipaddress=\'' + ipadd + '\' and enable =1' oraconn = oracon.openconnect(constr) numthreadrec = oracon.execSql(oraconn, sqlstmt) for numthread in numthreadrec: files_perrun = numthread[0] numthreadrec.close() oraconn.close() while (1): pythonRuns = commands.getoutput( "ps -ef | grep parser | grep -v grep | wc -l") if int(pythonRuns) < files_perrun + 1: p = Process(target=splitter) p.start() time.sleep(900) except: statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]), 'splitrunner')
def tabber(): oraconn = oracon.openconnect(constr) fl = statslog.logcreate('log/parser.log') try: statslog.logwrite(fl, 'Getting structure.....', 'tabber') sqlstmt = 'select distinct statsname from statsstructure' statsrec = oracon.execSql(oraconn, sqlstmt) statList = [] dtct = 0 for strec in statsrec: statList.append(strec[0]) statslog.logwrite(fl, 'Getting IP.....', 'tabber') ip = hostlib.retHostIP() statslog.logwrite(fl, 'Getting machine configuration.....', 'tabber') sqlstmt = 'select machineid,numberofthreads,dataprocessing_folder from statsprocessingmachine where ipaddress=\'' + ip + '\'' macrec = oracon.execSql(oraconn, sqlstmt) mid = 0 num_threads = 0 datafolder = '' for mrec in macrec: mid = mrec[0] num_threads = mrec[1] datafolder = mrec[2] statslog.logwrite(fl, 'Getting files to process.....', 'tabber') sqlstmt = 'select stats_splitfile_id,stats_splitfile_path,stats_splitfile_name,stats_split_file_type,stats_first_row_version from stats_split_files a,' sqlstmt += '(select distinct statsname,stats_first_row_version from statsstructure) b where machineid=' + str( mid) sqlstmt += ' and stats_split_file_type in (\'' + string.join( statList, '\',\'' ) + '\') and a.stats_split_file_type=b.statsname and a.File_process_status=0 ' sqlstmt += ' AND ROWNUM <=' + str( num_threads / 2) + ' order by stats_splitfile_id' procfiles = oracon.execSql(oraconn, sqlstmt) statslog.logwrite(fl, 'Recieved list of files.....', 'tabber') for procrec in procfiles: flpthlist = string.split(procrec[2], '.') inservserial = flpthlist[1] datepart = str(flpthlist[2]) + ' ' + str(flpthlist[3]) statslog.logwrite(fl, 'Starting file process for ' + procrec[2], 'tabber') p = Process(target=tabFile, args=( procrec[0], procrec[1], procrec[2], procrec[3], inservserial, datepart, procrec[4], datafolder, )) p.daemon = True p.start() sqlstmt = 'begin Update stats_split_files set FILE_PROCESS_STATUS=2 where stats_splitfile_id=' + str( procrec[0]) + '; commit; end;' oracon.execSql(oraconn, sqlstmt) except: statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]), 'tabber')
def splitFile(fileid, filename, filepath, datatype, inservserial, datafolder, machineid): lsconn = lconnect.connectMysql('ods', 'procuser', 'c@llhome', 'localhost') lscurr = lsconn.cursor() try: # Setting the path to the individual inserv splitfilepath = datafolder + '/' + str(inservserial) + '/' # statslog.logwrite(fl,'Processing file ..'+ filename); inputfile = filepath + '/' + filename # statslog.logwrite(fl,'Setup input file......'+inputfile); srcfl = open(inputfile) # statslog.logwrite(fl,'Reading file ..'+ filename); readalllines = srcfl.readlines() srcfl.close sqlstmt = 'select SPLIT_FILE_TYPE, SPLIT_FILE_SEARCH_TAG, SPLIT_FILE_SKIP_LINES, ifnull(SPLIT_FILE_END_TAG,\' \') SPLIT_FILE_END_TAG,' sqlstmt += 'SPLIT_FILE_LINE_SEPERATOR,STATSID from STAT_SPLIT_FILE_LOOKUP where datatype=\'' + datatype + '\'' # statslog.logwrite(fl,'Reading file Mysql structure for the input file...'); # statslog.logwrite(fl,sqlstmt); lscurr.execute(sqlstmt) # statslog.logwrite(fl,'Received structure...'); filenameList = filename.split('.') splitfllist = list() i = 0 for splt in filenameList: if i > 0: splitfllist.append(splt) i += 1 currlinenum = 0 readfileln = readalllines # statslog.logwrite(fl,'Reading os version...'); for ln in readfileln: if string.find(ln, 'Release version') >= 0: osver = string.strip(ln[15:len(ln)]) # statslog.logwrite(fl,'Looping through the structure...'); datrec = lscurr.fetchall() for dat in datrec: currspllist = list() readline = 0 readfileln = readalllines cpstat = 0 for ln in readfileln: if string.find(ln, dat[1]) >= 0: if readline == 0: spflnm = dat[0] + '.' + str( inservserial) + '.' + string.join( splitfllist, '.') cpstat = 1 if cpstat == 1: currspllist.append(ln) readline += 1 if string.strip(ln[0:len(dat[3])]) == string.strip(dat[3]): cpstat = 0 break if not ln: print 'ln is null' if currlinenum >= len(readfileln): cpstat = 0 break nextstr = readfileln[currlinenum + 1] print 'nexstr:' + nextstr if not nextstr or string.strip( nextstr[0:len(dat[3])]) == string.strip( dat[3]) or len( string.strip(nextstr) ) == 0 or string.find(nextstr, '<') >= 0: cpstat = 0 break currlinenum += 1 splafterskip = list() currline = 0 splafterskip.append(osver + '\n') if len(currspllist) > 0: # statslog.logwrite(fl,'Creating directory......'+splitfilepath); checkmakedir(splitfilepath, 'log/splitter_' + filename + '.log') # statslog.logwrite(fl,'Writing file ....'+splitfilepath+'/'+spflnm); spfl = open(splitfilepath + '/' + spflnm, 'w') for curr in currspllist: if currline >= dat[2] and currline < len(currspllist) - 1: if len(string.strip(curr)) > 0: if string.find(string.strip(curr), '-----') == -1: if curr: splafterskip.append(curr) currline += 1 spfl.write(string.join(splafterskip)) spfl.close() #sqlstmt='begin dataload.addsplitfile(inputfileid=>'+str(fileid)+',splitfiletype=>\''+lkp[0]+'\',splitfilename=>\''+spflnm+'\''; #sqlstmt+=',splitfilepath=>\''+splitfilepath+'\',statsid=>'+str(lkp[5])+'); end;'; sqlstmt = 'INSERT INTO STATS_SPLIT_FILES(STATS_FILEID, STATS_SPLITFILE_NAME, STATS_SPLITFILE_PATH, STATSID, STATS_SPLIT_FILE_TYPE) values ' sqlstmt += '(' + str( fileid ) + ',\'' + spflnm + '\',\'' + splitfilepath + '\',' + str( dat[5]) + ',\'' + dat[0] + '\')' # statslog.logwrite(fl,sqlstmt); lscurr.execute(sqlstmt) sqlstmt = 'Update STATSPROCESSTRANSACT set FILE_PROCESS_STATUS=1 where STATS_FILEID=' + str( fileid) lscurr.execute(sqlstmt) lscurr.close() lsconn.close() #os.remove('log/splitter_'+filename+'.log'); except: fl = statslog.logcreate('log/splitter_' + filename + '.log') statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1])) sqlstmt = 'Update STATSPROCESSTRANSACT set FILE_PROCESS_STATUS=3 where STATS_FILEID=' + str( fileid) lscurr.execute(sqlstmt) sqlstmt = 'delete from STATS_SPLIT_FILES where STATS_FILEID=' + str( fileid) lscurr.execute(sqlstmt) lscurr.close() lsconn.close()
def splitter(): oraconn = oracon.openconnect(constr) lconn = lconnect.connectMysql('ods', 'procuser', 'c@llhome', 'localhost') lcurr = lconn.cursor() fl = statslog.logcreate('log/splitter.log') try: statslog.logwrite(fl, 'Getting Host IP...\n') ipadd = hostlib.retHostIP() statslog.logwrite(fl, 'Getting number of threads...\n') sqlstmt = 'select machineid,numberofthreads,dataprocessing_folder,splitterloc from statsprocessingmachine where ipaddress=\'' + ipadd + '\' and enable =1' numthreadrec = oracon.execSql(oraconn, sqlstmt) for numthread in numthreadrec: numthreads = numthread[1] machineid = numthread[0] datafolder = numthread[3] numsplitterthreads = int(numthreads / 2) statslog.logwrite(fl, 'Getting number of datatypes...\n') sqlstmt = 'select distinct datatype from stat_split_file_lookup' datatyprec = oracon.execSql(oraconn, sqlstmt) dtct = 0 for dt in datatyprec: dtct += 1 numsplitthread = int(numsplitterthreads / dtct) datatyprec = oracon.execSql(oraconn, sqlstmt) lconn = lconnect.connectMysql('ods', 'procuser', 'c@llhome', 'localhost') lcurr = lconn.cursor() for dtype in datatyprec: sqlstmt = 'SELECT COUNT(1) FROM STATSPROCESSTRANSACT WHERE FILE_PROCESS_STATUS=0 and DATATYPE=\'' + dtype[ 0] + '\'' lcurr.execute(sqlstmt) reccount = lcurr.fetchall() numInMySql = 0 for rec in reccount: numInMySql = rec[0] numsplitterthreads = numsplitthread - numInMySql if numsplitterthreads > 0: statslog.logwrite(fl, 'Processing for... :' + dtype[0]) sqlstmt = 'SELECT STATS_FILEID,STATS_FILE_NAME,STATS_FILE_PATH,DATATYPE FROM STATSPROCESSTRANSACT ' sqlstmt += 'WHERE FILE_PROCESS_STATUS=0 AND MACHINEID=' + str( machineid) sqlstmt += ' AND ROWNUM <=' + str( numsplitterthreads ) + ' AND DATATYPE= \'' + dtype[0] + '\' ORDER BY STATS_FILEID' filerec = oracon.execSql(oraconn, sqlstmt) statslog.logwrite(fl, 'Inserting data into mysql...\n') sqlstmt = 'INSERT INTO STATSPROCESSTRANSACT (STATS_FILEID,STATS_FILE_NAME,STATS_FILE_PATH,DATATYPE) VALUES ' sql = [] for flrec in filerec: sql.append('(' + str(flrec[0]) + ',\'' + flrec[1] + '\',\'' + flrec[2] + '\',\'' + flrec[3] + '\')') sqlst = 'begin Update STATSPROCESSTRANSACT set FILE_PROCESS_STATUS=2 where STATS_FILEID=' + str( flrec[0]) + '; commit; end;' oracon.execSql(oraconn, sqlst) if len(sql) > 0: sqlstmt = sqlstmt + string.join(sql, ',') lcurr.execute(sqlstmt) lconn.close() oraconn.close() lconn = lconnect.connectMysql('ods', 'procuser', 'c@llhome', 'localhost') lcurr = lconn.cursor() statslog.logwrite(fl, 'Querying mysql...') sqlstmt = 'SELECT STATS_FILEID,STATS_FILE_NAME,STATS_FILE_PATH,DATATYPE FROM STATSPROCESSTRANSACT where ifnull(FILE_PROCESS_STATUS,0)=0 LIMIT ' + str( numsplitthread) lcurr.execute(sqlstmt) ctr = 0 statrow = lcurr.fetchall() pythonRuns = commands.getoutput( "ps -ef | grep python | grep -v grep | wc -l") if int(pythonRuns) - 3 < numsplitthread: residual = (numsplitthread - int(pythonRuns)) + 3 sqlstmt = 'SELECT STATS_FILEID,STATS_FILE_NAME,STATS_FILE_PATH,DATATYPE FROM STATSPROCESSTRANSACT where ifnull(FILE_PROCESS_STATUS,0)=0 LIMIT ' + str( residual) lcurr.execute(sqlstmt) ctr = 0 statrow = lcurr.fetchall() for statrec in statrow: statslog.logwrite(fl, 'Splitting file...:' + statrec[1]) flpthlist = string.split(statrec[2], '/') inservserial = flpthlist[len(flpthlist) - 2] statslog.logwrite(fl, 'Starting process for ... ' + statrec[1]) p = Process(target=splitFile, args=( statrec[0], statrec[1], statrec[2], statrec[3], inservserial, datafolder + '/splitter', machineid, )) p.daemon = True p.start() sqlstmt = 'Update STATSPROCESSTRANSACT set FILE_PROCESS_STATUS=2 where STATS_FILEID=' + str( statrec[0]) lcurr.execute(sqlstmt) ctr += 1 lcurr.close() lconn.close() os.remove('log/splitter.log') except: statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]))
def splitter(): oraconn = oracon.openconnect(constr) lconn = lconnect.connectMysql('ods', 'procuser', 'c@llhome', 'localhost') lcurr = lconn.cursor() try: sqlstmt = 'update STATSPROCESSTRANSACT set FILE_PROCESS_STATUS=0 where FILE_PROCESS_STATUS=2' lcurr.execute(sqlstmt) ipadd = hostlib.retHostIP() sqlstmt = 'select machineid,numberofthreads,dataprocessing_folder,splitterloc,number_of_files_per_run,delay_seconds from statsprocessingmachine where ipaddress=\'' + ipadd + '\' and enable =1' numthreadrec = oracon.execSql(oraconn, sqlstmt) for numthread in numthreadrec: numthreads = numthread[1] machineid = numthread[0] datafolder = numthread[3] files_perrun = numthread[4] delay_seconds = numthread[5] nummysqltreads = int(numthreads) numsplitterthreads = int(numthreads) sqlstmt = 'select distinct datatype from stat_split_file_lookup' datatyprec = oracon.execSql(oraconn, sqlstmt) #dtct=0; datatypes = [] for dt in datatyprec: #dtct+=1; datatypes.append(dt[0]) #numsplitthread=int(numsplitterthreads/dtct); #datatyprec=oracon.execSql(oraconn,sqlstmt); #lconn = lconnect.connectMysql('ods','procuser','c@llhome','localhost'); #lcurr=lconn.cursor(); sqlstmt = 'select count(1) from statsprocesstransact where machineid=' + str( machineid ) + ' and FILE_PROCESS_STATUS=0 and datatype in (\'' + string.join( datatypes, '\'' + ',' + '\'') + '\')' datatypct = oracon.execSql(oraconn, sqlstmt) for dtctrec in datatypct: tot = dtctrec[0] datatypelist = [] sqlstmt = 'select datatype,count(1) from statsprocesstransact where machineid=' + str( machineid ) + ' and FILE_PROCESS_STATUS=0 and datatype in (\'' + string.join( datatypes, '\'' + ',' + '\'') + '\') group by datatype' datatyprec = oracon.execSql(oraconn, sqlstmt) for dt in datatyprec: recperdatatype = int( ((dt[1] * 1.0) / (tot * 1.0)) * numsplitterthreads) datatypelist.append(dt[0] + ',' + str(recperdatatype)) for dtype in datatypelist: datatype = string.split(dtype, ',') sqlstmt = 'SELECT COUNT(1) FROM STATSPROCESSTRANSACT WHERE IFNULL(FILE_PROCESS_STATUS,0)=0 AND DATATYPE= \'' + datatype[ 0] + '\'' lcurr.execute(sqlstmt) reccount = lcurr.fetchall() numInMySql = 0 for rec in reccount: numInMySql = rec[0] numsplitterthreads = int(datatype[1]) - int(numInMySql) if numsplitterthreads > 0: sqlstmt = 'SELECT STATS_FILEID,STATS_FILE_NAME,STATS_FILE_PATH,DATATYPE FROM STATSPROCESSTRANSACT ' sqlstmt += 'WHERE FILE_PROCESS_STATUS=0 AND MACHINEID=' + str( machineid) sqlstmt += ' AND ROWNUM <=' + str( numsplitterthreads) + ' AND DATATYPE= \'' + datatype[ 0] + '\' ORDER BY STATS_FILEID' filerec = oracon.execSql(oraconn, sqlstmt) sqlstmt = 'INSERT INTO STATSPROCESSTRANSACT (STATS_FILEID,STATS_FILE_NAME,STATS_FILE_PATH,DATATYPE) VALUES ' sql = [] for flrec in filerec: sql.append('(' + str(flrec[0]) + ',\'' + flrec[1] + '\',\'' + flrec[2] + '\',\'' + flrec[3] + '\')') sqlst = 'begin Update STATSPROCESSTRANSACT set FILE_PROCESS_STATUS=2 where STATS_FILEID=' + str( flrec[0]) + '; commit; end;' oracon.execSql(oraconn, sqlst) if len(sql) > 0: sqlstmt = sqlstmt + string.join(sql, ',') lcurr.execute(sqlstmt) lconn.close() oraconn.close() lconn = lconnect.connectMysql('ods', 'procuser', 'c@llhome', 'localhost') lcurr = lconn.cursor() ctr = 0 pythonRuns = commands.getoutput( "ps -ef | grep parser | grep -v grep | wc -l") if int(pythonRuns) - 3 < nummysqltreads: residual = (nummysqltreads - int(pythonRuns)) sqlstmt = 'SELECT COUNT(1) FROM STATSPROCESSTRANSACT where ifnull(FILE_PROCESS_STATUS,0)=0' lcurr.execute(sqlstmt) totalcurr = lcurr.fetchall() for lcur in totalcurr: total_recs = lcur[0] sqlstmt = 'SELECT DATATYPE,COUNT(1) FROM STATSPROCESSTRANSACT where ifnull(FILE_PROCESS_STATUS,0)=0 GROUP BY DATATYPE' lcurr.execute(sqlstmt) datrec = lcurr.fetchall() for lcur in datrec: datatype = lcur[0] numrecs = int( round((lcur[1] * 1.0 / total_recs * 1.0) * residual * 1.0, 0)) sqlstmt = 'SELECT distinct STATS_FILEID,STATS_FILE_NAME,STATS_FILE_PATH,DATATYPE FROM STATSPROCESSTRANSACT where ifnull(FILE_PROCESS_STATUS,0)=0 and DATATYPE=\'' + datatype + '\' LIMIT ' + str( numrecs) lcurr.execute(sqlstmt) ctr = 0 statrow = lcurr.fetchall() for statrec in statrow: flpthlist = string.split(statrec[2], '/') inservserial = flpthlist[len(flpthlist) - 2] p = Process(target=splitFile, args=( statrec[0], statrec[1], statrec[2], statrec[3], inservserial, datafolder + '/splitter', machineid, )) p.daemon = True p.start() while ctr > files_perrun: time.sleep(delay_seconds) sqlstmt = 'SELECT COUNT(1) FROM STATSPROCESSTRANSACT WHERE FILE_PROCESS_STATUS=2' lcurr.execute(sqlstmt) rowRec = lcurr.fetchall() for rec in rowRec: ctr = rec[0] sqlstmt = 'Update STATSPROCESSTRANSACT set FILE_PROCESS_STATUS=2 where STATS_FILEID=' + str( statrec[0]) + ' and ifnull(FILE_PROCESS_STATUS,0)=0' lcurr.execute(sqlstmt) ctr += 1 lcurr.close() lconn.close() #os.remove('log/splitter.log'); except: fl = statslog.logcreate('log/splitter.log') statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]))
+ str(myRec[0])) sql += sqlstmt recount += 1 if recount >= 1000: recount = 0 sql = 'begin\n' + string.join( sqlArr, '\n') + '\n' + 'commit; end;' oracon.execSql(procConn, sql) totrec += recount recount = 0 sqlArr = [] if len(sqlArr) >= 0: sql = 'begin\n' + string.join( sqlArr, '\n') + '\n' + 'commit; end;' totrec += recount oracon.execSql(procConn, sql) recount = 0 for Mrec in MysqlArr: lcurr.execute(Mrec) numrec += 100000 lcurr.execute( 'delete from STATS_SPLIT_FILES where FILE_PROCESS_STATUS=1') procConn.close() time.sleep(900) except Exception: fl = statslog.logcreate("log/dataCopy.log") statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]))
def nfsShareStatus(): try: newdiskdt = [] newnodedt = [] diskArr = [] nodeArr = [] diskdt = subprocess.check_output("df -h|grep share", shell=True).split('\n') nodedt = subprocess.check_output("df -i|grep share", shell=True).split('\n') for dt in diskdt: if len(dt) > 4: strarr = dt.split(' ') #print "String Array is:" +str(strarr); newdt = [] for st in strarr: if st != '': newdt.append(st) newdiskdt.append(string.join(newdt, ' ')) for nt in nodedt: if len(nt) > 4: strarr = nt.split(' ') #print "String Array is:" +str(strarr); newndt = [] for st in strarr: if st != '': newndt.append(st) newnodedt.append(string.join(newndt, ' ')) for line in newdiskdt: mntpoints = line.split(' ') if len(mntpoints) > 4: mntpoint = mntpoints[4].replace('/share/', '') diskutil = mntpoints[3].strip('%') groupid = time.strftime('%Y%m%d%H%M') sqlstmt = ' insert /*+ append +*/ into omi_sharest_disk_status (MOUNTNAME, STATUS_POST_TIME, DISKSPACE_UTILIZATION, SNAPSHOT_GROUPID) values ' sqlstmt += '(\'' + str(mntpoint) + '\',sysdate,\'' + str( diskutil) + '\',\'' + str(groupid) + '\');' diskArr.append(sqlstmt) for line in newnodedt: mntpoints = line.split(' ') if len(mntpoints) > 4: mntpoint = mntpoints[4].replace('/share/', '') inodeutil = mntpoints[3].strip('%') groupid = time.strftime('%Y%m%d%H%M') sqlstmt = ' insert /*+ append +*/ into omi_sharest_inode_status (MOUNTNAME, STATUS_POST_TIME, INODE_UTILIZATION, SNAPSHOT_GROUPID) values ' sqlstmt += '(\'' + str(mntpoint) + '\',sysdate,\'' + str( inodeutil) + '\',\'' + str(groupid) + '\');' nodeArr.append(sqlstmt) oraconn = oracon.openconnect(constr) sqld = 'begin\n' + string.join(diskArr, '\n') + '\n' + 'commit; end;' oracon.execSql(oraconn, sqld) sqln = 'begin\n' + string.join(nodeArr, '\n') + '\n' + 'commit; end;' oracon.execSql(oraconn, sqln) oraconn.close() except: fl = statslog.logcreate('log/sharestutil.log') statslog.logwrite(fl, 'Error reported: ' + str(sys.exc_info()[1]))
def updateMetadata(): try: lsconn = lconnect.connectMysql('ods', 'procuser', 'c@llhome', 'localhost') lscurr = lsconn.cursor() oraconn = oracon.openconnect(constr) sqlstmt = 'SELECT STATSID, STATS_STRUCTURE_ID, STATSNAME, STATS_VERSION, STATS_FIRST_ROW_VERSION, STATS_SINGLE_ROW, STATS_END_OF_ROW FROM STATSSTRUCTURE ORDER BY 1' statrec = oracon.execSql(oraconn, sqlstmt) sqlstmt = 'SELECT DATATYPE, SPLIT_FILE_TYPE, SPLIT_FILE_SEARCH_TAG, SPLIT_FILE_SKIP_LINES, SPLIT_FILE_END_TAG, SPLIT_FILE_LINE_SEPERATOR, STATSID FROM STAT_SPLIT_FILE_LOOKUP' splitrec = oracon.execSql(oraconn, sqlstmt) sqlstmt = 'SELECT STATS_STRUCTURE_ID, PROCESS_SEQUENCE, FUNCTIONID FROM PROCESSLOGIC' processrec = oracon.execSql(oraconn, sqlstmt) sqlstmt = 'SELECT FUNCTIONID, FUNCTIONNAME, FUNCTIONPARAMETERS, FUNCTIONDESC FROM PROCESSFUNCTION' profunc = oracon.execSql(oraconn, sqlstmt) for rec in statrec: sqlstmt = 'SELECT COUNT(1) FROM STATSSTRUCTURE WHERE STATSID=' + str( rec[0]) + ' and STATS_STRUCTURE_ID=' + str(rec[1]) lscurr.execute(sqlstmt) recData = lscurr.fetchall() recCount = 0 for recdat in recData: recCount = recdat[0] if recCount == 0: if not rec[6]: end_row = '' else: end_row = rec[6] sqlstmt = 'INSERT INTO STATSSTRUCTURE (STATSID, STATS_STRUCTURE_ID, STATSNAME, STATS_VERSION, STATS_FIRST_ROW_VERSION, STATS_SINGLE_ROW, STATS_END_OF_ROW) VALUES ' sqlstmt += ' (' + str(rec[0]) + ',' + str( rec[1] ) + ',\'' + rec[2] + '\',\'' + rec[3] + '\',\'' + rec[ 4] + '\',\'' + rec[5] + '\',\'' + end_row + '\')' lscurr.execute(sqlstmt) for rec in splitrec: sqlstmt = 'SELECT COUNT(1) FROM STAT_SPLIT_FILE_LOOKUP WHERE DATATYPE=\'' + str( rec[0]) + '\' and SPLIT_FILE_TYPE=\'' + rec[1] + '\'' lscurr.execute(sqlstmt) recData = lscurr.fetchall() recCount = 0 for recdat in recData: recCount = recdat[0] if recCount == 0: if not rec[5]: line_sep = '' else: line_sep = rec[5] sqlstmt = 'INSERT INTO STAT_SPLIT_FILE_LOOKUP (DATATYPE, SPLIT_FILE_TYPE, SPLIT_FILE_SEARCH_TAG, SPLIT_FILE_SKIP_LINES, SPLIT_FILE_END_TAG, SPLIT_FILE_LINE_SEPERATOR, STATSID) ' sqlstmt += 'VALUES (\'' + str(rec[0]) + '\',\'' + str( rec[1]) + '\',\'' + rec[2] + '\',' + str( rec[3] ) + ',\'' + rec[4] + '\',\'' + line_sep + '\',' + str( rec[6]) + ')' lscurr.execute(sqlstmt) for rec in processrec: sqlstmt = 'SELECT COUNT(1) FROM PROCESSLOGIC WHERE STATS_STRUCTURE_ID=' + str( rec[0]) + ' and PROCESS_SEQUENCE=' + str( rec[1]) + ' and FUNCTIONID=' + str(rec[2]) #sqlstmt='SELECT COUNT(1) FROM PROCESSLOGIC WHERE STATS_STRUCTURE_ID='+str(rec[0]); lscurr.execute(sqlstmt) recData = lscurr.fetchall() recCount = 0 for recdat in recData: recCount = recdat[0] if recCount == 0: sqlstmt = 'INSERT INTO PROCESSLOGIC (STATS_STRUCTURE_ID, PROCESS_SEQUENCE, FUNCTIONID) ' sqlstmt += 'VALUES (' + str(rec[0]) + ',' + str( rec[1]) + ',' + str(rec[2]) + ')' lscurr.execute(sqlstmt) #else: ##fix logic if needed # sqlstmt='select STATS_STRUCTURE_ID, PROCESS_SEQUENCE, FUNCTIONID FROM PROCESSLOGIC WHERE STATS_STRUCTURE_ID='+str(rec[0])+' and PROCESS_SEQUENCE='+str(rec[1]); # lscurr.execute(sqlstmt); # dat=lscurr.fetchall(); # for dt in dat: # if dt[2] != rec[2]: # sqlstmt='update PROCESSLOGIC SET FUNCTIONID='+str(rec[2])+' WHERE STATS_STRUCTURE_ID='+str(rec[0])+' and PROCESS_SEQUENCE='+str(rec[1]); # lscurr.execute(sqlstmt); sqlstmt = 'SELECT STATS_STRUCTURE_ID, PROCESS_SEQUENCE, FUNCTIONID FROM PROCESSLOGIC' lscurr.execute(sqlstmt) mySqlDat = lscurr.fetchall() for myRec in mySqlDat: sqlstmt = 'SELECT COUNT(1) FROM PROCESSLOGIC where STATS_STRUCTURE_ID=' + str( myRec[0]) + ' AND PROCESS_SEQUENCE=' + str( myRec[1]) + ' AND FUNCTIONID=' + str(myRec[2]) processrec = oracon.execSql(oraconn, sqlstmt) for recdat in processrec: recCount = recdat[0] if recCount == 0: sqlstmt = 'delete from PROCESSLOGIC where STATS_STRUCTURE_ID=' + str( myRec[0]) + ' and PROCESS_SEQUENCE =' + str( myRec[1]) + ' and FUNCTIONID =' + str(myRec[2]) lscurr.execute(sqlstmt) for rec in profunc: sqlstmt = 'SELECT COUNT(1) FROM PROCESSFUNCTION WHERE FUNCTIONID=' + str( rec[0]) + ' and FUNCTIONNAME=\'' + str(rec[1]) + '\'' lscurr.execute(sqlstmt) recData = lscurr.fetchall() recCount = 0 for recdat in recData: recCount = recdat[0] if recCount == 0: sqlstmt = 'INSERT INTO PROCESSFUNCTION (FUNCTIONID, FUNCTIONNAME, FUNCTIONPARAMETERS, FUNCTIONDESC) ' sqlstmt += 'VALUES (' + str(rec[0]) + ',\'' + str( rec[1]) + '\',\'' + str(rec[2]) + '\',\'' + str( rec[3]) + '\')' lscurr.execute(sqlstmt) lscurr.close() lsconn.close() oraconn.close() except: fl = statslog.logcreate('log/copymeta.log') statslog.logwrite(fl, 'Error reported: ' + str(sys.exc_info()[1]))
def nfsUtildata(): try: newdiskdt = [] newnodedt = [] diskArr = [] nodeArr = [] diskdt = subprocess.check_output("df -h|grep ods", shell=True).split('\n') nodedt = subprocess.check_output("df -i|grep ods", shell=True).split('\n') for dt in diskdt: strarr = dt.split(' ') #print "String Array is:" +str(strarr); newdt = [] for st in strarr: if st != '': newdt.append(st) newdiskdt.append(string.join(newdt, ' ')) for nt in nodedt: strarr = nt.split(' ') #print "String Array is:" +str(strarr); newndt = [] for st in strarr: if st != '': newndt.append(st) newnodedt.append(string.join(newndt, ' ')) for line in newdiskdt: mntpoints = line.split(' ') if len(mntpoints) > 5: mntserver = mntpoints[0] mntpoint = mntpoints[5].strip('/') diskutil = mntpoints[4].strip('%') #dataloc=mntpoint #istring=("df -i|grep %s|awk '{print $5}'"%mntpoint); #inodeutil=commands.getoutput(istring); #sqlstmt=' insert /*+ append +*/ into omi_nfs_status (MOUNTNAME, MOUNTSERVER,STATUS_POST_TIME, DISKSPACE_UTILIZATION,INODE_UTILIZATION) values '; #sqlstmt+='(\''+str(mntpoint)+'\',\''+str(mntserver)+'\',sysdate,\''+str(diskutil)+'\',\''+str(inodeutil)+'\');'; sqlstmt = ' insert /*+ append +*/ into omi_nfsdisk_status (MOUNTNAME, MOUNTSERVER,STATUS_POST_TIME, DISKSPACE_UTILIZATION) values ' sqlstmt += '(\'' + str(mntpoint) + '\',\'' + str( mntserver) + '\',sysdate,\'' + str(diskutil) + '\');' diskArr.append(sqlstmt) for line in newnodedt: mntpoints = line.split(' ') if len(mntpoints) > 5: mntserver = mntpoints[0] mntpoint = mntpoints[5].strip('/') inodeutil = mntpoints[4].strip('%') sqlstmt = ' insert /*+ append +*/ into omi_nfsinode_status (MOUNTNAME, MOUNTSERVER,STATUS_POST_TIME, INODE_UTILIZATION) values ' sqlstmt += '(\'' + str(mntpoint) + '\',\'' + str( mntserver) + '\',sysdate,\'' + str(inodeutil) + '\');' nodeArr.append(sqlstmt) oraconn = oracon.openconnect(constr) sqld = 'begin\n' + string.join(diskArr, '\n') + '\n' + 'commit; end;' oracon.execSql(oraconn, sqld) sqln = 'begin\n' + string.join(nodeArr, '\n') + '\n' + 'commit; end;' oracon.execSql(oraconn, sqln) oraconn.close() except: fl = statslog.logcreate('log/nfsutil.log') statslog.logwrite(fl, 'Error reported: ' + str(sys.exc_info()[1]))
def processFile(STATS_SPLITFILE_ID, STATS_SPLITFILE_NAME, STATS_SPLITFILE_PATH, STATS_SPLITFILE_TYPE, INSERVSERIAL, DATAFOLDER, MACHINEID): try: fl = statslog.logcreate("log/" + STATS_SPLITFILE_NAME) lc = mysqlconn.connectMysql('ods', 'procuser', 'c@llhome', 'localhost') curr = lc.cursor() errlog = 'log/' + STATS_SPLITFILE_NAME + '.err' if versionExist(STATS_SPLITFILE_TYPE) == 1: version = prclib.getVersion(STATS_SPLITFILE_NAME, STATS_SPLITFILE_PATH) else: version = '2.3.1' sqlstmt = 'SELECT COUNT(1) FROM STATSSTRUCTURE WHERE STATS_VERSION=\'' + version + '\' and STATSNAME=\'' + STATS_SPLITFILE_TYPE + '\'' curr.execute(sqlstmt) dat = curr.fetchall() for datrec in dat: count = datrec[0] if count > 0: sqlstmt = 'SELECT STATS_STRUCTURE_ID FROM STATSSTRUCTURE WHERE STATS_VERSION=\'' + version + '\' and STATSNAME=\'' + STATS_SPLITFILE_TYPE + '\'' curr.execute(sqlstmt) verdat = curr.fetchall() STATS_STRUCTURE_ID = 0 for ver in verdat: STATS_STRUCTURE_ID = ver[0] if STATS_STRUCTURE_ID > 0: sqlstmt = 'SELECT PROCESS_SEQUENCE,FUNCTIONID,PARAMETER_VALUES FROM PROCESSLOGIC WHERE STATS_STRUCTURE_ID=' + str( STATS_STRUCTURE_ID) + ' ORDER BY PROCESS_SEQUENCE' curr.execute(sqlstmt) prcdat = curr.fetchall() linearray = prclib.readFile(STATS_SPLITFILE_NAME, STATS_SPLITFILE_PATH) for prc in prcdat: sqlstmt = 'SELECT FUNCTIONNAME FROM PROCESSFUNCTION WHERE FUNCTIONID=' + str( prc[1]) curr.execute(sqlstmt) funcNamerec = curr.fetchall() for fu in funcNamerec: functionName = fu[0] func = getattr(prclib, functionName) linearray = func(linearray) filepath = DATAFOLDER + '/' + str(INSERVSERIAL) + '/' checkmakedir(filepath, errlog) outputFl = open(filepath + STATS_SPLITFILE_NAME, 'w') outputFl.write(linearray) sqlstmt = 'insert into STATSOUTPUT(STATS_SPLITFILE_ID,STATS_OUTPUTFILE_NAME,STATS_OUTPUTFILE_PATH,STATS_FILE_CREATE_DATE,STATS_FILE_LOAD_STATUS,STATS_STRUCTURE_TYPE_ID) values ' sqlstmt += '(' + str( STATS_SPLITFILE_ID ) + ',\'' + STATS_SPLITFILE_NAME + '\',\'' + filepath + '\',sysdate(),0,' + str( STATS_STRUCTURE_ID) + ')' curr.execute(sqlstmt) sqlstmt = 'Update STATS_SPLIT_FILES set FILE_PROCESS_STATUS=1 where STATS_SPLITFILE_ID=' + str( STATS_SPLITFILE_ID) curr.execute(sqlstmt) curr.close() lc.close() else: sqlstmt = 'DELETE FROM STATS_SPLIT_FILES WHERE STATS_SPLITFILE_ID=' + str( STATS_SPLITFILE_ID) curr.execute(sqlstmt) curr.close() lc.close() else: fl = statslog.logcreate("log/" + STATS_SPLITFILE_NAME) statslog.logwrite( fl, "Error reported: Version " + version + " not yet supported") sqlstmt = 'Update STATS_SPLIT_FILES set FILE_PROCESS_STATUS=3 where STATS_SPLITFILE_ID=' + str( STATS_SPLITFILE_ID) lc = mysqlconn.connectMysql('ods', 'procuser', 'c@llhome', 'localhost') curr = lc.cursor() curr.execute(sqlstmt) curr.close() lc.close() except: fl = statslog.logcreate("log/" + STATS_SPLITFILE_NAME) statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1])) sqlstmt = 'Update STATS_SPLIT_FILES set FILE_PROCESS_STATUS=3 where STATS_SPLITFILE_ID=' + str( STATS_SPLITFILE_ID) lc = mysqlconn.connectMysql('ods', 'procuser', 'c@llhome', 'localhost') curr = lc.cursor() curr.execute(sqlstmt) curr.close() lc.close()
def procExec(): try: oraconn = oracon.openconnect(constr) lconn = mysqlconn.connectMysql('ods', 'procuser', 'c@llhome', 'localhost') lcurr = lconn.cursor() ipadd = hostlib.retHostIP() sqlstmt = 'select machineid,numberofthreads,processloc,number_of_files_per_run,delay_seconds from statsprocessingmachine where ipaddress=\'' + ipadd + '\' and enable =1' numthreadrec = oracon.execSql(oraconn, sqlstmt) for numthread in numthreadrec: numthreads = numthread[1] machineid = numthread[0] datafolder = numthread[2] files_per_run = numthread[3] delay_seconds = numthread[4] #statslog.logwrite(fl,str(numthreads)+':'+str(files_per_run)+':'+str(delay_seconds)); numthreads = int(numthreads) sqlstmt = 'select distinct statsname from STATSSTRUCTURE a,PROCESSLOGIC b where a.STATS_STRUCTURE_ID = b.STATS_STRUCTURE_ID' lcurr.execute(sqlstmt) recc = lcurr.fetchall() statList = [] for rec in recc: statList.append('\'' + rec[0] + '\'') statsStr = string.join(statList, ',') sqlstmt = 'update STATS_SPLIT_FILES set FILE_PROCESS_STATUS=0 WHERE FILE_PROCESS_STATUS=2' lcurr.execute(sqlstmt) sqlstmt = 'select count(1) from STATS_SPLIT_FILES where ifnull(FILE_PROCESS_STATUS,0)=0 and STATS_SPLIT_FILE_TYPE in (' + statsStr + ')' lcurr.execute(sqlstmt) recc = lcurr.fetchall() numRec = 0 for rec in recc: numRec = rec[0] numthreadstocopy = numthreads - numRec #statslog.logwrite(fl,str(numthreadstocopy)); if numthreadstocopy > 0: sqlstmt = '' rows = 1000 totrows = rows while totrows <= numthreadstocopy: sqlstmt = 'select STATS_FILEID,STATS_SPLITFILE_ID,STATS_SPLITFILE_NAME,STATS_SPLITFILE_PATH,STATSID,STATS_SPLIT_FILE_TYPE from STATS_SPLIT_FILES ' sqlstmt += ' where FILE_PROCESS_STATUS=0 and MACHINEID=' + str( machineid ) + ' and ROWNUM <=' + str( rows ) + ' and STATS_SPLIT_FILE_TYPE in (' + statsStr + ') order by STATS_SPLITFILE_ID desc' #statslog.logwrite(fl,sqlstmt); FileRec = oracon.execSql(oraconn, sqlstmt) sqlstmt = 'INSERT INTO STATS_SPLIT_FILES (STATS_FILEID,STATS_SPLITFILE_ID,STATS_SPLITFILE_NAME,STATS_SPLITFILE_PATH,STATSID,STATS_SPLIT_FILE_TYPE) VALUES ' sql = [] for flrec in FileRec: sql.append('(' + str(flrec[0]) + ',' + str(flrec[1]) + ',\'' + flrec[2] + '\',\'' + flrec[3] + '\',' + str(flrec[4]) + ',\'' + flrec[5] + '\')') sqlst = 'begin Update STATS_SPLIT_FILES set FILE_PROCESS_STATUS=2 where STATS_SPLITFILE_ID=' + str( flrec[1]) + '; commit; end;' oracon.execSql(oraconn, sqlst) if len(sql) > 0: sqlstmt += string.join(sql, ',') lcurr.execute(sqlstmt) totrows = totrows + rows pythonRuns = commands.getoutput( "ps -ef | grep parsetabber | grep -v grep | wc -l") while int(pythonRuns) - 3 <= files_per_run: sqlstmt = 'SELECT a.STATS_SPLITFILE_ID,a.STATS_SPLITFILE_NAME,a.STATS_SPLITFILE_PATH,STATS_SPLIT_FILE_TYPE from STATS_SPLIT_FILES a where ifnull(FILE_PROCESS_STATUS,0)=0 and STATS_SPLIT_FILE_TYPE in (' + statsStr + ') LIMIT ' + str( files_per_run - int(pythonRuns)) #statslog.logwrite(fl,sqlstmt); if (files_per_run - int(pythonRuns)) > 0: ctr = 0 lcurr = lconn.cursor() lcurr.execute(sqlstmt) filelist = lcurr.fetchall() for flrec in filelist: flpthlist = string.split(flrec[2], '/') inservserial = flpthlist[len(flpthlist) - 2] p = Process(target=processFile, args=( flrec[0], flrec[1], flrec[2], flrec[3], inservserial, datafolder, machineid, )) p.daemon = True p.start() ctr += 1 sqlstmt = 'Update STATS_SPLIT_FILES set FILE_PROCESS_STATUS=2 where STATS_SPLITFILE_ID=' + str( flrec[0]) lcurr.execute(sqlstmt) lcurr.close() if ctr == 0: break pythonRuns = commands.getoutput( "ps -ef | grep parsetabber | grep -v grep | wc -l") while int(pythonRuns) > files_per_run: time.sleep(delay_seconds) pythonRuns = commands.getoutput( "ps -ef | grep parsetabber | grep -v grep | wc -l") pythonRuns = commands.getoutput( "ps -ef | grep parsetabber | grep -v grep | wc -l") lconn.close() except: fl = statslog.logcreate("log/parser.log") statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]))
if (purpose == 'datamartextract' or purpose == 'sync'): st = os.statvfs('/') free = st.f_bavail * st.f_frsize total = st.f_blocks * st.f_frsize used = (st.f_blocks - st.f_bfree) * st.f_frsize spaceused = round((used * 1.0 / total * 1.0) * 100, 2) inodespace = 0 dataloc = str(hstname) oraconn = oracon.openconnect(constr) sqlst = 'begin insert into OMI_MACHINE_STATUS (MACHINEID,MACHINENAME,IPADDRESS,STATUS_POST_TIME,PURPOSE,NUM_PYTHON_THREADS,DATALOCATION,DATACOPY_STATUS,' sqlst += 'PARSER_STATUS,COPYMETA_STATUS,DATAMART_STATUS,TAGGER_STATUS,PROCESSED,TOBEPROCESSED,TOBECOPIED,FAILEDTOPROCESS,DISKSPACE_UTILIZATION,INODE_UTILIZATION) ' sqlst += 'values (\'' + str(macid) + '\',\'' + str( hstname) + '\',\'' + str(hstip) + '\',sysdate,\'' + mp[ 0] + '\',\'' + str(pythonRuns) + '\',\'' + str( dataloc) + '\',\'' + str(pdatacopy) + '\',' sqlst += '\'' + str(pparser) + '\',\'' + str(pcpmeta) + '\',\'' + str( pdatamart) + '\',\'' + str(ptagger) + '\',' + str( processed) + ',' + str(tobeProcessed) + ',' + str( recCount) + ',' sqlst += '' + str(errored) + ',\'' + str(spaceused) + '%\',\'' + str( inodespace) + '\'); commit; end;' oracon.execSql(oraconn, sqlst) #print "closing connection"; oraconn.close() except Exception: fl = statslog.logcreate("log/machineStatus.log") statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1])) time.sleep(900)