def checkmakedir(dname,fname):
try:
d=os.path.dirname(dname);
if not os.path.exists(d):
os.makedirs(d);
except:
fl=statslog.logcreate(fname);
statslog.logwrite(fl,"Error reported (create dir): "+str(sys.exc_info()[1]));
return;
def parserunner():
fl = statslog.logcreate('log/parserunner.log')
try:
while True:
p = Process(target=tabber)
p.start()
time.sleep(600)
except:
statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]),
'parserunner')
def splitrunner():
fl = statslog.logcreate('log/splitrunner.log')
try:
while (1):
p = Process(target=splitter)
p.start()
time.sleep(30)
except:
statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]),
'splitrunner')
def tabFile(fileid, filepath, filename, filetype, inservserial, datepart,
statsfirstrowversion, datafolder):
oraconn = oracon.openconnect(constr)
fl = statslog.logcreate('log/tabber_' + str(fileid) + '.log')
try:
outdir = datafolder + '/processed/' + str(inservserial) + '/'
checkmakedir(outdir)
dtsplit = string.split(datepart, ' ')
dt = string.join(dtsplit, '.')
outputfilename = filetype + '.' + str(inservserial) + '.' + str(
dt) + '.out'
outfile = open(outdir + outputfilename, 'w')
statslog.logwrite(fl, 'Opening file....' + filename)
infile = open(filepath + '/' + filename)
lines = infile.readlines()
infile.close()
datastruct = {}
if statsfirstrowversion == 'True':
if len(lines) > 1:
statslog.logwrite(fl, 'Generating data...', 'tabFile')
datarowstruct = {}
datarowstruct = generateData(lines, filetype, inservserial,
datepart, fl)
datarow = datarowstruct[0]
structid = datarowstruct[1]
outfile.write(datarow)
sqlstmt = 'begin dataload.ADDOUTPUTFILE (STATS_SPLITFILE_ID=>' + str(
fileid)
sqlstmt += ',STATS_OUTPUTFILE_NAME=>\'' + outputfilename + '\',STATS_OUTPUTFILE_PATH =>\'' + outdir + '\',STATS_STRUCTURE_ID=>' + str(
structid) + '); end;'
oracon.execSql(oraconn, sqlstmt)
sqlstmt = 'begin Update stats_split_files set FILE_PROCESS_STATUS=1,file_process_date=sysdate where stats_splitfile_id=' + str(
fileid) + '; commit; end;'
oracon.execSql(oraconn, sqlstmt)
statslog.logwrite(fl, 'Done processing :' + filename,
'tabFile')
except:
statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]),
'tabFile')
sqlstmt = 'begin Update stats_split_files set FILE_PROCESS_STATUS=3 where stats_splitfile_id=' + str(
fileid) + '; commit; end;'
oracon.execSql(oraconn, sqlstmt)
def splitrunner():
fl = statslog.logcreate('log/splitrunner.log')
try:
ipadd = hostlib.retHostIP()
sqlstmt = 'select number_of_files_per_run from statsprocessingmachine where ipaddress=\'' + ipadd + '\' and enable =1'
oraconn = oracon.openconnect(constr)
numthreadrec = oracon.execSql(oraconn, sqlstmt)
for numthread in numthreadrec:
files_perrun = numthread[0]
numthreadrec.close()
oraconn.close()
while (1):
pythonRuns = commands.getoutput(
"ps -ef | grep parser | grep -v grep | wc -l")
if int(pythonRuns) < files_perrun + 1:
p = Process(target=splitter)
p.start()
time.sleep(900)
except:
statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]),
'splitrunner')
def tabber():
oraconn = oracon.openconnect(constr)
fl = statslog.logcreate('log/parser.log')
try:
statslog.logwrite(fl, 'Getting structure.....', 'tabber')
sqlstmt = 'select distinct statsname from statsstructure'
statsrec = oracon.execSql(oraconn, sqlstmt)
statList = []
dtct = 0
for strec in statsrec:
statList.append(strec[0])
statslog.logwrite(fl, 'Getting IP.....', 'tabber')
ip = hostlib.retHostIP()
statslog.logwrite(fl, 'Getting machine configuration.....', 'tabber')
sqlstmt = 'select machineid,numberofthreads,dataprocessing_folder from statsprocessingmachine where ipaddress=\'' + ip + '\''
macrec = oracon.execSql(oraconn, sqlstmt)
mid = 0
num_threads = 0
datafolder = ''
for mrec in macrec:
mid = mrec[0]
num_threads = mrec[1]
datafolder = mrec[2]
statslog.logwrite(fl, 'Getting files to process.....', 'tabber')
sqlstmt = 'select stats_splitfile_id,stats_splitfile_path,stats_splitfile_name,stats_split_file_type,stats_first_row_version from stats_split_files a,'
sqlstmt += '(select distinct statsname,stats_first_row_version from statsstructure) b where machineid=' + str(
mid)
sqlstmt += ' and stats_split_file_type in (\'' + string.join(
statList, '\',\''
) + '\') and a.stats_split_file_type=b.statsname and a.File_process_status=0 '
sqlstmt += ' AND ROWNUM <=' + str(
num_threads / 2) + ' order by stats_splitfile_id'
procfiles = oracon.execSql(oraconn, sqlstmt)
statslog.logwrite(fl, 'Recieved list of files.....', 'tabber')
for procrec in procfiles:
flpthlist = string.split(procrec[2], '.')
inservserial = flpthlist[1]
datepart = str(flpthlist[2]) + ' ' + str(flpthlist[3])
statslog.logwrite(fl, 'Starting file process for ' + procrec[2],
'tabber')
p = Process(target=tabFile,
args=(
procrec[0],
procrec[1],
procrec[2],
procrec[3],
inservserial,
datepart,
procrec[4],
datafolder,
))
p.daemon = True
p.start()
sqlstmt = 'begin Update stats_split_files set FILE_PROCESS_STATUS=2 where stats_splitfile_id=' + str(
procrec[0]) + '; commit; end;'
oracon.execSql(oraconn, sqlstmt)
except:
statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]),
'tabber')
def splitFile(fileid, filename, filepath, datatype, inservserial, datafolder,
machineid):
lsconn = lconnect.connectMysql('ods', 'procuser', 'c@llhome', 'localhost')
lscurr = lsconn.cursor()
try:
# Setting the path to the individual inserv
splitfilepath = datafolder + '/' + str(inservserial) + '/'
# statslog.logwrite(fl,'Processing file ..'+ filename);
inputfile = filepath + '/' + filename
# statslog.logwrite(fl,'Setup input file......'+inputfile);
srcfl = open(inputfile)
# statslog.logwrite(fl,'Reading file ..'+ filename);
readalllines = srcfl.readlines()
srcfl.close
sqlstmt = 'select SPLIT_FILE_TYPE, SPLIT_FILE_SEARCH_TAG, SPLIT_FILE_SKIP_LINES, ifnull(SPLIT_FILE_END_TAG,\' \') SPLIT_FILE_END_TAG,'
sqlstmt += 'SPLIT_FILE_LINE_SEPERATOR,STATSID from STAT_SPLIT_FILE_LOOKUP where datatype=\'' + datatype + '\''
# statslog.logwrite(fl,'Reading file Mysql structure for the input file...');
# statslog.logwrite(fl,sqlstmt);
lscurr.execute(sqlstmt)
# statslog.logwrite(fl,'Received structure...');
filenameList = filename.split('.')
splitfllist = list()
i = 0
for splt in filenameList:
if i > 0:
splitfllist.append(splt)
i += 1
currlinenum = 0
readfileln = readalllines
# statslog.logwrite(fl,'Reading os version...');
for ln in readfileln:
if string.find(ln, 'Release version') >= 0:
osver = string.strip(ln[15:len(ln)])
# statslog.logwrite(fl,'Looping through the structure...');
datrec = lscurr.fetchall()
for dat in datrec:
currspllist = list()
readline = 0
readfileln = readalllines
cpstat = 0
for ln in readfileln:
if string.find(ln, dat[1]) >= 0:
if readline == 0:
spflnm = dat[0] + '.' + str(
inservserial) + '.' + string.join(
splitfllist, '.')
cpstat = 1
if cpstat == 1:
currspllist.append(ln)
readline += 1
if string.strip(ln[0:len(dat[3])]) == string.strip(dat[3]):
cpstat = 0
break
if not ln:
print 'ln is null'
if currlinenum >= len(readfileln):
cpstat = 0
break
nextstr = readfileln[currlinenum + 1]
print 'nexstr:' + nextstr
if not nextstr or string.strip(
nextstr[0:len(dat[3])]) == string.strip(
dat[3]) or len(
string.strip(nextstr)
) == 0 or string.find(nextstr, '<') >= 0:
cpstat = 0
break
currlinenum += 1
splafterskip = list()
currline = 0
splafterskip.append(osver + '\n')
if len(currspllist) > 0:
# statslog.logwrite(fl,'Creating directory......'+splitfilepath);
checkmakedir(splitfilepath,
'log/splitter_' + filename + '.log')
# statslog.logwrite(fl,'Writing file ....'+splitfilepath+'/'+spflnm);
spfl = open(splitfilepath + '/' + spflnm, 'w')
for curr in currspllist:
if currline >= dat[2] and currline < len(currspllist) - 1:
if len(string.strip(curr)) > 0:
if string.find(string.strip(curr), '-----') == -1:
if curr:
splafterskip.append(curr)
currline += 1
spfl.write(string.join(splafterskip))
spfl.close()
#sqlstmt='begin dataload.addsplitfile(inputfileid=>'+str(fileid)+',splitfiletype=>\''+lkp[0]+'\',splitfilename=>\''+spflnm+'\'';
#sqlstmt+=',splitfilepath=>\''+splitfilepath+'\',statsid=>'+str(lkp[5])+'); end;';
sqlstmt = 'INSERT INTO STATS_SPLIT_FILES(STATS_FILEID, STATS_SPLITFILE_NAME, STATS_SPLITFILE_PATH, STATSID, STATS_SPLIT_FILE_TYPE) values '
sqlstmt += '(' + str(
fileid
) + ',\'' + spflnm + '\',\'' + splitfilepath + '\',' + str(
dat[5]) + ',\'' + dat[0] + '\')'
# statslog.logwrite(fl,sqlstmt);
lscurr.execute(sqlstmt)
sqlstmt = 'Update STATSPROCESSTRANSACT set FILE_PROCESS_STATUS=1 where STATS_FILEID=' + str(
fileid)
lscurr.execute(sqlstmt)
lscurr.close()
lsconn.close()
#os.remove('log/splitter_'+filename+'.log');
except:
fl = statslog.logcreate('log/splitter_' + filename + '.log')
statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]))
sqlstmt = 'Update STATSPROCESSTRANSACT set FILE_PROCESS_STATUS=3 where STATS_FILEID=' + str(
fileid)
lscurr.execute(sqlstmt)
sqlstmt = 'delete from STATS_SPLIT_FILES where STATS_FILEID=' + str(
fileid)
lscurr.execute(sqlstmt)
lscurr.close()
lsconn.close()
def splitter():
oraconn = oracon.openconnect(constr)
lconn = lconnect.connectMysql('ods', 'procuser', 'c@llhome', 'localhost')
lcurr = lconn.cursor()
fl = statslog.logcreate('log/splitter.log')
try:
statslog.logwrite(fl, 'Getting Host IP...\n')
ipadd = hostlib.retHostIP()
statslog.logwrite(fl, 'Getting number of threads...\n')
sqlstmt = 'select machineid,numberofthreads,dataprocessing_folder,splitterloc from statsprocessingmachine where ipaddress=\'' + ipadd + '\' and enable =1'
numthreadrec = oracon.execSql(oraconn, sqlstmt)
for numthread in numthreadrec:
numthreads = numthread[1]
machineid = numthread[0]
datafolder = numthread[3]
numsplitterthreads = int(numthreads / 2)
statslog.logwrite(fl, 'Getting number of datatypes...\n')
sqlstmt = 'select distinct datatype from stat_split_file_lookup'
datatyprec = oracon.execSql(oraconn, sqlstmt)
dtct = 0
for dt in datatyprec:
dtct += 1
numsplitthread = int(numsplitterthreads / dtct)
datatyprec = oracon.execSql(oraconn, sqlstmt)
lconn = lconnect.connectMysql('ods', 'procuser', 'c@llhome',
'localhost')
lcurr = lconn.cursor()
for dtype in datatyprec:
sqlstmt = 'SELECT COUNT(1) FROM STATSPROCESSTRANSACT WHERE FILE_PROCESS_STATUS=0 and DATATYPE=\'' + dtype[
0] + '\''
lcurr.execute(sqlstmt)
reccount = lcurr.fetchall()
numInMySql = 0
for rec in reccount:
numInMySql = rec[0]
numsplitterthreads = numsplitthread - numInMySql
if numsplitterthreads > 0:
statslog.logwrite(fl, 'Processing for... :' + dtype[0])
sqlstmt = 'SELECT STATS_FILEID,STATS_FILE_NAME,STATS_FILE_PATH,DATATYPE FROM STATSPROCESSTRANSACT '
sqlstmt += 'WHERE FILE_PROCESS_STATUS=0 AND MACHINEID=' + str(
machineid)
sqlstmt += ' AND ROWNUM <=' + str(
numsplitterthreads
) + ' AND DATATYPE= \'' + dtype[0] + '\' ORDER BY STATS_FILEID'
filerec = oracon.execSql(oraconn, sqlstmt)
statslog.logwrite(fl, 'Inserting data into mysql...\n')
sqlstmt = 'INSERT INTO STATSPROCESSTRANSACT (STATS_FILEID,STATS_FILE_NAME,STATS_FILE_PATH,DATATYPE) VALUES '
sql = []
for flrec in filerec:
sql.append('(' + str(flrec[0]) + ',\'' + flrec[1] +
'\',\'' + flrec[2] + '\',\'' + flrec[3] + '\')')
sqlst = 'begin Update STATSPROCESSTRANSACT set FILE_PROCESS_STATUS=2 where STATS_FILEID=' + str(
flrec[0]) + '; commit; end;'
oracon.execSql(oraconn, sqlst)
if len(sql) > 0:
sqlstmt = sqlstmt + string.join(sql, ',')
lcurr.execute(sqlstmt)
lconn.close()
oraconn.close()
lconn = lconnect.connectMysql('ods', 'procuser', 'c@llhome',
'localhost')
lcurr = lconn.cursor()
statslog.logwrite(fl, 'Querying mysql...')
sqlstmt = 'SELECT STATS_FILEID,STATS_FILE_NAME,STATS_FILE_PATH,DATATYPE FROM STATSPROCESSTRANSACT where ifnull(FILE_PROCESS_STATUS,0)=0 LIMIT ' + str(
numsplitthread)
lcurr.execute(sqlstmt)
ctr = 0
statrow = lcurr.fetchall()
pythonRuns = commands.getoutput(
"ps -ef | grep python | grep -v grep | wc -l")
if int(pythonRuns) - 3 < numsplitthread:
residual = (numsplitthread - int(pythonRuns)) + 3
sqlstmt = 'SELECT STATS_FILEID,STATS_FILE_NAME,STATS_FILE_PATH,DATATYPE FROM STATSPROCESSTRANSACT where ifnull(FILE_PROCESS_STATUS,0)=0 LIMIT ' + str(
residual)
lcurr.execute(sqlstmt)
ctr = 0
statrow = lcurr.fetchall()
for statrec in statrow:
statslog.logwrite(fl, 'Splitting file...:' + statrec[1])
flpthlist = string.split(statrec[2], '/')
inservserial = flpthlist[len(flpthlist) - 2]
statslog.logwrite(fl, 'Starting process for ... ' + statrec[1])
p = Process(target=splitFile,
args=(
statrec[0],
statrec[1],
statrec[2],
statrec[3],
inservserial,
datafolder + '/splitter',
machineid,
))
p.daemon = True
p.start()
sqlstmt = 'Update STATSPROCESSTRANSACT set FILE_PROCESS_STATUS=2 where STATS_FILEID=' + str(
statrec[0])
lcurr.execute(sqlstmt)
ctr += 1
lcurr.close()
lconn.close()
os.remove('log/splitter.log')
except:
statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]))
def splitter():
oraconn = oracon.openconnect(constr)
lconn = lconnect.connectMysql('ods', 'procuser', 'c@llhome', 'localhost')
lcurr = lconn.cursor()
try:
sqlstmt = 'update STATSPROCESSTRANSACT set FILE_PROCESS_STATUS=0 where FILE_PROCESS_STATUS=2'
lcurr.execute(sqlstmt)
ipadd = hostlib.retHostIP()
sqlstmt = 'select machineid,numberofthreads,dataprocessing_folder,splitterloc,number_of_files_per_run,delay_seconds from statsprocessingmachine where ipaddress=\'' + ipadd + '\' and enable =1'
numthreadrec = oracon.execSql(oraconn, sqlstmt)
for numthread in numthreadrec:
numthreads = numthread[1]
machineid = numthread[0]
datafolder = numthread[3]
files_perrun = numthread[4]
delay_seconds = numthread[5]
nummysqltreads = int(numthreads)
numsplitterthreads = int(numthreads)
sqlstmt = 'select distinct datatype from stat_split_file_lookup'
datatyprec = oracon.execSql(oraconn, sqlstmt)
#dtct=0;
datatypes = []
for dt in datatyprec:
#dtct+=1;
datatypes.append(dt[0])
#numsplitthread=int(numsplitterthreads/dtct);
#datatyprec=oracon.execSql(oraconn,sqlstmt);
#lconn = lconnect.connectMysql('ods','procuser','c@llhome','localhost');
#lcurr=lconn.cursor();
sqlstmt = 'select count(1) from statsprocesstransact where machineid=' + str(
machineid
) + ' and FILE_PROCESS_STATUS=0 and datatype in (\'' + string.join(
datatypes, '\'' + ',' + '\'') + '\')'
datatypct = oracon.execSql(oraconn, sqlstmt)
for dtctrec in datatypct:
tot = dtctrec[0]
datatypelist = []
sqlstmt = 'select datatype,count(1) from statsprocesstransact where machineid=' + str(
machineid
) + ' and FILE_PROCESS_STATUS=0 and datatype in (\'' + string.join(
datatypes, '\'' + ',' + '\'') + '\') group by datatype'
datatyprec = oracon.execSql(oraconn, sqlstmt)
for dt in datatyprec:
recperdatatype = int(
((dt[1] * 1.0) / (tot * 1.0)) * numsplitterthreads)
datatypelist.append(dt[0] + ',' + str(recperdatatype))
for dtype in datatypelist:
datatype = string.split(dtype, ',')
sqlstmt = 'SELECT COUNT(1) FROM STATSPROCESSTRANSACT WHERE IFNULL(FILE_PROCESS_STATUS,0)=0 AND DATATYPE= \'' + datatype[
0] + '\''
lcurr.execute(sqlstmt)
reccount = lcurr.fetchall()
numInMySql = 0
for rec in reccount:
numInMySql = rec[0]
numsplitterthreads = int(datatype[1]) - int(numInMySql)
if numsplitterthreads > 0:
sqlstmt = 'SELECT STATS_FILEID,STATS_FILE_NAME,STATS_FILE_PATH,DATATYPE FROM STATSPROCESSTRANSACT '
sqlstmt += 'WHERE FILE_PROCESS_STATUS=0 AND MACHINEID=' + str(
machineid)
sqlstmt += ' AND ROWNUM <=' + str(
numsplitterthreads) + ' AND DATATYPE= \'' + datatype[
0] + '\' ORDER BY STATS_FILEID'
filerec = oracon.execSql(oraconn, sqlstmt)
sqlstmt = 'INSERT INTO STATSPROCESSTRANSACT (STATS_FILEID,STATS_FILE_NAME,STATS_FILE_PATH,DATATYPE) VALUES '
sql = []
for flrec in filerec:
sql.append('(' + str(flrec[0]) + ',\'' + flrec[1] +
'\',\'' + flrec[2] + '\',\'' + flrec[3] + '\')')
sqlst = 'begin Update STATSPROCESSTRANSACT set FILE_PROCESS_STATUS=2 where STATS_FILEID=' + str(
flrec[0]) + '; commit; end;'
oracon.execSql(oraconn, sqlst)
if len(sql) > 0:
sqlstmt = sqlstmt + string.join(sql, ',')
lcurr.execute(sqlstmt)
lconn.close()
oraconn.close()
lconn = lconnect.connectMysql('ods', 'procuser', 'c@llhome',
'localhost')
lcurr = lconn.cursor()
ctr = 0
pythonRuns = commands.getoutput(
"ps -ef | grep parser | grep -v grep | wc -l")
if int(pythonRuns) - 3 < nummysqltreads:
residual = (nummysqltreads - int(pythonRuns))
sqlstmt = 'SELECT COUNT(1) FROM STATSPROCESSTRANSACT where ifnull(FILE_PROCESS_STATUS,0)=0'
lcurr.execute(sqlstmt)
totalcurr = lcurr.fetchall()
for lcur in totalcurr:
total_recs = lcur[0]
sqlstmt = 'SELECT DATATYPE,COUNT(1) FROM STATSPROCESSTRANSACT where ifnull(FILE_PROCESS_STATUS,0)=0 GROUP BY DATATYPE'
lcurr.execute(sqlstmt)
datrec = lcurr.fetchall()
for lcur in datrec:
datatype = lcur[0]
numrecs = int(
round((lcur[1] * 1.0 / total_recs * 1.0) * residual * 1.0,
0))
sqlstmt = 'SELECT distinct STATS_FILEID,STATS_FILE_NAME,STATS_FILE_PATH,DATATYPE FROM STATSPROCESSTRANSACT where ifnull(FILE_PROCESS_STATUS,0)=0 and DATATYPE=\'' + datatype + '\' LIMIT ' + str(
numrecs)
lcurr.execute(sqlstmt)
ctr = 0
statrow = lcurr.fetchall()
for statrec in statrow:
flpthlist = string.split(statrec[2], '/')
inservserial = flpthlist[len(flpthlist) - 2]
p = Process(target=splitFile,
args=(
statrec[0],
statrec[1],
statrec[2],
statrec[3],
inservserial,
datafolder + '/splitter',
machineid,
))
p.daemon = True
p.start()
while ctr > files_perrun:
time.sleep(delay_seconds)
sqlstmt = 'SELECT COUNT(1) FROM STATSPROCESSTRANSACT WHERE FILE_PROCESS_STATUS=2'
lcurr.execute(sqlstmt)
rowRec = lcurr.fetchall()
for rec in rowRec:
ctr = rec[0]
sqlstmt = 'Update STATSPROCESSTRANSACT set FILE_PROCESS_STATUS=2 where STATS_FILEID=' + str(
statrec[0]) + ' and ifnull(FILE_PROCESS_STATUS,0)=0'
lcurr.execute(sqlstmt)
ctr += 1
lcurr.close()
lconn.close()
#os.remove('log/splitter.log');
except:
fl = statslog.logcreate('log/splitter.log')
statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]))
+ str(myRec[0]))
sql += sqlstmt
recount += 1
if recount >= 1000:
recount = 0
sql = 'begin\n' + string.join(
sqlArr, '\n') + '\n' + 'commit; end;'
oracon.execSql(procConn, sql)
totrec += recount
recount = 0
sqlArr = []
if len(sqlArr) >= 0:
sql = 'begin\n' + string.join(
sqlArr, '\n') + '\n' + 'commit; end;'
totrec += recount
oracon.execSql(procConn, sql)
recount = 0
for Mrec in MysqlArr:
lcurr.execute(Mrec)
numrec += 100000
lcurr.execute(
'delete from STATS_SPLIT_FILES where FILE_PROCESS_STATUS=1')
procConn.close()
time.sleep(900)
except Exception:
fl = statslog.logcreate("log/dataCopy.log")
statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]))
def nfsShareStatus():
try:
newdiskdt = []
newnodedt = []
diskArr = []
nodeArr = []
diskdt = subprocess.check_output("df -h|grep share",
shell=True).split('\n')
nodedt = subprocess.check_output("df -i|grep share",
shell=True).split('\n')
for dt in diskdt:
if len(dt) > 4:
strarr = dt.split(' ')
#print "String Array is:" +str(strarr);
newdt = []
for st in strarr:
if st != '':
newdt.append(st)
newdiskdt.append(string.join(newdt, ' '))
for nt in nodedt:
if len(nt) > 4:
strarr = nt.split(' ')
#print "String Array is:" +str(strarr);
newndt = []
for st in strarr:
if st != '':
newndt.append(st)
newnodedt.append(string.join(newndt, ' '))
for line in newdiskdt:
mntpoints = line.split(' ')
if len(mntpoints) > 4:
mntpoint = mntpoints[4].replace('/share/', '')
diskutil = mntpoints[3].strip('%')
groupid = time.strftime('%Y%m%d%H%M')
sqlstmt = ' insert /*+ append +*/ into omi_sharest_disk_status (MOUNTNAME, STATUS_POST_TIME, DISKSPACE_UTILIZATION, SNAPSHOT_GROUPID) values '
sqlstmt += '(\'' + str(mntpoint) + '\',sysdate,\'' + str(
diskutil) + '\',\'' + str(groupid) + '\');'
diskArr.append(sqlstmt)
for line in newnodedt:
mntpoints = line.split(' ')
if len(mntpoints) > 4:
mntpoint = mntpoints[4].replace('/share/', '')
inodeutil = mntpoints[3].strip('%')
groupid = time.strftime('%Y%m%d%H%M')
sqlstmt = ' insert /*+ append +*/ into omi_sharest_inode_status (MOUNTNAME, STATUS_POST_TIME, INODE_UTILIZATION, SNAPSHOT_GROUPID) values '
sqlstmt += '(\'' + str(mntpoint) + '\',sysdate,\'' + str(
inodeutil) + '\',\'' + str(groupid) + '\');'
nodeArr.append(sqlstmt)
oraconn = oracon.openconnect(constr)
sqld = 'begin\n' + string.join(diskArr, '\n') + '\n' + 'commit; end;'
oracon.execSql(oraconn, sqld)
sqln = 'begin\n' + string.join(nodeArr, '\n') + '\n' + 'commit; end;'
oracon.execSql(oraconn, sqln)
oraconn.close()
except:
fl = statslog.logcreate('log/sharestutil.log')
statslog.logwrite(fl, 'Error reported: ' + str(sys.exc_info()[1]))
def updateMetadata():
try:
lsconn = lconnect.connectMysql('ods', 'procuser', 'c@llhome',
'localhost')
lscurr = lsconn.cursor()
oraconn = oracon.openconnect(constr)
sqlstmt = 'SELECT STATSID, STATS_STRUCTURE_ID, STATSNAME, STATS_VERSION, STATS_FIRST_ROW_VERSION, STATS_SINGLE_ROW, STATS_END_OF_ROW FROM STATSSTRUCTURE ORDER BY 1'
statrec = oracon.execSql(oraconn, sqlstmt)
sqlstmt = 'SELECT DATATYPE, SPLIT_FILE_TYPE, SPLIT_FILE_SEARCH_TAG, SPLIT_FILE_SKIP_LINES, SPLIT_FILE_END_TAG, SPLIT_FILE_LINE_SEPERATOR, STATSID FROM STAT_SPLIT_FILE_LOOKUP'
splitrec = oracon.execSql(oraconn, sqlstmt)
sqlstmt = 'SELECT STATS_STRUCTURE_ID, PROCESS_SEQUENCE, FUNCTIONID FROM PROCESSLOGIC'
processrec = oracon.execSql(oraconn, sqlstmt)
sqlstmt = 'SELECT FUNCTIONID, FUNCTIONNAME, FUNCTIONPARAMETERS, FUNCTIONDESC FROM PROCESSFUNCTION'
profunc = oracon.execSql(oraconn, sqlstmt)
for rec in statrec:
sqlstmt = 'SELECT COUNT(1) FROM STATSSTRUCTURE WHERE STATSID=' + str(
rec[0]) + ' and STATS_STRUCTURE_ID=' + str(rec[1])
lscurr.execute(sqlstmt)
recData = lscurr.fetchall()
recCount = 0
for recdat in recData:
recCount = recdat[0]
if recCount == 0:
if not rec[6]:
end_row = ''
else:
end_row = rec[6]
sqlstmt = 'INSERT INTO STATSSTRUCTURE (STATSID, STATS_STRUCTURE_ID, STATSNAME, STATS_VERSION, STATS_FIRST_ROW_VERSION, STATS_SINGLE_ROW, STATS_END_OF_ROW) VALUES '
sqlstmt += ' (' + str(rec[0]) + ',' + str(
rec[1]
) + ',\'' + rec[2] + '\',\'' + rec[3] + '\',\'' + rec[
4] + '\',\'' + rec[5] + '\',\'' + end_row + '\')'
lscurr.execute(sqlstmt)
for rec in splitrec:
sqlstmt = 'SELECT COUNT(1) FROM STAT_SPLIT_FILE_LOOKUP WHERE DATATYPE=\'' + str(
rec[0]) + '\' and SPLIT_FILE_TYPE=\'' + rec[1] + '\''
lscurr.execute(sqlstmt)
recData = lscurr.fetchall()
recCount = 0
for recdat in recData:
recCount = recdat[0]
if recCount == 0:
if not rec[5]:
line_sep = ''
else:
line_sep = rec[5]
sqlstmt = 'INSERT INTO STAT_SPLIT_FILE_LOOKUP (DATATYPE, SPLIT_FILE_TYPE, SPLIT_FILE_SEARCH_TAG, SPLIT_FILE_SKIP_LINES, SPLIT_FILE_END_TAG, SPLIT_FILE_LINE_SEPERATOR, STATSID) '
sqlstmt += 'VALUES (\'' + str(rec[0]) + '\',\'' + str(
rec[1]) + '\',\'' + rec[2] + '\',' + str(
rec[3]
) + ',\'' + rec[4] + '\',\'' + line_sep + '\',' + str(
rec[6]) + ')'
lscurr.execute(sqlstmt)
for rec in processrec:
sqlstmt = 'SELECT COUNT(1) FROM PROCESSLOGIC WHERE STATS_STRUCTURE_ID=' + str(
rec[0]) + ' and PROCESS_SEQUENCE=' + str(
rec[1]) + ' and FUNCTIONID=' + str(rec[2])
#sqlstmt='SELECT COUNT(1) FROM PROCESSLOGIC WHERE STATS_STRUCTURE_ID='+str(rec[0]);
lscurr.execute(sqlstmt)
recData = lscurr.fetchall()
recCount = 0
for recdat in recData:
recCount = recdat[0]
if recCount == 0:
sqlstmt = 'INSERT INTO PROCESSLOGIC (STATS_STRUCTURE_ID, PROCESS_SEQUENCE, FUNCTIONID) '
sqlstmt += 'VALUES (' + str(rec[0]) + ',' + str(
rec[1]) + ',' + str(rec[2]) + ')'
lscurr.execute(sqlstmt)
#else:
##fix logic if needed
# sqlstmt='select STATS_STRUCTURE_ID, PROCESS_SEQUENCE, FUNCTIONID FROM PROCESSLOGIC WHERE STATS_STRUCTURE_ID='+str(rec[0])+' and PROCESS_SEQUENCE='+str(rec[1]);
# lscurr.execute(sqlstmt);
# dat=lscurr.fetchall();
# for dt in dat:
# if dt[2] != rec[2]:
# sqlstmt='update PROCESSLOGIC SET FUNCTIONID='+str(rec[2])+' WHERE STATS_STRUCTURE_ID='+str(rec[0])+' and PROCESS_SEQUENCE='+str(rec[1]);
# lscurr.execute(sqlstmt);
sqlstmt = 'SELECT STATS_STRUCTURE_ID, PROCESS_SEQUENCE, FUNCTIONID FROM PROCESSLOGIC'
lscurr.execute(sqlstmt)
mySqlDat = lscurr.fetchall()
for myRec in mySqlDat:
sqlstmt = 'SELECT COUNT(1) FROM PROCESSLOGIC where STATS_STRUCTURE_ID=' + str(
myRec[0]) + ' AND PROCESS_SEQUENCE=' + str(
myRec[1]) + ' AND FUNCTIONID=' + str(myRec[2])
processrec = oracon.execSql(oraconn, sqlstmt)
for recdat in processrec:
recCount = recdat[0]
if recCount == 0:
sqlstmt = 'delete from PROCESSLOGIC where STATS_STRUCTURE_ID=' + str(
myRec[0]) + ' and PROCESS_SEQUENCE =' + str(
myRec[1]) + ' and FUNCTIONID =' + str(myRec[2])
lscurr.execute(sqlstmt)
for rec in profunc:
sqlstmt = 'SELECT COUNT(1) FROM PROCESSFUNCTION WHERE FUNCTIONID=' + str(
rec[0]) + ' and FUNCTIONNAME=\'' + str(rec[1]) + '\''
lscurr.execute(sqlstmt)
recData = lscurr.fetchall()
recCount = 0
for recdat in recData:
recCount = recdat[0]
if recCount == 0:
sqlstmt = 'INSERT INTO PROCESSFUNCTION (FUNCTIONID, FUNCTIONNAME, FUNCTIONPARAMETERS, FUNCTIONDESC) '
sqlstmt += 'VALUES (' + str(rec[0]) + ',\'' + str(
rec[1]) + '\',\'' + str(rec[2]) + '\',\'' + str(
rec[3]) + '\')'
lscurr.execute(sqlstmt)
lscurr.close()
lsconn.close()
oraconn.close()
except:
fl = statslog.logcreate('log/copymeta.log')
statslog.logwrite(fl, 'Error reported: ' + str(sys.exc_info()[1]))
def nfsUtildata():
try:
newdiskdt = []
newnodedt = []
diskArr = []
nodeArr = []
diskdt = subprocess.check_output("df -h|grep ods",
shell=True).split('\n')
nodedt = subprocess.check_output("df -i|grep ods",
shell=True).split('\n')
for dt in diskdt:
strarr = dt.split(' ')
#print "String Array is:" +str(strarr);
newdt = []
for st in strarr:
if st != '':
newdt.append(st)
newdiskdt.append(string.join(newdt, ' '))
for nt in nodedt:
strarr = nt.split(' ')
#print "String Array is:" +str(strarr);
newndt = []
for st in strarr:
if st != '':
newndt.append(st)
newnodedt.append(string.join(newndt, ' '))
for line in newdiskdt:
mntpoints = line.split(' ')
if len(mntpoints) > 5:
mntserver = mntpoints[0]
mntpoint = mntpoints[5].strip('/')
diskutil = mntpoints[4].strip('%')
#dataloc=mntpoint
#istring=("df -i|grep %s|awk '{print $5}'"%mntpoint);
#inodeutil=commands.getoutput(istring);
#sqlstmt=' insert /*+ append +*/ into omi_nfs_status (MOUNTNAME, MOUNTSERVER,STATUS_POST_TIME, DISKSPACE_UTILIZATION,INODE_UTILIZATION) values ';
#sqlstmt+='(\''+str(mntpoint)+'\',\''+str(mntserver)+'\',sysdate,\''+str(diskutil)+'\',\''+str(inodeutil)+'\');';
sqlstmt = ' insert /*+ append +*/ into omi_nfsdisk_status (MOUNTNAME, MOUNTSERVER,STATUS_POST_TIME, DISKSPACE_UTILIZATION) values '
sqlstmt += '(\'' + str(mntpoint) + '\',\'' + str(
mntserver) + '\',sysdate,\'' + str(diskutil) + '\');'
diskArr.append(sqlstmt)
for line in newnodedt:
mntpoints = line.split(' ')
if len(mntpoints) > 5:
mntserver = mntpoints[0]
mntpoint = mntpoints[5].strip('/')
inodeutil = mntpoints[4].strip('%')
sqlstmt = ' insert /*+ append +*/ into omi_nfsinode_status (MOUNTNAME, MOUNTSERVER,STATUS_POST_TIME, INODE_UTILIZATION) values '
sqlstmt += '(\'' + str(mntpoint) + '\',\'' + str(
mntserver) + '\',sysdate,\'' + str(inodeutil) + '\');'
nodeArr.append(sqlstmt)
oraconn = oracon.openconnect(constr)
sqld = 'begin\n' + string.join(diskArr, '\n') + '\n' + 'commit; end;'
oracon.execSql(oraconn, sqld)
sqln = 'begin\n' + string.join(nodeArr, '\n') + '\n' + 'commit; end;'
oracon.execSql(oraconn, sqln)
oraconn.close()
except:
fl = statslog.logcreate('log/nfsutil.log')
statslog.logwrite(fl, 'Error reported: ' + str(sys.exc_info()[1]))
def processFile(STATS_SPLITFILE_ID, STATS_SPLITFILE_NAME, STATS_SPLITFILE_PATH,
STATS_SPLITFILE_TYPE, INSERVSERIAL, DATAFOLDER, MACHINEID):
try:
fl = statslog.logcreate("log/" + STATS_SPLITFILE_NAME)
lc = mysqlconn.connectMysql('ods', 'procuser', 'c@llhome', 'localhost')
curr = lc.cursor()
errlog = 'log/' + STATS_SPLITFILE_NAME + '.err'
if versionExist(STATS_SPLITFILE_TYPE) == 1:
version = prclib.getVersion(STATS_SPLITFILE_NAME,
STATS_SPLITFILE_PATH)
else:
version = '2.3.1'
sqlstmt = 'SELECT COUNT(1) FROM STATSSTRUCTURE WHERE STATS_VERSION=\'' + version + '\' and STATSNAME=\'' + STATS_SPLITFILE_TYPE + '\''
curr.execute(sqlstmt)
dat = curr.fetchall()
for datrec in dat:
count = datrec[0]
if count > 0:
sqlstmt = 'SELECT STATS_STRUCTURE_ID FROM STATSSTRUCTURE WHERE STATS_VERSION=\'' + version + '\' and STATSNAME=\'' + STATS_SPLITFILE_TYPE + '\''
curr.execute(sqlstmt)
verdat = curr.fetchall()
STATS_STRUCTURE_ID = 0
for ver in verdat:
STATS_STRUCTURE_ID = ver[0]
if STATS_STRUCTURE_ID > 0:
sqlstmt = 'SELECT PROCESS_SEQUENCE,FUNCTIONID,PARAMETER_VALUES FROM PROCESSLOGIC WHERE STATS_STRUCTURE_ID=' + str(
STATS_STRUCTURE_ID) + ' ORDER BY PROCESS_SEQUENCE'
curr.execute(sqlstmt)
prcdat = curr.fetchall()
linearray = prclib.readFile(STATS_SPLITFILE_NAME,
STATS_SPLITFILE_PATH)
for prc in prcdat:
sqlstmt = 'SELECT FUNCTIONNAME FROM PROCESSFUNCTION WHERE FUNCTIONID=' + str(
prc[1])
curr.execute(sqlstmt)
funcNamerec = curr.fetchall()
for fu in funcNamerec:
functionName = fu[0]
func = getattr(prclib, functionName)
linearray = func(linearray)
filepath = DATAFOLDER + '/' + str(INSERVSERIAL) + '/'
checkmakedir(filepath, errlog)
outputFl = open(filepath + STATS_SPLITFILE_NAME, 'w')
outputFl.write(linearray)
sqlstmt = 'insert into STATSOUTPUT(STATS_SPLITFILE_ID,STATS_OUTPUTFILE_NAME,STATS_OUTPUTFILE_PATH,STATS_FILE_CREATE_DATE,STATS_FILE_LOAD_STATUS,STATS_STRUCTURE_TYPE_ID) values '
sqlstmt += '(' + str(
STATS_SPLITFILE_ID
) + ',\'' + STATS_SPLITFILE_NAME + '\',\'' + filepath + '\',sysdate(),0,' + str(
STATS_STRUCTURE_ID) + ')'
curr.execute(sqlstmt)
sqlstmt = 'Update STATS_SPLIT_FILES set FILE_PROCESS_STATUS=1 where STATS_SPLITFILE_ID=' + str(
STATS_SPLITFILE_ID)
curr.execute(sqlstmt)
curr.close()
lc.close()
else:
sqlstmt = 'DELETE FROM STATS_SPLIT_FILES WHERE STATS_SPLITFILE_ID=' + str(
STATS_SPLITFILE_ID)
curr.execute(sqlstmt)
curr.close()
lc.close()
else:
fl = statslog.logcreate("log/" + STATS_SPLITFILE_NAME)
statslog.logwrite(
fl,
"Error reported: Version " + version + " not yet supported")
sqlstmt = 'Update STATS_SPLIT_FILES set FILE_PROCESS_STATUS=3 where STATS_SPLITFILE_ID=' + str(
STATS_SPLITFILE_ID)
lc = mysqlconn.connectMysql('ods', 'procuser', 'c@llhome',
'localhost')
curr = lc.cursor()
curr.execute(sqlstmt)
curr.close()
lc.close()
except:
fl = statslog.logcreate("log/" + STATS_SPLITFILE_NAME)
statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]))
sqlstmt = 'Update STATS_SPLIT_FILES set FILE_PROCESS_STATUS=3 where STATS_SPLITFILE_ID=' + str(
STATS_SPLITFILE_ID)
lc = mysqlconn.connectMysql('ods', 'procuser', 'c@llhome', 'localhost')
curr = lc.cursor()
curr.execute(sqlstmt)
curr.close()
lc.close()
def procExec():
try:
oraconn = oracon.openconnect(constr)
lconn = mysqlconn.connectMysql('ods', 'procuser', 'c@llhome',
'localhost')
lcurr = lconn.cursor()
ipadd = hostlib.retHostIP()
sqlstmt = 'select machineid,numberofthreads,processloc,number_of_files_per_run,delay_seconds from statsprocessingmachine where ipaddress=\'' + ipadd + '\' and enable =1'
numthreadrec = oracon.execSql(oraconn, sqlstmt)
for numthread in numthreadrec:
numthreads = numthread[1]
machineid = numthread[0]
datafolder = numthread[2]
files_per_run = numthread[3]
delay_seconds = numthread[4]
#statslog.logwrite(fl,str(numthreads)+':'+str(files_per_run)+':'+str(delay_seconds));
numthreads = int(numthreads)
sqlstmt = 'select distinct statsname from STATSSTRUCTURE a,PROCESSLOGIC b where a.STATS_STRUCTURE_ID = b.STATS_STRUCTURE_ID'
lcurr.execute(sqlstmt)
recc = lcurr.fetchall()
statList = []
for rec in recc:
statList.append('\'' + rec[0] + '\'')
statsStr = string.join(statList, ',')
sqlstmt = 'update STATS_SPLIT_FILES set FILE_PROCESS_STATUS=0 WHERE FILE_PROCESS_STATUS=2'
lcurr.execute(sqlstmt)
sqlstmt = 'select count(1) from STATS_SPLIT_FILES where ifnull(FILE_PROCESS_STATUS,0)=0 and STATS_SPLIT_FILE_TYPE in (' + statsStr + ')'
lcurr.execute(sqlstmt)
recc = lcurr.fetchall()
numRec = 0
for rec in recc:
numRec = rec[0]
numthreadstocopy = numthreads - numRec
#statslog.logwrite(fl,str(numthreadstocopy));
if numthreadstocopy > 0:
sqlstmt = ''
rows = 1000
totrows = rows
while totrows <= numthreadstocopy:
sqlstmt = 'select STATS_FILEID,STATS_SPLITFILE_ID,STATS_SPLITFILE_NAME,STATS_SPLITFILE_PATH,STATSID,STATS_SPLIT_FILE_TYPE from STATS_SPLIT_FILES '
sqlstmt += ' where FILE_PROCESS_STATUS=0 and MACHINEID=' + str(
machineid
) + ' and ROWNUM <=' + str(
rows
) + ' and STATS_SPLIT_FILE_TYPE in (' + statsStr + ') order by STATS_SPLITFILE_ID desc'
#statslog.logwrite(fl,sqlstmt);
FileRec = oracon.execSql(oraconn, sqlstmt)
sqlstmt = 'INSERT INTO STATS_SPLIT_FILES (STATS_FILEID,STATS_SPLITFILE_ID,STATS_SPLITFILE_NAME,STATS_SPLITFILE_PATH,STATSID,STATS_SPLIT_FILE_TYPE) VALUES '
sql = []
for flrec in FileRec:
sql.append('(' + str(flrec[0]) + ',' + str(flrec[1]) +
',\'' + flrec[2] + '\',\'' + flrec[3] + '\',' +
str(flrec[4]) + ',\'' + flrec[5] + '\')')
sqlst = 'begin Update STATS_SPLIT_FILES set FILE_PROCESS_STATUS=2 where STATS_SPLITFILE_ID=' + str(
flrec[1]) + '; commit; end;'
oracon.execSql(oraconn, sqlst)
if len(sql) > 0:
sqlstmt += string.join(sql, ',')
lcurr.execute(sqlstmt)
totrows = totrows + rows
pythonRuns = commands.getoutput(
"ps -ef | grep parsetabber | grep -v grep | wc -l")
while int(pythonRuns) - 3 <= files_per_run:
sqlstmt = 'SELECT a.STATS_SPLITFILE_ID,a.STATS_SPLITFILE_NAME,a.STATS_SPLITFILE_PATH,STATS_SPLIT_FILE_TYPE from STATS_SPLIT_FILES a where ifnull(FILE_PROCESS_STATUS,0)=0 and STATS_SPLIT_FILE_TYPE in (' + statsStr + ') LIMIT ' + str(
files_per_run - int(pythonRuns))
#statslog.logwrite(fl,sqlstmt);
if (files_per_run - int(pythonRuns)) > 0:
ctr = 0
lcurr = lconn.cursor()
lcurr.execute(sqlstmt)
filelist = lcurr.fetchall()
for flrec in filelist:
flpthlist = string.split(flrec[2], '/')
inservserial = flpthlist[len(flpthlist) - 2]
p = Process(target=processFile,
args=(
flrec[0],
flrec[1],
flrec[2],
flrec[3],
inservserial,
datafolder,
machineid,
))
p.daemon = True
p.start()
ctr += 1
sqlstmt = 'Update STATS_SPLIT_FILES set FILE_PROCESS_STATUS=2 where STATS_SPLITFILE_ID=' + str(
flrec[0])
lcurr.execute(sqlstmt)
lcurr.close()
if ctr == 0:
break
pythonRuns = commands.getoutput(
"ps -ef | grep parsetabber | grep -v grep | wc -l")
while int(pythonRuns) > files_per_run:
time.sleep(delay_seconds)
pythonRuns = commands.getoutput(
"ps -ef | grep parsetabber | grep -v grep | wc -l")
pythonRuns = commands.getoutput(
"ps -ef | grep parsetabber | grep -v grep | wc -l")
lconn.close()
except:
fl = statslog.logcreate("log/parser.log")
statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]))
if (purpose == 'datamartextract' or purpose == 'sync'):
st = os.statvfs('/')
free = st.f_bavail * st.f_frsize
total = st.f_blocks * st.f_frsize
used = (st.f_blocks - st.f_bfree) * st.f_frsize
spaceused = round((used * 1.0 / total * 1.0) * 100, 2)
inodespace = 0
dataloc = str(hstname)
oraconn = oracon.openconnect(constr)
sqlst = 'begin insert into OMI_MACHINE_STATUS (MACHINEID,MACHINENAME,IPADDRESS,STATUS_POST_TIME,PURPOSE,NUM_PYTHON_THREADS,DATALOCATION,DATACOPY_STATUS,'
sqlst += 'PARSER_STATUS,COPYMETA_STATUS,DATAMART_STATUS,TAGGER_STATUS,PROCESSED,TOBEPROCESSED,TOBECOPIED,FAILEDTOPROCESS,DISKSPACE_UTILIZATION,INODE_UTILIZATION) '
sqlst += 'values (\'' + str(macid) + '\',\'' + str(
hstname) + '\',\'' + str(hstip) + '\',sysdate,\'' + mp[
0] + '\',\'' + str(pythonRuns) + '\',\'' + str(
dataloc) + '\',\'' + str(pdatacopy) + '\','
sqlst += '\'' + str(pparser) + '\',\'' + str(pcpmeta) + '\',\'' + str(
pdatamart) + '\',\'' + str(ptagger) + '\',' + str(
processed) + ',' + str(tobeProcessed) + ',' + str(
recCount) + ','
sqlst += '' + str(errored) + ',\'' + str(spaceused) + '%\',\'' + str(
inodespace) + '\'); commit; end;'
oracon.execSql(oraconn, sqlst)
#print "closing connection";
oraconn.close()
except Exception:
fl = statslog.logcreate("log/machineStatus.log")
statslog.logwrite(fl, "Error reported: " + str(sys.exc_info()[1]))
time.sleep(900)
