def check_dir(dp, dl, suf=''):
flst = fu.getDirFiles (dp,suf)
for f in flst:
fn = '%s/%s' % (dp,f)
r,b = treadCSV(fn)
log.info('Processed file %s rows %d bad %d' % (f,len(r),len(b)))
if len(b) > 0:
fu.createFile ('%s.bad' % fn, b)
log.error('Created file %s.bad' % fn)
def _procOpsReg(self):
concatFls = (
"Transcripts_OpsRegion.txt",
"Transcripts_OpsRegion_Canada.txt",
"Transcripts_OpsRegion_Central.txt",
"Transcripts_OpsRegion_Northeast.txt",
"Transcripts_OpsRegion_Southeast.txt",
"Transcripts_OpsRegion_West.txt",
)
concatFlsFP = []
for fn in concatFls:
concatFlsFP.append("%s/%s" % (self.ib.workDir, fn))
self.log.debug("concatFlsFP = ", concatFlsFP)
# Remove Concatenated file:
rc = fu.delFile(concatFlsFP[0])
self.log.debug("Removing %s\trc=%s" % (concatFlsFP[0], rc))
self.log.debug("Creating HDR file %s" % concatFlsFP[0])
rc = fu.createFile(concatFlsFP[0], self.ib.opsRegHdrRow)
if rc != 0:
self.log.error("Could not create file %s" % (concatFlsFP[0]))
return 2
# Concatenate the files
rc = fu.concatFile(concatFlsFP)
self.log.info("concatFile rc = %s" % rc)
return rc
def _crtSchemaScript(args,schema_script,table_name):
# Create NZ script to replace existing non print ascii
sql_c= bld_nzsql_cmd(args,table_name)
log.debug("sql_c = %s" % sql_c)
rc = fu.createFile(schema_script,sql_c)
if rc == 0 : os.chmod(schema_script , 0777)
return rc
def createCSV(dm):
data = []
dks = sorted(dm.keys())
for k in dks:
data.append(' %s,\tLabel,\tBytes,\tTimestamp,\tChecksum\n' % k)
lst = dm[k]
for m in lst:
data.append('%s\n' % m)
return (fu.createFile(fn,data))
def procIncFiles(self):
if len(self.workFiles) < 1 :
self.log.error('No files to process')
return 1
self.workFiles.sort()
rc = 0
# Files in the working directory:
i = 1
self.log.debug('Will Process a total of %d file(s) ' % len(self.workFiles))
for fnp in self.workFiles:
self.log.info('\nProcessing File (%d) => %s ' % (i,fnp))
fn = fu.getFileBaseName(fnp)
if self.verifyCSVFlg is True:
r,b = fu.readCSV(fnp, FLDxROW, SEP)
if len(b) > 0 :
fbad = '%s/%s.bad' % (self.ib.badDir,fn)
rc = fu.createFile(fbad , b)
self.log.error("No of %d bad row(s) on %s" % (len(b),fnp))
self.log.error("Creating file %s rc = %s" % (fbad,rc))
self.log.debug("Bad rows = , ", b)
return 5
if len(r) == 0 :
self.log.error("No rows to process on file %s" % fnp)
return 6
t = '%s/%s' % (self.ib.workDir , self.ib.srcFile[0])
rc = fu.moveFile(fnp, t)
self.log.info('Renaming %s to %s rc = %s' % (fnp,t,rc))
if rc != 0 :
self.log.error('Could not move File %s to %s' % (fnp,t))
return 7
# Invoke workflow.
rc = self._wkfCodeOvrd()
if rc != 0 :
self.log.error('Running %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
if self.exitOnError:
self.log.debug('ExitOnError is TRUE rc = %s' % (rc))
return rc
else :
self.log.info('Ran %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
r = fu.delFile(t)
self.log.debug('Deleting File %s rc = %s' % (t,r))
i+=1
return rc
def procXMLFiles(self):
rc = 0; r = False ; x = [] ; cdrRes = [] # List that contains all the responses.
if len(self.cdrFilestoProc) < 1 :
self.log.error('No incoming CDR files to process : cdrFilestoProc')
return 1
pxmlCDR = pxml.ProcXMLCDR(self.log)
self.log.debug("Files to process %s" % ''.join(self.cdrFilestoProc))
for fn in self.cdrFilestoProc:
self.log.debug("processing %s" % fn)
r = pxmlCDR.isValidXML(fn)
if r == False: rc+=1
else:
x = pxmlCDR.parseCDR(fn)
ftn = fu.getFileBaseName(fn)
if x is None or len(x) != 3:
self.log.error('Error Parsing %s ' % fn)
tgt = '%s/%s.%s' % (self.ib.badDir,ftn,self.ts)
else:
self.log.debug('fn=%s rfId=%s rCode=%s desc=%s' % (fn,x[0],x[1],x[2]))
r = su.toInt(x[1])
if r != 0 :
self.log.error('fn=%s rfId=%s rCode=%s desc=%s' % (fn,x[0],x[1],x[2]))
tgt = '%s/%s.%s' % (self.ib.badDir,ftn,self.ts)
rc+=1
else :
cdrRes.append('%s %s %s\n' % (x[0],self.ib.FS,x[1]))
tgt = '%s/%s.%s' % (self.ib.archDir,ftn,self.ts)
# Move files
r = fu.moveFile(fn, tgt)
if r == 0 : self.log.info( 'mv %s to %s' % (fn,tgt))
else : self.log.error('mv %s to %s r = %s ' % (fn,tgt,r))
self.log.debug("cdrRes No of elements %d " % len(cdrRes))
if len(cdrRes) < 1:
self.lo.error('cdrRes Did not find any valid element !')
return 2
fn = '%s/CDR_%s.txt' % (self.ib.cdrOutDir, self.ts)
r = fu.createFile(fn,cdrRes)
if r == 0: self.log.info('Created file %s with CDR responses' % fn)
else :
self.log.error('Could not create file %s with CDR responses' % fn)
rc=1
return rc
def splitMainCntFile(self):
#Get Header.
fnp = '%s/%s' % (self.ib.workDir,self.srcFile)
s = fu.readFile(fnp)
hdr = su.remSpacefrStr(s)
if len(hdr) != 38 :
self.log.error('Invalid len= %d for %s. Need to be 38\nhdr=%s' % (len(hdr),fnp,hdr))
return 1
self.log.debug('hdr = %s' % hdr)
# Get counts
po870D30 = su.toInt(hdr[0:8])
po875D30 = su.toInt(hdr[8:16])
po880D30 = su.toInt(hdr[16:24])
po225D15 = su.toInt(hdr[24:32])
rdate = hdr[32:]
self.log.debug('po870D30=%s po875D30=%s po880D30=%s po225D15=%s rdate=%s' % (po870D30, po875D30,po880D30,po225D15,rdate))
# Sanity check.
if (po870D30 is None or po875D30 is None or po880D30 is None or po225D15 is None or su.isValidDate(rdate,'%m%d%y') == False) :
self.log.error('po870D30=%s po875D30=%s po880D30=%s po225D15=%s rdate=%s' % (po870D30, po875D30,po880D30,po225D15,rdate))
return 2
# Create count files.
rc = 0
r = fu.createFile('%s/po870d30comp.cnt' % self.ib.workDir, '%08d%s' % (po870D30,rdate)) ; rc+= r
self.log.info (' Creating file %s/po870d30comp.cnt rc = %s' % (self.ib.workDir, r))
r = fu.createFile('%s/po875D30.cnt' % self.ib.workDir, '%08d%s' % (po875D30,rdate)) ; rc+= r
self.log.info (' Creating file %s/po875d30.cnt rc = %s' % (self.ib.workDir, r))
r = fu.createFile('%s/po880D30.cnt' % self.ib.workDir, '%08d%s' % (po880D30,rdate)) ; rc+= r
self.log.info (' Creating file %s/po880d30.cnt rc = %s' % (self.ib.workDir, r))
r = fu.createFile('%s/po225D15.cnt' % self.ib.workDir, '%08d%s' % (po225D15,rdate)) ; rc+= r
self.log.info (' Creating file %s/po225d15.cnt rc = %s' % (self.ib.workDir, r))
return rc
def getLock(self):
# Check if a valid process lock exists !
rc = self._chkIfValidLock()
if rc is True:
self.log.warn('Cannot Create %s. Process is currently running!!' % self.lckFn)
return False
rc = fu.createFile(self.lckFn,str(self.pid))
if rc == 0 :
self.log.info('Created lock %s for PID = %s' % (self.lckFn,self.pid))
self.lock = True
return True
else :
self.log.error('Could not create Lock %s for PID = %s' % (self.lckFn,self.pid))
return False
def sendTotForApproval(self):
pegBill = 'C:\\apps\\pegbillDet.csv'
print "In send counts(*)"
rc = self._getPegBillTot()
if rc != 0 : return rc
rprDet = self._getNZDS(da.selPegBillMthDeanDetQry)
if rprDet is not None and len(rprDet) > 0 : detDS = self._procDetDS(rprDet)
else :
self.log.error('Querying RO returned 0')
return 1
rv = fu.createFile(pegBill, detDS)
if rv != 0 : self.log.error('Could not create file %s' % pegBill)
else : self.log.info( 'Created file %s' % pegBill)
return 0
def getLock(self):
# Check if a valid process lock exists !
rc = self._chkIfValidLock()
if rc is True:
self.log.warn('Cannot Create %s. Process is currently running!!' %
self.lckFn)
return False
rc = fu.createFile(self.lckFn, str(self.pid))
if rc == 0:
self.log.info('Created lock %s for PID = %s' %
(self.lckFn, self.pid))
self.lock = True
return True
else:
self.log.error('Could not create Lock %s for PID = %s' %
(self.lckFn, self.pid))
return False
def getCanCPI(self):
lf = '%s/%s/cancpi%s.html' % (self.ib.shareDir,self.landDir, self.ts)
tf = '%s/%s/%s' % (self.ib.shareDir,self.tgtDir,self.ib.cancpifile)
self.log.debug('landing file = %s target file = %s' % (lf,tf))
rc = fu.get_url_data(url_can, lf)
if rc == 'SUCCESS' :
self.log.info('Created temp file %s ' % lf)
d = px.parseCanCPI(lf)
if len(d) < 1 :
self.log.warn('No data was retrieved for Canada')
rc = 2
else:
self.log.debug('data len %s from %s' % (len(d),lf) )
rc = fu.createFile(tf, d)
self.log.debug('Create file %s rc =%s ' % (tf,rc))
else :
self.log.error('%s ' % rc)
rc = 1
return rc
def _insNZLoad(self,val,seqid):
ldata = []
for s,u,p,d in val :
ldata.append("%s,%s,%s,%s\n" % (s,u,p,d ))
data = ''.join(ldata)
# Create CSV file
fn = '%s/%s.csv' % (self.ib.dataDir,seqid)
fl = '%s/tis_fb_carr_pmpt_imprt_dtl_%s.%s.nzlog' % (self.ib.dataDir,seqid,self.ib.db)
bf = '%s/tis_fb_carr_pmpt_imprt_dtl_%s.%s.nzbad' % (self.ib.dataDir,seqid,self.ib.db)
rc = fu.createFile(fn, data)
self.log.debug('File Creation %s/%s.csv rc = %s' % (d,seqid,rc))
if rc != 0 :
self.log.error('File Creation %s/%s.csv rc = %s' % (d,seqid,rc))
return rc
# Invoke nzload command
nz_cmd = "%s/nzload -host %s -u %s -pw %s -db %s -t tis_fb_carr_pmpt_imprt_dtl -delim '%s' -df %s -lf %s -bf %s" % (self.ib.nzbin,
self.ib.dbserver,
self.ib.user,
self.ib.pwd,
self.ib.db,
self.ib.delim, fn,fl,bf)
self.log.debug('nz_cmd = %s' % nz_cmd)
rc,rmsg = proc.runSync(nz_cmd,self.log)
if rc != 0 :
self.log.error('cmd %s rc = %s -> msg= %s' % (nz_cmd,rc,rmsg))
return rc
# Delete csv file (based on option)
if self.delCSVFileFlg is True:
r = fu.delFile(fn)
self.log.debug('Deleting %s rc = %s ' % (fn,r))
if r != 0 : self.log.error('Cannot Delete %s ' % fn )
return rc
def checkFileCols(self,flst,FLD_SEP,strp=' \t\n\r'):
rc = 0
for fnp in flst:
bfnm = fu.getFileBaseName(fnp)
colc = self.ib.FileColCnt.get(bfnm)
self.log.debug('fkey = %s colc= %s fp = %s ' %(bfnm,colc,fnp))
if colc is not None:
x,b = fu.readCSV(fnp,colc,FLD_SEP,strp)
if len(b) < 1 :
self.log.debug('Columns number [%d] match on file %s ' % (colc,fnp))
else :
rc += 1
badf = '%s/%s%s.bad' % (self.ib.badDir,bfnm,su.getTimeSTamp())
f = fu.createFile(badf,b)
if f == 0 : self.log.error('Columns number (%d) did not match on %s.\nPlease see Bad file %s .' % (colc,fnp,badf))
else :
self.log.error('Columns number (%d) did not match on %s.\n.COULD NOT create file %s .' % (colc,fnp,badf))
self.log.error('BAD Rows===========\n', b)
else:
rc += 1
self.log.error("Did not find Column Count for %s. Unable to verify Column Numbers !" % bfnm)
return rc
def procIncFiles(self):
if len(self.workFiles) < 1:
self.log.error("No files to process")
return 1
ctlFile = "%s/%s.ctl" % (self.ib.ctlDir, self.appName)
self.workFiles.sort()
rc = 0
# Files in the working directory:
i = 1
self.log.debug("Will Process a total of %d file(s) " % len(self.workFiles))
for fnp in self.workFiles:
self.log.info("\nProcessing File (%d) => %s " % (i, fnp))
# Get date run from 1st filename
fn = fu.getFileBaseName(fnp)
cur_dayr = self._getDateRunStr(fn)
if cur_dayr is None:
self.log.error("No Date String %s " % cur_dayr)
return 1
fmt = "%Y%m"
date = "%s%s" % (cur_dayr[0:4], cur_dayr[4:6])
rc = su.isValidDate(date, fmt)
if rc is False:
self.log.error("Invalid Date %s on file %s " % (date, fn))
return 2
self.fileDate = date
self.log.debug("self.fileDate = %s" % (self.fileDate))
if self.checkNextRunFlg is True:
# Get Previous Run Info. File should contain one line only : YYYYMM from storage.
prev_dayr = self._getCtlFile()
if prev_dayr is None:
return 3
pd, pr = self._getMonth(prev_dayr, DP_LEN)
if pd is None:
return 4
# rc = self._chkNextRun(cur_dayr,prev_dayr,pd,pr,RUN_PER_MTH)
rc = psch.getNextRunDate(pd, cur_dayr, "Mthly", self.log)
if rc != 0:
self.log.error("self._chkNextRun rc = %s" % rc)
return rc
if self.verifyCSVFlg is True:
r, b = fu.readCSV(fnp, FLDxROW, SEP)
if len(b) > 0:
fbad = "%s/%s.bad" % (self.ib.badDir, fn)
rc = fu.createFile(fbad, b)
self.log.error("No of %d bad row(s) on %s" % (len(b), fnp))
self.log.error("Creating file %s rc = %s" % (fbad, rc))
self.log.debug("Bad rows = , ", b)
return 5
if len(r) == 0:
self.log.error("No rows to process on file %s" % fnp)
return 6
t = "%s/%s" % (self.ib.workDir, self.ib.srcFile[0])
rc = fu.moveFile(fnp, t)
self.log.info("Renaming %s to %s rc = %s" % (fnp, t, rc))
if rc != 0:
self.log.error("Could not move File %s to %s" % (fnp, t))
return 7
# Invoke workflow.
rc = self._wkfIMSSftyLoc()
if rc != 0:
self.log.error("Running %s.%s rc = %s" % (self.ib.fld, self.ib.wkf, rc))
if self.exitOnError:
self.log.debug("ExitOnError is TRUE rc = %s" % (rc))
return rc
else:
self.log.info("Ran %s.%s rc = %s" % (self.ib.fld, self.ib.wkf, rc))
# Loading Staging Succeeded. Update the control file.
rc = fu.updFile(ctlFile, cur_dayr)
if rc == 0:
if self.checkNextRunFlg:
self.log.info(
"Updated Cur Load Date from %s to %s , Control File %s" % (prev_dayr, cur_dayr, ctlFile)
)
else:
self.log.info("Overwriting Cur Load Date to %s , Control File %s" % (cur_dayr, ctlFile))
else:
self.log.error("Could not Update Load Date %s, Control File %s rc = %s" % (cur_dayr, ctlFile, rc))
return rc
r = fu.delFile(t)
self.log.debug("Deleting File %s rc = %s" % (t, r))
i += 1
return rc