def procHierSCOS(self):
# Check for file existence
src = '%s/%s/%s' % (self.ib.shareDir,self.outDir,self.ib.outFile)
r = fu.fileExists(src)
if r == False :
self.log.error('File %s does not exist' % src)
return 1
# Move File to output dir
dt = su.getTodayDtStr('%y%m%d')
self.outFile = 'SCS_Org_Hierarchy%s.csv' % dt
outFile = '%s/%s/%s/%s' % (self.ib.shareDir,self.outDir,self.ib.outDirname,self.outFile)
r = fu.moveFile(src,outFile)
if r != 0 :
self.log.error("Could not move %s to %s " % (src,outFile))
self.log.debug('Moved src = %s to self.outFile = %s RC = %s' % (src,outFile,r))
# Archive File .
arcfn = '%s/%s/%s/%s-%s' % (self.ib.shareDir,self.outDir,self.outArchDir,self.ts,self.outFile)
r = fu.copyFile(outFile,arcfn)
if r != 0 :
self.log.error("Could not copy %s to %s " % (outFile,arcfn))
self.log.debug('Copied src = %s to racfn = %s RC = %s' % (outFile,arcfn,r))
return 0
def _chkIfValidLock(self):
rc = True
# Check if there is a lock file.
if fu.fileExists(self.lckFn):
sPid = fu.readFile(self.lckFn)
pid = su.toInt(sPid)
self.log.info('Lock File %s EXISTS. sPid = %s , pid = %s' %
(self.lckFn, sPid, pid))
# Check if file has a valid PID (number)
if pid is None:
rc = fu.delFile(self.lckFn)
if rc == 0:
self.log.info('Removed File %s' % (self.lckFn))
else:
self.log.error('Could not removed File %s' % (self.lckFn))
return False
# If pid is a valid number, check if the process is running ...
rc = ps.isProcRunning(pid, self.log)
self.log.debug('isProcRunning returned %s' % (rc))
return rc
# No lock file exists.
else:
return False
def getStateFile(self):
fn = '%s/%s.state' % (self.ib.ctlDir, self.appName)
if fu.fileExists(fn) is False:
self.log.error('State File:%s does not exist' % fn)
return None
self.log.info('Reading State File :%s' % fn)
return fu.readFile(fn).strip()
def _getCtlFile(self):
fn = '%s/%s.ctl' % (self.ib.ctlDir, self.appName)
if fu.fileExists(fn) is False:
self.log.error('Control File:%s does not exist' % fn)
return None
self.log.info('Loading Control File:%s' % fn)
return fu.readFile(fn).strip()
def chkTrailer (self,fL,trl_date):
rc = 0
for fnp in fL:
if fu.fileExists(fnp) is False:
self.log.error('File %s does not exists ' % fnp)
continue
if fu.getFilseSize(fnp) == 0 :
self.log.error('File %s is Empty !' % fnp)
continue
r = self._checkTrailer(fnp,trl_date)
if r != 0 : rc = 1
return rc
def chkHeader(self,fL):
procFiles = []
for fnp in fL:
if fu.fileExists(fnp) is False:
self.log.error('File %s does not exists ' % fnp)
continue
if fu.getFilseSize(fnp) == 0 :
self.log.error('File %s is Empty !' % fnp)
continue
rc = self._checkHeader(fnp)
if rc == 0 : procFiles.append(fnp)
return procFiles
def getOptFiles(self):
rc = 0
for f in self.ib.srcOptFile:
fnp = '%s/%s/%s' % (self.ib.shareDir,self.landDir,f)
self.incFiles.append(fnp) # Need to be in the incset to be moved to working dir.
if fu.fileExists(fnp):
self.log.debug('fnp exists %s' % fnp)
else :
r = fu.crtEmpFile(fnp)
if r == True : self.log.info ('Touching File %s' % fnp)
else :
self.log.Error ('Touching File %s' % fnp)
rc+=1
return rc
def _chkFiles(self, fls):
rc = 0
if len(fls) < 1 : return 1
for fnp in fls:
if fu.fileExists(fnp):
if fu.isDir(fnp):
self.log.error('%s is a directory' % fnp)
rc += 1
else:
self.log.info('%s exists' % fnp)
else :
self.log.error('%s does not exist' % fnp)
rc += 1
return rc
def createRecXMLFile(self,fxml,recPid):
# read from serialized object.
frec = '%s/%s/%s%s.rec' % (self.cb.recDir,recPid,self.cb.appName,recPid) # Serialized REC object
if not fu.fileExists(frec) :
self.log.error('Recovery File %s does not exist! Will not be able to generate recovery plan' % frec)
return -1
self.log.info('REC Recovery File : %s' % frec)
grpbl = fu.loadFSDB(frec)
self.log.debug("GRPBL =", grpbl)
if grpbl is None : return -1
# Create an xml recovery File
self.log.info('XML Recovery File : %s' % fxml)
rj = pxml.RecJobXML(fxml,self.cb.pid,self.log)
rc = rj.writeXmlFile(grpbl)
return rc
def recMRCJob(self):
cb = self._configBean()
self.log.debug('Config Bean = %s' % cb)
pid = su.toInt(self.recPid)
if pid is None or pid < 2:
self.log.error('Recovery PID needs to be numeric and > 1. Current %s' % self.recPid)
return 2
fxml = '%s/%s/%s%s.xml' % (cb.recDir,self.recPid,cb.appName,self.recPid)
self.log.info('Execute Plan file %s' % fxml)
pj = pjl.ProcessJobs(cb,self.log)
if not fu.fileExists(fxml):
rc = pj.createRecXMLFile(fxml,self.recPid)
if rc != 0 :
self.log.error("Could not create recovery file %s" % fxml)
return 1
else : self.log.info('File %s exits. Delete if you want to recreate !')
grpbl = pj.loadExecPlan(fxml)
if len(grpbl) < 1 :
self.log.error('Nothing to process!. Check Contents of %s' % fxml)
return 1
# Give the user the option to review and change the Execution plan file for recovery purposes.
if not self.forceRec:
pj.printExecPlan(grpbl)
r = su.getResponse('Please Enter Yes to Continue, any other key to exit.\n','Yes')
if r is False: return -1
# Read again, since file could be modified by the user !!
grpbl = pj.loadExecPlan(fxml)
if len(grpbl) < 1 :
self.log.error('Nothing to process!. Check Contents of %s' % fxml)
return 1
self.log.info('Reloaded %s ' % fxml)
rc = pj.mainProc(fxml,grpbl)
return rc
def chkErrFile(self):
rc = 1
fnp = '%s/%s' % (self.ib.tgtDir,self.ib.errorFileName)
self.log.info('Checking for file %s' % (fnp))
if fu.fileExists(fnp):
fileSz = fu.getFilseSize(fnp)
if fileSz == 0 : return 0
else:
self.log.error('filename %s does not exist!' % (fnp))
if self.notifUsersFlg is True: self._notifyUsers()
# At this point file exist and size > 0. Let's Check for blank lines!
rc = fu.remBlankLines(fnp)
self.log.info('Found File %s size = %d(bytes). Checked for non blank lines = %d.' % (fnp,fileSz,rc))
if rc != 0 :
self.log.error ('Error File created : filename %s. Lines = %d ' % (fnp,rc))
if self.notifUsersFlg is True:
r = self._notifyUsers(rc)
self.log.info ('Notifying Users rc= %s' % (r) )
return rc
def notifyUsers(self):
self.files = []
self.subj = 'Monthly details for Canadian Vehicles repaired in US'
self.text = 'Please find attached the monthly details for Canadian vehicles repaired in US. Please review and let us know if any issues. Thanks..!!'
self.rc = 0
for tf in self.tgtFiles:
tfp = '%s/%s/%s' % (self.ib.shareDir,self.tgtDir,tf)
if fu.fileExists(tfp):
self.files.append(tfp)
else :
self.log.error('File %s does not exist' % tfp)
self.log.info('Attaching ', self.files)
if len(self.files) < 1:
self.log.error('No attachments where found !')
return 1
rc = self._notifyAppUsers('plain')
self.log.debug('rc = %s SUBJ %s \t MSG %s' % (rc,self.subj, self.text))
return rc
def _chkIfValidLock(self):
rc = True
# Check if there is a lock file.
if fu.fileExists(self.lckFn):
sPid = fu.readFile(self.lckFn)
pid = su.toInt(sPid)
self.log.info('Lock File %s EXISTS. sPid = %s , pid = %s' % (self.lckFn,sPid,pid))
# Check if file has a valid PID (number)
if pid is None :
rc = fu.delFile(self.lckFn)
if rc == 0 :
self.log.info('Removed File %s' % (self.lckFn))
else:
self.log.error('Could not removed File %s' % (self.lckFn))
return False
# If pid is a valid number, check if the process is running ...
rc = ps.isProcRunning(pid,self.log)
self.log.debug('isProcRunning returned %s' % (rc))
return rc
# No lock file exists.
else: return False
def procCortFiles(self):
dataFile = []
ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)
setn = len (self.procCort)
self.log.info( ' ---- Starting Processing. Total of %d iteration(s) ----' % setn)
fileset = self.procCort.keys()
fileset.sort()
self.log.debug('Process Order = ' , fileset)
for fn in fileset:
f = '%s/%s' % (self.ib.landDir,fn)
if not fu.fileExists(f):
self.log.error('File %s have not been downloaded!' % f)
return 1
# Processing loop.
for fn in fileset:
self.log.info( 'Processing file %s/%s ' % (self.ib.landDir,fn))
cur_dayr = self._getDateRunStr(fn)
if cur_dayr is None :
self.log.error('cur_dayr is None. No Date Run String ')
return 2
# cur_dayr = YYYYMMDD from filename . eg. Cortera_Trx_Extract_20130502.txt
# pdate format YYYY-MM-DD
pdate = '%s-%s-%s' % (cur_dayr[:4],cur_dayr[4:6],cur_dayr[6:8])
self.log.debug('cur_dayr = %s pdate = %s' % (cur_dayr,pdate))
rc = su.isValidDate(pdate,'%Y-%m-%d')
if rc is False :
self.log.error('Invalid Date %s on file %s ' % (pdate,fn))
return 2
self.log.debug('self.checkNextRunFlg is %s' % self.checkNextRunFlg)
if self.checkNextRunFlg is True:
prev_dayr = self._getCtlFile()
pd,pr = self._getDay(prev_dayr,DP_LEN)
rc = self._chkNextRun(cur_dayr,prev_dayr,pd,pr,RUN_PER_DAY)
if rc != 0 :
self.log.error("self._chkNextRun rc = %s" % rc)
return rc
# Cortera
t = '%s/%s' % (self.ib.workDir,self.ib.srcFile[1])
fnp = '%s/%s' % (self.ib.landDir,fn)
cl = self.procCort[fn]
rc = self._chkDataRecCnt(fnp,cl)
if rc != 0 : return rc
rc = fu.copyFile(fnp, t)
if rc != 0 :
self.log.error('Could not copy File %s to %s' % (fnp,t))
return 5
self.log.info ('Copying File from %s to %s' % (fnp,t))
rc = self.archGenFiles([fnp,], self.ts)
self.log.info ('Archiving file rc = %s' % rc)
# Invoke workflow.
rc = pi.runWkflWait(self.ib,self.log)
if rc != 0 :
self.log.error('Running %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
if self.exitOnError:
self.log.debug('ExitOnError is TRUE rc = %s' % (rc))
return rc
else :
self.log.info('Ran %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
# Update Control table in Netezza with date.
rc = self._updLeaseCrdTbl(pdate)
if rc != 0 :
self.log.error('No rows were updated for process date %s' % pdate)
return rc
# Loading Staging Succeeded. Update the control file.
rc = fu.updFile(ctlFile,cur_dayr)
if rc == 0 :
if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile))
else : self.log.info('Overwriting Cur Load Date to %s , Control File %s' % (cur_dayr, ctlFile))
else :
self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc))
return rc
r = fu.delFile(t)
self.log.info('Removing %s rc = %s ' % (t,r))
# Delete BU Region.
fnp = '%s/%s' % (self.ib.workDir,self.ib.srcFile[0]) # BU_region
rc = fu.delFile(fnp)
self.log.info('Removing %s rc = %s ' % (fnp,rc))
return rc
