def procHierSCOS(self): # Check for file existence src = '%s/%s/%s' % (self.ib.shareDir,self.outDir,self.ib.outFile) r = fu.fileExists(src) if r == False : self.log.error('File %s does not exist' % src) return 1 # Move File to output dir dt = su.getTodayDtStr('%y%m%d') self.outFile = 'SCS_Org_Hierarchy%s.csv' % dt outFile = '%s/%s/%s/%s' % (self.ib.shareDir,self.outDir,self.ib.outDirname,self.outFile) r = fu.moveFile(src,outFile) if r != 0 : self.log.error("Could not move %s to %s " % (src,outFile)) self.log.debug('Moved src = %s to self.outFile = %s RC = %s' % (src,outFile,r)) # Archive File . arcfn = '%s/%s/%s/%s-%s' % (self.ib.shareDir,self.outDir,self.outArchDir,self.ts,self.outFile) r = fu.copyFile(outFile,arcfn) if r != 0 : self.log.error("Could not copy %s to %s " % (outFile,arcfn)) self.log.debug('Copied src = %s to racfn = %s RC = %s' % (outFile,arcfn,r)) return 0
def _chkIfValidLock(self): rc = True # Check if there is a lock file. if fu.fileExists(self.lckFn): sPid = fu.readFile(self.lckFn) pid = su.toInt(sPid) self.log.info('Lock File %s EXISTS. sPid = %s , pid = %s' % (self.lckFn, sPid, pid)) # Check if file has a valid PID (number) if pid is None: rc = fu.delFile(self.lckFn) if rc == 0: self.log.info('Removed File %s' % (self.lckFn)) else: self.log.error('Could not removed File %s' % (self.lckFn)) return False # If pid is a valid number, check if the process is running ... rc = ps.isProcRunning(pid, self.log) self.log.debug('isProcRunning returned %s' % (rc)) return rc # No lock file exists. else: return False
def getStateFile(self): fn = '%s/%s.state' % (self.ib.ctlDir, self.appName) if fu.fileExists(fn) is False: self.log.error('State File:%s does not exist' % fn) return None self.log.info('Reading State File :%s' % fn) return fu.readFile(fn).strip()
def _getCtlFile(self): fn = '%s/%s.ctl' % (self.ib.ctlDir, self.appName) if fu.fileExists(fn) is False: self.log.error('Control File:%s does not exist' % fn) return None self.log.info('Loading Control File:%s' % fn) return fu.readFile(fn).strip()
def chkTrailer (self,fL,trl_date): rc = 0 for fnp in fL: if fu.fileExists(fnp) is False: self.log.error('File %s does not exists ' % fnp) continue if fu.getFilseSize(fnp) == 0 : self.log.error('File %s is Empty !' % fnp) continue r = self._checkTrailer(fnp,trl_date) if r != 0 : rc = 1 return rc
def chkHeader(self,fL): procFiles = [] for fnp in fL: if fu.fileExists(fnp) is False: self.log.error('File %s does not exists ' % fnp) continue if fu.getFilseSize(fnp) == 0 : self.log.error('File %s is Empty !' % fnp) continue rc = self._checkHeader(fnp) if rc == 0 : procFiles.append(fnp) return procFiles
def getOptFiles(self): rc = 0 for f in self.ib.srcOptFile: fnp = '%s/%s/%s' % (self.ib.shareDir,self.landDir,f) self.incFiles.append(fnp) # Need to be in the incset to be moved to working dir. if fu.fileExists(fnp): self.log.debug('fnp exists %s' % fnp) else : r = fu.crtEmpFile(fnp) if r == True : self.log.info ('Touching File %s' % fnp) else : self.log.Error ('Touching File %s' % fnp) rc+=1 return rc
def _chkFiles(self, fls): rc = 0 if len(fls) < 1 : return 1 for fnp in fls: if fu.fileExists(fnp): if fu.isDir(fnp): self.log.error('%s is a directory' % fnp) rc += 1 else: self.log.info('%s exists' % fnp) else : self.log.error('%s does not exist' % fnp) rc += 1 return rc
def createRecXMLFile(self,fxml,recPid): # read from serialized object. frec = '%s/%s/%s%s.rec' % (self.cb.recDir,recPid,self.cb.appName,recPid) # Serialized REC object if not fu.fileExists(frec) : self.log.error('Recovery File %s does not exist! Will not be able to generate recovery plan' % frec) return -1 self.log.info('REC Recovery File : %s' % frec) grpbl = fu.loadFSDB(frec) self.log.debug("GRPBL =", grpbl) if grpbl is None : return -1 # Create an xml recovery File self.log.info('XML Recovery File : %s' % fxml) rj = pxml.RecJobXML(fxml,self.cb.pid,self.log) rc = rj.writeXmlFile(grpbl) return rc
def recMRCJob(self): cb = self._configBean() self.log.debug('Config Bean = %s' % cb) pid = su.toInt(self.recPid) if pid is None or pid < 2: self.log.error('Recovery PID needs to be numeric and > 1. Current %s' % self.recPid) return 2 fxml = '%s/%s/%s%s.xml' % (cb.recDir,self.recPid,cb.appName,self.recPid) self.log.info('Execute Plan file %s' % fxml) pj = pjl.ProcessJobs(cb,self.log) if not fu.fileExists(fxml): rc = pj.createRecXMLFile(fxml,self.recPid) if rc != 0 : self.log.error("Could not create recovery file %s" % fxml) return 1 else : self.log.info('File %s exits. Delete if you want to recreate !') grpbl = pj.loadExecPlan(fxml) if len(grpbl) < 1 : self.log.error('Nothing to process!. Check Contents of %s' % fxml) return 1 # Give the user the option to review and change the Execution plan file for recovery purposes. if not self.forceRec: pj.printExecPlan(grpbl) r = su.getResponse('Please Enter Yes to Continue, any other key to exit.\n','Yes') if r is False: return -1 # Read again, since file could be modified by the user !! grpbl = pj.loadExecPlan(fxml) if len(grpbl) < 1 : self.log.error('Nothing to process!. Check Contents of %s' % fxml) return 1 self.log.info('Reloaded %s ' % fxml) rc = pj.mainProc(fxml,grpbl) return rc
def chkErrFile(self): rc = 1 fnp = '%s/%s' % (self.ib.tgtDir,self.ib.errorFileName) self.log.info('Checking for file %s' % (fnp)) if fu.fileExists(fnp): fileSz = fu.getFilseSize(fnp) if fileSz == 0 : return 0 else: self.log.error('filename %s does not exist!' % (fnp)) if self.notifUsersFlg is True: self._notifyUsers() # At this point file exist and size > 0. Let's Check for blank lines! rc = fu.remBlankLines(fnp) self.log.info('Found File %s size = %d(bytes). Checked for non blank lines = %d.' % (fnp,fileSz,rc)) if rc != 0 : self.log.error ('Error File created : filename %s. Lines = %d ' % (fnp,rc)) if self.notifUsersFlg is True: r = self._notifyUsers(rc) self.log.info ('Notifying Users rc= %s' % (r) ) return rc
def notifyUsers(self): self.files = [] self.subj = 'Monthly details for Canadian Vehicles repaired in US' self.text = 'Please find attached the monthly details for Canadian vehicles repaired in US. Please review and let us know if any issues. Thanks..!!' self.rc = 0 for tf in self.tgtFiles: tfp = '%s/%s/%s' % (self.ib.shareDir,self.tgtDir,tf) if fu.fileExists(tfp): self.files.append(tfp) else : self.log.error('File %s does not exist' % tfp) self.log.info('Attaching ', self.files) if len(self.files) < 1: self.log.error('No attachments where found !') return 1 rc = self._notifyAppUsers('plain') self.log.debug('rc = %s SUBJ %s \t MSG %s' % (rc,self.subj, self.text)) return rc
def _chkIfValidLock(self): rc = True # Check if there is a lock file. if fu.fileExists(self.lckFn): sPid = fu.readFile(self.lckFn) pid = su.toInt(sPid) self.log.info('Lock File %s EXISTS. sPid = %s , pid = %s' % (self.lckFn,sPid,pid)) # Check if file has a valid PID (number) if pid is None : rc = fu.delFile(self.lckFn) if rc == 0 : self.log.info('Removed File %s' % (self.lckFn)) else: self.log.error('Could not removed File %s' % (self.lckFn)) return False # If pid is a valid number, check if the process is running ... rc = ps.isProcRunning(pid,self.log) self.log.debug('isProcRunning returned %s' % (rc)) return rc # No lock file exists. else: return False
def procCortFiles(self): dataFile = [] ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName) setn = len (self.procCort) self.log.info( ' ---- Starting Processing. Total of %d iteration(s) ----' % setn) fileset = self.procCort.keys() fileset.sort() self.log.debug('Process Order = ' , fileset) for fn in fileset: f = '%s/%s' % (self.ib.landDir,fn) if not fu.fileExists(f): self.log.error('File %s have not been downloaded!' % f) return 1 # Processing loop. for fn in fileset: self.log.info( 'Processing file %s/%s ' % (self.ib.landDir,fn)) cur_dayr = self._getDateRunStr(fn) if cur_dayr is None : self.log.error('cur_dayr is None. No Date Run String ') return 2 # cur_dayr = YYYYMMDD from filename . eg. Cortera_Trx_Extract_20130502.txt # pdate format YYYY-MM-DD pdate = '%s-%s-%s' % (cur_dayr[:4],cur_dayr[4:6],cur_dayr[6:8]) self.log.debug('cur_dayr = %s pdate = %s' % (cur_dayr,pdate)) rc = su.isValidDate(pdate,'%Y-%m-%d') if rc is False : self.log.error('Invalid Date %s on file %s ' % (pdate,fn)) return 2 self.log.debug('self.checkNextRunFlg is %s' % self.checkNextRunFlg) if self.checkNextRunFlg is True: prev_dayr = self._getCtlFile() pd,pr = self._getDay(prev_dayr,DP_LEN) rc = self._chkNextRun(cur_dayr,prev_dayr,pd,pr,RUN_PER_DAY) if rc != 0 : self.log.error("self._chkNextRun rc = %s" % rc) return rc # Cortera t = '%s/%s' % (self.ib.workDir,self.ib.srcFile[1]) fnp = '%s/%s' % (self.ib.landDir,fn) cl = self.procCort[fn] rc = self._chkDataRecCnt(fnp,cl) if rc != 0 : return rc rc = fu.copyFile(fnp, t) if rc != 0 : self.log.error('Could not copy File %s to %s' % (fnp,t)) return 5 self.log.info ('Copying File from %s to %s' % (fnp,t)) rc = self.archGenFiles([fnp,], self.ts) self.log.info ('Archiving file rc = %s' % rc) # Invoke workflow. rc = pi.runWkflWait(self.ib,self.log) if rc != 0 : self.log.error('Running %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc)) if self.exitOnError: self.log.debug('ExitOnError is TRUE rc = %s' % (rc)) return rc else : self.log.info('Ran %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc)) # Update Control table in Netezza with date. rc = self._updLeaseCrdTbl(pdate) if rc != 0 : self.log.error('No rows were updated for process date %s' % pdate) return rc # Loading Staging Succeeded. Update the control file. rc = fu.updFile(ctlFile,cur_dayr) if rc == 0 : if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile)) else : self.log.info('Overwriting Cur Load Date to %s , Control File %s' % (cur_dayr, ctlFile)) else : self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc)) return rc r = fu.delFile(t) self.log.info('Removing %s rc = %s ' % (t,r)) # Delete BU Region. fnp = '%s/%s' % (self.ib.workDir,self.ib.srcFile[0]) # BU_region rc = fu.delFile(fnp) self.log.info('Removing %s rc = %s ' % (fnp,rc)) return rc