def getNextRunDate(pd, cd, scf, log, sch=[]): rc = -1 log.debug("pd = %s ,cd = %s,scf = %s" % (pd, cd, scf)) if scf == "Mthly": r = su.isValidDate(pd, "%Y%m") else: r = su.isValidDate(pd, "%Y%m%d") if r is False: log.error("prevDayRun = %s is invalid !" % pd) return 1 try: nrd = eval("_get%sNextRunDate(pd,sch)" % (scf)) log.debug("_get%sNextRunDate ret = %s " % (scf, nrd)) if nrd == cd: return 0 log.debug("errLen = %d " % errLen) if nrd < errLen: log.error("%s" % errmsg[nrd]) else: log.error( "Invalid Run date. Previous Ran date = %s . Next Ran Date should be %s and not %s" % (pd, nrd, cd) ) except NameError: log.error("Invalid Schedule Frequency %s " % scf) rc = 10 return rc
def _getDay(self,dayr,dpLen): d = None if (len(dayr) != dpLen) : self.log.debug('dayr=%s (%d) len does not match DP_LEN %s ' % (dayr,len(dayr),dpLen)) return d, -1 d= dayr[:8] rc = su.isValidDate(d, '%Y%m%d') if rc is False : d = None return d,1
def getFileProcDate(self,fnp): tr = fu.getLastLine(fnp) tok = tr.split() self.log.debug('Last line = ', tr , ' tokens ', tok) if len(tok) != 4 : self.log.error('filename %s === \tInvalid Number of tokens.It has %d need to be 4 !' % (fnp,len(tok))) return None if su.isValidDate(tok[1],'%Y%m%d') is False : self.log.error('fn %s === \tInvalid date %s!' % (fnp,tok[1])) return None return tok[1]
def _getDayRun(self,dayr,dpLen): d = None; r = -1 if (len(dayr) != dpLen) : self.log.debug('dayr=%s (%d) len does not match DP_LEN %s ' % (dayr,len(dayr),dpLen)) return d,r d= dayr[:8]; r = dayr[9:10] rc = su.isValidDate(d, '%Y%m%d') if rc is False : d = None rc = su.toInt(r) if rc is None : r = -1 else : r = rc return d,r
def _checkTrailer(self,fnp,trl_date): self.log.info("Filename : %s " % fnp) fd = -1; ln =-1; tr=-1; rowc=-1; dt=-1; lc=-1;tok = [] try: # Check if valid Date & truncate last line (trailer Line) tr = fu.getLastLine(fnp,1).rstrip() # Trailer Line tok = tr.split() self.log.debug('Last line = ', tr , ' tokens ', tok) if len(tok) != 4 : self.log.error('filename %s === \tInvalid Number of tokens.It has %d need to be 4 !' % (fnp,len(tok))) return 1 if tok[0] != 'TRAILER': self.log.error('filename %s === \tInvalid TRAILER LINE.It need to start with TRAILER insted of !' % (fnp,tok[0])) return 2 dt = tok[1] if su.isValidDate(dt,'%Y%m%d') is False : self.log.error('fn %s === \tInvalid date %s!' % (fnp,dt)) return 3 rowc = su.toInt(tok[3]) if rowc is None: self.log.error('fn %s === \tInvalid row count %s!' % (fnp,tok[3])) return 4 if dt != trl_date: self.log.error('fn %s === \tInvalid Trailer ran date of %s. Should be %s ' % (fnp,dt,trl_date)) return 5 # Check Number of lines ln = fu.getLines(fnp) # Number of lines w/o trailer. dc = int(ln) - rowc # ln file has 2 extra lines header/trailer, but at this point trailer rec had been removed. self.log.debug('fn %s === \tFile record count %s trailer = %s diff %s' % (fnp,ln,rowc,dc)) if dc != 0 : self.log.error('fn %s === \tFile record count %s do not match trailer = %s # of records diff = %d!' % (fnp,ln,rowc,dc)) return 6 return 0 # Invalid datatypes fields for verification process !! except: self.log.error(' ==EXCEP %s ' % (sys.exc_info()[1])) return 5 finally: self.log.debug('fn %s == date %s\t lines %s' % (fnp,fd,ln)) self.log.debug('trl %s == date %s\t lines %s' % (tr,dt,rowc))
def _chkMthlyTrailer(self,fnp,fdr): # Check if valid Date & truncate last line (trailer Line) tr = fu.getLastLine(fnp,1).rstrip() # Trailer Line dt, rowc = tr[:6],int(tr[6:]) date = '%s%s' % (dt[:4],dt[4:]) if su.isValidDate(date,'%Y%m') is False : self.log.error('fn %s === \tInvalid date %s!' % (fnp,date)) return -1 fd = fdr[:6] if dt != fd : self.log.error('fn %s === \tFile Date = %s do not match trailer = %s!' % (fnp,fd,dt)) return -2 return rowc
def procIncFiles(self): if len(self.workFiles) < 1 : self.log.error('No files to process') return 1 ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName) self.workFiles.sort() rc = 0 # Files in the working directory: i = 1 self.log.debug('Will Process a total of %d file(s) ' % len(self.workFiles)) for fnp in self.workFiles: self.log.info('\nProcessing File (%d) => %s ' % (i,fnp)) # Get date run from 1st filename fn = fu.getFileBaseName(fnp) cur_dayr = self._getDateRunStr(fn) if cur_dayr is None : self.log.error('No Date String %s ' % cur_dayr) return 1 date = '%s/%s/%s' % (cur_dayr[4:6],cur_dayr[6:8],cur_dayr[:4]) rc = su.isValidDate(date) if rc is False : self.log.error('Invalid Date %s on file %s ' % (date,fn)) return 2 self.fileDate = date self.log.debug('self.fileDate = %s' % (self.fileDate)) if self.checkNextRunFlg is True: # Get Previous Run Info. File should contain one line only : YYYYMMDD from storage. prev_dayr = self._getCtlFile() if prev_dayr is None : return 3 pd,pr = self._getDay(prev_dayr,DP_LEN) rc = self._chkNextRun(cur_dayr,prev_dayr,pd,pr,RUN_PER_DAY) if rc != 0 : self.log.error("self._chkNextRun rc = %s" % rc) return rc procFiles = self.chkTrailer([fnp,],fn,cur_dayr) if len(procFiles) != self.FILE_SET_LEN : self.log.error("chkTrailer Files that were OK ", procFiles) return 4 t = '%s/%s' % (self.ib.workDir , self.ib.srcFile) rc = fu.moveFile(fnp, t) if rc != 0 : self.log.error('Could not move File %s to %s' % (fnp,t)) return 5 # Invoke workflow. rc = pi.runWkflWait(self.ib,self.log) if rc != 0 : self.log.error('Running %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc)) if self.exitOnError: self.log.debug('ExitOnError is TRUE rc = %s' % (rc)) return rc else : self.log.info('Ran %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc)) # PostCheck Every File, before updating flag: if self.postCheckFlg is True : rc = self._postCheck() if rc != 0 : self.log.warn('Post Check Failed !!! Did not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc)) return rc # Loading Staging Succeeded. Update the control file. rc = fu.updFile(ctlFile,cur_dayr) if rc == 0 : if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile)) else : self.log.info('Overwriting Cur Load Date to %s , Control File %s' % (cur_dayr, ctlFile)) else : self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc)) return rc # r = fu.delFile(t) i+=1 return rc
def splitMainCntFile(self): #Get Header. fnp = '%s/%s' % (self.ib.workDir,self.srcFile) s = fu.readFile(fnp) hdr = su.remSpacefrStr(s) if len(hdr) != 38 : self.log.error('Invalid len= %d for %s. Need to be 38\nhdr=%s' % (len(hdr),fnp,hdr)) return 1 self.log.debug('hdr = %s' % hdr) # Get counts po870D30 = su.toInt(hdr[0:8]) po875D30 = su.toInt(hdr[8:16]) po880D30 = su.toInt(hdr[16:24]) po225D15 = su.toInt(hdr[24:32]) rdate = hdr[32:] self.log.debug('po870D30=%s po875D30=%s po880D30=%s po225D15=%s rdate=%s' % (po870D30, po875D30,po880D30,po225D15,rdate)) # Sanity check. if (po870D30 is None or po875D30 is None or po880D30 is None or po225D15 is None or su.isValidDate(rdate,'%m%d%y') == False) : self.log.error('po870D30=%s po875D30=%s po880D30=%s po225D15=%s rdate=%s' % (po870D30, po875D30,po880D30,po225D15,rdate)) return 2 # Create count files. rc = 0 r = fu.createFile('%s/po870d30comp.cnt' % self.ib.workDir, '%08d%s' % (po870D30,rdate)) ; rc+= r self.log.info (' Creating file %s/po870d30comp.cnt rc = %s' % (self.ib.workDir, r)) r = fu.createFile('%s/po875D30.cnt' % self.ib.workDir, '%08d%s' % (po875D30,rdate)) ; rc+= r self.log.info (' Creating file %s/po875d30.cnt rc = %s' % (self.ib.workDir, r)) r = fu.createFile('%s/po880D30.cnt' % self.ib.workDir, '%08d%s' % (po880D30,rdate)) ; rc+= r self.log.info (' Creating file %s/po880d30.cnt rc = %s' % (self.ib.workDir, r)) r = fu.createFile('%s/po225D15.cnt' % self.ib.workDir, '%08d%s' % (po225D15,rdate)) ; rc+= r self.log.info (' Creating file %s/po225d15.cnt rc = %s' % (self.ib.workDir, r)) return rc
def procCortFiles(self): dataFile = [] ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName) setn = len (self.procCort) self.log.info( ' ---- Starting Processing. Total of %d iteration(s) ----' % setn) fileset = self.procCort.keys() fileset.sort() self.log.debug('Process Order = ' , fileset) for fn in fileset: f = '%s/%s' % (self.ib.landDir,fn) if not fu.fileExists(f): self.log.error('File %s have not been downloaded!' % f) return 1 # Processing loop. for fn in fileset: self.log.info( 'Processing file %s/%s ' % (self.ib.landDir,fn)) cur_dayr = self._getDateRunStr(fn) if cur_dayr is None : self.log.error('cur_dayr is None. No Date Run String ') return 2 # cur_dayr = YYYYMMDD from filename . eg. Cortera_Trx_Extract_20130502.txt # pdate format YYYY-MM-DD pdate = '%s-%s-%s' % (cur_dayr[:4],cur_dayr[4:6],cur_dayr[6:8]) self.log.debug('cur_dayr = %s pdate = %s' % (cur_dayr,pdate)) rc = su.isValidDate(pdate,'%Y-%m-%d') if rc is False : self.log.error('Invalid Date %s on file %s ' % (pdate,fn)) return 2 self.log.debug('self.checkNextRunFlg is %s' % self.checkNextRunFlg) if self.checkNextRunFlg is True: prev_dayr = self._getCtlFile() pd,pr = self._getDay(prev_dayr,DP_LEN) rc = self._chkNextRun(cur_dayr,prev_dayr,pd,pr,RUN_PER_DAY) if rc != 0 : self.log.error("self._chkNextRun rc = %s" % rc) return rc # Cortera t = '%s/%s' % (self.ib.workDir,self.ib.srcFile[1]) fnp = '%s/%s' % (self.ib.landDir,fn) cl = self.procCort[fn] rc = self._chkDataRecCnt(fnp,cl) if rc != 0 : return rc rc = fu.copyFile(fnp, t) if rc != 0 : self.log.error('Could not copy File %s to %s' % (fnp,t)) return 5 self.log.info ('Copying File from %s to %s' % (fnp,t)) rc = self.archGenFiles([fnp,], self.ts) self.log.info ('Archiving file rc = %s' % rc) # Invoke workflow. rc = pi.runWkflWait(self.ib,self.log) if rc != 0 : self.log.error('Running %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc)) if self.exitOnError: self.log.debug('ExitOnError is TRUE rc = %s' % (rc)) return rc else : self.log.info('Ran %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc)) # Update Control table in Netezza with date. rc = self._updLeaseCrdTbl(pdate) if rc != 0 : self.log.error('No rows were updated for process date %s' % pdate) return rc # Loading Staging Succeeded. Update the control file. rc = fu.updFile(ctlFile,cur_dayr) if rc == 0 : if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile)) else : self.log.info('Overwriting Cur Load Date to %s , Control File %s' % (cur_dayr, ctlFile)) else : self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc)) return rc r = fu.delFile(t) self.log.info('Removing %s rc = %s ' % (t,r)) # Delete BU Region. fnp = '%s/%s' % (self.ib.workDir,self.ib.srcFile[0]) # BU_region rc = fu.delFile(fnp) self.log.info('Removing %s rc = %s ' % (fnp,rc)) return rc
def procIncFiles(self): if len(self.workFiles) < 1 : self.log.error('No files to process') return 1 ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName) self.workFiles.sort() rc = 0 # Files in the working directory: i = 1 self.log.debug('Will Process a total of %d file(s) ' % len(self.workFiles)) for fnp in self.workFiles: self.log.info('\nProcessing File (%d) => %s ' % (i,fnp)) # Get date run from 1st filename fn = fu.getFileBaseName(fnp) cur_dayr = self._getDateRunStr(fn) if cur_dayr is None : self.log.error('No Date String %s ' % cur_dayr) return 1 date = '%s%s' % (cur_dayr[:4],cur_dayr[4:6]) rc = su.isValidDate(date,'%Y%m') if rc is False : self.log.error('Invalid Date %s on file %s ' % (date,fn)) return 2 self.fileDate = date self.log.debug('self.fileDate = %s' % (self.fileDate)) ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName) self.log.debug('self.checkNextRunFlg is %s' % self.checkNextRunFlg) prev_dayr = self._getCtlFile() if self.checkNextRunFlg is True: if prev_dayr is None or prev_dayr.strip() == '': self.log.error("Could not find control file or No Data") return -1 rc = psc.getNextRunDate(prev_dayr, cur_dayr, SCH_FREQ, self.log,sch) if rc != 0 : self.log.error("self._chkNextRun rc = %s" % rc) return rc procFiles = self.chkTrailer([fnp,],fn,cur_dayr) if len(procFiles) != self.FILE_SET_LEN : self.log.error("chkTrailer Files that were OK ", procFiles) return 4 t = '%s/%s' % (self.ib.workDir , self.ib.srcFile) self.log.debug('fnp =%s Move to %s ' % (fnp,t)) rc = fu.moveFile(fnp, t) if rc != 0 : self.log.error('Could not move File %s to %s' % (fnp,t)) return 5 self.log.info('mv src file fnp %s -> t %s' % (fnp,t)) # Invoke workflow. rc = pi.runWkflWait(self.ib,self.log) if rc != 0 : self.log.error('Running %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc)) if self.exitOnError: self.log.debug('ExitOnError is TRUE rc = %s' % (rc)) return rc else : self.log.info('Ran %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc)) # Loading Staging Succeeded. Update the control file. rc = fu.updFile(ctlFile,cur_dayr) if rc == 0 : if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile)) else : self.log.info('Overwriting Cur Load Date to %s , Control File %s' % (cur_dayr, ctlFile)) else : self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc)) return rc i+=1 return rc
def procIncFiles(self): if len(self.workFiles) < 1: self.log.error("No files to process") return 1 ctlFile = "%s/%s.ctl" % (self.ib.ctlDir, self.appName) self.workFiles.sort() rc = 0 # Files in the working directory: i = 1 self.log.debug("Will Process a total of %d file(s) " % len(self.workFiles)) for fnp in self.workFiles: self.log.info("\nProcessing File (%d) => %s " % (i, fnp)) # Get date run from 1st filename fn = fu.getFileBaseName(fnp) cur_dayr = self._getDateRunStr(fn) if cur_dayr is None: self.log.error("No Date String %s " % cur_dayr) return 1 fmt = "%Y%m" date = "%s%s" % (cur_dayr[0:4], cur_dayr[4:6]) rc = su.isValidDate(date, fmt) if rc is False: self.log.error("Invalid Date %s on file %s " % (date, fn)) return 2 self.fileDate = date self.log.debug("self.fileDate = %s" % (self.fileDate)) if self.checkNextRunFlg is True: # Get Previous Run Info. File should contain one line only : YYYYMM from storage. prev_dayr = self._getCtlFile() if prev_dayr is None: return 3 pd, pr = self._getMonth(prev_dayr, DP_LEN) if pd is None: return 4 # rc = self._chkNextRun(cur_dayr,prev_dayr,pd,pr,RUN_PER_MTH) rc = psch.getNextRunDate(pd, cur_dayr, "Mthly", self.log) if rc != 0: self.log.error("self._chkNextRun rc = %s" % rc) return rc if self.verifyCSVFlg is True: r, b = fu.readCSV(fnp, FLDxROW, SEP) if len(b) > 0: fbad = "%s/%s.bad" % (self.ib.badDir, fn) rc = fu.createFile(fbad, b) self.log.error("No of %d bad row(s) on %s" % (len(b), fnp)) self.log.error("Creating file %s rc = %s" % (fbad, rc)) self.log.debug("Bad rows = , ", b) return 5 if len(r) == 0: self.log.error("No rows to process on file %s" % fnp) return 6 t = "%s/%s" % (self.ib.workDir, self.ib.srcFile[0]) rc = fu.moveFile(fnp, t) self.log.info("Renaming %s to %s rc = %s" % (fnp, t, rc)) if rc != 0: self.log.error("Could not move File %s to %s" % (fnp, t)) return 7 # Invoke workflow. rc = self._wkfIMSSftyLoc() if rc != 0: self.log.error("Running %s.%s rc = %s" % (self.ib.fld, self.ib.wkf, rc)) if self.exitOnError: self.log.debug("ExitOnError is TRUE rc = %s" % (rc)) return rc else: self.log.info("Ran %s.%s rc = %s" % (self.ib.fld, self.ib.wkf, rc)) # Loading Staging Succeeded. Update the control file. rc = fu.updFile(ctlFile, cur_dayr) if rc == 0: if self.checkNextRunFlg: self.log.info( "Updated Cur Load Date from %s to %s , Control File %s" % (prev_dayr, cur_dayr, ctlFile) ) else: self.log.info("Overwriting Cur Load Date to %s , Control File %s" % (cur_dayr, ctlFile)) else: self.log.error("Could not Update Load Date %s, Control File %s rc = %s" % (cur_dayr, ctlFile, rc)) return rc r = fu.delFile(t) self.log.debug("Deleting File %s rc = %s" % (t, r)) i += 1 return rc