Exemplo n.º 1
0
def getNextRunDate(pd, cd, scf, log, sch=[]):
    rc = -1
    log.debug("pd = %s ,cd = %s,scf = %s" % (pd, cd, scf))
    if scf == "Mthly":
        r = su.isValidDate(pd, "%Y%m")
    else:
        r = su.isValidDate(pd, "%Y%m%d")
    if r is False:
        log.error("prevDayRun = %s is invalid !" % pd)
        return 1
    try:
        nrd = eval("_get%sNextRunDate(pd,sch)" % (scf))
        log.debug("_get%sNextRunDate ret = %s " % (scf, nrd))
        if nrd == cd:
            return 0
        log.debug("errLen = %d " % errLen)
        if nrd < errLen:
            log.error("%s" % errmsg[nrd])
        else:
            log.error(
                "Invalid Run date. Previous Ran date = %s . Next Ran Date should be %s and not %s" % (pd, nrd, cd)
            )

    except NameError:
        log.error("Invalid Schedule Frequency %s " % scf)
        rc = 10

    return rc
Exemplo n.º 2
0
 def _getDay(self,dayr,dpLen):
     
     d = None
     if (len(dayr) != dpLen) : 
         self.log.debug('dayr=%s (%d) len does not match DP_LEN %s ' % (dayr,len(dayr),dpLen))
         return d, -1
     
     d= dayr[:8]
     
     rc = su.isValidDate(d, '%Y%m%d')
     if rc is False   : d = None
     return d,1
Exemplo n.º 3
0
 def getFileProcDate(self,fnp):
     tr  = fu.getLastLine(fnp)
     tok = tr.split()
     self.log.debug('Last line = ', tr , ' tokens ', tok)
     if len(tok) != 4 :
         self.log.error('filename %s === \tInvalid Number of tokens.It has %d need to be 4 !' % (fnp,len(tok)))
         return None
         
     if su.isValidDate(tok[1],'%Y%m%d') is False : 
         self.log.error('fn %s === \tInvalid date %s!' % (fnp,tok[1]))
         return None
     
     return tok[1]
Exemplo n.º 4
0
 def _getDayRun(self,dayr,dpLen):
     
     d = None; r = -1
     if (len(dayr) != dpLen) : 
         self.log.debug('dayr=%s (%d) len does not match DP_LEN %s ' % (dayr,len(dayr),dpLen))
         return d,r
     
     d= dayr[:8]; r = dayr[9:10]
     
     rc = su.isValidDate(d, '%Y%m%d')
     if rc is False   : d = None
     rc = su.toInt(r) 
     if rc is None    : r = -1
     else             : r = rc
     return d,r
Exemplo n.º 5
0
 def _checkTrailer(self,fnp,trl_date):
     self.log.info("Filename : %s " %  fnp)
     fd = -1; ln =-1; tr=-1; rowc=-1; dt=-1; lc=-1;tok = []
     try:
         # Check if valid Date & truncate last line (trailer Line)
         tr         = fu.getLastLine(fnp,1).rstrip()     # Trailer Line
         tok        = tr.split()
         self.log.debug('Last line = ', tr , ' tokens ', tok)
         if len(tok) != 4 :
             self.log.error('filename %s === \tInvalid Number of tokens.It has %d need to be 4 !' % (fnp,len(tok)))
             return 1
         
         if tok[0] != 'TRAILER':
             self.log.error('filename %s === \tInvalid TRAILER LINE.It need to start with TRAILER insted of !' % (fnp,tok[0]))
             return 2
         
         dt = tok[1]
         if su.isValidDate(dt,'%Y%m%d') is False : 
             self.log.error('fn %s === \tInvalid date %s!' % (fnp,dt))
             return 3
         
         rowc = su.toInt(tok[3])
         if rowc is None:
             self.log.error('fn %s === \tInvalid row count %s!' % (fnp,tok[3]))
             return 4
         
         if dt != trl_date:
             self.log.error('fn %s === \tInvalid Trailer ran date of %s. Should be %s  ' % (fnp,dt,trl_date))
             return 5
                 
         # Check Number of lines 
         ln  = fu.getLines(fnp)      # Number of lines w/o trailer.
         dc =  int(ln) - rowc        # ln file has 2 extra lines header/trailer, but at this point trailer rec had been removed.
         self.log.debug('fn %s === \tFile record count %s trailer = %s diff %s' % (fnp,ln,rowc,dc))
         if  dc != 0 :
             self.log.error('fn %s === \tFile record count %s do not match trailer = %s # of records diff = %d!' % (fnp,ln,rowc,dc))
             return 6
         
         return 0
     
     # Invalid datatypes fields for verification process !!
     except:            
         self.log.error(' ==EXCEP %s ' % (sys.exc_info()[1]))
         return 5
     
     finally:
         self.log.debug('fn  %s == date %s\t lines %s' % (fnp,fd,ln))
         self.log.debug('trl %s == date %s\t lines %s' % (tr,dt,rowc))
Exemplo n.º 6
0
    def _chkMthlyTrailer(self,fnp,fdr):
 
        # Check if valid Date & truncate last line (trailer Line)
        tr         = fu.getLastLine(fnp,1).rstrip()     # Trailer Line
        dt, rowc = tr[:6],int(tr[6:])
        date = '%s%s' % (dt[:4],dt[4:])
        if su.isValidDate(date,'%Y%m') is False : 
            self.log.error('fn %s === \tInvalid date %s!' % (fnp,date))
            return -1
              
        fd  = fdr[:6]
        if dt != fd :
            self.log.error('fn %s === \tFile Date = %s do not match trailer = %s!' % (fnp,fd,dt))
            return -2     
        
        return rowc
Exemplo n.º 7
0
    def procIncFiles(self):

        if len(self.workFiles) < 1 : 
            self.log.error('No files to process')
            return 1
        
        ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)
        
        self.workFiles.sort()
        rc = 0
        
        # Files in the working directory:
        i = 1
        self.log.debug('Will Process a total of %d file(s) ' % len(self.workFiles))
        for fnp in self.workFiles:
            self.log.info('\nProcessing File (%d) =>  %s ' % (i,fnp))
 
            # Get date run  from 1st filename
            fn    = fu.getFileBaseName(fnp)
            cur_dayr = self._getDateRunStr(fn)
            
            if cur_dayr is None : 
                self.log.error('No  Date String %s ' % cur_dayr) 
                return 1
 
            date = '%s/%s/%s' % (cur_dayr[4:6],cur_dayr[6:8],cur_dayr[:4])

            rc = su.isValidDate(date)
            if rc is False :
                self.log.error('Invalid Date %s on file %s ' % (date,fn))
                return 2
            
            self.fileDate =  date
            self.log.debug('self.fileDate = %s' % (self.fileDate))
            
            if self.checkNextRunFlg is True:
                # Get Previous Run Info. File should contain one line only :  YYYYMMDD from storage.
                prev_dayr = self._getCtlFile()
                if prev_dayr is None : return 3
                
                pd,pr     = self._getDay(prev_dayr,DP_LEN)
                
                rc        = self._chkNextRun(cur_dayr,prev_dayr,pd,pr,RUN_PER_DAY)
                if rc != 0 : 
                    self.log.error("self._chkNextRun rc = %s" % rc)
                    return rc  

            procFiles = self.chkTrailer([fnp,],fn,cur_dayr)
            
            if len(procFiles) != self.FILE_SET_LEN : 
                self.log.error("chkTrailer Files that were OK ",  procFiles)
                return 4
                      
            t = '%s/%s' % (self.ib.workDir , self.ib.srcFile)
            rc = fu.moveFile(fnp, t)
            if rc != 0 : 
                self.log.error('Could not move File %s to %s' % (fnp,t))
                return 5
                                    
            # Invoke workflow.   
            rc = pi.runWkflWait(self.ib,self.log)  
            if rc != 0 : 
                self.log.error('Running  %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
                if self.exitOnError: 
                    self.log.debug('ExitOnError is TRUE rc = %s' % (rc))
                    return rc
            else : 
                self.log.info('Ran  %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
            
            # PostCheck Every File, before updating flag:
            if self.postCheckFlg is True :
                rc = self._postCheck()
                if rc != 0 :
                    self.log.warn('Post Check Failed !!! Did not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc))
                    return rc
                
            # Loading Staging Succeeded. Update the control file.
            rc = fu.updFile(ctlFile,cur_dayr)               
            if rc == 0 :
                if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to  %s , Control File %s' % (prev_dayr,cur_dayr,   ctlFile))
                else                   : self.log.info('Overwriting Cur Load Date to  %s , Control File %s' % (cur_dayr,   ctlFile))
            else       : 
                self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc))
                return rc 
                            
            # r = fu.delFile(t) 
            i+=1  
        
        return rc
Exemplo n.º 8
0
    def splitMainCntFile(self):
        #Get Header.
        fnp = '%s/%s' % (self.ib.workDir,self.srcFile)
        s   = fu.readFile(fnp)
        hdr = su.remSpacefrStr(s)
        if len(hdr) != 38 :
            self.log.error('Invalid len= %d  for %s. Need to be 38\nhdr=%s' % (len(hdr),fnp,hdr))
            return 1
    
        self.log.debug('hdr = %s' % hdr)
        
        # Get counts 
        po870D30 = su.toInt(hdr[0:8])
        po875D30 = su.toInt(hdr[8:16])
        po880D30 = su.toInt(hdr[16:24])
        po225D15 = su.toInt(hdr[24:32])
        rdate    = hdr[32:]
        
        self.log.debug('po870D30=%s po875D30=%s po880D30=%s po225D15=%s rdate=%s' % (po870D30, po875D30,po880D30,po225D15,rdate))
        
        # Sanity check.
        if  (po870D30 is None or po875D30 is None or po880D30 is None or po225D15 is None or su.isValidDate(rdate,'%m%d%y') == False) :
            self.log.error('po870D30=%s po875D30=%s po880D30=%s po225D15=%s rdate=%s' % (po870D30, po875D30,po880D30,po225D15,rdate))
            return 2
        
        # Create count files.
        rc = 0 
        r = fu.createFile('%s/po870d30comp.cnt' % self.ib.workDir, '%08d%s' % (po870D30,rdate))  ; rc+= r
        self.log.info (' Creating file %s/po870d30comp.cnt rc = %s' % (self.ib.workDir, r))
        r = fu.createFile('%s/po875D30.cnt' % self.ib.workDir, '%08d%s' % (po875D30,rdate))  ; rc+= r
        self.log.info (' Creating file %s/po875d30.cnt rc = %s' % (self.ib.workDir, r))
        r = fu.createFile('%s/po880D30.cnt' % self.ib.workDir, '%08d%s' % (po880D30,rdate))  ; rc+= r
        self.log.info (' Creating file %s/po880d30.cnt rc = %s' % (self.ib.workDir, r))
        r = fu.createFile('%s/po225D15.cnt' % self.ib.workDir, '%08d%s' % (po225D15,rdate))  ; rc+= r
        self.log.info (' Creating file %s/po225d15.cnt rc = %s' % (self.ib.workDir, r))

        return rc 
Exemplo n.º 9
0
    def procCortFiles(self):
            
        dataFile = []    
        ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)     
        
        setn  = len (self.procCort)     
        self.log.info( ' ---- Starting Processing. Total of  %d iteration(s) ----' % setn)            
        fileset = self.procCort.keys()
        fileset.sort()
        self.log.debug('Process Order = ' , fileset)

        for fn in fileset:
            f = '%s/%s' % (self.ib.landDir,fn)
            if not fu.fileExists(f):
                 self.log.error('File %s have not been downloaded!' % f)
                 return 1
       
        # Processing loop. 
        for fn in fileset:
            self.log.info( 'Processing file %s/%s ' % (self.ib.landDir,fn))
            cur_dayr = self._getDateRunStr(fn)
            if cur_dayr is None : 
                self.log.error('cur_dayr is None. No Date Run String ') 
                return 2
    
            # cur_dayr = YYYYMMDD from filename . eg. Cortera_Trx_Extract_20130502.txt
            # pdate format YYYY-MM-DD
            pdate = '%s-%s-%s' % (cur_dayr[:4],cur_dayr[4:6],cur_dayr[6:8])
           
            self.log.debug('cur_dayr = %s pdate = %s' % (cur_dayr,pdate))     
            rc = su.isValidDate(pdate,'%Y-%m-%d')
            if rc is False :
                self.log.error('Invalid Date %s on file %s ' % (pdate,fn))
                return 2
   

            self.log.debug('self.checkNextRunFlg is %s' %  self.checkNextRunFlg)
            if self.checkNextRunFlg is True:
                prev_dayr = self._getCtlFile()
                pd,pr     = self._getDay(prev_dayr,DP_LEN)
                rc        = self._chkNextRun(cur_dayr,prev_dayr,pd,pr,RUN_PER_DAY)
                if rc != 0 : 
                    self.log.error("self._chkNextRun rc = %s" % rc)
                    return rc

            # Cortera
            t   = '%s/%s' % (self.ib.workDir,self.ib.srcFile[1])
            fnp = '%s/%s' % (self.ib.landDir,fn)
            cl = self.procCort[fn]
            
            rc = self._chkDataRecCnt(fnp,cl)
            if rc != 0 : return rc
          
            rc = fu.copyFile(fnp, t)
            if rc != 0 : 
                self.log.error('Could not copy File %s to %s' % (fnp,t))
                return 5
            self.log.info ('Copying File from %s to %s' % (fnp,t))
 
            rc = self.archGenFiles([fnp,], self.ts)                           
            self.log.info ('Archiving file rc =  %s' % rc)
 
            # Invoke workflow.   
            rc = pi.runWkflWait(self.ib,self.log)  
            if rc != 0 : 
                self.log.error('Running  %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
                if self.exitOnError: 
                    self.log.debug('ExitOnError is TRUE rc = %s' % (rc))
                    return rc
            else : 
                self.log.info('Ran  %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
            
            # Update Control table in Netezza with date.  
            rc = self._updLeaseCrdTbl(pdate)
            if rc != 0 :
                self.log.error('No rows were updated for process date %s' % pdate)
                return rc
                 
            # Loading Staging Succeeded. Update the control file.
            rc = fu.updFile(ctlFile,cur_dayr)               
            if rc == 0 :
                if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to  %s , Control File %s' % (prev_dayr,cur_dayr,   ctlFile))
                else                   : self.log.info('Overwriting Cur Load Date to  %s , Control File %s' % (cur_dayr,   ctlFile))
            else       : 
                self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc))
                return rc 
            
                     
            r = fu.delFile(t) 
            self.log.info('Removing %s rc = %s ' % (t,r))    
        
        # Delete BU Region. 
        fnp = '%s/%s' % (self.ib.workDir,self.ib.srcFile[0])    # BU_region       
        rc = fu.delFile(fnp)   
        self.log.info('Removing %s rc = %s ' % (fnp,rc))         
                  
        return rc
Exemplo n.º 10
0
    def procIncFiles(self):

        if len(self.workFiles) < 1 : 
            self.log.error('No files to process')
            return 1
        
        ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)
        
        self.workFiles.sort()
        rc = 0
        
        # Files in the working directory:
        i = 1
        self.log.debug('Will Process a total of %d file(s) ' % len(self.workFiles))
        for fnp in self.workFiles:
            self.log.info('\nProcessing File (%d) =>  %s ' % (i,fnp))
 
            # Get date run  from 1st filename
            fn    = fu.getFileBaseName(fnp)
            cur_dayr = self._getDateRunStr(fn)
            
            if cur_dayr is None : 
                self.log.error('No  Date String %s ' % cur_dayr) 
                return 1
 
            date = '%s%s' % (cur_dayr[:4],cur_dayr[4:6])  

            rc = su.isValidDate(date,'%Y%m')
            if rc is False :
                self.log.error('Invalid Date %s on file %s ' % (date,fn))
                return 2
            
            self.fileDate =  date
            self.log.debug('self.fileDate = %s' % (self.fileDate))
 
            ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)                
            self.log.debug('self.checkNextRunFlg is %s' %  self.checkNextRunFlg)
            prev_dayr = self._getCtlFile()
            
            if self.checkNextRunFlg is True:
                
                if prev_dayr is None or prev_dayr.strip() == '': 
                    self.log.error("Could not find control file or No Data")
                    return -1
                
                rc = psc.getNextRunDate(prev_dayr, cur_dayr, SCH_FREQ, self.log,sch)
                if rc != 0 : 
                    self.log.error("self._chkNextRun rc = %s" % rc)
                    return rc
     
            procFiles = self.chkTrailer([fnp,],fn,cur_dayr)
            
            if len(procFiles) != self.FILE_SET_LEN : 
                self.log.error("chkTrailer Files that were OK ",  procFiles)
                return 4
                      
            t = '%s/%s' % (self.ib.workDir , self.ib.srcFile)
            self.log.debug('fnp =%s Move to %s ' % (fnp,t))
            rc = fu.moveFile(fnp, t)
            if rc != 0 : 
                self.log.error('Could not move File %s to %s' % (fnp,t))
                return 5
            
            self.log.info('mv src file fnp %s -> t %s' % (fnp,t))                        
            # Invoke workflow.   
            rc = pi.runWkflWait(self.ib,self.log)  
            if rc != 0 : 
                self.log.error('Running  %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
                if self.exitOnError: 
                    self.log.debug('ExitOnError is TRUE rc = %s' % (rc))
                    return rc
            else : 
                self.log.info('Ran  %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
            
            # Loading Staging Succeeded. Update the control file.
            rc = fu.updFile(ctlFile,cur_dayr)               
            if rc == 0 :
                if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to  %s , Control File %s' % (prev_dayr,cur_dayr,   ctlFile))
                else                   : self.log.info('Overwriting Cur Load Date to  %s , Control File %s' % (cur_dayr,   ctlFile))
            else       : 
                self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc))
                return rc 
            
            i+=1  
        
        return rc
Exemplo n.º 11
0
    def procIncFiles(self):

        if len(self.workFiles) < 1:
            self.log.error("No files to process")
            return 1

        ctlFile = "%s/%s.ctl" % (self.ib.ctlDir, self.appName)

        self.workFiles.sort()
        rc = 0

        # Files in the working directory:
        i = 1
        self.log.debug("Will Process a total of %d file(s) " % len(self.workFiles))
        for fnp in self.workFiles:
            self.log.info("\nProcessing File (%d) =>  %s " % (i, fnp))

            # Get date run  from 1st filename
            fn = fu.getFileBaseName(fnp)
            cur_dayr = self._getDateRunStr(fn)

            if cur_dayr is None:
                self.log.error("No  Date String %s " % cur_dayr)
                return 1

            fmt = "%Y%m"
            date = "%s%s" % (cur_dayr[0:4], cur_dayr[4:6])

            rc = su.isValidDate(date, fmt)
            if rc is False:
                self.log.error("Invalid Date %s on file %s " % (date, fn))
                return 2

            self.fileDate = date
            self.log.debug("self.fileDate = %s" % (self.fileDate))

            if self.checkNextRunFlg is True:
                # Get Previous Run Info. File should contain one line only :  YYYYMM from storage.
                prev_dayr = self._getCtlFile()
                if prev_dayr is None:
                    return 3

                pd, pr = self._getMonth(prev_dayr, DP_LEN)
                if pd is None:
                    return 4

                # rc = self._chkNextRun(cur_dayr,prev_dayr,pd,pr,RUN_PER_MTH)
                rc = psch.getNextRunDate(pd, cur_dayr, "Mthly", self.log)
                if rc != 0:
                    self.log.error("self._chkNextRun rc = %s" % rc)
                    return rc

            if self.verifyCSVFlg is True:
                r, b = fu.readCSV(fnp, FLDxROW, SEP)

                if len(b) > 0:
                    fbad = "%s/%s.bad" % (self.ib.badDir, fn)
                    rc = fu.createFile(fbad, b)
                    self.log.error("No of %d bad row(s) on %s" % (len(b), fnp))
                    self.log.error("Creating file %s rc = %s" % (fbad, rc))
                    self.log.debug("Bad rows = , ", b)
                    return 5

                if len(r) == 0:
                    self.log.error("No rows to process on file %s" % fnp)
                    return 6

            t = "%s/%s" % (self.ib.workDir, self.ib.srcFile[0])
            rc = fu.moveFile(fnp, t)
            self.log.info("Renaming %s to %s rc = %s" % (fnp, t, rc))
            if rc != 0:
                self.log.error("Could not move File %s to %s" % (fnp, t))
                return 7

            # Invoke workflow.
            rc = self._wkfIMSSftyLoc()
            if rc != 0:
                self.log.error("Running  %s.%s rc = %s" % (self.ib.fld, self.ib.wkf, rc))
                if self.exitOnError:
                    self.log.debug("ExitOnError is TRUE rc = %s" % (rc))
                    return rc
            else:
                self.log.info("Ran  %s.%s rc = %s" % (self.ib.fld, self.ib.wkf, rc))

            # Loading Staging Succeeded. Update the control file.
            rc = fu.updFile(ctlFile, cur_dayr)
            if rc == 0:
                if self.checkNextRunFlg:
                    self.log.info(
                        "Updated Cur Load Date from %s to  %s , Control File %s" % (prev_dayr, cur_dayr, ctlFile)
                    )
                else:
                    self.log.info("Overwriting Cur Load Date to  %s , Control File %s" % (cur_dayr, ctlFile))
            else:
                self.log.error("Could not Update Load Date %s, Control File %s rc = %s" % (cur_dayr, ctlFile, rc))
                return rc

            r = fu.delFile(t)
            self.log.debug("Deleting File %s rc = %s" % (t, r))
            i += 1

        return rc