Esempio n. 1
0
    def procVehPOLIWkly(self):
        ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)   
        prev_dayr = self._getCtlFile()             
        self.log.debug('Ctl File is %s self.checkNextRunFlg is %s prev_dayr=%s' %  (ctlFile,self.checkNextRunFlg,prev_dayr))
        
        if self.checkNextRunFlg is True:
            
            if prev_dayr is None or prev_dayr.strip() == '': 
                self.log.error("Could not find control file or No Data")
                return -1
            
            rc = psc.getNextRunDate(prev_dayr, cur_dayr, SCH_FREQ, self.log)
            if rc != 0 : 
                self.log.error("self._chkNextRun rc = %s" % rc)
                return rc
 
        # Run workflows
        if self._wkfVehPOWkly()       != 0 : return 1
     
        # Loading Staging Succeeded. Update the control file.
        rc = fu.updFile(ctlFile,cur_dayr)               
        if rc == 0 :
            if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to  %s , Control File %s'     % (prev_dayr,cur_dayr, ctlFile))
            else                   : self.log.info('Overwriting Cur Load Date from %s to  %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile))
        else       : 
            self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc))
            return rc 
        
        for i in range (len(self.srcFile)):   
            t   = '%s/%s' % (self.ib.workDir,self.srcFile[i]) 
            r = fu.delFile(t) 
            self.log.info('Removing %s rc = %s ' % (t,r))     
        return rc
Esempio n. 2
0
    def procIncFiles(self):
     
        ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)
        self.log.debug('self.checkNextRunFlg is %s. Loading Control File %s ' %  (self.checkNextRunFlg,ctlFile))
        if self.checkNextRunFlg is True:
                
            # Get Previous Run Info. File should contain one line only :  YYYYMMDD_R from storage.
            prev_dayr = self._getCtlFile()      
            if prev_dayr is None or prev_dayr.strip() == '':
                self.log.error("Could not find control file or No Data")
                return -1

            rc = psc.getNextRunDate(prev_dayr, cur_dayr, SCH_FREQ, self.log)
            if rc != 0 : 
                self.log.error("self._chkNextRun rc = %s" % rc)
                return rc  
            
        self.log.debug('self.runWkfFlowFlg is %s' %  self.runWkfFlowFlg)
        if self.runWkfFlowFlg == True:
            rc = self.wflSdlAribaDly()
            if rc != 0 : return rc 

            # End to End Loading Succeeded. Update the control file only after wkf ran.
            rc = fu.updFile(ctlFile,cur_dayr)
            if rc == 0 : self.log.info('Updated Cur Load Date %s, Control File %s' % (cur_dayr,ctlFile))
            else       :
                self.log.error('Could not Update Load Date %s, Control File rc = %s' % (cur_dayr,ctlFile,rc))
                return rc
        return rc   
Esempio n. 3
0
    def procClaims(self):
        ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)                
        self.log.debug('self.checkNextRunFlg is %s' %  self.checkNextRunFlg)
        prev_dayr = self._getCtlFile()
        
        if self.checkNextRunFlg is True:
            
            if prev_dayr is None or prev_dayr.strip() == '': 
                self.log.error("Could not find control file or No Data")
                return -1
            
            rc = psc.getNextRunDate(prev_dayr, cur_dayr, SCH_FREQ, self.log,sch)
            if rc != 0 : 
                self.log.error("self._chkNextRun rc = %s" % rc)
                return rc
 
        # Run workflows
        if self._wkf_claims_mthly()      != 0 : return 1
     
        # Loading Staging Succeeded. Update the control file.
        rc = fu.updFile(ctlFile,cur_dayr)               
        if rc == 0 :
            if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to  %s , Control File %s'     % (prev_dayr,cur_dayr, ctlFile))
            else                   : self.log.info('Overwriting Cur Load Date from %s to  %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile))
        else       : 
            self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc))
        
        return rc 
Esempio n. 4
0
    def procLocationDim(self):
        ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)                
        self.log.debug('self.checkNextRunFlg is %s' %  self.checkNextRunFlg)
        prev_dayr = self._getCtlFile()
        
        if self.checkNextRunFlg is True:
            
            if prev_dayr is None or prev_dayr.strip() == '': 
                self.log.error("Could not find control file or No Data")
                return -1
            
            rc = psc.getNextRunDate(prev_dayr, cur_dayr, SCH_FREQ, self.log,sch)
            if rc != 0 : 
                self.log.error("self._chkNextRun rc = %s" % rc)
                return rc
 
        # Run workflows
        if self.loc_cbu_mthly_flg: 
            if self._wkf_loc_cbu_mthly()    != 0 : return 1
        if self.loc_geo_mthly_flg  :
            if self._wkf_loc_geo_mthly()    != 0 : return 1
        if self.loc_area_mthly_flg  :    
            if self._wkf_loc_area_mthly()   != 0 : return 1
        if self.loc_rfp_mthly_flg :    
            if self._wkf_loc_rfp_mthly()    != 0 : return 1
        if self.loc_admin_mthly_flg :    
            if self._wkf_loc_admin_mthly()  != 0 : return 1
        if self.loc_walker_mthly_flg:    
            if self._wkf_loc_walker_mthly() != 0 : return 1
        if self.loc_team_mthly_flg  :    
            if self._wkf_loc_team_mthly()   != 0 : return 1    
        if self.emp_dim_mthly_flg   :    
            if self.getFtpFiles()          != 0 : return 1
            if self.getIncFiles()      != 0 : return 1
            if self.cpSrcToTgtFiles()     != 0 : return 1
            rc = self.archFiles()
            self.log.info("archFilesTS() rc = %s" % rc )
            if self._wkf_emp_dim_mthly()   != 0 : return 1
     
        # Loading Staging Succeeded. Update the control file.
        rc = fu.updFile(ctlFile,cur_dayr)               
        if rc == 0 :
            if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to  %s , Control File %s'     % (prev_dayr,cur_dayr, ctlFile))
            else                   : self.log.info('Overwriting Cur Load Date from %s to  %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile))
        else       : 
            self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc))
        
        return rc 
Esempio n. 5
0
    def procCompFiles(self):
        ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)                
        self.log.debug('self.checkNextRunFlg is %s' %  self.checkNextRunFlg)
        prev_dayr = self._getCtlFile()
        if self.checkNextRunFlg is True:
            
            if prev_dayr is None or prev_dayr.strip() == '': 
                self.log.error("Could not find control file or No Data")
                return -1
            
            rc = psc.getNextRunDate(prev_dayr, cur_dayr, SCH_FREQ, self.log,sch)
            if rc != 0 : 
                self.log.error("self._chkNextRun rc = %s" % rc)
                return rc
 
        if self.checkFileRowsFlg is True:
            for f in self.trigFiles:
                self.log.debug('Processing %s (self.workfile)' % f)
                rc = self._chkHeader(f,self.ib.rdayoffset) 
                if rc != 0 : return rc
            
 
        # Invoke workflow.   
        #if self._wkfVehCompDly() != 0 : return 1
               
        # Loading Staging Succeeded. Update the control file.
        rc = fu.updFile(ctlFile,cur_dayr)               
        if rc == 0 :
            if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to  %s , Control File %s'     % (prev_dayr,cur_dayr, ctlFile))
            else                   : self.log.info('Overwriting Cur Load Date from %s to  %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile))
        else       : 
            self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc))
            return rc 
        
        for i in range (len(self.srcFile)):   
            t   = '%s/%s' % (self.ib.workDir,self.srcFile[i]) 
            r = fu.delFile(t) 
            self.log.info('Removing %s rc = %s ' % (t,r))     
        return rc
Esempio n. 6
0
    def procVehOSWkly(self):
        ctlFile = "%s/%s.ctl" % (self.ib.ctlDir, self.appName)
        prev_dayr = self._getCtlFile()
        self.log.debug(
            "Ctl File is %s self.checkNextRunFlg is %s prev_dayr=%s" % (ctlFile, self.checkNextRunFlg, prev_dayr)
        )

        if self.checkNextRunFlg is True:

            if prev_dayr is None or prev_dayr.strip() == "":
                self.log.error("Could not find control file or No Data")
                return -1

            rc = psc.getNextRunDate(prev_dayr, cur_dayr, SCH_FREQ, self.log)
            self.log.debug(" getNextRunDate rc = %s " % rc)
            if rc != 0:
                self.log.error("self._chkNextRun rc = %s" % rc)
                return rc

        # Run workflow
        if self._wkfVehOrdStatWkly() != 0:
            return 1

        # Loading Staging Succeeded. Update the control file.
        rc = fu.updFile(ctlFile, cur_dayr)
        if rc == 0:
            if self.checkNextRunFlg:
                self.log.info("Updated Cur Load Date from %s to  %s , Control File %s" % (prev_dayr, cur_dayr, ctlFile))
            else:
                self.log.info(
                    "Overwriting Cur Load Date from %s to  %s , Control File %s" % (prev_dayr, cur_dayr, ctlFile)
                )
        else:
            self.log.error("Could not Update Load Date %s, Control File %s rc = %s" % (cur_dayr, ctlFile, rc))
            return rc

        return rc
Esempio n. 7
0
    def wFinDetail(self):
        
        # Update State File, for next run. Wkf has a final command task that wilol UPdate the file to RUN.
        # If cmd task does not ran we know wkf is not successful and should not start.
        
        fn = '%s/%s.state' % (self.ib.ctlDir, self.appName)
        dt = '%s-EXECUTING' % su.getTimeSTamp('%d%02d%02d:%02d:%02d')
        self.log.debug("fn = %s Updating TO ==> dt = %s" % (fn,dt))
        rc = fu.updFile(fn,dt)
        self.log.info('Updating state returned %s' % rc)
        cmd = '%s/pstate.sh findetailmf  EXECUTING' % self.ib.appDir
        p.runSync(cmd,self.log)

        
        # Start WKF execution
        self.ib.fld     = 'Financial'
        self.ib.wkf     = 'wkf_FIN_DETAIL'
        
        rc = pi.runWkflWait(self.ib,self.log) 
        if rc != 0 : 
            self.log.error('Running  %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
        else : 
            self.log.info('Running  %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
        return rc
Esempio n. 8
0
    def procIncFiles(self):

        if len(self.workFiles) < 1 : 
            self.log.error('No files to process')
            return 1
        
        ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)
        
        self.workFiles.sort()
        rc = 0
        
        # Files in the working directory:
        i = 1
        self.log.debug('Will Process a total of %d file(s) ' % len(self.workFiles))
        for fnp in self.workFiles:
            self.log.info('\nProcessing File (%d) =>  %s ' % (i,fnp))
 
            # Get date run  from 1st filename
            fn    = fu.getFileBaseName(fnp)
            cur_dayr = self._getDateRunStr(fn)
            
            if cur_dayr is None : 
                self.log.error('No  Date String %s ' % cur_dayr) 
                return 1
 
            date = '%s/%s/%s' % (cur_dayr[4:6],cur_dayr[6:8],cur_dayr[:4])

            rc = su.isValidDate(date)
            if rc is False :
                self.log.error('Invalid Date %s on file %s ' % (date,fn))
                return 2
            
            self.fileDate =  date
            self.log.debug('self.fileDate = %s' % (self.fileDate))
            
            if self.checkNextRunFlg is True:
                # Get Previous Run Info. File should contain one line only :  YYYYMMDD from storage.
                prev_dayr = self._getCtlFile()
                if prev_dayr is None : return 3
                
                pd,pr     = self._getDay(prev_dayr,DP_LEN)
                
                rc        = self._chkNextRun(cur_dayr,prev_dayr,pd,pr,RUN_PER_DAY)
                if rc != 0 : 
                    self.log.error("self._chkNextRun rc = %s" % rc)
                    return rc  

            procFiles = self.chkTrailer([fnp,],fn,cur_dayr)
            
            if len(procFiles) != self.FILE_SET_LEN : 
                self.log.error("chkTrailer Files that were OK ",  procFiles)
                return 4
                      
            t = '%s/%s' % (self.ib.workDir , self.ib.srcFile)
            rc = fu.moveFile(fnp, t)
            if rc != 0 : 
                self.log.error('Could not move File %s to %s' % (fnp,t))
                return 5
                                    
            # Invoke workflow.   
            rc = pi.runWkflWait(self.ib,self.log)  
            if rc != 0 : 
                self.log.error('Running  %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
                if self.exitOnError: 
                    self.log.debug('ExitOnError is TRUE rc = %s' % (rc))
                    return rc
            else : 
                self.log.info('Ran  %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
            
            # PostCheck Every File, before updating flag:
            if self.postCheckFlg is True :
                rc = self._postCheck()
                if rc != 0 :
                    self.log.warn('Post Check Failed !!! Did not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc))
                    return rc
                
            # Loading Staging Succeeded. Update the control file.
            rc = fu.updFile(ctlFile,cur_dayr)               
            if rc == 0 :
                if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to  %s , Control File %s' % (prev_dayr,cur_dayr,   ctlFile))
                else                   : self.log.info('Overwriting Cur Load Date to  %s , Control File %s' % (cur_dayr,   ctlFile))
            else       : 
                self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc))
                return rc 
                            
            # r = fu.delFile(t) 
            i+=1  
        
        return rc
Esempio n. 9
0
    def procCortFiles(self):
            
        dataFile = []    
        ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)     
        
        setn  = len (self.procCort)     
        self.log.info( ' ---- Starting Processing. Total of  %d iteration(s) ----' % setn)            
        fileset = self.procCort.keys()
        fileset.sort()
        self.log.debug('Process Order = ' , fileset)

        for fn in fileset:
            f = '%s/%s' % (self.ib.landDir,fn)
            if not fu.fileExists(f):
                 self.log.error('File %s have not been downloaded!' % f)
                 return 1
       
        # Processing loop. 
        for fn in fileset:
            self.log.info( 'Processing file %s/%s ' % (self.ib.landDir,fn))
            cur_dayr = self._getDateRunStr(fn)
            if cur_dayr is None : 
                self.log.error('cur_dayr is None. No Date Run String ') 
                return 2
    
            # cur_dayr = YYYYMMDD from filename . eg. Cortera_Trx_Extract_20130502.txt
            # pdate format YYYY-MM-DD
            pdate = '%s-%s-%s' % (cur_dayr[:4],cur_dayr[4:6],cur_dayr[6:8])
           
            self.log.debug('cur_dayr = %s pdate = %s' % (cur_dayr,pdate))     
            rc = su.isValidDate(pdate,'%Y-%m-%d')
            if rc is False :
                self.log.error('Invalid Date %s on file %s ' % (pdate,fn))
                return 2
   

            self.log.debug('self.checkNextRunFlg is %s' %  self.checkNextRunFlg)
            if self.checkNextRunFlg is True:
                prev_dayr = self._getCtlFile()
                pd,pr     = self._getDay(prev_dayr,DP_LEN)
                rc        = self._chkNextRun(cur_dayr,prev_dayr,pd,pr,RUN_PER_DAY)
                if rc != 0 : 
                    self.log.error("self._chkNextRun rc = %s" % rc)
                    return rc

            # Cortera
            t   = '%s/%s' % (self.ib.workDir,self.ib.srcFile[1])
            fnp = '%s/%s' % (self.ib.landDir,fn)
            cl = self.procCort[fn]
            
            rc = self._chkDataRecCnt(fnp,cl)
            if rc != 0 : return rc
          
            rc = fu.copyFile(fnp, t)
            if rc != 0 : 
                self.log.error('Could not copy File %s to %s' % (fnp,t))
                return 5
            self.log.info ('Copying File from %s to %s' % (fnp,t))
 
            rc = self.archGenFiles([fnp,], self.ts)                           
            self.log.info ('Archiving file rc =  %s' % rc)
 
            # Invoke workflow.   
            rc = pi.runWkflWait(self.ib,self.log)  
            if rc != 0 : 
                self.log.error('Running  %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
                if self.exitOnError: 
                    self.log.debug('ExitOnError is TRUE rc = %s' % (rc))
                    return rc
            else : 
                self.log.info('Ran  %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
            
            # Update Control table in Netezza with date.  
            rc = self._updLeaseCrdTbl(pdate)
            if rc != 0 :
                self.log.error('No rows were updated for process date %s' % pdate)
                return rc
                 
            # Loading Staging Succeeded. Update the control file.
            rc = fu.updFile(ctlFile,cur_dayr)               
            if rc == 0 :
                if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to  %s , Control File %s' % (prev_dayr,cur_dayr,   ctlFile))
                else                   : self.log.info('Overwriting Cur Load Date to  %s , Control File %s' % (cur_dayr,   ctlFile))
            else       : 
                self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc))
                return rc 
            
                     
            r = fu.delFile(t) 
            self.log.info('Removing %s rc = %s ' % (t,r))    
        
        # Delete BU Region. 
        fnp = '%s/%s' % (self.ib.workDir,self.ib.srcFile[0])    # BU_region       
        rc = fu.delFile(fnp)   
        self.log.info('Removing %s rc = %s ' % (fnp,rc))         
                  
        return rc
Esempio n. 10
0
    def procIncFiles(self):

        if len(self.workFiles) < 1 : 
            self.log.error('No files to process')
            return 1
        
        ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)
        
        self.workFiles.sort()
        rc = 0
        
        # Files in the working directory:
        i = 1
        self.log.debug('Will Process a total of %d file(s) ' % len(self.workFiles))
        for fnp in self.workFiles:
            self.log.info('\nProcessing File (%d) =>  %s ' % (i,fnp))
 
            # Get date run  from 1st filename
            fn    = fu.getFileBaseName(fnp)
            cur_dayr = self._getDateRunStr(fn)
            
            if cur_dayr is None : 
                self.log.error('No  Date String %s ' % cur_dayr) 
                return 1
 
            date = '%s%s' % (cur_dayr[:4],cur_dayr[4:6])  

            rc = su.isValidDate(date,'%Y%m')
            if rc is False :
                self.log.error('Invalid Date %s on file %s ' % (date,fn))
                return 2
            
            self.fileDate =  date
            self.log.debug('self.fileDate = %s' % (self.fileDate))
 
            ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)                
            self.log.debug('self.checkNextRunFlg is %s' %  self.checkNextRunFlg)
            prev_dayr = self._getCtlFile()
            
            if self.checkNextRunFlg is True:
                
                if prev_dayr is None or prev_dayr.strip() == '': 
                    self.log.error("Could not find control file or No Data")
                    return -1
                
                rc = psc.getNextRunDate(prev_dayr, cur_dayr, SCH_FREQ, self.log,sch)
                if rc != 0 : 
                    self.log.error("self._chkNextRun rc = %s" % rc)
                    return rc
     
            procFiles = self.chkTrailer([fnp,],fn,cur_dayr)
            
            if len(procFiles) != self.FILE_SET_LEN : 
                self.log.error("chkTrailer Files that were OK ",  procFiles)
                return 4
                      
            t = '%s/%s' % (self.ib.workDir , self.ib.srcFile)
            self.log.debug('fnp =%s Move to %s ' % (fnp,t))
            rc = fu.moveFile(fnp, t)
            if rc != 0 : 
                self.log.error('Could not move File %s to %s' % (fnp,t))
                return 5
            
            self.log.info('mv src file fnp %s -> t %s' % (fnp,t))                        
            # Invoke workflow.   
            rc = pi.runWkflWait(self.ib,self.log)  
            if rc != 0 : 
                self.log.error('Running  %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
                if self.exitOnError: 
                    self.log.debug('ExitOnError is TRUE rc = %s' % (rc))
                    return rc
            else : 
                self.log.info('Ran  %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
            
            # Loading Staging Succeeded. Update the control file.
            rc = fu.updFile(ctlFile,cur_dayr)               
            if rc == 0 :
                if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to  %s , Control File %s' % (prev_dayr,cur_dayr,   ctlFile))
                else                   : self.log.info('Overwriting Cur Load Date to  %s , Control File %s' % (cur_dayr,   ctlFile))
            else       : 
                self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc))
                return rc 
            
            i+=1  
        
        return rc
Esempio n. 11
0
    def procIncFiles(self):

        # 'B' : self.getIncSetFiles, populates self.incFileSet, which is [filename][sets]  
        if len(self.incFileSet) != self.FILE_SET_LEN:
            self.log.error("Invalid Len for incFileSet = %d " % len(self.incFileSet))
            return 1
        
        ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)
        
        i = 0
        # Get complete number of fileset to process. It is an Array of 5 elem.
        # Each element is a bucket containing a list of similar files (n numebr of runs)  
        # FileSet[i] is a bucket that contains file(s) for the same table.
        # FileSet[i] is already sorted.
        # e.g.
        #[ $PATH:/Ap_sce5100_20120811_3.txt'
        #  $PATH:/Ap_sce5100_20120812_1.txt'
        #  $PATH:/Ap_sce5100_20120812_2.txt'
        #  $PATH:/Ap_sce5100_20120813_1.txt']           
        self.log.debug("self.incFileSet ",  self.incFileSet  , " len = " , len (self.incFileSet) )
  
        # Get the minimun len from each file(s) with the same pattern (self.incFileSet). It will tell us how many complete FS we have.
        setn = min(len(self.incFileSet[0]), len(self.incFileSet[1]),len(self.incFileSet[2]),len(self.incFileSet[3]),len(self.incFileSet[4]))
        self.log.info( ' ---- Starting Processing. Total of  %d iteration(s) ----' % setn)
        
        while  i < setn:
            
            self.incFiles = self.incFileSet[0][i], self.incFileSet[1][i],self.incFileSet[2][i],self.incFileSet[3][i],self.incFileSet[4][i]
            self.log.debug(' iter= ', i, '\tincFiles =', self.incFiles)
            
            i+=1
            
            if  len (self.incFiles) != self.FILE_SET_LEN : 
                self.log.error('Invalid FileSet len = %d should be = %d' % (len (self.incFiles),self.FILE_SET_LEN)) 
                return 1
            
            # Get date run  from 1st filename
            fn    = fu.getFileBaseName(self.incFiles[0])
            cur_dayr = self._getDateRunStr(fn)
            
            if cur_dayr is None : 
                self.log.error('No Date Run String %s ' % cur_dayr) 
                return 1
            
            rc = self.chkCompleteSet(cur_dayr,self.incFiles)
            if rc != 0 : 
                self.log.error("chkCompleteSet() rc = %s" % rc)
                return rc  
            
            self.log.debug('self.checkNextRunFlg is %s' %  self.checkNextRunFlg)
            if self.checkNextRunFlg is True:
                
                # Get Previous Run Info. File should contain one line only :  YYYYMMDD_R from storage.
                prev_dayr = self._getCtlFile()
                #pd,pr    = self._getDay(prev_dayr,DP_LEN)
                
                if prev_dayr is None or prev_dayr.strip() == '':
                    self.log.error("Could not find control file or No Data")
                    return -1

                #rc = self._chkNextRun(cur_dayr,prev_dayr,pd,pr,RUN_PER_DAY)
                rc = psc.getNextRunDate(prev_dayr, cur_dayr, SCH_FREQ, self.log,sch)
                if rc != 0 : 
                    self.log.error("self._chkNextRun rc = %s" % rc)
                    return rc  
            
            rc = self.cpFileToWorkDir() 
            if rc != 0 : 
                self.log.error(" cpFileToWorkDir() rc = %s" % rc)
                return rc  
            
            rc = self.archFiles()
            if rc != 0 : 
                self.log.error(" archFiles() rc = %s" % rc)
                return rc 
            
            procFiles = self.chkTrailer(self.workFiles,fn,cur_dayr)
            if len(procFiles) != self.FILE_SET_LEN : 
                self.log.error("chkTrailer Files that were OK ",  procFiles)
                return 1
            
            # At this  point all files are valid for processing and filenames are sorted.                    
            for fnp in procFiles:
                fn = fu.getFileBaseName(fnp)
                f  = self._getFileProcName(fn)
                t  = '%s/%s' % (self.ib.workDir , f)
                rc = fu.moveFile(fnp, t)
                if rc == 0 :
                    self.log.info('Renaming File %s to %s' % (fnp,f))
                else: 
                    self.log.error('Could not rename File %s to %s' % (fnp,t))
                    continue
                        
            #rc = 0      # Remove after testing 
                  
            # Invoke workflow(s).    
            self.log.debug('self.runWkfFlowFlg is %s' %  self.runWkfFlowFlg)
            if self.runWkfFlowFlg == True:
                rc = self.wFinDetailStg()
                if rc != 0 : return rc 
                
                rc = self.wFinDetail()
                if rc != 0 : return rc 

                # End to End Loading Succeeded. Update the control file.
                rc = fu.updFile(ctlFile,cur_dayr)
                if rc == 0 : self.log.info('Updated Cur Load Date %s, Control File %s' % (cur_dayr,ctlFile))
                else       :
                    self.log.error('Could not Update Load Date %s, Control File rc = %s' % (cur_dayr,ctlFile,rc))
                    return rc
                    
            #r = fu.delFile(t)
            
        return rc   
Esempio n. 12
0
    def procIncFiles(self):

        if len(self.workFiles) < 1:
            self.log.error("No files to process")
            return 1

        ctlFile = "%s/%s.ctl" % (self.ib.ctlDir, self.appName)

        self.workFiles.sort()
        rc = 0

        # Files in the working directory:
        i = 1
        self.log.debug("Will Process a total of %d file(s) " % len(self.workFiles))
        for fnp in self.workFiles:
            self.log.info("\nProcessing File (%d) =>  %s " % (i, fnp))

            # Get date run  from 1st filename
            fn = fu.getFileBaseName(fnp)
            cur_dayr = self._getDateRunStr(fn)

            if cur_dayr is None:
                self.log.error("No  Date String %s " % cur_dayr)
                return 1

            fmt = "%Y%m"
            date = "%s%s" % (cur_dayr[0:4], cur_dayr[4:6])

            rc = su.isValidDate(date, fmt)
            if rc is False:
                self.log.error("Invalid Date %s on file %s " % (date, fn))
                return 2

            self.fileDate = date
            self.log.debug("self.fileDate = %s" % (self.fileDate))

            if self.checkNextRunFlg is True:
                # Get Previous Run Info. File should contain one line only :  YYYYMM from storage.
                prev_dayr = self._getCtlFile()
                if prev_dayr is None:
                    return 3

                pd, pr = self._getMonth(prev_dayr, DP_LEN)
                if pd is None:
                    return 4

                # rc = self._chkNextRun(cur_dayr,prev_dayr,pd,pr,RUN_PER_MTH)
                rc = psch.getNextRunDate(pd, cur_dayr, "Mthly", self.log)
                if rc != 0:
                    self.log.error("self._chkNextRun rc = %s" % rc)
                    return rc

            if self.verifyCSVFlg is True:
                r, b = fu.readCSV(fnp, FLDxROW, SEP)

                if len(b) > 0:
                    fbad = "%s/%s.bad" % (self.ib.badDir, fn)
                    rc = fu.createFile(fbad, b)
                    self.log.error("No of %d bad row(s) on %s" % (len(b), fnp))
                    self.log.error("Creating file %s rc = %s" % (fbad, rc))
                    self.log.debug("Bad rows = , ", b)
                    return 5

                if len(r) == 0:
                    self.log.error("No rows to process on file %s" % fnp)
                    return 6

            t = "%s/%s" % (self.ib.workDir, self.ib.srcFile[0])
            rc = fu.moveFile(fnp, t)
            self.log.info("Renaming %s to %s rc = %s" % (fnp, t, rc))
            if rc != 0:
                self.log.error("Could not move File %s to %s" % (fnp, t))
                return 7

            # Invoke workflow.
            rc = self._wkfIMSSftyLoc()
            if rc != 0:
                self.log.error("Running  %s.%s rc = %s" % (self.ib.fld, self.ib.wkf, rc))
                if self.exitOnError:
                    self.log.debug("ExitOnError is TRUE rc = %s" % (rc))
                    return rc
            else:
                self.log.info("Ran  %s.%s rc = %s" % (self.ib.fld, self.ib.wkf, rc))

            # Loading Staging Succeeded. Update the control file.
            rc = fu.updFile(ctlFile, cur_dayr)
            if rc == 0:
                if self.checkNextRunFlg:
                    self.log.info(
                        "Updated Cur Load Date from %s to  %s , Control File %s" % (prev_dayr, cur_dayr, ctlFile)
                    )
                else:
                    self.log.info("Overwriting Cur Load Date to  %s , Control File %s" % (cur_dayr, ctlFile))
            else:
                self.log.error("Could not Update Load Date %s, Control File %s rc = %s" % (cur_dayr, ctlFile, rc))
                return rc

            r = fu.delFile(t)
            self.log.debug("Deleting File %s rc = %s" % (t, r))
            i += 1

        return rc