示例#1
0
    def procIncFiles(self):

        if len(self.workFiles) < 1 : 
            self.log.error('No files to process')
            return 1
        
        self.workFiles.sort()
        rc = 0
        
        # Files in the working directory:
        i = 1
        self.log.debug('Will Process a total of %d file(s) ' % len(self.workFiles))
        for fnp in self.workFiles:
            self.log.info('\nProcessing File (%d) =>  %s ' % (i,fnp))
            fn    = fu.getFileBaseName(fnp)
            
            if self.verifyCSVFlg is True:    
                r,b = fu.readCSV(fnp, FLDxROW, SEP)   
                
                if len(b) > 0 :  
                    fbad = '%s/%s.bad' % (self.ib.badDir,fn)                 
                    rc = fu.createFile(fbad , b)
                    self.log.error("No of %d bad row(s) on %s" % (len(b),fnp))
                    self.log.error("Creating file %s rc = %s" % (fbad,rc))
                    self.log.debug("Bad rows = , ", b)
                    return 5
                
                if len(r) == 0 :
                    self.log.error("No rows to process on file %s" % fnp)
                    return 6 
                               
            t = '%s/%s' % (self.ib.workDir , self.ib.srcFile[0])
            rc = fu.moveFile(fnp, t)
            self.log.info('Renaming %s to %s rc = %s' %  (fnp,t,rc))
            if rc != 0 : 
                self.log.error('Could not move File %s to %s' % (fnp,t))
                return 7
                                    
            # Invoke workflow.   
            rc = self._wkfCodeOvrd()  
            if rc != 0 : 
                self.log.error('Running  %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
                if self.exitOnError: 
                    self.log.debug('ExitOnError is TRUE rc = %s' % (rc))
                    return rc
            else : 
                self.log.info('Ran  %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
                           
                            
            r = fu.delFile(t) 
            self.log.debug('Deleting File %s rc = %s' % (t,r))
            i+=1  
        
        return rc
示例#2
0
def loadCredFile(fn, lt, uflg = False):
    import os.path
  
    if not os.path.exists(fn) :
        mg.logger.error('File %s does not exits ' % fn)
        return (mm.FILE_NOT_EXIST,[],[])
    
    if not os.path.isfile(fn) :
        mg.logger.error('%s is not a file' % fn)
        return (mm.NOT_FILE_OBJ, [], [])
    
    mg.logger.info( "Loading  = %s" % (fn))
    rs,bad = readCSV(fn,4)                      # rs + bad + records read
    rej=loadBulkCred(rs,lt,uflg)
    return len(rs), bad, rej
示例#3
0
    def checkFileCols(self,flst,FLD_SEP,strp=' \t\n\r'):
        rc = 0
        for fnp in flst:
            bfnm   = fu.getFileBaseName(fnp)  
            colc   = self.ib.FileColCnt.get(bfnm)
            self.log.debug('fkey = %s colc= %s fp = %s ' %(bfnm,colc,fnp))
            if colc is not None:
                    x,b = fu.readCSV(fnp,colc,FLD_SEP,strp)
                    if len(b) < 1 : 
                        self.log.debug('Columns number [%d] match on file %s ' % (colc,fnp)) 
                    else :
                        rc += 1         
                        badf = '%s/%s%s.bad' % (self.ib.badDir,bfnm,su.getTimeSTamp())
                        f = fu.createFile(badf,b)
                        if f == 0 : self.log.error('Columns number (%d) did not match on %s.\nPlease see Bad file %s .' % (colc,fnp,badf))
                        else      :
                            self.log.error('Columns number (%d) did not match on %s.\n.COULD NOT  create file  %s .' % (colc,fnp,badf))
                            self.log.error('BAD Rows===========\n', b)     
            else:
                rc += 1
                self.log.error("Did not find Column Count for %s. Unable to verify Column Numbers !" % bfnm)   

        return rc
示例#4
0
    def procIncFiles(self):

        if len(self.workFiles) < 1:
            self.log.error("No files to process")
            return 1

        ctlFile = "%s/%s.ctl" % (self.ib.ctlDir, self.appName)

        self.workFiles.sort()
        rc = 0

        # Files in the working directory:
        i = 1
        self.log.debug("Will Process a total of %d file(s) " % len(self.workFiles))
        for fnp in self.workFiles:
            self.log.info("\nProcessing File (%d) =>  %s " % (i, fnp))

            # Get date run  from 1st filename
            fn = fu.getFileBaseName(fnp)
            cur_dayr = self._getDateRunStr(fn)

            if cur_dayr is None:
                self.log.error("No  Date String %s " % cur_dayr)
                return 1

            fmt = "%Y%m"
            date = "%s%s" % (cur_dayr[0:4], cur_dayr[4:6])

            rc = su.isValidDate(date, fmt)
            if rc is False:
                self.log.error("Invalid Date %s on file %s " % (date, fn))
                return 2

            self.fileDate = date
            self.log.debug("self.fileDate = %s" % (self.fileDate))

            if self.checkNextRunFlg is True:
                # Get Previous Run Info. File should contain one line only :  YYYYMM from storage.
                prev_dayr = self._getCtlFile()
                if prev_dayr is None:
                    return 3

                pd, pr = self._getMonth(prev_dayr, DP_LEN)
                if pd is None:
                    return 4

                # rc = self._chkNextRun(cur_dayr,prev_dayr,pd,pr,RUN_PER_MTH)
                rc = psch.getNextRunDate(pd, cur_dayr, "Mthly", self.log)
                if rc != 0:
                    self.log.error("self._chkNextRun rc = %s" % rc)
                    return rc

            if self.verifyCSVFlg is True:
                r, b = fu.readCSV(fnp, FLDxROW, SEP)

                if len(b) > 0:
                    fbad = "%s/%s.bad" % (self.ib.badDir, fn)
                    rc = fu.createFile(fbad, b)
                    self.log.error("No of %d bad row(s) on %s" % (len(b), fnp))
                    self.log.error("Creating file %s rc = %s" % (fbad, rc))
                    self.log.debug("Bad rows = , ", b)
                    return 5

                if len(r) == 0:
                    self.log.error("No rows to process on file %s" % fnp)
                    return 6

            t = "%s/%s" % (self.ib.workDir, self.ib.srcFile[0])
            rc = fu.moveFile(fnp, t)
            self.log.info("Renaming %s to %s rc = %s" % (fnp, t, rc))
            if rc != 0:
                self.log.error("Could not move File %s to %s" % (fnp, t))
                return 7

            # Invoke workflow.
            rc = self._wkfIMSSftyLoc()
            if rc != 0:
                self.log.error("Running  %s.%s rc = %s" % (self.ib.fld, self.ib.wkf, rc))
                if self.exitOnError:
                    self.log.debug("ExitOnError is TRUE rc = %s" % (rc))
                    return rc
            else:
                self.log.info("Ran  %s.%s rc = %s" % (self.ib.fld, self.ib.wkf, rc))

            # Loading Staging Succeeded. Update the control file.
            rc = fu.updFile(ctlFile, cur_dayr)
            if rc == 0:
                if self.checkNextRunFlg:
                    self.log.info(
                        "Updated Cur Load Date from %s to  %s , Control File %s" % (prev_dayr, cur_dayr, ctlFile)
                    )
                else:
                    self.log.info("Overwriting Cur Load Date to  %s , Control File %s" % (cur_dayr, ctlFile))
            else:
                self.log.error("Could not Update Load Date %s, Control File %s rc = %s" % (cur_dayr, ctlFile, rc))
                return rc

            r = fu.delFile(t)
            self.log.debug("Deleting File %s rc = %s" % (t, r))
            i += 1

        return rc