def relLock(self): if self.lock: try: fu.delFile(self.lckFn) self.log.info('Released lock: %s for pid %s' % (self.lckFn,self.pid)) return 0 except: self.log.error("==EXCEP %s %s" % (sys.exc_type,sys.exc_value)) return 1 else: self.log.warn('No lock to release for: %s for pid %s' % (self.lckFn,self.pid)) return 0
def procVehPOLIWkly(self): ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName) prev_dayr = self._getCtlFile() self.log.debug('Ctl File is %s self.checkNextRunFlg is %s prev_dayr=%s' % (ctlFile,self.checkNextRunFlg,prev_dayr)) if self.checkNextRunFlg is True: if prev_dayr is None or prev_dayr.strip() == '': self.log.error("Could not find control file or No Data") return -1 rc = psc.getNextRunDate(prev_dayr, cur_dayr, SCH_FREQ, self.log) if rc != 0 : self.log.error("self._chkNextRun rc = %s" % rc) return rc # Run workflows if self._wkfVehPOWkly() != 0 : return 1 # Loading Staging Succeeded. Update the control file. rc = fu.updFile(ctlFile,cur_dayr) if rc == 0 : if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile)) else : self.log.info('Overwriting Cur Load Date from %s to %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile)) else : self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc)) return rc for i in range (len(self.srcFile)): t = '%s/%s' % (self.ib.workDir,self.srcFile[i]) r = fu.delFile(t) self.log.info('Removing %s rc = %s ' % (t,r)) return rc
def _procOpsReg(self): concatFls = ( "Transcripts_OpsRegion.txt", "Transcripts_OpsRegion_Canada.txt", "Transcripts_OpsRegion_Central.txt", "Transcripts_OpsRegion_Northeast.txt", "Transcripts_OpsRegion_Southeast.txt", "Transcripts_OpsRegion_West.txt", ) concatFlsFP = [] for fn in concatFls: concatFlsFP.append("%s/%s" % (self.ib.workDir, fn)) self.log.debug("concatFlsFP = ", concatFlsFP) # Remove Concatenated file: rc = fu.delFile(concatFlsFP[0]) self.log.debug("Removing %s\trc=%s" % (concatFlsFP[0], rc)) self.log.debug("Creating HDR file %s" % concatFlsFP[0]) rc = fu.createFile(concatFlsFP[0], self.ib.opsRegHdrRow) if rc != 0: self.log.error("Could not create file %s" % (concatFlsFP[0])) return 2 # Concatenate the files rc = fu.concatFile(concatFlsFP) self.log.info("concatFile rc = %s" % rc) return rc
def relLock(self): if self.lock: try: fu.delFile(self.lckFn) self.log.info('Released lock: %s for pid %s' % (self.lckFn, self.pid)) return 0 except: self.log.error("==EXCEP %s %s" % (sys.exc_type, sys.exc_value)) return 1 else: self.log.warn('No lock to release for: %s for pid %s' % (self.lckFn, self.pid)) return 0
def _chkIfValidLock(self): rc = True # Check if there is a lock file. if fu.fileExists(self.lckFn): sPid = fu.readFile(self.lckFn) pid = su.toInt(sPid) self.log.info('Lock File %s EXISTS. sPid = %s , pid = %s' % (self.lckFn, sPid, pid)) # Check if file has a valid PID (number) if pid is None: rc = fu.delFile(self.lckFn) if rc == 0: self.log.info('Removed File %s' % (self.lckFn)) else: self.log.error('Could not removed File %s' % (self.lckFn)) return False # If pid is a valid number, check if the process is running ... rc = ps.isProcRunning(pid, self.log) self.log.debug('isProcRunning returned %s' % (rc)) return rc # No lock file exists. else: return False
def procEmpBenefit(self): # Run workflows if self._wkfEmpBenefitonDem() != 0: return 1 for i in range(len(self.srcFile)): t = "%s/%s" % (self.ib.workDir, self.srcFile[i]) r = fu.delFile(t) self.log.info("Removing %s rc = %s " % (t, r)) return 0
def procIncFiles(self): if len(self.workFiles) < 1 : self.log.error('No files to process') return 1 self.workFiles.sort() rc = 0 # Files in the working directory: i = 1 self.log.debug('Will Process a total of %d file(s) ' % len(self.workFiles)) for fnp in self.workFiles: self.log.info('\nProcessing File (%d) => %s ' % (i,fnp)) fn = fu.getFileBaseName(fnp) if self.verifyCSVFlg is True: r,b = fu.readCSV(fnp, FLDxROW, SEP) if len(b) > 0 : fbad = '%s/%s.bad' % (self.ib.badDir,fn) rc = fu.createFile(fbad , b) self.log.error("No of %d bad row(s) on %s" % (len(b),fnp)) self.log.error("Creating file %s rc = %s" % (fbad,rc)) self.log.debug("Bad rows = , ", b) return 5 if len(r) == 0 : self.log.error("No rows to process on file %s" % fnp) return 6 t = '%s/%s' % (self.ib.workDir , self.ib.srcFile[0]) rc = fu.moveFile(fnp, t) self.log.info('Renaming %s to %s rc = %s' % (fnp,t,rc)) if rc != 0 : self.log.error('Could not move File %s to %s' % (fnp,t)) return 7 # Invoke workflow. rc = self._wkfCodeOvrd() if rc != 0 : self.log.error('Running %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc)) if self.exitOnError: self.log.debug('ExitOnError is TRUE rc = %s' % (rc)) return rc else : self.log.info('Ran %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc)) r = fu.delFile(t) self.log.debug('Deleting File %s rc = %s' % (t,r)) i+=1 return rc
def _remFiles(self, flp): rc = 0 if len(flp) < 1 : self.log.warn('File list to delete is empty !') return 1 #fns = fl.split(',') for fnp in flp: r = fu.delFile(fnp) if r != 0 : rc += 1 self.log.error ("Could not delete %s r = %s " % (fnp, r)) else : self.log.info ("Removed %s r = %d " % (fnp, r)) return rc
def checkApprovFile(self): fl = fu.getFileName(self.ib.landDir, self.ib.fileName) if len(fl) == 0: i = 1 self.log.info('Approval File %s/%s does not exists. Will start in %s sec' % (self.ib.landDir, self.ib.fileName, self.ib.waitAppFile)) wi = int(self.ib.waitFileAppIter) while (i <= wi): time.sleep(float(self.ib.waitAppFile)) fl = fu.getFileName(self.ib.landDir, self.ib.fileName) self.log.info('Iteration %d out of %d ' % (i, wi)) if len(fl) > 0 : break else : i += 1 if len(fl) == 0 : return 1 else: rc = fu.delFile(fl[0]) self.log.info('Removing %s , rc = %s ' % (fl[0], rc)) return 0
def procCompFiles(self): ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName) self.log.debug('self.checkNextRunFlg is %s' % self.checkNextRunFlg) prev_dayr = self._getCtlFile() if self.checkNextRunFlg is True: if prev_dayr is None or prev_dayr.strip() == '': self.log.error("Could not find control file or No Data") return -1 rc = psc.getNextRunDate(prev_dayr, cur_dayr, SCH_FREQ, self.log,sch) if rc != 0 : self.log.error("self._chkNextRun rc = %s" % rc) return rc if self.checkFileRowsFlg is True: for f in self.trigFiles: self.log.debug('Processing %s (self.workfile)' % f) rc = self._chkHeader(f,self.ib.rdayoffset) if rc != 0 : return rc # Invoke workflow. #if self._wkfVehCompDly() != 0 : return 1 # Loading Staging Succeeded. Update the control file. rc = fu.updFile(ctlFile,cur_dayr) if rc == 0 : if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile)) else : self.log.info('Overwriting Cur Load Date from %s to %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile)) else : self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc)) return rc for i in range (len(self.srcFile)): t = '%s/%s' % (self.ib.workDir,self.srcFile[i]) r = fu.delFile(t) self.log.info('Removing %s rc = %s ' % (t,r)) return rc
def _insNZLoad(self,val,seqid): ldata = [] for s,u,p,d in val : ldata.append("%s,%s,%s,%s\n" % (s,u,p,d )) data = ''.join(ldata) # Create CSV file fn = '%s/%s.csv' % (self.ib.dataDir,seqid) fl = '%s/tis_fb_carr_pmpt_imprt_dtl_%s.%s.nzlog' % (self.ib.dataDir,seqid,self.ib.db) bf = '%s/tis_fb_carr_pmpt_imprt_dtl_%s.%s.nzbad' % (self.ib.dataDir,seqid,self.ib.db) rc = fu.createFile(fn, data) self.log.debug('File Creation %s/%s.csv rc = %s' % (d,seqid,rc)) if rc != 0 : self.log.error('File Creation %s/%s.csv rc = %s' % (d,seqid,rc)) return rc # Invoke nzload command nz_cmd = "%s/nzload -host %s -u %s -pw %s -db %s -t tis_fb_carr_pmpt_imprt_dtl -delim '%s' -df %s -lf %s -bf %s" % (self.ib.nzbin, self.ib.dbserver, self.ib.user, self.ib.pwd, self.ib.db, self.ib.delim, fn,fl,bf) self.log.debug('nz_cmd = %s' % nz_cmd) rc,rmsg = proc.runSync(nz_cmd,self.log) if rc != 0 : self.log.error('cmd %s rc = %s -> msg= %s' % (nz_cmd,rc,rmsg)) return rc # Delete csv file (based on option) if self.delCSVFileFlg is True: r = fu.delFile(fn) self.log.debug('Deleting %s rc = %s ' % (fn,r)) if r != 0 : self.log.error('Cannot Delete %s ' % fn ) return rc
def _chkIfValidLock(self): rc = True # Check if there is a lock file. if fu.fileExists(self.lckFn): sPid = fu.readFile(self.lckFn) pid = su.toInt(sPid) self.log.info('Lock File %s EXISTS. sPid = %s , pid = %s' % (self.lckFn,sPid,pid)) # Check if file has a valid PID (number) if pid is None : rc = fu.delFile(self.lckFn) if rc == 0 : self.log.info('Removed File %s' % (self.lckFn)) else: self.log.error('Could not removed File %s' % (self.lckFn)) return False # If pid is a valid number, check if the process is running ... rc = ps.isProcRunning(pid,self.log) self.log.debug('isProcRunning returned %s' % (rc)) return rc # No lock file exists. else: return False
def procCortFiles(self): dataFile = [] ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName) setn = len (self.procCort) self.log.info( ' ---- Starting Processing. Total of %d iteration(s) ----' % setn) fileset = self.procCort.keys() fileset.sort() self.log.debug('Process Order = ' , fileset) for fn in fileset: f = '%s/%s' % (self.ib.landDir,fn) if not fu.fileExists(f): self.log.error('File %s have not been downloaded!' % f) return 1 # Processing loop. for fn in fileset: self.log.info( 'Processing file %s/%s ' % (self.ib.landDir,fn)) cur_dayr = self._getDateRunStr(fn) if cur_dayr is None : self.log.error('cur_dayr is None. No Date Run String ') return 2 # cur_dayr = YYYYMMDD from filename . eg. Cortera_Trx_Extract_20130502.txt # pdate format YYYY-MM-DD pdate = '%s-%s-%s' % (cur_dayr[:4],cur_dayr[4:6],cur_dayr[6:8]) self.log.debug('cur_dayr = %s pdate = %s' % (cur_dayr,pdate)) rc = su.isValidDate(pdate,'%Y-%m-%d') if rc is False : self.log.error('Invalid Date %s on file %s ' % (pdate,fn)) return 2 self.log.debug('self.checkNextRunFlg is %s' % self.checkNextRunFlg) if self.checkNextRunFlg is True: prev_dayr = self._getCtlFile() pd,pr = self._getDay(prev_dayr,DP_LEN) rc = self._chkNextRun(cur_dayr,prev_dayr,pd,pr,RUN_PER_DAY) if rc != 0 : self.log.error("self._chkNextRun rc = %s" % rc) return rc # Cortera t = '%s/%s' % (self.ib.workDir,self.ib.srcFile[1]) fnp = '%s/%s' % (self.ib.landDir,fn) cl = self.procCort[fn] rc = self._chkDataRecCnt(fnp,cl) if rc != 0 : return rc rc = fu.copyFile(fnp, t) if rc != 0 : self.log.error('Could not copy File %s to %s' % (fnp,t)) return 5 self.log.info ('Copying File from %s to %s' % (fnp,t)) rc = self.archGenFiles([fnp,], self.ts) self.log.info ('Archiving file rc = %s' % rc) # Invoke workflow. rc = pi.runWkflWait(self.ib,self.log) if rc != 0 : self.log.error('Running %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc)) if self.exitOnError: self.log.debug('ExitOnError is TRUE rc = %s' % (rc)) return rc else : self.log.info('Ran %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc)) # Update Control table in Netezza with date. rc = self._updLeaseCrdTbl(pdate) if rc != 0 : self.log.error('No rows were updated for process date %s' % pdate) return rc # Loading Staging Succeeded. Update the control file. rc = fu.updFile(ctlFile,cur_dayr) if rc == 0 : if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile)) else : self.log.info('Overwriting Cur Load Date to %s , Control File %s' % (cur_dayr, ctlFile)) else : self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc)) return rc r = fu.delFile(t) self.log.info('Removing %s rc = %s ' % (t,r)) # Delete BU Region. fnp = '%s/%s' % (self.ib.workDir,self.ib.srcFile[0]) # BU_region rc = fu.delFile(fnp) self.log.info('Removing %s rc = %s ' % (fnp,rc)) return rc
def preProcTMapFiles(self): rc = 0 tgtFiles = [] dataFile = self.chkHeader(self.workFiles) dfl = len(dataFile) if dfl != len(self.ib.srcFile): self.log.error( "File Set to process (%d) is not complete. Expecting a total of %d. Check config file!" % (dfl, len(self.ib.srcFile)) ) self.log.debug("datafiles to process = ", dataFile) return 1 # Set Header row for OPS Transcripts File(s). self.log.debug("Getting hdr from file %s" % self.workFiles[2]) self.ib.opsRegHdrRow = self.getHdrRow(self.workFiles[2]) if self.ib.opsRegHdrRow is None or len(self.ib.opsRegHdrRow) < 50: self.log.error( "File %s does not exists or Hdr is too small hdr = %s" % (self.workFiles[2], self.ib.opsRegHdrRow) ) return 1 i = 0 while i < dfl: tgt = "%s/%s" % (self.ib.workDir, self.ib.srcFile[i]) rc = fu.delFile(tgt) self.log.debug("Removing %s\trc=%s" % (tgt, rc)) rc = fu.moveFile(dataFile[i], tgt) self.log.info("Moving File From %s\t\t\n TO %s" % (dataFile[i], tgt)) if rc != 0: self.log.error("cannot move File from %s to %s" % (dataFile[i], tgt)) return rc if i < 2: offset = _TalentMapFile.HDR_ROW - 1 # Remove first 3 rows for all Transcript Files. Keep HDR else: offset = _TalentMapFile.HDR_ROW # Remove first 4 rows for all OPS Training Files rc = fu.remFileLine(tgt, offset) self.log.debug("Removing %d lines from %s" % (offset, tgt)) if rc != 0: self.log.error("Issue removing lines on %s " % tgt) return rc tgtFiles.append(tgt) i += 1 # Verify Column Numbers: # if self.checkFileColsFlag is True: # rc = self.checkFileCols(tgtFiles,FLD_SEP,' \n\r') # if rc != 0: # self.log.error('Issue with column number. PLease check bad directory under %s' % self.ib.badDir) # return rc return rc
def procIncFiles(self): if len(self.workFiles) < 1: self.log.error("No files to process") return 1 ctlFile = "%s/%s.ctl" % (self.ib.ctlDir, self.appName) self.workFiles.sort() rc = 0 # Files in the working directory: i = 1 self.log.debug("Will Process a total of %d file(s) " % len(self.workFiles)) for fnp in self.workFiles: self.log.info("\nProcessing File (%d) => %s " % (i, fnp)) # Get date run from 1st filename fn = fu.getFileBaseName(fnp) cur_dayr = self._getDateRunStr(fn) if cur_dayr is None: self.log.error("No Date String %s " % cur_dayr) return 1 fmt = "%Y%m" date = "%s%s" % (cur_dayr[0:4], cur_dayr[4:6]) rc = su.isValidDate(date, fmt) if rc is False: self.log.error("Invalid Date %s on file %s " % (date, fn)) return 2 self.fileDate = date self.log.debug("self.fileDate = %s" % (self.fileDate)) if self.checkNextRunFlg is True: # Get Previous Run Info. File should contain one line only : YYYYMM from storage. prev_dayr = self._getCtlFile() if prev_dayr is None: return 3 pd, pr = self._getMonth(prev_dayr, DP_LEN) if pd is None: return 4 # rc = self._chkNextRun(cur_dayr,prev_dayr,pd,pr,RUN_PER_MTH) rc = psch.getNextRunDate(pd, cur_dayr, "Mthly", self.log) if rc != 0: self.log.error("self._chkNextRun rc = %s" % rc) return rc if self.verifyCSVFlg is True: r, b = fu.readCSV(fnp, FLDxROW, SEP) if len(b) > 0: fbad = "%s/%s.bad" % (self.ib.badDir, fn) rc = fu.createFile(fbad, b) self.log.error("No of %d bad row(s) on %s" % (len(b), fnp)) self.log.error("Creating file %s rc = %s" % (fbad, rc)) self.log.debug("Bad rows = , ", b) return 5 if len(r) == 0: self.log.error("No rows to process on file %s" % fnp) return 6 t = "%s/%s" % (self.ib.workDir, self.ib.srcFile[0]) rc = fu.moveFile(fnp, t) self.log.info("Renaming %s to %s rc = %s" % (fnp, t, rc)) if rc != 0: self.log.error("Could not move File %s to %s" % (fnp, t)) return 7 # Invoke workflow. rc = self._wkfIMSSftyLoc() if rc != 0: self.log.error("Running %s.%s rc = %s" % (self.ib.fld, self.ib.wkf, rc)) if self.exitOnError: self.log.debug("ExitOnError is TRUE rc = %s" % (rc)) return rc else: self.log.info("Ran %s.%s rc = %s" % (self.ib.fld, self.ib.wkf, rc)) # Loading Staging Succeeded. Update the control file. rc = fu.updFile(ctlFile, cur_dayr) if rc == 0: if self.checkNextRunFlg: self.log.info( "Updated Cur Load Date from %s to %s , Control File %s" % (prev_dayr, cur_dayr, ctlFile) ) else: self.log.info("Overwriting Cur Load Date to %s , Control File %s" % (cur_dayr, ctlFile)) else: self.log.error("Could not Update Load Date %s, Control File %s rc = %s" % (cur_dayr, ctlFile, rc)) return rc r = fu.delFile(t) self.log.debug("Deleting File %s rc = %s" % (t, r)) i += 1 return rc