def relLock(self):
if self.lock:
try:
fu.delFile(self.lckFn)
self.log.info('Released lock: %s for pid %s' % (self.lckFn,self.pid))
return 0
except:
self.log.error("==EXCEP %s %s" % (sys.exc_type,sys.exc_value))
return 1
else:
self.log.warn('No lock to release for: %s for pid %s' % (self.lckFn,self.pid))
return 0
def procVehPOLIWkly(self):
ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)
prev_dayr = self._getCtlFile()
self.log.debug('Ctl File is %s self.checkNextRunFlg is %s prev_dayr=%s' % (ctlFile,self.checkNextRunFlg,prev_dayr))
if self.checkNextRunFlg is True:
if prev_dayr is None or prev_dayr.strip() == '':
self.log.error("Could not find control file or No Data")
return -1
rc = psc.getNextRunDate(prev_dayr, cur_dayr, SCH_FREQ, self.log)
if rc != 0 :
self.log.error("self._chkNextRun rc = %s" % rc)
return rc
# Run workflows
if self._wkfVehPOWkly() != 0 : return 1
# Loading Staging Succeeded. Update the control file.
rc = fu.updFile(ctlFile,cur_dayr)
if rc == 0 :
if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile))
else : self.log.info('Overwriting Cur Load Date from %s to %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile))
else :
self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc))
return rc
for i in range (len(self.srcFile)):
t = '%s/%s' % (self.ib.workDir,self.srcFile[i])
r = fu.delFile(t)
self.log.info('Removing %s rc = %s ' % (t,r))
return rc
def _procOpsReg(self):
concatFls = (
"Transcripts_OpsRegion.txt",
"Transcripts_OpsRegion_Canada.txt",
"Transcripts_OpsRegion_Central.txt",
"Transcripts_OpsRegion_Northeast.txt",
"Transcripts_OpsRegion_Southeast.txt",
"Transcripts_OpsRegion_West.txt",
)
concatFlsFP = []
for fn in concatFls:
concatFlsFP.append("%s/%s" % (self.ib.workDir, fn))
self.log.debug("concatFlsFP = ", concatFlsFP)
# Remove Concatenated file:
rc = fu.delFile(concatFlsFP[0])
self.log.debug("Removing %s\trc=%s" % (concatFlsFP[0], rc))
self.log.debug("Creating HDR file %s" % concatFlsFP[0])
rc = fu.createFile(concatFlsFP[0], self.ib.opsRegHdrRow)
if rc != 0:
self.log.error("Could not create file %s" % (concatFlsFP[0]))
return 2
# Concatenate the files
rc = fu.concatFile(concatFlsFP)
self.log.info("concatFile rc = %s" % rc)
return rc
def relLock(self):
if self.lock:
try:
fu.delFile(self.lckFn)
self.log.info('Released lock: %s for pid %s' %
(self.lckFn, self.pid))
return 0
except:
self.log.error("==EXCEP %s %s" % (sys.exc_type, sys.exc_value))
return 1
else:
self.log.warn('No lock to release for: %s for pid %s' %
(self.lckFn, self.pid))
return 0
def _chkIfValidLock(self):
rc = True
# Check if there is a lock file.
if fu.fileExists(self.lckFn):
sPid = fu.readFile(self.lckFn)
pid = su.toInt(sPid)
self.log.info('Lock File %s EXISTS. sPid = %s , pid = %s' %
(self.lckFn, sPid, pid))
# Check if file has a valid PID (number)
if pid is None:
rc = fu.delFile(self.lckFn)
if rc == 0:
self.log.info('Removed File %s' % (self.lckFn))
else:
self.log.error('Could not removed File %s' % (self.lckFn))
return False
# If pid is a valid number, check if the process is running ...
rc = ps.isProcRunning(pid, self.log)
self.log.debug('isProcRunning returned %s' % (rc))
return rc
# No lock file exists.
else:
return False
def procEmpBenefit(self):
# Run workflows
if self._wkfEmpBenefitonDem() != 0:
return 1
for i in range(len(self.srcFile)):
t = "%s/%s" % (self.ib.workDir, self.srcFile[i])
r = fu.delFile(t)
self.log.info("Removing %s rc = %s " % (t, r))
return 0
def procIncFiles(self):
if len(self.workFiles) < 1 :
self.log.error('No files to process')
return 1
self.workFiles.sort()
rc = 0
# Files in the working directory:
i = 1
self.log.debug('Will Process a total of %d file(s) ' % len(self.workFiles))
for fnp in self.workFiles:
self.log.info('\nProcessing File (%d) => %s ' % (i,fnp))
fn = fu.getFileBaseName(fnp)
if self.verifyCSVFlg is True:
r,b = fu.readCSV(fnp, FLDxROW, SEP)
if len(b) > 0 :
fbad = '%s/%s.bad' % (self.ib.badDir,fn)
rc = fu.createFile(fbad , b)
self.log.error("No of %d bad row(s) on %s" % (len(b),fnp))
self.log.error("Creating file %s rc = %s" % (fbad,rc))
self.log.debug("Bad rows = , ", b)
return 5
if len(r) == 0 :
self.log.error("No rows to process on file %s" % fnp)
return 6
t = '%s/%s' % (self.ib.workDir , self.ib.srcFile[0])
rc = fu.moveFile(fnp, t)
self.log.info('Renaming %s to %s rc = %s' % (fnp,t,rc))
if rc != 0 :
self.log.error('Could not move File %s to %s' % (fnp,t))
return 7
# Invoke workflow.
rc = self._wkfCodeOvrd()
if rc != 0 :
self.log.error('Running %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
if self.exitOnError:
self.log.debug('ExitOnError is TRUE rc = %s' % (rc))
return rc
else :
self.log.info('Ran %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
r = fu.delFile(t)
self.log.debug('Deleting File %s rc = %s' % (t,r))
i+=1
return rc
def _remFiles(self, flp):
rc = 0
if len(flp) < 1 :
self.log.warn('File list to delete is empty !')
return 1
#fns = fl.split(',')
for fnp in flp:
r = fu.delFile(fnp)
if r != 0 :
rc += 1
self.log.error ("Could not delete %s r = %s " % (fnp, r))
else :
self.log.info ("Removed %s r = %d " % (fnp, r))
return rc
def checkApprovFile(self):
fl = fu.getFileName(self.ib.landDir, self.ib.fileName)
if len(fl) == 0:
i = 1
self.log.info('Approval File %s/%s does not exists. Will start in %s sec' % (self.ib.landDir, self.ib.fileName, self.ib.waitAppFile))
wi = int(self.ib.waitFileAppIter)
while (i <= wi):
time.sleep(float(self.ib.waitAppFile))
fl = fu.getFileName(self.ib.landDir, self.ib.fileName)
self.log.info('Iteration %d out of %d ' % (i, wi))
if len(fl) > 0 : break
else : i += 1
if len(fl) == 0 : return 1
else:
rc = fu.delFile(fl[0])
self.log.info('Removing %s , rc = %s ' % (fl[0], rc))
return 0
def procCompFiles(self):
ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)
self.log.debug('self.checkNextRunFlg is %s' % self.checkNextRunFlg)
prev_dayr = self._getCtlFile()
if self.checkNextRunFlg is True:
if prev_dayr is None or prev_dayr.strip() == '':
self.log.error("Could not find control file or No Data")
return -1
rc = psc.getNextRunDate(prev_dayr, cur_dayr, SCH_FREQ, self.log,sch)
if rc != 0 :
self.log.error("self._chkNextRun rc = %s" % rc)
return rc
if self.checkFileRowsFlg is True:
for f in self.trigFiles:
self.log.debug('Processing %s (self.workfile)' % f)
rc = self._chkHeader(f,self.ib.rdayoffset)
if rc != 0 : return rc
# Invoke workflow.
#if self._wkfVehCompDly() != 0 : return 1
# Loading Staging Succeeded. Update the control file.
rc = fu.updFile(ctlFile,cur_dayr)
if rc == 0 :
if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile))
else : self.log.info('Overwriting Cur Load Date from %s to %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile))
else :
self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc))
return rc
for i in range (len(self.srcFile)):
t = '%s/%s' % (self.ib.workDir,self.srcFile[i])
r = fu.delFile(t)
self.log.info('Removing %s rc = %s ' % (t,r))
return rc
def _insNZLoad(self,val,seqid):
ldata = []
for s,u,p,d in val :
ldata.append("%s,%s,%s,%s\n" % (s,u,p,d ))
data = ''.join(ldata)
# Create CSV file
fn = '%s/%s.csv' % (self.ib.dataDir,seqid)
fl = '%s/tis_fb_carr_pmpt_imprt_dtl_%s.%s.nzlog' % (self.ib.dataDir,seqid,self.ib.db)
bf = '%s/tis_fb_carr_pmpt_imprt_dtl_%s.%s.nzbad' % (self.ib.dataDir,seqid,self.ib.db)
rc = fu.createFile(fn, data)
self.log.debug('File Creation %s/%s.csv rc = %s' % (d,seqid,rc))
if rc != 0 :
self.log.error('File Creation %s/%s.csv rc = %s' % (d,seqid,rc))
return rc
# Invoke nzload command
nz_cmd = "%s/nzload -host %s -u %s -pw %s -db %s -t tis_fb_carr_pmpt_imprt_dtl -delim '%s' -df %s -lf %s -bf %s" % (self.ib.nzbin,
self.ib.dbserver,
self.ib.user,
self.ib.pwd,
self.ib.db,
self.ib.delim, fn,fl,bf)
self.log.debug('nz_cmd = %s' % nz_cmd)
rc,rmsg = proc.runSync(nz_cmd,self.log)
if rc != 0 :
self.log.error('cmd %s rc = %s -> msg= %s' % (nz_cmd,rc,rmsg))
return rc
# Delete csv file (based on option)
if self.delCSVFileFlg is True:
r = fu.delFile(fn)
self.log.debug('Deleting %s rc = %s ' % (fn,r))
if r != 0 : self.log.error('Cannot Delete %s ' % fn )
return rc
def _chkIfValidLock(self):
rc = True
# Check if there is a lock file.
if fu.fileExists(self.lckFn):
sPid = fu.readFile(self.lckFn)
pid = su.toInt(sPid)
self.log.info('Lock File %s EXISTS. sPid = %s , pid = %s' % (self.lckFn,sPid,pid))
# Check if file has a valid PID (number)
if pid is None :
rc = fu.delFile(self.lckFn)
if rc == 0 :
self.log.info('Removed File %s' % (self.lckFn))
else:
self.log.error('Could not removed File %s' % (self.lckFn))
return False
# If pid is a valid number, check if the process is running ...
rc = ps.isProcRunning(pid,self.log)
self.log.debug('isProcRunning returned %s' % (rc))
return rc
# No lock file exists.
else: return False
def procCortFiles(self):
dataFile = []
ctlFile = '%s/%s.ctl' % (self.ib.ctlDir,self.appName)
setn = len (self.procCort)
self.log.info( ' ---- Starting Processing. Total of %d iteration(s) ----' % setn)
fileset = self.procCort.keys()
fileset.sort()
self.log.debug('Process Order = ' , fileset)
for fn in fileset:
f = '%s/%s' % (self.ib.landDir,fn)
if not fu.fileExists(f):
self.log.error('File %s have not been downloaded!' % f)
return 1
# Processing loop.
for fn in fileset:
self.log.info( 'Processing file %s/%s ' % (self.ib.landDir,fn))
cur_dayr = self._getDateRunStr(fn)
if cur_dayr is None :
self.log.error('cur_dayr is None. No Date Run String ')
return 2
# cur_dayr = YYYYMMDD from filename . eg. Cortera_Trx_Extract_20130502.txt
# pdate format YYYY-MM-DD
pdate = '%s-%s-%s' % (cur_dayr[:4],cur_dayr[4:6],cur_dayr[6:8])
self.log.debug('cur_dayr = %s pdate = %s' % (cur_dayr,pdate))
rc = su.isValidDate(pdate,'%Y-%m-%d')
if rc is False :
self.log.error('Invalid Date %s on file %s ' % (pdate,fn))
return 2
self.log.debug('self.checkNextRunFlg is %s' % self.checkNextRunFlg)
if self.checkNextRunFlg is True:
prev_dayr = self._getCtlFile()
pd,pr = self._getDay(prev_dayr,DP_LEN)
rc = self._chkNextRun(cur_dayr,prev_dayr,pd,pr,RUN_PER_DAY)
if rc != 0 :
self.log.error("self._chkNextRun rc = %s" % rc)
return rc
# Cortera
t = '%s/%s' % (self.ib.workDir,self.ib.srcFile[1])
fnp = '%s/%s' % (self.ib.landDir,fn)
cl = self.procCort[fn]
rc = self._chkDataRecCnt(fnp,cl)
if rc != 0 : return rc
rc = fu.copyFile(fnp, t)
if rc != 0 :
self.log.error('Could not copy File %s to %s' % (fnp,t))
return 5
self.log.info ('Copying File from %s to %s' % (fnp,t))
rc = self.archGenFiles([fnp,], self.ts)
self.log.info ('Archiving file rc = %s' % rc)
# Invoke workflow.
rc = pi.runWkflWait(self.ib,self.log)
if rc != 0 :
self.log.error('Running %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
if self.exitOnError:
self.log.debug('ExitOnError is TRUE rc = %s' % (rc))
return rc
else :
self.log.info('Ran %s.%s rc = %s' % (self.ib.fld,self.ib.wkf,rc))
# Update Control table in Netezza with date.
rc = self._updLeaseCrdTbl(pdate)
if rc != 0 :
self.log.error('No rows were updated for process date %s' % pdate)
return rc
# Loading Staging Succeeded. Update the control file.
rc = fu.updFile(ctlFile,cur_dayr)
if rc == 0 :
if self.checkNextRunFlg: self.log.info('Updated Cur Load Date from %s to %s , Control File %s' % (prev_dayr,cur_dayr, ctlFile))
else : self.log.info('Overwriting Cur Load Date to %s , Control File %s' % (cur_dayr, ctlFile))
else :
self.log.error('Could not Update Load Date %s, Control File %s rc = %s' % (cur_dayr,ctlFile,rc))
return rc
r = fu.delFile(t)
self.log.info('Removing %s rc = %s ' % (t,r))
# Delete BU Region.
fnp = '%s/%s' % (self.ib.workDir,self.ib.srcFile[0]) # BU_region
rc = fu.delFile(fnp)
self.log.info('Removing %s rc = %s ' % (fnp,rc))
return rc
def preProcTMapFiles(self):
rc = 0
tgtFiles = []
dataFile = self.chkHeader(self.workFiles)
dfl = len(dataFile)
if dfl != len(self.ib.srcFile):
self.log.error(
"File Set to process (%d) is not complete. Expecting a total of %d. Check config file!"
% (dfl, len(self.ib.srcFile))
)
self.log.debug("datafiles to process = ", dataFile)
return 1
# Set Header row for OPS Transcripts File(s).
self.log.debug("Getting hdr from file %s" % self.workFiles[2])
self.ib.opsRegHdrRow = self.getHdrRow(self.workFiles[2])
if self.ib.opsRegHdrRow is None or len(self.ib.opsRegHdrRow) < 50:
self.log.error(
"File %s does not exists or Hdr is too small hdr = %s" % (self.workFiles[2], self.ib.opsRegHdrRow)
)
return 1
i = 0
while i < dfl:
tgt = "%s/%s" % (self.ib.workDir, self.ib.srcFile[i])
rc = fu.delFile(tgt)
self.log.debug("Removing %s\trc=%s" % (tgt, rc))
rc = fu.moveFile(dataFile[i], tgt)
self.log.info("Moving File From %s\t\t\n TO %s" % (dataFile[i], tgt))
if rc != 0:
self.log.error("cannot move File from %s to %s" % (dataFile[i], tgt))
return rc
if i < 2:
offset = _TalentMapFile.HDR_ROW - 1 # Remove first 3 rows for all Transcript Files. Keep HDR
else:
offset = _TalentMapFile.HDR_ROW # Remove first 4 rows for all OPS Training Files
rc = fu.remFileLine(tgt, offset)
self.log.debug("Removing %d lines from %s" % (offset, tgt))
if rc != 0:
self.log.error("Issue removing lines on %s " % tgt)
return rc
tgtFiles.append(tgt)
i += 1
# Verify Column Numbers:
# if self.checkFileColsFlag is True:
# rc = self.checkFileCols(tgtFiles,FLD_SEP,' \n\r')
# if rc != 0:
# self.log.error('Issue with column number. PLease check bad directory under %s' % self.ib.badDir)
# return rc
return rc
def procIncFiles(self):
if len(self.workFiles) < 1:
self.log.error("No files to process")
return 1
ctlFile = "%s/%s.ctl" % (self.ib.ctlDir, self.appName)
self.workFiles.sort()
rc = 0
# Files in the working directory:
i = 1
self.log.debug("Will Process a total of %d file(s) " % len(self.workFiles))
for fnp in self.workFiles:
self.log.info("\nProcessing File (%d) => %s " % (i, fnp))
# Get date run from 1st filename
fn = fu.getFileBaseName(fnp)
cur_dayr = self._getDateRunStr(fn)
if cur_dayr is None:
self.log.error("No Date String %s " % cur_dayr)
return 1
fmt = "%Y%m"
date = "%s%s" % (cur_dayr[0:4], cur_dayr[4:6])
rc = su.isValidDate(date, fmt)
if rc is False:
self.log.error("Invalid Date %s on file %s " % (date, fn))
return 2
self.fileDate = date
self.log.debug("self.fileDate = %s" % (self.fileDate))
if self.checkNextRunFlg is True:
# Get Previous Run Info. File should contain one line only : YYYYMM from storage.
prev_dayr = self._getCtlFile()
if prev_dayr is None:
return 3
pd, pr = self._getMonth(prev_dayr, DP_LEN)
if pd is None:
return 4
# rc = self._chkNextRun(cur_dayr,prev_dayr,pd,pr,RUN_PER_MTH)
rc = psch.getNextRunDate(pd, cur_dayr, "Mthly", self.log)
if rc != 0:
self.log.error("self._chkNextRun rc = %s" % rc)
return rc
if self.verifyCSVFlg is True:
r, b = fu.readCSV(fnp, FLDxROW, SEP)
if len(b) > 0:
fbad = "%s/%s.bad" % (self.ib.badDir, fn)
rc = fu.createFile(fbad, b)
self.log.error("No of %d bad row(s) on %s" % (len(b), fnp))
self.log.error("Creating file %s rc = %s" % (fbad, rc))
self.log.debug("Bad rows = , ", b)
return 5
if len(r) == 0:
self.log.error("No rows to process on file %s" % fnp)
return 6
t = "%s/%s" % (self.ib.workDir, self.ib.srcFile[0])
rc = fu.moveFile(fnp, t)
self.log.info("Renaming %s to %s rc = %s" % (fnp, t, rc))
if rc != 0:
self.log.error("Could not move File %s to %s" % (fnp, t))
return 7
# Invoke workflow.
rc = self._wkfIMSSftyLoc()
if rc != 0:
self.log.error("Running %s.%s rc = %s" % (self.ib.fld, self.ib.wkf, rc))
if self.exitOnError:
self.log.debug("ExitOnError is TRUE rc = %s" % (rc))
return rc
else:
self.log.info("Ran %s.%s rc = %s" % (self.ib.fld, self.ib.wkf, rc))
# Loading Staging Succeeded. Update the control file.
rc = fu.updFile(ctlFile, cur_dayr)
if rc == 0:
if self.checkNextRunFlg:
self.log.info(
"Updated Cur Load Date from %s to %s , Control File %s" % (prev_dayr, cur_dayr, ctlFile)
)
else:
self.log.info("Overwriting Cur Load Date to %s , Control File %s" % (cur_dayr, ctlFile))
else:
self.log.error("Could not Update Load Date %s, Control File %s rc = %s" % (cur_dayr, ctlFile, rc))
return rc
r = fu.delFile(t)
self.log.debug("Deleting File %s rc = %s" % (t, r))
i += 1
return rc
