def procHierSCOS(self):
# Check for file existence
src = '%s/%s/%s' % (self.ib.shareDir,self.outDir,self.ib.outFile)
r = fu.fileExists(src)
if r == False :
self.log.error('File %s does not exist' % src)
return 1
# Move File to output dir
dt = su.getTodayDtStr('%y%m%d')
self.outFile = 'SCS_Org_Hierarchy%s.csv' % dt
outFile = '%s/%s/%s/%s' % (self.ib.shareDir,self.outDir,self.ib.outDirname,self.outFile)
r = fu.moveFile(src,outFile)
if r != 0 :
self.log.error("Could not move %s to %s " % (src,outFile))
self.log.debug('Moved src = %s to self.outFile = %s RC = %s' % (src,outFile,r))
# Archive File .
arcfn = '%s/%s/%s/%s-%s' % (self.ib.shareDir,self.outDir,self.outArchDir,self.ts,self.outFile)
r = fu.copyFile(outFile,arcfn)
if r != 0 :
self.log.error("Could not copy %s to %s " % (outFile,arcfn))
self.log.debug('Copied src = %s to racfn = %s RC = %s' % (outFile,arcfn,r))
return 0
def _chkInfaRdyMthly(self,job):
rs = self._chkInfaRdy(job)
if rs is None or len(rs) != 1 : return -1
d = rs[0][0]; dt = d[:-2]
todm = su.getTodayDtStr('%Y%m')
self.log.debug('d = %s dt = %s todm = %s' % (d,dt,todm))
if dt == todm : return 0
return 1
def _chkInfaRdyDly(self,job):
rs = self._chkInfaRdy(job)
if rs is None or len(rs) != 1 : return -1
dt = rs[0][0]
todm = su.getTodayDtStr('%Y%m%d')
self.log.debug('dt = %s todm = %s' % (dt,todm))
if dt == todm : return 0
return 1
def chkMssqlRdyDly(self,job):
qryStr = ds.selMssqlPredQry % job
self.log.debug('qryStr = %s' % qryStr)
rs = self._chkMssqlRdy(qryStr)
if rs is None or len(rs) != 1 : return -1
dt = rs[0][0]
todm = su.getTodayDtStr('%Y%m%d')
self.log.debug('dt = %s todm = %s' % (dt,todm))
if dt == todm : return 0
return 1
def getLastWorkDay(self,lwd):
mtyr = su.getTodayDtStr(fmt='%m%Y')
m = mtyr[:2] ; y = mtyr[2:]
qry = ds.lastworkDay % (m,y,m,y,lwd)
self.log.debug('qry = %s ' % qry)
rs = self._getNZDS(qry)
self.log.debug('rs' , rs)
if rs is None : return None
if len(rs) != 1 : return -1
if len(rs[0]) != 1 : return -4
return rs[0][0]
def isLastWorkDayWarn(self):
rc = -2
w = self.getLastWorkDay(self.ib.lastworkday)
if w is None :
self.log.error(' wkday = %s : Please check DB connectivity table' % w)
return -5
wd = su.getDtStr(w,'%Y-%m-%d')
d=su.getTodayDtStr('%Y-%m-%d')
self.log.debug('self.ib.lastworkday = %s date_day = %s today = %s' % (self.ib.lastworkday,w,d))
if d == wd : rc = 0
else : rc = RET_WARN
return rc
def _updLeaseCrdTbl(self,pdate):
if pdate is None or len(pdate) != 10 :
self.log.debug('Invalid pdate %s ' % (pdate))
return 1
cdate = su.getTodayDtStr(fmt='%Y-%m-%d')
if cdate == pdate:
self.log.debug('curr_date %s = pdate %s. No need to update table.' % (cdate,pdate))
return 0
self.log.info("Updating table setting curr_date %s to pdate = %s " % (cdate, pdate))
rc =self._getNZDS(da.updLCRStgCortExtQry % (pdate,cdate) , po='UPD')
self.log.info('Update Qry returned %s' % rc)
if rc > 0 : return 0
return 2
def _build2XML(self, grpbl):
root = ET.Element('PROC_GRP')
comment = ET.Comment('For Recovery PID %s. Generated on %s ' % (self.pid, su.getTodayDtStr('%m/%d/%Y %H:%M:%S')))
root.append(comment)
for g in grpbl:
self.log.debug(" name = " , g.name , " order = ", g.order , " active= ", g.active , " exit_on_err = " , g.exitErr)
root.append(ET.Comment('Group %s' % g.name))
group = ET.SubElement(root, 'group', name = g.name, order = '%s' % g.order, active= '%s' % g.active, exit_on_err = '%s' % g.exitErr )
for p in g.pipelines:
self.log.debug("\t name = " , p.name , " order = ", p.order , " active= ", p.active , " exit_on_err = " , p.exitErr)
pipel = ET.SubElement(group, 'pipeline',name = p.name, order = '%s' % p.order, active= '%s' % p.active, exit_on_err = '%s' % p.exitErr )
for t in p.tasks:
self._setElemTask(pipel,t)
return root
def _chkSrcReady(self):
qryStr = "select max(MeasureDate) MonthDate , 1 isServiceCallsReady from dbo.edw_ServiceCalls"
self.ib.user = self.ib.ms_user; self.ib.pwd = self.ib.ms_pwd; self.ib.dbserver = self.ib.ms_dbserver ; self.ib.db = self.ib.ms_db
rs = self._getMSSQLNatDS(qryStr)
self.log.debug('qryStr = ', qryStr , ' rs ', rs)
if rs is None or len(rs) != 1 : return -1
isRdy = rs[0][1]
rdtm = su.getDtStr(rs[0][0],'%m%Y')
todm = su.getTodayDtStr('%m%Y')
nmth = su.getMonthPlusStr(1,rdtm,'%m%Y')
self.log.debug("isRdy = %s rdtm=%s todm=%s nmth=%s" % (isRdy,rdtm,todm,nmth))
if isRdy == 1 and nmth == todm : # and sysdate - 1
self.log.info ("Ready to Proceed : rdtm = %s isRdy = %s " % (rdtm,isRdy))
return 0
return 1
def _getROPOPred(self,tbn):
qryStr = ds.selFinRCOpenROPOQry % tbn
rs = self._chkMssqlRdy(qryStr)
self.log.debug('rs = ' , rs)
# rs = [(u'ConsolidatedSHPOD ', '20150310', 6240165)]
if rs is None or len(rs) < 1 or len(rs[0]) != 3 : return -1
tbn = rs[0][0]; dt = rs[0][1] ; rowc = rs[0][2] ;
self.log.info('tbn = %s dt = %s rowc = %s ' % (tbn,dt,rowc))
# Check if date current date
todm = su.getTodayDtStr('%Y%m%d')
self.log.debug('dt = %s todm = %s' % (dt,todm))
if dt != todm : return 1
# Check if number of rows for today are > 0
irowc = su.toInt(rowc)
self.log.debug('irowc = %s' % (irowc))
if irowc is None or irowc < 1 : return 2
return 0
def _chkSrcReady(self):
qryStr = """ select MonthDate, CAST(isSourcingReady AS int)
from dbo.DataReady
WHERE MonthDate = (select MAX(MonthDate) from dbo.DataReady)"""
self.ib.user = self.ib.ms_user; self.ib.pwd = self.ib.ms_pwd; self.ib.dbserver = self.ib.ms_dbserver ; self.ib.db = self.ib.ms_db
rs = self._getMSSQLNatDS(qryStr)
self.log.debug('qryStr = ', qryStr , ' rs ', rs)
if len(rs) != 1 : return -1
isRdy = rs[0][1]
rdtm = su.getDtStr(rs[0][0],'%m%Y')
todm = su.getTodayDtStr('%m%Y')
nmth = su.getMonthPlusStr(1,rdtm,'%m%Y')
self.log.debug("isRdy = %s rdtm=%s todm=%s nmth=%s" % (isRdy,rdtm,todm,nmth))
if isRdy == 1 and nmth == todm : # and sysdate - 1 = dt
self.log.info ("Ready to Proceed : rdtm = %s isRdy = %s " % (rdtm,isRdy))
return 0
return 1
def _chkHeader(self,fnp,rdoff):
RUN_DATE = su.getTodayDtStr(fmt='%y%m%d')
rdayoffset = su.toInt(rdoff)
hdr = fu.readFile(fnp)
if len(hdr) != 14 :
self.log.error('Invalid len= %d for %s. Need to be 14\nhdr=%s' % (len(hdr),fnp,hdr))
return 1
rowc = su.toInt(hdr[0:8])
hdr_date = hdr[8:] # Run Date from file.
dt = su.getDayMinusStr(rdayoffset, RUN_DATE, '%Y%m%d')
# Check run date.
if dt != hdr_date:
self.log.warn('fn %s === \tInvalid Header ran date of %s. Should be %s ' % (fnp,dt,hdr_date))
if rdayoffset < 1 :
self.log.error('Could not attempt next day since rdayoffset is %s < 1' % rdayoffset)
return 2
dt = su.getDayMinusStr(rdayoffset - 1, RUN_DATE, '%Y%m%d')
# Check run date + 1.
if dt != hdr_date:
self.log.error('fn %s === \tInvalid Header ran date of %s. Should be %s ' % (fnp,dt,hdr_date))
return 3
# Check Number of lines
ln = fu.getLines(fnp) # Number of lines w/o trailer.
dc = int(ln) - rowc # ln file has 2 extra lines header/trailer, but at this point trailer rec had been removed.
self.log.debug('fn %s === \tFile record count %s trailer = %s diff %s' % (fnp,ln,rowc,dc))
if dc != 0 :
self.log.error('fn %s === \tFile record count %s do not match trailer = %s # of records diff = %d!' % (fnp,ln,rowc,dc))
return 4
return 0
from apps.infbaseapp import _InfaBaseApp
# Mandatory to define self.cmdStep
# method _getNextRunDate is sensitive to schedule changes !
RUN_PER_DAY = 1 # Daily runs.
DP_LEN = len('YYYYMMDD')
# Schedules
#SCH_FREQ = 'WDay'
SCH_FREQ = 'Cust'
sch = ('Tue','Wed','Thu','Fri','Sat')
cur_dayr = su.getTodayDtStr('%Y%m%d')
class VehFifthWkday(_InfaBaseApp):
exitOnError = True
def __init__(self):
super(VehFifthWkday,self).__init__()
self.landDir = 'SrcFiles/vehicle'
self.incFileSet = [] # Incoming Files. Contains full path name.
self.incFiles = []
self.workFiles = [] # Files that were moved to the working dir (ideally same than incSetFile).
self.RowCnt = -1
self.srcFile = ('PO225D15.dat','PO875D30.dat') # File that Informatica expects. Alphabetical.
self.ib.fileName = r"'P.PO225D15.UMPK(0)','P.PO875D30.WHOUSE.CURR.STD'"
self.checkNextRunFlg = False
import os #, os.path
import sys
import socket
import time
import logging
import common.simpmail as sm
import common.lockfile as lck
import utils.fileutils as fu
import utils.strutils as su
from common.loghdl import getLogHandler
RET_WARN = 101
RUN_DATE = su.getTodayDtStr(fmt='%Y%m%d')
# Empty Container
class BaseAppBean:
pass
class _BaseApp(object):
hostname = socket.gethostname()
def __init__(self):
self.appName = self.__class__.__name__.lower()
self.log = logging.getLogger(self.appName)
self.suf = 'cfg'
import utils.fileutils as fu
import utils.strutils as su
import utils.filetransf as ft
import procdata.procinfa as pi
from apps.infbaseapp import _InfaBaseApp
# Mandatory to define self.cmdStep
# method _getNextRunDate is sensitive to schedule changes !
RUN_PER_DAY = 1 # Daily runs.
DP_LEN = len("YYYYMM")
# Schedules
SCH_FREQ = "Mthly"
sch = ()
cur_dayr = su.getTodayDtStr("%Y%m")
class EmpBenefitonDem(_InfaBaseApp):
exitOnError = True
def __init__(self):
super(EmpBenefitonDem, self).__init__()
self.landDir = "SrcFiles/employee"
self.incFileSet = [] # Incoming Files. Contains full path name.
self.incFiles = []
self.workFiles = [] # Files that were moved to the working dir (ideally same than incSetFile).
self.RowCnt = -1
self.srcFile = ("hp400jnm_sap_dwext.dat", "hrcoord5.csv") # File that Informatica expects. Alphabetical.
self.ib.fileName = r"'P.HP400JNM.SAP.DW.EXTRACT',hrcoord5.csv"
def __init__(self):
super(TalentMap, self).__init__()
self.landDir = "SrcFiles/talentmap"
self.incFileSet = [] # Incoming Files. Contains full path name.
self.workFiles = [] # Files that were moved to the working dir (ideally same than incFiles).
self.RowCnt = -1
self.ib.fld = "TalentMap"
self.ib.wkf = "wkf_talentMap"
self.ib.srcFile = (
"All_Training.txt",
"All_Training_Providers.txt",
"Transcripts_OpsRegion_Canada.txt",
"Transcripts_OpsRegion_Central.txt",
"Transcripts_OpsRegion_Northeast.txt",
"Transcripts_OpsRegion_Southeast.txt",
"Transcripts_OpsRegion_West.txt",
) # File that Informatica expects. Alphabetical.
self.ib.FileColCnt = {
"All_Training.txt": 12,
"All_Training_Providers.txt": 2,
"Transcripts_OpsRegion_Canada.txt": 18,
"Transcripts_OpsRegion_Central.txt": 18,
"Transcripts_OpsRegion_Northeast.txt": 18,
"Transcripts_OpsRegion_Southeast.txt": 18,
"Transcripts_OpsRegion_West.txt": 18,
"Tops_Plan_Current.csv": 13,
"FMS_Qualifications.csv": 8,
}
self.ib.opsRegHdrRow = ""
self.fileDate = ""
self.FILE_SET_LEN = 4
self.procCort = {}
self.procBuReg = {}
self.renameFtpFilesTgtFlg = False
self.ts = su.getTimeSTamp()
# Allowable commands for this application. Make sure to Set
self.cmdStep = {
"A": self.getLock,
"B": self.renFtpFilesTgtFlg,
"C": self.getTMapftpFiles, # FTP Files based on regex.
"D": self.getIncSetFiles, # Populates self.incSetFiles. Incoming Files.
"E": self.setIncFilePvt, # Run after setting self.incFiles
"F": self.cpFileToWorkDir, # Copies FileSet and sets self.workFiles (full path)
"G": self.archFiles,
"H": self.chkFileColsFlg,
"I": self.preProcTMapFiles, # Check that headers/record count, number of fields.
"J": self.procTMapFiles,
"K": self.procCustFiles,
"L": self.crtTrigFile,
"M": self.wkfTalentMapProc,
}
# Infa Environmental variables/
self.infaEnvVar = {
"PMCMD": "mg.pmcmd",
"INFA_USER": "******",
"INFA_XPWD": "self.ib.rep_xpwd",
"DOMAIN": "self.ib.dom_name",
"INT_SERV": "self.ib.IS",
"INFA_SHARE": "self.ib.shareDir",
"INFA_APP_CFG": "self.ib.cfgDir",
"INFA_APP_LCK": "self.ib.lckDir",
"INFA_APP_CTL": "self.ib.ctlDir",
}
self.fnd = su.getTodayDtStr("%m_%d_%Y")
# FTP is expecting the following env variables, which should not be in a config file.
# os.environ['RXFILE' ] = '.*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*.TXT' # data file(s). Use for MGET
os.environ["FILE"] = "Name of control file" # data file(s). Use for GET
os.environ["RXFILE"] = (
"Transcripts_OpsRegion_.*_[0-9]*_%s_[0-9]*_[0-9]*.TXT" % self.fnd
) # data file(s). Use for MGET
import common.log4py as log4py
import common.simpmail as sm
import common.lockfile as lck
import utils.fileutils as fu
import utils.strutils as su
import datastore.dbapp as ds
import mainglob as mg # Do not remove.
from datastore.dbutil import NZODBC, MSSQLODBC,MSSQLNat, DBOracle, getDSConnStr
from common.loghdl import getLogHandler
RET_WARN = 101
RUN_DATE = su.getTodayDtStr(fmt='%Y%m%d')
# Empty Container
class InfaBaseAppBean:
pass
class _InfaBaseApp(object):
'''
classdocs
'''
# Set of diagnostics commands to run
hostname = socket.gethostname()
exitOnError = False # default.
def __init__(self):
self.appName = self.__class__.__name__.lower()
