def procHierSCOS(self): # Check for file existence src = '%s/%s/%s' % (self.ib.shareDir,self.outDir,self.ib.outFile) r = fu.fileExists(src) if r == False : self.log.error('File %s does not exist' % src) return 1 # Move File to output dir dt = su.getTodayDtStr('%y%m%d') self.outFile = 'SCS_Org_Hierarchy%s.csv' % dt outFile = '%s/%s/%s/%s' % (self.ib.shareDir,self.outDir,self.ib.outDirname,self.outFile) r = fu.moveFile(src,outFile) if r != 0 : self.log.error("Could not move %s to %s " % (src,outFile)) self.log.debug('Moved src = %s to self.outFile = %s RC = %s' % (src,outFile,r)) # Archive File . arcfn = '%s/%s/%s/%s-%s' % (self.ib.shareDir,self.outDir,self.outArchDir,self.ts,self.outFile) r = fu.copyFile(outFile,arcfn) if r != 0 : self.log.error("Could not copy %s to %s " % (outFile,arcfn)) self.log.debug('Copied src = %s to racfn = %s RC = %s' % (outFile,arcfn,r)) return 0
def _chkInfaRdyMthly(self,job): rs = self._chkInfaRdy(job) if rs is None or len(rs) != 1 : return -1 d = rs[0][0]; dt = d[:-2] todm = su.getTodayDtStr('%Y%m') self.log.debug('d = %s dt = %s todm = %s' % (d,dt,todm)) if dt == todm : return 0 return 1
def _chkInfaRdyDly(self,job): rs = self._chkInfaRdy(job) if rs is None or len(rs) != 1 : return -1 dt = rs[0][0] todm = su.getTodayDtStr('%Y%m%d') self.log.debug('dt = %s todm = %s' % (dt,todm)) if dt == todm : return 0 return 1
def chkMssqlRdyDly(self,job): qryStr = ds.selMssqlPredQry % job self.log.debug('qryStr = %s' % qryStr) rs = self._chkMssqlRdy(qryStr) if rs is None or len(rs) != 1 : return -1 dt = rs[0][0] todm = su.getTodayDtStr('%Y%m%d') self.log.debug('dt = %s todm = %s' % (dt,todm)) if dt == todm : return 0 return 1
def getLastWorkDay(self,lwd): mtyr = su.getTodayDtStr(fmt='%m%Y') m = mtyr[:2] ; y = mtyr[2:] qry = ds.lastworkDay % (m,y,m,y,lwd) self.log.debug('qry = %s ' % qry) rs = self._getNZDS(qry) self.log.debug('rs' , rs) if rs is None : return None if len(rs) != 1 : return -1 if len(rs[0]) != 1 : return -4 return rs[0][0]
def isLastWorkDayWarn(self): rc = -2 w = self.getLastWorkDay(self.ib.lastworkday) if w is None : self.log.error(' wkday = %s : Please check DB connectivity table' % w) return -5 wd = su.getDtStr(w,'%Y-%m-%d') d=su.getTodayDtStr('%Y-%m-%d') self.log.debug('self.ib.lastworkday = %s date_day = %s today = %s' % (self.ib.lastworkday,w,d)) if d == wd : rc = 0 else : rc = RET_WARN return rc
def _updLeaseCrdTbl(self,pdate): if pdate is None or len(pdate) != 10 : self.log.debug('Invalid pdate %s ' % (pdate)) return 1 cdate = su.getTodayDtStr(fmt='%Y-%m-%d') if cdate == pdate: self.log.debug('curr_date %s = pdate %s. No need to update table.' % (cdate,pdate)) return 0 self.log.info("Updating table setting curr_date %s to pdate = %s " % (cdate, pdate)) rc =self._getNZDS(da.updLCRStgCortExtQry % (pdate,cdate) , po='UPD') self.log.info('Update Qry returned %s' % rc) if rc > 0 : return 0 return 2
def _build2XML(self, grpbl): root = ET.Element('PROC_GRP') comment = ET.Comment('For Recovery PID %s. Generated on %s ' % (self.pid, su.getTodayDtStr('%m/%d/%Y %H:%M:%S'))) root.append(comment) for g in grpbl: self.log.debug(" name = " , g.name , " order = ", g.order , " active= ", g.active , " exit_on_err = " , g.exitErr) root.append(ET.Comment('Group %s' % g.name)) group = ET.SubElement(root, 'group', name = g.name, order = '%s' % g.order, active= '%s' % g.active, exit_on_err = '%s' % g.exitErr ) for p in g.pipelines: self.log.debug("\t name = " , p.name , " order = ", p.order , " active= ", p.active , " exit_on_err = " , p.exitErr) pipel = ET.SubElement(group, 'pipeline',name = p.name, order = '%s' % p.order, active= '%s' % p.active, exit_on_err = '%s' % p.exitErr ) for t in p.tasks: self._setElemTask(pipel,t) return root
def _chkSrcReady(self): qryStr = "select max(MeasureDate) MonthDate , 1 isServiceCallsReady from dbo.edw_ServiceCalls" self.ib.user = self.ib.ms_user; self.ib.pwd = self.ib.ms_pwd; self.ib.dbserver = self.ib.ms_dbserver ; self.ib.db = self.ib.ms_db rs = self._getMSSQLNatDS(qryStr) self.log.debug('qryStr = ', qryStr , ' rs ', rs) if rs is None or len(rs) != 1 : return -1 isRdy = rs[0][1] rdtm = su.getDtStr(rs[0][0],'%m%Y') todm = su.getTodayDtStr('%m%Y') nmth = su.getMonthPlusStr(1,rdtm,'%m%Y') self.log.debug("isRdy = %s rdtm=%s todm=%s nmth=%s" % (isRdy,rdtm,todm,nmth)) if isRdy == 1 and nmth == todm : # and sysdate - 1 self.log.info ("Ready to Proceed : rdtm = %s isRdy = %s " % (rdtm,isRdy)) return 0 return 1
def _getROPOPred(self,tbn): qryStr = ds.selFinRCOpenROPOQry % tbn rs = self._chkMssqlRdy(qryStr) self.log.debug('rs = ' , rs) # rs = [(u'ConsolidatedSHPOD ', '20150310', 6240165)] if rs is None or len(rs) < 1 or len(rs[0]) != 3 : return -1 tbn = rs[0][0]; dt = rs[0][1] ; rowc = rs[0][2] ; self.log.info('tbn = %s dt = %s rowc = %s ' % (tbn,dt,rowc)) # Check if date current date todm = su.getTodayDtStr('%Y%m%d') self.log.debug('dt = %s todm = %s' % (dt,todm)) if dt != todm : return 1 # Check if number of rows for today are > 0 irowc = su.toInt(rowc) self.log.debug('irowc = %s' % (irowc)) if irowc is None or irowc < 1 : return 2 return 0
def _chkSrcReady(self): qryStr = """ select MonthDate, CAST(isSourcingReady AS int) from dbo.DataReady WHERE MonthDate = (select MAX(MonthDate) from dbo.DataReady)""" self.ib.user = self.ib.ms_user; self.ib.pwd = self.ib.ms_pwd; self.ib.dbserver = self.ib.ms_dbserver ; self.ib.db = self.ib.ms_db rs = self._getMSSQLNatDS(qryStr) self.log.debug('qryStr = ', qryStr , ' rs ', rs) if len(rs) != 1 : return -1 isRdy = rs[0][1] rdtm = su.getDtStr(rs[0][0],'%m%Y') todm = su.getTodayDtStr('%m%Y') nmth = su.getMonthPlusStr(1,rdtm,'%m%Y') self.log.debug("isRdy = %s rdtm=%s todm=%s nmth=%s" % (isRdy,rdtm,todm,nmth)) if isRdy == 1 and nmth == todm : # and sysdate - 1 = dt self.log.info ("Ready to Proceed : rdtm = %s isRdy = %s " % (rdtm,isRdy)) return 0 return 1
def _chkHeader(self,fnp,rdoff): RUN_DATE = su.getTodayDtStr(fmt='%y%m%d') rdayoffset = su.toInt(rdoff) hdr = fu.readFile(fnp) if len(hdr) != 14 : self.log.error('Invalid len= %d for %s. Need to be 14\nhdr=%s' % (len(hdr),fnp,hdr)) return 1 rowc = su.toInt(hdr[0:8]) hdr_date = hdr[8:] # Run Date from file. dt = su.getDayMinusStr(rdayoffset, RUN_DATE, '%Y%m%d') # Check run date. if dt != hdr_date: self.log.warn('fn %s === \tInvalid Header ran date of %s. Should be %s ' % (fnp,dt,hdr_date)) if rdayoffset < 1 : self.log.error('Could not attempt next day since rdayoffset is %s < 1' % rdayoffset) return 2 dt = su.getDayMinusStr(rdayoffset - 1, RUN_DATE, '%Y%m%d') # Check run date + 1. if dt != hdr_date: self.log.error('fn %s === \tInvalid Header ran date of %s. Should be %s ' % (fnp,dt,hdr_date)) return 3 # Check Number of lines ln = fu.getLines(fnp) # Number of lines w/o trailer. dc = int(ln) - rowc # ln file has 2 extra lines header/trailer, but at this point trailer rec had been removed. self.log.debug('fn %s === \tFile record count %s trailer = %s diff %s' % (fnp,ln,rowc,dc)) if dc != 0 : self.log.error('fn %s === \tFile record count %s do not match trailer = %s # of records diff = %d!' % (fnp,ln,rowc,dc)) return 4 return 0
from apps.infbaseapp import _InfaBaseApp # Mandatory to define self.cmdStep # method _getNextRunDate is sensitive to schedule changes ! RUN_PER_DAY = 1 # Daily runs. DP_LEN = len('YYYYMMDD') # Schedules #SCH_FREQ = 'WDay' SCH_FREQ = 'Cust' sch = ('Tue','Wed','Thu','Fri','Sat') cur_dayr = su.getTodayDtStr('%Y%m%d') class VehFifthWkday(_InfaBaseApp): exitOnError = True def __init__(self): super(VehFifthWkday,self).__init__() self.landDir = 'SrcFiles/vehicle' self.incFileSet = [] # Incoming Files. Contains full path name. self.incFiles = [] self.workFiles = [] # Files that were moved to the working dir (ideally same than incSetFile). self.RowCnt = -1 self.srcFile = ('PO225D15.dat','PO875D30.dat') # File that Informatica expects. Alphabetical. self.ib.fileName = r"'P.PO225D15.UMPK(0)','P.PO875D30.WHOUSE.CURR.STD'" self.checkNextRunFlg = False
import os #, os.path import sys import socket import time import logging import common.simpmail as sm import common.lockfile as lck import utils.fileutils as fu import utils.strutils as su from common.loghdl import getLogHandler RET_WARN = 101 RUN_DATE = su.getTodayDtStr(fmt='%Y%m%d') # Empty Container class BaseAppBean: pass class _BaseApp(object): hostname = socket.gethostname() def __init__(self): self.appName = self.__class__.__name__.lower() self.log = logging.getLogger(self.appName) self.suf = 'cfg'
import utils.fileutils as fu import utils.strutils as su import utils.filetransf as ft import procdata.procinfa as pi from apps.infbaseapp import _InfaBaseApp # Mandatory to define self.cmdStep # method _getNextRunDate is sensitive to schedule changes ! RUN_PER_DAY = 1 # Daily runs. DP_LEN = len("YYYYMM") # Schedules SCH_FREQ = "Mthly" sch = () cur_dayr = su.getTodayDtStr("%Y%m") class EmpBenefitonDem(_InfaBaseApp): exitOnError = True def __init__(self): super(EmpBenefitonDem, self).__init__() self.landDir = "SrcFiles/employee" self.incFileSet = [] # Incoming Files. Contains full path name. self.incFiles = [] self.workFiles = [] # Files that were moved to the working dir (ideally same than incSetFile). self.RowCnt = -1 self.srcFile = ("hp400jnm_sap_dwext.dat", "hrcoord5.csv") # File that Informatica expects. Alphabetical. self.ib.fileName = r"'P.HP400JNM.SAP.DW.EXTRACT',hrcoord5.csv"
def __init__(self): super(TalentMap, self).__init__() self.landDir = "SrcFiles/talentmap" self.incFileSet = [] # Incoming Files. Contains full path name. self.workFiles = [] # Files that were moved to the working dir (ideally same than incFiles). self.RowCnt = -1 self.ib.fld = "TalentMap" self.ib.wkf = "wkf_talentMap" self.ib.srcFile = ( "All_Training.txt", "All_Training_Providers.txt", "Transcripts_OpsRegion_Canada.txt", "Transcripts_OpsRegion_Central.txt", "Transcripts_OpsRegion_Northeast.txt", "Transcripts_OpsRegion_Southeast.txt", "Transcripts_OpsRegion_West.txt", ) # File that Informatica expects. Alphabetical. self.ib.FileColCnt = { "All_Training.txt": 12, "All_Training_Providers.txt": 2, "Transcripts_OpsRegion_Canada.txt": 18, "Transcripts_OpsRegion_Central.txt": 18, "Transcripts_OpsRegion_Northeast.txt": 18, "Transcripts_OpsRegion_Southeast.txt": 18, "Transcripts_OpsRegion_West.txt": 18, "Tops_Plan_Current.csv": 13, "FMS_Qualifications.csv": 8, } self.ib.opsRegHdrRow = "" self.fileDate = "" self.FILE_SET_LEN = 4 self.procCort = {} self.procBuReg = {} self.renameFtpFilesTgtFlg = False self.ts = su.getTimeSTamp() # Allowable commands for this application. Make sure to Set self.cmdStep = { "A": self.getLock, "B": self.renFtpFilesTgtFlg, "C": self.getTMapftpFiles, # FTP Files based on regex. "D": self.getIncSetFiles, # Populates self.incSetFiles. Incoming Files. "E": self.setIncFilePvt, # Run after setting self.incFiles "F": self.cpFileToWorkDir, # Copies FileSet and sets self.workFiles (full path) "G": self.archFiles, "H": self.chkFileColsFlg, "I": self.preProcTMapFiles, # Check that headers/record count, number of fields. "J": self.procTMapFiles, "K": self.procCustFiles, "L": self.crtTrigFile, "M": self.wkfTalentMapProc, } # Infa Environmental variables/ self.infaEnvVar = { "PMCMD": "mg.pmcmd", "INFA_USER": "******", "INFA_XPWD": "self.ib.rep_xpwd", "DOMAIN": "self.ib.dom_name", "INT_SERV": "self.ib.IS", "INFA_SHARE": "self.ib.shareDir", "INFA_APP_CFG": "self.ib.cfgDir", "INFA_APP_LCK": "self.ib.lckDir", "INFA_APP_CTL": "self.ib.ctlDir", } self.fnd = su.getTodayDtStr("%m_%d_%Y") # FTP is expecting the following env variables, which should not be in a config file. # os.environ['RXFILE' ] = '.*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*.TXT' # data file(s). Use for MGET os.environ["FILE"] = "Name of control file" # data file(s). Use for GET os.environ["RXFILE"] = ( "Transcripts_OpsRegion_.*_[0-9]*_%s_[0-9]*_[0-9]*.TXT" % self.fnd ) # data file(s). Use for MGET
import common.log4py as log4py import common.simpmail as sm import common.lockfile as lck import utils.fileutils as fu import utils.strutils as su import datastore.dbapp as ds import mainglob as mg # Do not remove. from datastore.dbutil import NZODBC, MSSQLODBC,MSSQLNat, DBOracle, getDSConnStr from common.loghdl import getLogHandler RET_WARN = 101 RUN_DATE = su.getTodayDtStr(fmt='%Y%m%d') # Empty Container class InfaBaseAppBean: pass class _InfaBaseApp(object): ''' classdocs ''' # Set of diagnostics commands to run hostname = socket.gethostname() exitOnError = False # default. def __init__(self): self.appName = self.__class__.__name__.lower()