def _setConnParam(self):
#rc = _BaseFT._getFtpEnvVars(self)
rc = self._getFtpEnvVars()
if rc != 0 : return rc
ftpVerb = su.toInt(self.ftpVerb)
if ftpVerb is None:
self.log.error('FTP_VERB %s needs to be an integer. Setting to default %d' % (self.ftpVerb,FTP_VERBOSE))
self.ftpVerb = FTP_VERBOSE
else:
self.ftpVerb = 0 if ( ftpVerb < 0 or ftpVerb > 2 ) else ftpVerb
self.log.debug('FTP_VERB : %d ' % self.ftpVerb)
ftpPort = su.toInt(self.ftpPort)
if ftpPort is None:
self.log.error('FTP_PORT : %s needs to be an integer.Setting to Default %d ' % (self.ftpPort,FTP_PORT))
self.ftpPort = FTP_PORT
else:
self.ftpPort = ftpPort
self.log.debug('FTP_PORT : %s' % (self.ftpPort))
ftpTimeout = su.toFloat(self.ftpTimeout)
if ftpTimeout is None:
self.ftpTimeout = FTP_TIMEOUT
self.log.error('FTP_TIMEOUT : %s needs to be a float .Setting to Default %s' % (self.ftpTimeout, FTP_TIMEOUT))
else:
self.ftpTimeout = ftpTimeout
self.log.debug('FTP_TIMEOUT : %s' % (self.ftpTimeout))
return rc
def start_load(args,table_list):
for t in table_list:
table_name = t.strip(' \t\n\r')
export_script = os.path.join(args['script'],'exp_'+ args['gp_db_sche'] + '.' + table_name + '.sql')
data_filename = os.path.join(args['data'], args['gp_db_sche'] + '.' + table_name + '.csv')
log_filename = os.path.join(args['nzlog'], args['gp_db_sche'] + '.' + table_name + '.log')
log.debug ('export_script = %s\ndata_filename = %s\ntable_name = %s\nlog_filename = %s log_filename' % (export_script,
data_filename,
table_name,
log_filename))
source_count = _getGPCnt(args,table_name)
if su.toInt(source_count) < 1 :
log.warn('No rows to load from GP %s.%s ' % (args['gp_db_sche'],table_name))
continue
log.info('GP Source Table %s\tcount=%s' % (table_name,source_count))
schema_script = os.path.join(args['script'],'schm_' + args['gp_db_sche'] + '.' + table_name + '.sql')
rc =_crtSchemaScript(args,schema_script,table_name)
if rc == 0 :
log.debug('Created Schema script %s' % (schema_script))
else :
log.error('Creating Schema script %s' % (schema_script))
continue
rc = _runGPExpScript(args,schema_script,export_script,table_name)
nz_cnt = _getNZCnt(args,table_name)
log.debug('Record count in Netezza (pre-import) = %s'%(nz_cnt))
if su.toInt(nz_cnt) > 0 :
rc = _delNZRows(args,table_name)
if rc == 0 : log.debug('Table %s has been truncated..' % table_name)
else : log.error('Table %s cannot be truncated..' % table_name)
nz_cnt = _getNZCnt(args,table_name)
log.debug('Record count in Netezza after delete = %s'%(nz_cnt))
if export_script == '/infa/product/pc951/server/infa_shared_conv/SrcFiles/gp_nz_migration/script/exp_sdl.aps_stg_spcases.sql':
export_script = '/infa/product/pc951/server/infa_shared_conv/SrcFiles/gp_nz_migration/script/exp_sdl.aps_stg_spcases_bkp.sql'
log.debug('export_script = %s' % export_script)
rc = _runNZImpScript(args,export_script,table_name)
target_count = _getNZCnt(args,table_name)
log.info('NZ Target Table %s\tcount=%s'%(table_name,target_count))
if source_count <> target_count:
log.error('Loading %s. Please check log %s'%(args['gp_db_sche'] + table_name ,log_filename))
else:
log.debug('Loaded successfully')
return 0 # EO Need to add .
def _setIBVarsInt():
n = su.toInt(ib.GGS_lag)
if n == None:
log.error("GGS_LAG_MAX Needs to be an integer. Current setting %s" % ib.GGS_lag)
return 1
ib.GGS_lag = n
n = su.toInt(ib.GGS_chkpt)
if n == None:
log.error("GGS_LAG_MAX Needs to be an integer. Current setting %s" % ib.GGS_lag)
return 1
ib.GGS_chkpt = n
return 0
def delCarrPrompt(self):
val = []
rc = self._connToDB('NZODBC')
if rc != 0 :
self.log.error('Error Connecting to DB = %s' % rc)
return rc
self.log.info('Deleting records older than %s' % self.ib.numdays)
val.append(su.toInt(self.ib.numdays))
sql = "delete from tis_fb_carr_pmpt_imprt where proc_flag = 1 and date_time < current_date - ?"
rc = self.dbh.exeQry(sql,val)
self.log.debug('qry %s rc= %s ' % (sql,rc))
if rc < 0 :
self.log.error('qry = %s rc = %s' % (sql,rc))
return rc
#print "del records from tis_fb_carr_pmpt_imprt rc ", rc
sql = "delete from tis_fb_carr_pmpt_imprt_dtl where seqid > 0 and date_time < current_date - ?"
rc = self.dbh.exeQry(sql,val)
self.log.debug('qry %s rc= %s ' % (sql,rc))
if rc < 0 :
self.log.error('qry = %s rc = %s' % (sql,rc))
return rc
return 0
def _checkIncWildFiles(fn):
# Get file
fnp = fu.getNewestFile(ib.landDir, fn)
log.debug("Newest File ", fnp)
if fnp is None:
log.error("No files were found for %s/%s " % (ib.landDir, fn))
return 1
# Check file last modification time
tm = fu.chkFileModTime(fnp)
if float(tm) > float(ib.fileAge):
log.error("file %s age %.1f secs is older than fileAge %s secs" % (fnp, tm, ib.fileAge))
return 2
log.info("file %s age %.1f secs is newer than fileAge %s secs" % (fnp, tm, ib.fileAge))
fms = su.toInt(ib.fileMinSize)
log.debug("fms = %s ib.fileMinSize %s" % (fms, ib.fileMinSize))
if fms is None:
return 3
# Check File Size in bytes, only if fileMinSize > -1
if fms > -1:
fs = fu.getFilseSize(fnp)
log.debug("size = %s minsize = %s fnp = %s " % (fs, fms, fnp))
if fs < fms:
log.error("File fnp %s %s(bytes) < %s fileMinSize" % (fnp, fs, fms))
return 4
return 0
def postCheck(self):
rc = 0
for fn,rcnt in self.srcCount.items():
tblNm = self.ctlTbl[fn]
if tblNm == '' : continue
rs = self._getNZDS(da.selFinCntlRecCnt % (tblNm,tblNm))
self.log.debug('tblNm=', tblNm ,'rs', rs)
if rs is not None and len(rs) > 0:
dbcnt = su.toInt(rs[0][0])
self.log.debug ('File = %s Fcnt = %s Tbl = %s Tcnt = %s' % (fn,rcnt,tblNm,dbcnt))
if dbcnt is not None and dbcnt >= 0:
if rcnt - dbcnt == 0 :
self.log.info('Count for %s %d Matches' % (fn,rcnt))
else :
self.log.error('Count for %s = %d Does Not Match Table %s = %d' % (fn,rcnt,tblNm,dbcnt))
rc+=1
else:
self.log.error("Count for File %s = Table %s Mismatch" % (fn, tblNm))
rc+=1
return rc
def dispExePlan(self):
cb = self._configBean()
self.log.debug('Config Bean = %s' % cb)
pid = su.toInt(self.recPid)
if pid is None :
self.error('Invalid PID %s' % pid)
return -1
if pid == 0:
fxml = '%s/%s.xml' % (cb.confDir,cb.appName)
self.log.info('Displaying Main Execution plan %s' % fxml)
else :
fxml = '%s/%s/%s%s.xml' % (cb.recDir,self.recPid,cb.appName,self.recPid)
self.log.info('Displaying Recovery Execution plan %s' % fxml)
pj = pjl.ProcessJobs(cb,self.log)
grpbl = pj.loadExecPlan(fxml)
if len(grpbl) < 1 :
self.log.error('Nothing to process!. Check Contents of %s' % fxml)
return 1
pj.printExecPlan(grpbl)
return 0
def _chkIfValidLock(self):
rc = True
# Check if there is a lock file.
if fu.fileExists(self.lckFn):
sPid = fu.readFile(self.lckFn)
pid = su.toInt(sPid)
self.log.info('Lock File %s EXISTS. sPid = %s , pid = %s' %
(self.lckFn, sPid, pid))
# Check if file has a valid PID (number)
if pid is None:
rc = fu.delFile(self.lckFn)
if rc == 0:
self.log.info('Removed File %s' % (self.lckFn))
else:
self.log.error('Could not removed File %s' % (self.lckFn))
return False
# If pid is a valid number, check if the process is running ...
rc = ps.isProcRunning(pid, self.log)
self.log.debug('isProcRunning returned %s' % (rc))
return rc
# No lock file exists.
else:
return False
def setEnvDir(options):
args = {}
tbl_samp_row = 0
tbl_samp_pct = 0
# Env vars.
try:
args['gp_nz_home'] = os.environ['GP_NZ_HOME' ]
# GP
args['gp_db_host'] = os.environ['GP_DB_HOST' ]
args['gp_db_user'] = os.environ['GP_DB_USER' ]
args['gp_db_port'] = os.environ['GP_DB_PORT' ]
args['gp_db_env' ] = os.environ['GP_DB_ENV' ]
args['gp_db_sche'] = os.environ['GP_DB_SCHE' ]
#NZ
args['nz_db_env' ] = os.environ['NZ_DB_ENV' ]
args['nz_db_host'] = os.environ['NZ_DB_HOST' ]
args['nz_db_user'] = os.environ['NZ_DB_USER' ]
args['nz_db_pwd' ] = os.environ['NZ_DB_PWD' ]
args['nz_db_port'] = os.environ['NZ_DB_PORT' ]
#Verification of result sets
if os.environ.has_key('TBL_SAMP_PCT'): tbl_samp_pct = su.toInt(os.environ['TBL_SAMP_PCT'])
args['tbl_samp_pct'] = tbl_samp_pct
if os.environ.has_key('TBL_SAMP_ROW'): tbl_samp_row = su.toInt(os.environ['TBL_SAMP_ROW'])
args['tbl_samp_row'] = tbl_samp_row
if tbl_samp_pct > 0 and tbl_samp_row:
log.error('TBL_SAMP_PCT = %s TBL_SAMP_ROW = %s are Mutually exclusive param.\nWill do sampling : %s %% .' % (tbl_samp_pct,tbl_samp_row,tbl_samp_pct))
if options.config != None:
args['config'] = options.config
else:
args['config'] = '%s/config/gp_table.ini'%(args['gp_nz_home'])
# Directory settings.
args['data' ] = '%s/data' %(args['gp_nz_home'])
args['log' ] = '%s/log' %(args['gp_nz_home'])
args['script' ] = '%s/script' %(args['gp_nz_home'])
args['nzlog' ] = '%s/nzlog' %(args['gp_nz_home'])
args['nzbad' ] = '%s/nzbad' %(args['gp_nz_home'])
except:
log.error("ENV var not set %s %s\n" % (sys.exc_type, sys.exc_info()[1]))
finally:
log.debug('args %s ' % args)
return args
def procXMLFiles(self):
rc = 0; r = False ; x = [] ; cdrRes = [] # List that contains all the responses.
if len(self.cdrFilestoProc) < 1 :
self.log.error('No incoming CDR files to process : cdrFilestoProc')
return 1
pxmlCDR = pxml.ProcXMLCDR(self.log)
self.log.debug("Files to process %s" % ''.join(self.cdrFilestoProc))
for fn in self.cdrFilestoProc:
self.log.debug("processing %s" % fn)
r = pxmlCDR.isValidXML(fn)
if r == False: rc+=1
else:
x = pxmlCDR.parseCDR(fn)
ftn = fu.getFileBaseName(fn)
if x is None or len(x) != 3:
self.log.error('Error Parsing %s ' % fn)
tgt = '%s/%s.%s' % (self.ib.badDir,ftn,self.ts)
else:
self.log.debug('fn=%s rfId=%s rCode=%s desc=%s' % (fn,x[0],x[1],x[2]))
r = su.toInt(x[1])
if r != 0 :
self.log.error('fn=%s rfId=%s rCode=%s desc=%s' % (fn,x[0],x[1],x[2]))
tgt = '%s/%s.%s' % (self.ib.badDir,ftn,self.ts)
rc+=1
else :
cdrRes.append('%s %s %s\n' % (x[0],self.ib.FS,x[1]))
tgt = '%s/%s.%s' % (self.ib.archDir,ftn,self.ts)
# Move files
r = fu.moveFile(fn, tgt)
if r == 0 : self.log.info( 'mv %s to %s' % (fn,tgt))
else : self.log.error('mv %s to %s r = %s ' % (fn,tgt,r))
self.log.debug("cdrRes No of elements %d " % len(cdrRes))
if len(cdrRes) < 1:
self.lo.error('cdrRes Did not find any valid element !')
return 2
fn = '%s/CDR_%s.txt' % (self.ib.cdrOutDir, self.ts)
r = fu.createFile(fn,cdrRes)
if r == 0: self.log.info('Created file %s with CDR responses' % fn)
else :
self.log.error('Could not create file %s with CDR responses' % fn)
rc=1
return rc