def init(self):
DwServiceCore.init(self)
Logger.init()
self.ubaSqlPath = self.getDwCoreInstance().SystemPath(
'basePath') + '/uba_log/uba_sql'
def process(self):
Logger.init()
pars = self.getFormatParameter()
u'日期'
parsDate = pars.get('date')
if (parsDate != None):
if (parsDate == "today"):
date = self.getRegisterInstance('dateModel').getToday()
elif (parsDate == "tomorrow"):
date = self.getRegisterInstance('dateModel').getTomorrow()
elif (parsDate == "yesterday"):
date = self.getRegisterInstance('dateModel').getYesterday()
else:
date = parsDate
else:
u'默认是昨天日期 ,格式: 20151010'
date = self.getRegisterInstance('dateModel').getYesterday()
print date
self.schedulerJob(date)
self.minireportJob(date)
self.hiveMetadata(date)
self.dataWarehouse(date)
self.hdfsMonitor(date)
def runDwSqlProcess(self, parsMap):
status = False
try :
Logger.init()
sqlFile = parsMap.get('sql')
# sql file 文件
if (sqlFile == None):
Logger.info("sql 仓库文件不存在")
exit(1)
# 日期
parsDate = parsMap.get('date')
if (parsDate != None ):
if (parsDate == "today") :
date = self.getRegisterInstance('dateModel').getToday()
elif (parsDate == "tomorrow") :
date = self.getRegisterInstance('dateModel').getTomorrow()
elif (parsDate == "yesterday") :
date = self.getRegisterInstance('dateModel').getYesterday()
else :
date = parsDate
else :
# 默认是昨天日期 ,格式: 20151010
date = self.getRegisterInstance('dateModel').getYesterday()
# 服务器类型
serverType = parsMap.get('serverType')
if (serverType == None ):
Logger.info("serverType : hive or spark")
exit(1)
# 是否是 dw 仓库的 sql 文件
isDwSql = parsMap.get("isDwSql")
# 读取 sql 文件内容，并且格式化好时间
if (isDwSql == None):
sqlContent = self.getDwSqlContent(parsMap.get('sql'), date)
elif(isDwSql == "yes"):
sqlContent = self.getDwSqlContent(parsMap.get('sql'), date)
elif(isDwSql == "no"):
sqlContent = self.getSqlContent(parsMap.get('sql'), date)
else:
Logger.info("isDwSql 参数: [yes|no]")
exit(1)
if (serverType == 'hive'):
status = self.runSqlByHive(sqlContent, parsMap.get('runEnv'))
elif (serverType == 'spark'):
status = self.runSqlBySpark(sqlContent)
except Exception,ex:
log = "异常存储过程: "
log += " -> " + str(Exception) + ":" + str(ex)
Logger.info(log)
def process(self):
Logger.init()
# 录入参数:
parsMap = self.getFormatParameter()
# 执行控制
self.dwSqlServcie = DwSql()
# 日志类型
logType = parsMap.get('logType')
if (logType == None or logType == '') :
# APP 主题 LOG
self.dwAppAccessLogStatus = self.dwAppAccessLog(parsMap)
self.dwAppActionDetailLogStatus = self.dwAppActionDetailLog(parsMap)
# WEB 主题 Log
self.dwWebVisitTrafficLogStatus = self.dwWebVisitTrafficLog(parsMap)
self.dwWebActionDetailLogStatus = self.dwWebActionDetailLog(parsMap)
elif (logType == 'dw_app_access_log' ):
self.dwAppAccessLogStatus = self.dwAppAccessLog(parsMap)
elif (logType == 'dw_app_action_detail_log' ):
self.dwAppActionDetailLogStatus = self.dwAppActionDetailLog(parsMap)
elif (logType == 'dw_web_visit_traffic_log' ):
self.dwWebVisitTrafficLogStatus = sself.dwWebVisitTrafficLog(parsMap)
elif (logType == 'dw_web_action_detail_log' ):
self.dwWebActionDetailLogStatus = self.dwWebActionDetailLog(parsMap)
def run(self):
Logger.init()
# 抽取类型(全量、增量)
curExtractType = self.getExtractType()
# 全量抽取
if (curExtractType == ExtractMysql.COMPLETE):
self.extractCompleteAction()
# 增量抽取
elif (curExtractType == ExtractMysql.INCREMENTAL):
self.extractIncrementalAction()
def run(self):
Logger.init()
# 目标数据库不存在
if (self.isExistsGatherTable() == False):
# 初始化数据表
self.initGatherTable()
else:
sourceTableNewFields = self.getSourceTableNewFields()
# 如果 source table 有新增的字段
if (len(sourceTableNewFields) > 0):
# 增加新的字段
self.alterGatherTableField(sourceTableNewFields)
# 导入
self.sourceTableToGatherTable()
else:
# 导入
self.sourceTableToGatherTable()
def process(self):
Logger.init()
# 解析参数
parameter = self.getFormatParameter()
dbName = parameter.get('dbName')
tbName = parameter.get('tbName')
date = parameter.get('date')
if (date == None):
date = self.getRegisterInstance('dateModel').getYesterdayByYmd()
# snapshot 指定数据表
if (dbName != None and tbName != None):
self.snapshotTable(dbName,tbName,date)
# snapshot 所有表
else:
self.snapshotTableAll(date)
def process(self):
Logger.init()
# 解析参数
parameter = self.getFormatParameter()
# 运行类型
runType = parameter.get('runType')
# 串行抽取
if (runType == 'liste'):
self.extractMysqlTableListe()
# 并行抽取
elif (runType == 'thread'):
self.extractMysqlTableThread()
# 指定抽取数据表抽取
elif (parameter.get('sourceDb') != None
and parameter.get('sourceTb') != None):
self.extractMysqlTableIndependent(parameter)
# 测试
else:
self.extractMysqlTableTest()