def run_etl_main(startdate, enddate, q1, q2, model=None):
info("start_run_etl_main:%s-%s,model=%s" %
(str(startdate), str(enddate), model))
d1 = datetime.now()
etldate = int(startdate)
Config().etldate = etldate
while int(etldate) <= int(enddate):
d1 = datetime.now()
Config().etldate = etldate
Config().stretldate = util.tostrdate(etldate)
info("start_run_etl_main while :%s,model=%s" % (str(etldate), model))
if model == 'L':
info("run loan_hook")
LoanHook().loadtofact(q1, q2)
elif model == 'D':
info("run deposit_hook")
DepositHook().loadtofact(q1, q2)
elif model == 'E':
info("run ebank_hook")
EbankHook().loadtofact(q1, q2)
elif model == 'F':
info("run finances_hook")
FinancesHook().loadtofact(q1, q2)
elif model == 'S':
info("run stock_hook")
StockHook().loadtofact(q1, q2)
elif model == 'I':
info("run insurance_hook")
InsuranceHook().loadtofact(q1, q2)
elif model == 'C':
info("run creditcard_hook")
CreditcardHook().loadtofact(q1, q2)
else:
info("run loan_hook")
LoanHook().loadtofact(q1, q2)
info("run deposit_hook")
DepositHook().loadtofact(q1, q2)
info("run ebank_hook")
EbankHook().loadtofact(q1, q2)
info("run finances_hook")
FinancesHook().loadtofact(q1, q2)
info("run stock_hook")
StockHook().loadtofact(q1, q2)
info("run insurance_hook")
InsuranceHook().loadtofact(q1, q2)
info("run creditcard_hook")
CreditcardHook().loadtofact(q1, q2)
etldate = int(daycalc(etldate, 1))
info("finish_run_etl_main:%s,model=%s" % (str(startdate), model))
def run_etl_main(startdate, enddate, q1, q2, model=None):
info("start_run_etl_main:%s,model=%s" % (str(startdate), model))
d1 = datetime.now()
etldate = int(startdate)
Config().etldate = etldate
while int(etldate) <= int(enddate):
d1 = datetime.now()
Config().etldate = etldate
Config().stretldate = util.tostrdate(etldate)
if model == 'S':
info("run Staff")
Staff().loadtofact(q1, q1)
elif model == 'O':
info("run Custcore")
Custcore().loadtofact(q1, q1)
elif model == 'C':
info("run Customer")
Customer().loadtofact(q1, q1)
elif model == 'R':
info("run Custcredit")
Custcredit().loadtofact(q1, q1)
elif model == 'U':
info("run Custcredit")
Custupdate().loadtofact(q1, q1)
elif model == 'OCR':
info("run Customer")
Customer().loadtofact(q1, q1)
hisdict = Customer().hisdict
info("run Custcore")
Custcore().loadtofact(q1, q1, hisdict)
hisdict = Customer().hisdict
info("run Custcredit")
Custcredit().loadtofact(q1, q1, hisdict)
else:
info("run Staff")
Staff().loadtofact(q1, q1)
info("run Customer")
Customer().loadtofact(q1, q1)
info("run Custcore")
Custcore().loadtofact(q1, q1)
info("run Custcredit")
Custcredit().loadtofact(q1, q1)
etldate = int(daycalc(etldate, 1))
info("finish_run_etl_main:%s,model=%s" % (str(startdate), model))
if len(rs) > 1000:
for ir in rs:
qwork.put((insertsql, ir))
rs = []
qwork = que[count % 2]
load_dims(qwork) #进程切换
count = count + 1
if len(rs) > 0:
for ir in rs:
q1.put((insertsql, ir))
rs = []
def loadtofact(self, q1, q2):
info("贷款合同")
self.files2fact(LoanCon(), q1, q2) #
if __name__ == '__main__':
arglen = len(sys.argv)
if arglen != 3:
print "please input python %s yyyyyMMdd yyyyMMdd " % (sys.argv[0])
else:
startdate = sys.argv[1]
enddate = sys.argv[2]
etldate = int(startdate)
while etldate <= int(enddate):
Config().etldate = etldate
Config().stretldate = util.tostrdate(etldate)
loanCon().loadtofact(None, None)
etldate = int(daycalc(etldate, 1))
def run_etl_main(startdate, enddate, q1, q2, model=None):
print "load dims "
info("start_run_etl_main:%s,model=%s" % (str(startdate), model))
d1 = datetime.now()
etldate = int(startdate)
Config().etldate = etldate
#load_dims(q1,q2,q3)
#TBD CCRD MPUR跑,跑完打开 plwu
load_dims(q2)
print "load dims ,times=", datetime.now() - d1
print "run_etl_man step1..."
print etldate
if model == 'E':
util.delete_contract(etldate, int(enddate), '丰收e支付')
util.delete_contract(etldate, int(enddate), '新丰收e支付')
elif model == 'L':
util.delete_contract(etldate, int(enddate), '贷款合同')
elif model == 'T':
util.delete_contract(etldate, int(enddate), '自助终端')
elif model == 'A':
util.delete_contract(etldate, int(enddate), 'ATM')
elif model == 'P':
util.delete_contract(etldate, int(enddate), 'POS')
elif model == 'Z':
util.delete_contract(etldate, int(enddate), '支付宝快捷支付')
elif model == 'C':
util.delete_creditbad2(etldate, int(enddate))
elif model == 'CM':
util.delete_creditmpur(etldate, int(enddate))
util.delete_creditbad2(etldate, int(enddate)) #TBD
else:
util.delete_contract(etldate, int(enddate), '丰收e支付')
util.delete_contract(etldate, int(enddate), '新丰收e支付')
util.delete_contract(etldate, int(enddate), '贷款合同')
util.delete_contract(etldate, int(enddate), '支付宝快捷支付')
#util.delete_creditmpur(etldate,int(enddate))
#util.delete_creditbad2(etldate,int(enddate)) #TBD
while int(etldate) <= int(enddate):
print "run_etl_main step2...", etldate
info("run_etl_main step2:%d,model=%s" % (int(etldate), model))
d1 = datetime.now()
Config().etldate = etldate
Config().stretldate = util.tostrdate(etldate)
if model == 'E':
info("run EPAY")
EPay().loadtofact(q1, q1)
elif model == 'L':
info("run LOANCONTRACT")
loanCon().loadtofact(q1, q1)
elif model == 'CM':
info("run CCRDMPUR")
elif model == 'T':
info("run TERMINAL")
Terminal().loadtofact(q1, q1)
elif model == 'A':
info("run ATM")
ATM().loadtofact(q1, q1)
elif model == 'P':
info("run POS")
POS().loadtofact(q1, q1)
elif model == 'Z':
info("run ZFBKJ")
ZFBKJ().loadtofact(q1, q1)
else:
info("run EPAY")
EPay().loadtofact(q1, q1)
info("run CCRDACCT")
info("run ZFBKJ")
ZFBKJ().loadtofact(q1, q1)
loanCon().loadtofact(q1, q1) #贷款合同
etldate = int(daycalc(etldate, 1))
util.fix_seq_id('D_CUST_CONTRACT', 'D_CUST_CONTRACT_SEQ')
info("finish_run_etl_main:%s,model=%s" % (str(startdate), model))
def run_etl_main(startdate, enddate, q1, q2, model=None):
print "load dims "
info("start_run_etl_main:%s,model=%s" % (str(startdate), model))
d1 = datetime.now()
etldate = int(startdate)
Config().etldate = etldate
#load_dims(q1,q2,q3)
load_dims(q2)
print "load dims ,times=", datetime.now() - d1
print "run_etl_man step1..."
print etldate
if model == 'C':
util.delete_fcustview(etldate, int(enddate))
elif model == 'F':
util.delete_financialsale(etldate, int(enddate))
elif model == 'T':
util.delete_blta(etldate, int(enddate))
elif model == 'BILL':
util.delete_fbillacount(etldate, int(enddate))
elif model == 'GAS':
util.delete_gasaccount(etldate, int(enddate))
else:
#util.delete_fcustview(etldate,int(enddate))
util.delete_financialsale(etldate, int(enddate))
util.delete_fcacctjrnl(etldate, int(enddate))
util.delete_fbillacount(etldate, int(enddate))
while int(etldate) <= int(enddate):
print "run_etl_main step2...", etldate
d1 = datetime.now()
Config().etldate = etldate
Config().stretldate = util.tostrdate(etldate)
if model == 'C':
info("run CUSTVIEW")
CustView().loaddbtofact(q1, q1)
elif model == 'F':
info("run financialsale")
FinancialSale().loadtofact(q2, q2)
elif model == 'T':
info("run FTransAtion")
FTransAtion().loadtofact(q2, q2)
elif model == 'BILL':
info("run Fbill")
FBillAccount().loadtofact(q2, q2)
elif model == 'GAS':
info("run gas")
GAS().loadtofact(q2, q2)
GAS_BRANCH().loadtofact(q2, q2)
GAS_DO6().loadtofact(q2, q2)
GAS_KJ().loadtofact(q2, q2)
else:
#info("run CUSTVIEW")
#CustView().loaddbtofact(q1,q1)
info("run financialsale")
FinancialSale().loadtofact(q2, q2)
info("run FTransAtion")
FTransAtion().loadtofact(q2, q2)
info("run Fbill")
FBillAccount().loadtofact(q2, q2)
#update_org_manage_bal(etldate)
#update_ltd(etldate,etldate)
#update_orgid(etldate,etldate)
etldate = int(daycalc(etldate, 1))
util.fix_seq_id('D_ACCOUNT', 'D_ACCOUNT_SEQ_NEW')
q1.put(None)
q2.put(None)
#q3.put(None)
info("finish_run_etl_main:%s,model=%s" % (str(startdate), model))
def __init__(self, etldate):
self.db = util.DBConnect()
self.etldate = etldate
self.stretldate = util.tostrdate(self.etldate)
self.para = util.get_etl_date(etldate)
self.data_func()
def run_etl_main(startdate, enddate, q1, q2, model=None):
info("start_run_etl_main:%s,model=%s" % (str(startdate), model))
d1 = datetime.now()
etldate = int(startdate)
Config().etldate = etldate
#load_dims(q1,q2,q3)
load_dims(q2)
if model == 'C':
util.delete_fcustview(etldate, int(enddate))
elif model == 'F':
util.delete_financialsale(etldate, int(enddate))
elif model == 'T':
util.delete_fcacctjrnl(etldate, int(enddate))
elif model == 'POS':
util.delete_pos_insert(etldate, int(enddate))
elif model == 'CORP': #国际业务按客户
util.delete_ebills_qry_settlement_corp(etldate, int(enddate))
elif model == 'CASH_INFO': #现金调拨信息信息
util.delete_core_bhfmcmrd(etldate, int(enddate))
elif model == 'CASH_DIR': #现金调拨方向
util.delete_CORE_BHFMCMRM_DIRECT(etldate, int(enddate))
elif model == 'ATM':
util.delete_atm_insert(etldate, int(enddate))
elif model == 'FTBHB':
util.delete_financialsale(etldate, int(enddate))
util.delete_fcacctjrnl(etldate, int(enddate))
util.delete_fbillacount(etldate, int(enddate))
util.delete_blfm(etldate, int(enddate))
util.delete_blfmmtrn(etldate, int(enddate))
elif model == 'BILL':
util.delete_fbillacount(etldate, int(enddate))
elif model == 'HXHK': #核销清单
util.delete_blfm(etldate, int(enddate))
elif model == 'BLFMM': #逾期还款清单
util.delete_blfmmtrn(etldate, int(enddate))
else:
pass
while int(etldate) <= int(enddate):
print "run_etl_main step2...", etldate
d1 = datetime.now()
Config().etldate = etldate
Config().stretldate = util.tostrdate(etldate)
if model == 'C':
info("run CUSTVIEW")
CustView().loaddbtofact(q1, q1)
elif model == 'F':
info("run financialsale")
FinancialSale().loadtofact(q2, q2)
elif model == 'T':
info("run FTransAtion")
FTransAtion().loadtofact(q2, q2)
elif model == 'CORP': #国际业务按客户
qry_settlement_corp().loadtofact(q2, q2)
elif model == 'CASH_INFO': #现金调拨信息信息
corp_bhfmcmrd().loadtofact(q2, q2)
elif model == 'CASH_DIR': #现金调拨方向
core_bhfmcmrm_direct().loadtofact(q2, q2)
elif model == 'POS': #新增POS助农
PosInsert().loadtofact(q2, q2)
elif model == 'ATM': #新增ATM助农
AtmInsert().loadtofact(q2, q2)
elif model == 'FTBHB':
info("run financialsale")
FinancialSale().loadtofact(q2, q2)
info("run FTransAtion")
FTransAtion().loadtofact(q2, q2)
info("run Fbill")
FBillAccount().loadtofact(q2, q2)
info("run Blfmcnaf")
Blfmcnaf().loadtofact(q2, q2)
info("run Blfmmtrn")
Blfmmtrn().loadtofact(q2, q2)
elif model == 'BILL':
info("run Fbill")
FBillAccount().loadtofact(q2, q2)
elif model == 'HXHK': #核销清单 F_CORE_BLFMCNAF
Blfmcnaf().loadtofact(q2, q2)
elif model == 'BLFMM': ##逾期还款清单 F_CORE_BLFMMTRN
Blfmmtrn().loadtofact(q2, q2)
else:
pass
etldate = int(daycalc(etldate, 1))
util.fix_seq_id('D_ACCOUNT', 'D_ACCOUNT_SEQ_NEW')
info("finish_run_etl_main:%s,model=%s" % (str(startdate), model))
def __init__(self, etldate=None):
self.db = util.DBConnect()
self.etldate = etldate
self.stretldate = util.tostrdate(self.etldate)
self.para = util.get_common_parameter(self.db)
if etldate is not None: self.para['TJRQ'] = etldate
