def run_etl_main(startdate, enddate, q1, q2, model=None): info("start_run_etl_main:%s-%s,model=%s" % (str(startdate), str(enddate), model)) d1 = datetime.now() etldate = int(startdate) Config().etldate = etldate while int(etldate) <= int(enddate): d1 = datetime.now() Config().etldate = etldate Config().stretldate = util.tostrdate(etldate) info("start_run_etl_main while :%s,model=%s" % (str(etldate), model)) if model == 'L': info("run loan_hook") LoanHook().loadtofact(q1, q2) elif model == 'D': info("run deposit_hook") DepositHook().loadtofact(q1, q2) elif model == 'E': info("run ebank_hook") EbankHook().loadtofact(q1, q2) elif model == 'F': info("run finances_hook") FinancesHook().loadtofact(q1, q2) elif model == 'S': info("run stock_hook") StockHook().loadtofact(q1, q2) elif model == 'I': info("run insurance_hook") InsuranceHook().loadtofact(q1, q2) elif model == 'C': info("run creditcard_hook") CreditcardHook().loadtofact(q1, q2) else: info("run loan_hook") LoanHook().loadtofact(q1, q2) info("run deposit_hook") DepositHook().loadtofact(q1, q2) info("run ebank_hook") EbankHook().loadtofact(q1, q2) info("run finances_hook") FinancesHook().loadtofact(q1, q2) info("run stock_hook") StockHook().loadtofact(q1, q2) info("run insurance_hook") InsuranceHook().loadtofact(q1, q2) info("run creditcard_hook") CreditcardHook().loadtofact(q1, q2) etldate = int(daycalc(etldate, 1)) info("finish_run_etl_main:%s,model=%s" % (str(startdate), model))
def run_etl_main(startdate, enddate, q1, q2, model=None): info("start_run_etl_main:%s,model=%s" % (str(startdate), model)) d1 = datetime.now() etldate = int(startdate) Config().etldate = etldate while int(etldate) <= int(enddate): d1 = datetime.now() Config().etldate = etldate Config().stretldate = util.tostrdate(etldate) if model == 'S': info("run Staff") Staff().loadtofact(q1, q1) elif model == 'O': info("run Custcore") Custcore().loadtofact(q1, q1) elif model == 'C': info("run Customer") Customer().loadtofact(q1, q1) elif model == 'R': info("run Custcredit") Custcredit().loadtofact(q1, q1) elif model == 'U': info("run Custcredit") Custupdate().loadtofact(q1, q1) elif model == 'OCR': info("run Customer") Customer().loadtofact(q1, q1) hisdict = Customer().hisdict info("run Custcore") Custcore().loadtofact(q1, q1, hisdict) hisdict = Customer().hisdict info("run Custcredit") Custcredit().loadtofact(q1, q1, hisdict) else: info("run Staff") Staff().loadtofact(q1, q1) info("run Customer") Customer().loadtofact(q1, q1) info("run Custcore") Custcore().loadtofact(q1, q1) info("run Custcredit") Custcredit().loadtofact(q1, q1) etldate = int(daycalc(etldate, 1)) info("finish_run_etl_main:%s,model=%s" % (str(startdate), model))
if len(rs) > 1000: for ir in rs: qwork.put((insertsql, ir)) rs = [] qwork = que[count % 2] load_dims(qwork) #进程切换 count = count + 1 if len(rs) > 0: for ir in rs: q1.put((insertsql, ir)) rs = [] def loadtofact(self, q1, q2): info("贷款合同") self.files2fact(LoanCon(), q1, q2) # if __name__ == '__main__': arglen = len(sys.argv) if arglen != 3: print "please input python %s yyyyyMMdd yyyyMMdd " % (sys.argv[0]) else: startdate = sys.argv[1] enddate = sys.argv[2] etldate = int(startdate) while etldate <= int(enddate): Config().etldate = etldate Config().stretldate = util.tostrdate(etldate) loanCon().loadtofact(None, None) etldate = int(daycalc(etldate, 1))
def run_etl_main(startdate, enddate, q1, q2, model=None): print "load dims " info("start_run_etl_main:%s,model=%s" % (str(startdate), model)) d1 = datetime.now() etldate = int(startdate) Config().etldate = etldate #load_dims(q1,q2,q3) #TBD CCRD MPUR跑,跑完打开 plwu load_dims(q2) print "load dims ,times=", datetime.now() - d1 print "run_etl_man step1..." print etldate if model == 'E': util.delete_contract(etldate, int(enddate), '丰收e支付') util.delete_contract(etldate, int(enddate), '新丰收e支付') elif model == 'L': util.delete_contract(etldate, int(enddate), '贷款合同') elif model == 'T': util.delete_contract(etldate, int(enddate), '自助终端') elif model == 'A': util.delete_contract(etldate, int(enddate), 'ATM') elif model == 'P': util.delete_contract(etldate, int(enddate), 'POS') elif model == 'Z': util.delete_contract(etldate, int(enddate), '支付宝快捷支付') elif model == 'C': util.delete_creditbad2(etldate, int(enddate)) elif model == 'CM': util.delete_creditmpur(etldate, int(enddate)) util.delete_creditbad2(etldate, int(enddate)) #TBD else: util.delete_contract(etldate, int(enddate), '丰收e支付') util.delete_contract(etldate, int(enddate), '新丰收e支付') util.delete_contract(etldate, int(enddate), '贷款合同') util.delete_contract(etldate, int(enddate), '支付宝快捷支付') #util.delete_creditmpur(etldate,int(enddate)) #util.delete_creditbad2(etldate,int(enddate)) #TBD while int(etldate) <= int(enddate): print "run_etl_main step2...", etldate info("run_etl_main step2:%d,model=%s" % (int(etldate), model)) d1 = datetime.now() Config().etldate = etldate Config().stretldate = util.tostrdate(etldate) if model == 'E': info("run EPAY") EPay().loadtofact(q1, q1) elif model == 'L': info("run LOANCONTRACT") loanCon().loadtofact(q1, q1) elif model == 'CM': info("run CCRDMPUR") elif model == 'T': info("run TERMINAL") Terminal().loadtofact(q1, q1) elif model == 'A': info("run ATM") ATM().loadtofact(q1, q1) elif model == 'P': info("run POS") POS().loadtofact(q1, q1) elif model == 'Z': info("run ZFBKJ") ZFBKJ().loadtofact(q1, q1) else: info("run EPAY") EPay().loadtofact(q1, q1) info("run CCRDACCT") info("run ZFBKJ") ZFBKJ().loadtofact(q1, q1) loanCon().loadtofact(q1, q1) #贷款合同 etldate = int(daycalc(etldate, 1)) util.fix_seq_id('D_CUST_CONTRACT', 'D_CUST_CONTRACT_SEQ') info("finish_run_etl_main:%s,model=%s" % (str(startdate), model))
def run_etl_main(startdate, enddate, q1, q2, model=None): print "load dims " info("start_run_etl_main:%s,model=%s" % (str(startdate), model)) d1 = datetime.now() etldate = int(startdate) Config().etldate = etldate #load_dims(q1,q2,q3) load_dims(q2) print "load dims ,times=", datetime.now() - d1 print "run_etl_man step1..." print etldate if model == 'C': util.delete_fcustview(etldate, int(enddate)) elif model == 'F': util.delete_financialsale(etldate, int(enddate)) elif model == 'T': util.delete_blta(etldate, int(enddate)) elif model == 'BILL': util.delete_fbillacount(etldate, int(enddate)) elif model == 'GAS': util.delete_gasaccount(etldate, int(enddate)) else: #util.delete_fcustview(etldate,int(enddate)) util.delete_financialsale(etldate, int(enddate)) util.delete_fcacctjrnl(etldate, int(enddate)) util.delete_fbillacount(etldate, int(enddate)) while int(etldate) <= int(enddate): print "run_etl_main step2...", etldate d1 = datetime.now() Config().etldate = etldate Config().stretldate = util.tostrdate(etldate) if model == 'C': info("run CUSTVIEW") CustView().loaddbtofact(q1, q1) elif model == 'F': info("run financialsale") FinancialSale().loadtofact(q2, q2) elif model == 'T': info("run FTransAtion") FTransAtion().loadtofact(q2, q2) elif model == 'BILL': info("run Fbill") FBillAccount().loadtofact(q2, q2) elif model == 'GAS': info("run gas") GAS().loadtofact(q2, q2) GAS_BRANCH().loadtofact(q2, q2) GAS_DO6().loadtofact(q2, q2) GAS_KJ().loadtofact(q2, q2) else: #info("run CUSTVIEW") #CustView().loaddbtofact(q1,q1) info("run financialsale") FinancialSale().loadtofact(q2, q2) info("run FTransAtion") FTransAtion().loadtofact(q2, q2) info("run Fbill") FBillAccount().loadtofact(q2, q2) #update_org_manage_bal(etldate) #update_ltd(etldate,etldate) #update_orgid(etldate,etldate) etldate = int(daycalc(etldate, 1)) util.fix_seq_id('D_ACCOUNT', 'D_ACCOUNT_SEQ_NEW') q1.put(None) q2.put(None) #q3.put(None) info("finish_run_etl_main:%s,model=%s" % (str(startdate), model))
def __init__(self, etldate): self.db = util.DBConnect() self.etldate = etldate self.stretldate = util.tostrdate(self.etldate) self.para = util.get_etl_date(etldate) self.data_func()
def run_etl_main(startdate, enddate, q1, q2, model=None): info("start_run_etl_main:%s,model=%s" % (str(startdate), model)) d1 = datetime.now() etldate = int(startdate) Config().etldate = etldate #load_dims(q1,q2,q3) load_dims(q2) if model == 'C': util.delete_fcustview(etldate, int(enddate)) elif model == 'F': util.delete_financialsale(etldate, int(enddate)) elif model == 'T': util.delete_fcacctjrnl(etldate, int(enddate)) elif model == 'POS': util.delete_pos_insert(etldate, int(enddate)) elif model == 'CORP': #国际业务按客户 util.delete_ebills_qry_settlement_corp(etldate, int(enddate)) elif model == 'CASH_INFO': #现金调拨信息信息 util.delete_core_bhfmcmrd(etldate, int(enddate)) elif model == 'CASH_DIR': #现金调拨方向 util.delete_CORE_BHFMCMRM_DIRECT(etldate, int(enddate)) elif model == 'ATM': util.delete_atm_insert(etldate, int(enddate)) elif model == 'FTBHB': util.delete_financialsale(etldate, int(enddate)) util.delete_fcacctjrnl(etldate, int(enddate)) util.delete_fbillacount(etldate, int(enddate)) util.delete_blfm(etldate, int(enddate)) util.delete_blfmmtrn(etldate, int(enddate)) elif model == 'BILL': util.delete_fbillacount(etldate, int(enddate)) elif model == 'HXHK': #核销清单 util.delete_blfm(etldate, int(enddate)) elif model == 'BLFMM': #逾期还款清单 util.delete_blfmmtrn(etldate, int(enddate)) else: pass while int(etldate) <= int(enddate): print "run_etl_main step2...", etldate d1 = datetime.now() Config().etldate = etldate Config().stretldate = util.tostrdate(etldate) if model == 'C': info("run CUSTVIEW") CustView().loaddbtofact(q1, q1) elif model == 'F': info("run financialsale") FinancialSale().loadtofact(q2, q2) elif model == 'T': info("run FTransAtion") FTransAtion().loadtofact(q2, q2) elif model == 'CORP': #国际业务按客户 qry_settlement_corp().loadtofact(q2, q2) elif model == 'CASH_INFO': #现金调拨信息信息 corp_bhfmcmrd().loadtofact(q2, q2) elif model == 'CASH_DIR': #现金调拨方向 core_bhfmcmrm_direct().loadtofact(q2, q2) elif model == 'POS': #新增POS助农 PosInsert().loadtofact(q2, q2) elif model == 'ATM': #新增ATM助农 AtmInsert().loadtofact(q2, q2) elif model == 'FTBHB': info("run financialsale") FinancialSale().loadtofact(q2, q2) info("run FTransAtion") FTransAtion().loadtofact(q2, q2) info("run Fbill") FBillAccount().loadtofact(q2, q2) info("run Blfmcnaf") Blfmcnaf().loadtofact(q2, q2) info("run Blfmmtrn") Blfmmtrn().loadtofact(q2, q2) elif model == 'BILL': info("run Fbill") FBillAccount().loadtofact(q2, q2) elif model == 'HXHK': #核销清单 F_CORE_BLFMCNAF Blfmcnaf().loadtofact(q2, q2) elif model == 'BLFMM': ##逾期还款清单 F_CORE_BLFMMTRN Blfmmtrn().loadtofact(q2, q2) else: pass etldate = int(daycalc(etldate, 1)) util.fix_seq_id('D_ACCOUNT', 'D_ACCOUNT_SEQ_NEW') info("finish_run_etl_main:%s,model=%s" % (str(startdate), model))
def __init__(self, etldate=None): self.db = util.DBConnect() self.etldate = etldate self.stretldate = util.tostrdate(self.etldate) self.para = util.get_common_parameter(self.db) if etldate is not None: self.para['TJRQ'] = etldate