def starrun(etldate): db = DBConnect() try: db.cursor.execute("delete from MB_PB_ACCINF where date_id = ?", (etldate)) db.conn.commit() db.cursor.execute( "delete from ODS_EBILLS_PA_QUOTEPRICE where date_id = ?", (etldate)) db.conn.commit() finally: db.closeDB() sm = StarManage(1, []) sm.start_dim_process() sm.start_fact_process() sm.setDimQueue() sm.start() StarBase().files2fact2(MBPBACCINF(), sm) StarBase().files2fact2(PA_QUOTEPRICE(), sm) sm.finish() db = DBConnect() try: usql = """ merge into MB_PB_ACCINF f using (select distinct account_no,cst_id,cst_no from d_account aa where aa.account_class in ('定期分户','活期分户') ) a on f.date_id = ? and a.account_no =f.MAIF_ACCNO when matched then update set f.cst_no=a.cst_no,f.cst_id=a.cst_id """ db.cursor.execute(usql, (etldate)) db.conn.commit() finally: db.closeDB()
def starun(etldate): db = DBConnect() try: print "delete from F_ACCOUNT_BALANCE", etldate #db.cursor.execute("delete from F_ACCOUNT_BALANCE f where ACCOUNT_CLASS='D' and date_id= ?",(etldate)) #db.cursor.execute("delete from F_ACCOUNT_BALANCE f where ACCOUNT_CLASS='H' and date_id= ?",(etldate)) #db.cursor.execute("delete from F_ACCOUNT_BALANCE f where ACCOUNT_CLASS='C' and date_id= ?",(etldate)) db.cursor.execute("delete from F_ACCOUNT_BALANCE f where date_id= ?", (etldate)) db.conn.commit() print "finish delete from F_ACCOUNT_BALANCE", etldate finally: db.closeDB() lastdate = int(util.daycalc(etldate, -1)) #if etldate != FIRSTDATE: # mergeallfile(etldate) act2 = DimAccount2() act2.start() ac = DimAccountGid() dim1 = DimAccountTypeExtend() dim2 = DimGroup() sm = StarManage(10, [dim1, dim2, ac, act2]) sm.start_dim_process() sm.start_fact_process() sm.setDimQueue() sm.start() ds3 = HQAB() if etldate >= ds3.firstday: StarBase().files2fact2(ds3, sm) if etldate >= ds3.firstday: put_ds_data(ds3, sm, etldate) sm.restart_fact_process() ds1 = DQAB() if etldate >= ds1.firstday: StarBase().files2fact2(ds1, sm) if etldate >= ds1.firstday: put_ds_data(ds1, sm, etldate) sm.restart_fact_process() ds2 = DQABC() if etldate >= ds2.firstday: StarBase().files2fact2(ds2, sm) if etldate >= ds2.firstday: put_ds_data(ds2, sm, etldate) sm.restart_fact_process() sm.finish()
def starrun(etldate): if etldate != 20170101: #第一天,手动生成 mergeall_ccrdfile(etldate) db = DBConnect() try: db.cursor.execute("delete from F_TRANSACTION f where date_id= ?", (etldate)) db.conn.commit() finally: db.closeDB() acct = CCRDACCT(etldate).loadfile2dict() def card_key(row): k = row[0].replace(".", "").strip() return k cards = CCRDCARD(etldate, "ALL").loadfile2dict2([0], card_key) manager = multiprocessing.Manager() dg = DimAccountGid() dim1 = DimAccount2() dim1.start() dim2 = DimTransAction() dim3 = DimTransActionType() #遍历acct sm = StarManage(2, [dim1, dim2, dim3, dg]) sm.manager = manager sm.start_dim_process() sm.start_fact_process() sm.setDimQueue() sm.start() for k in acct: row = acct[k] k = k.replace(".", "").strip() card = cards.get(k) ac = None sn = 0 if card is not None: for c in card: if ac is None: ac = c if c["MASTER_NBR"] == c['CARD_NBR'] and int( Decimal(c["ISSUE_NBR"])) >= sn: ac = c if ac is not None: row["CARD_NO"] = ac["MASTER_NBR"] #row["CST_NAME"] = row["CST_NAME"].decode("gbk").encode("utf8").strip() #row["ACCOUNT_NAME"] = row["CST_NAME"].decode("gbk").encode("utf8").strip() dim1.find_dim_id_by_ccrd(row) StarBase().files2fact2(Jorj(), sm) sm.finish() dim1.finish()
def starrun(etldate): if etldate != 20170101: #第一天,手动生成 mergeall_ccrdfile(etldate) db = DBConnect() try: db.cursor.execute( "delete from F_CREDIT_MPUR_20161031 where date_id = ?", (etldate)) db.conn.commit() finally: db.closeDB() sm = StarManage(1, []) sm.start_dim_process() sm.start_fact_process() sm.setDimQueue() sm.start() StarBase().files2fact2(CcrdMpur(), sm) sm.finish()
def load_loan_price_view(etldate): db = util.DBConnect() try: db.cursor.execute("delete from GAS_BI_CUX_LOAN_CHECK_DTL_V where date_id = ?",(etldate)) db.conn.commit() finally: db.closeDB() ''' 导入总账系统下发的贷款视图 ''' prices = query_loan_para_price(etldate) db = util.DBConnect() try: sql2 = """ SELECT A.ACCOUNT_NO,F.MANAGE_ID,t.GUA_TP_NAME,f.year_pdt FROM F_BALANCE F inner join d_account_type t on t.id = f.ACCOUNT_TYPE_ID INNER JOIN D_ACCOUNT A ON A.ID = F.ACCOUNT_ID WHERE F.DATE_ID=? AND F.ACCT_TYPE='4' """ db.cursor.execute(sql2, etldate ) row = db.cursor.fetchone() datas = {} desc = db.cursor.description ld = len(desc) sm = StarManage(2,[ DimStarPrice() ]) sm.start_dim_process() sm.start_fact_process() sm.setDimQueue() sm.start() while row: data = { desc[x][0] : row[x] for x in range(ld) } dim_id = find_loan_price_id(sm, prices, data) datas[ row[0] ] = (row[1],dim_id,row[3]) row = db.cursor.fetchone() loan = BICUXLOAN() loan.manages = datas StarBase().files2fact2(loan, sm) sm.finish() finally: db.closeDB()
def starun(etldate): db = DBConnect() try: db.cursor.execute("delete from F_CREDIT_BAD where date_id= ?", (etldate)) db.conn.commit() finally: db.closeDB() lastdate = int(util.daycalc(etldate, -1)) sm = StarManage(2, []) #sm.start_dim_process() sm.start_fact_process() #sm.setDimQueue() sm.start() ds1 = CcrdAcct2() StarBase().files2fact2(ds1, sm) sm.finish()
def starrun(etldate): db = DBConnect() try: db.cursor.execute( "delete from F_EBILLS_BU_TRANSACTIONINFO f where date_id= ?", (etldate)) db.conn.commit() finally: db.closeDB() manager = multiprocessing.Manager() dim1 = DimBuTransaction() dim2 = DimEbillsTranType() sm = StarManage(1, [dim1, dim2]) sm.manager = manager sm.start_dim_process() sm.start_fact_process() sm.setDimQueue() sm.start() StarBase().files2fact2(BuTransaction(), sm) sm.finish() snapshot(etldate)
def busi_hook(etldate): dg = DimAccountGid() dim1 = DimAccountNew() dim1.start() sm = StarManage(1, [dg, dim1]) sm.start_dim_process() sm.start_fact_process() sm.setDimQueue() sm.start() ''' info("run loan_hook star") StarBase().files2fact(Loan_Contract(), sm) sm.restart_fact_process() info("run loan_hook end") info("run deposit_hook") StarBase().files2fact(Deposit_hq(), sm) sm.restart_fact_process() info("run deposit_dq") StarBase().files2fact(Deposit_dq(), sm) sm.restart_fact_process() info("run deposit_hook end") info("run ebank_hook star") StarBase().files2fact(Ebank_kjzf(), sm) sm.restart_fact_process() StarBase().files2fact(Ebank_epay(), sm) sm.restart_fact_process() StarBase().files2fact(Ebank_zfb(), sm) sm.restart_fact_process() StarBase().files2fact(Ebank_pb(), sm) sm.restart_fact_process() StarBase().files2fact(Ebank_mb(), sm) sm.restart_fact_process() StarBase().files2fact(Ebank_cb(), sm) sm.restart_fact_process() #StarBase().files2fact(Ebank_etc(), sm) #sm.restart_fact_process() info("run finances_hook") StarBase().files2fact(Finances(), sm) sm.restart_fact_process() info("run stock_hook") StarBase().files2fact(Stock(), sm) sm.restart_fact_process() info("run insurance_hook") StarBase().files2fact(Insurance(), sm) sm.restart_fact_process() info("run creditcard_hook") StarBase().files2fact(Creditcard(), sm) sm.restart_fact_process() ''' info("run ParentAccount") StarBase().files2fact(ParentAccount(), sm) sm.restart_fact_process() sm.finish()
def starrun(etldate): sm = StarManage([]) StarBase().files2fact(Deposit_hq(), sm)