def check(wdbconn, dbsrc, dbdst, conf, count): ''' 检测转换后的数据的正确性 ''' check_cond = conf.get('check') if check_cond is None: sys.stderr.write(u'无法校验数据,需要在配置文件中指定 check 字段') return tbfrom, tbto = dbsrc + '.' + conf['from'], dbdst + '.' + conf['to'] where = conf.get('where', '1') update_by = conf.get('update_by') from convertor import Convertor c = Convertor(conf['map'], conf.get('const'), wdbconn) progress = make_progress(wdbconn, tbfrom, where, count) check_where = conf.get('check_where', '1') check_src_use_where = conf.get('check_src_use_where', False) if not check_src_use_where: where = '1' rand_sql = make_check_rand_sql(tbto, check_where) limit = 0 ncomm = 0 while limit < count: # 从目标数据随机取出一条数据 with wdbconn.cursor() as wcsr: wcsr.execute(rand_sql) dst_data = wcsr.fetchone() if dst_data is None: sys.stderr.write(u'数据不足,无法检测') break # 获取对应的源数据 src_sql = make_check_src_sql(c.keys, tbfrom, where, dst_data, check_cond) wcsr.execute(src_sql) src_data = wcsr.fetchall() for sd in src_data: src_dst_data = c.process(sd) # 比较双方的数据 if compare(src_dst_data, dst_data): ncomm += 1 break progress.update(1) limit += 1 progress.close() ndiff = count - ncomm print(u'''随机检测条数 %d 相同 %d 不同 %d 正确率 %.2f 错误率 %.2f''' % (count, ncomm, ndiff, ncomm * 100.0 / count, ndiff * 100.0 / count))
def convert(wdbconn, rdbconn, dbsrc, dbdst, conf): limit = max(conf.get('limit', 0), 0) tbfrom, tbto = dbsrc + '.' + conf['from'], dbdst + '.' + conf['to'] where = conf.get('where', '1') update_by = conf.get('update_by') from convertor import Convertor c = Convertor(conf['map'], conf.get('const'), wdbconn) progress = make_progress(wdbconn, tbfrom, where, limit) rcsr = rdbconn.cursor() srcsql = make_src_sql(c.keys, tbfrom, where) rcsr.execute(srcsql) # 获取数据表 offset, step = 0, 200 while limit == 0 or offset < limit: with wdbconn.cursor() as wcsr: srcdata = rcsr.fetchmany(step) # 因为每条 sql 可能都不一样,所以不能使用 executemany for sd in srcdata: dstdata = c.process(sd) dstsql = make_dst_sql(dstdata, tbto, update_by) # print(dstsql) if dstsql: wcsr.execute(dstsql, tuple(dstdata.values())) wdbconn.commit() progress.update(step) if len(srcdata) < step: break offset += step rcsr.close() progress.close()