def check(wdbconn, dbsrc, dbdst, conf, count):
'''
检测转换后的数据的正确性
'''
check_cond = conf.get('check')
if check_cond is None:
sys.stderr.write(u'无法校验数据，需要在配置文件中指定 check 字段')
return
tbfrom, tbto = dbsrc + '.' + conf['from'], dbdst + '.' + conf['to']
where = conf.get('where', '1')
update_by = conf.get('update_by')
from convertor import Convertor
c = Convertor(conf['map'], conf.get('const'), wdbconn)
progress = make_progress(wdbconn, tbfrom, where, count)
check_where = conf.get('check_where', '1')
check_src_use_where = conf.get('check_src_use_where', False)
if not check_src_use_where:
where = '1'
rand_sql = make_check_rand_sql(tbto, check_where)
limit = 0
ncomm = 0
while limit < count:
# 从目标数据随机取出一条数据
with wdbconn.cursor() as wcsr:
wcsr.execute(rand_sql)
dst_data = wcsr.fetchone()
if dst_data is None:
sys.stderr.write(u'数据不足，无法检测')
break
# 获取对应的源数据
src_sql = make_check_src_sql(c.keys, tbfrom, where, dst_data,
check_cond)
wcsr.execute(src_sql)
src_data = wcsr.fetchall()
for sd in src_data:
src_dst_data = c.process(sd)
# 比较双方的数据
if compare(src_dst_data, dst_data):
ncomm += 1
break
progress.update(1)
limit += 1
progress.close()
ndiff = count - ncomm
print(u'''随机检测条数 %d
相同 %d
不同 %d
正确率 %.2f
错误率 %.2f''' % (count, ncomm, ndiff, ncomm * 100.0 / count,
ndiff * 100.0 / count))
def convert(wdbconn, rdbconn, dbsrc, dbdst, conf):
limit = max(conf.get('limit', 0), 0)
tbfrom, tbto = dbsrc + '.' + conf['from'], dbdst + '.' + conf['to']
where = conf.get('where', '1')
update_by = conf.get('update_by')
from convertor import Convertor
c = Convertor(conf['map'], conf.get('const'), wdbconn)
progress = make_progress(wdbconn, tbfrom, where, limit)
rcsr = rdbconn.cursor()
srcsql = make_src_sql(c.keys, tbfrom, where)
rcsr.execute(srcsql)
# 获取数据表
offset, step = 0, 200
while limit == 0 or offset < limit:
with wdbconn.cursor() as wcsr:
srcdata = rcsr.fetchmany(step)
# 因为每条 sql 可能都不一样，所以不能使用 executemany
for sd in srcdata:
dstdata = c.process(sd)
dstsql = make_dst_sql(dstdata, tbto, update_by)
# print(dstsql)
if dstsql:
wcsr.execute(dstsql, tuple(dstdata.values()))
wdbconn.commit()
progress.update(step)
if len(srcdata) < step:
break
offset += step
rcsr.close()
progress.close()
