if record.findKeyExist('migrate'):
# 存在该记录, 取出处理的数据量
handleNum = record.getHandledNum('migrate')
if handleNum != total:
startIndex = handleNum
sum = handleNum
print("%s startIndex= %d, sum = %d" % (tableName, startIndex, sum))
# 读取数据
while startIndex < total:
limit = everyCount
if startIndex + limit > total:
limit = total - startIndex
# 获取数据
count, data = mongoDb.getDataFromDocument(tableName, startIndex, limit)
#print("dataLen=%d" % count)
# 插入数据到mongodb
mongDbPro.insertMany(tableName, data)
sum += count
print("表%s插入数据量sum=%d" % (tableName, sum))
# 更新处理数据入库
record.updateHandledNum('migrate', sum)
startIndex += limit
if sum != total:
print("获取数据总量不正确")
else:
print("%s插入全部数据" % tableName)
# 后期记录表中的记录
if record.findKeyExist("record"):
# 存在该记录, 取出处理的数据量
name_list.extend(record.getHandledNum("record"))
collection_names = mongoDb.getDocument("tb_vehicle_status_his")
print("all collections: {}".format(collection_names))
last_list = []
for name in collection_names:
if name in collection_names:
if name.startswith('tb_vehicle_status_his_20_06_'
) and name != "tb_vehicle_status_his_20_06_12":
print("更新表格：", name)
mongoDb.updateTime(name)
name_list.append(name)
record.updateHandledNum("record", name_list)
else:
last_list.append(name)
for name in last_list:
if name in collection_names:
if name != "tb_vehicle_status_his_20_06_12":
print("更新表格：", name)
mongoDb.updateTime(name)
name_list.append(name)
record.updateHandledNum("record", name_list)
from record.record import Record
if __name__ == '__main__':
r = Record('test.db')
print(r.findKeyExist('tb_vehicle_status_his_20_05_16'))
r.updateHandledNum('tb_vehicle_status_his_20_05_16', 10)
print(r.getHandledNum('tb_vehicle_status_his_20_05_16'))
print(r.findKeyExist('tb_vehicle_status_his_20_05_16'))
r.updateHandledNum('tb_vehicle_status_his_20_05_16', 20)
print(r.getHandledNum('tb_vehicle_status_his_20_05_16'))
handleNum = record.getHandledNum(tableName.replace('_', ''))
if handleNum == total:
# 全部数据已处理
continue
else:
startIndex = handleNum
sum = handleNum
print("%s startIndex= %d, sum = %d" % (tableName, startIndex, sum))
while startIndex < total:
limit = everyCount
if startIndex + limit > total:
limit = total - startIndex
# 获取数据
count, data = sqlDb.getDataFromTable(tableName, startIndex, limit)
# print("dataLen=%d" % count)
# 插入数据到mongodb
mongoDb.insertMany(tableName, data)
sum += count
print("表%s插入数据量sum=%d" % (tableName, sum))
# 更新处理数据入库
record.updateHandledNum(tableName.replace('_', ''), sum)
startIndex += limit
if sum != total:
print("获取数据总量不正确")
else:
print("%s插入全部数据" % tableName)