mongDbPro = MongoClient(mongoConnPro)
mongoDb = MongoClient(mongoConn)
record = Record(recordPath)
print("开始迁移表：%s" % tableName)
total = mongoDb.getCount(tableName)
print("%s数据总量：%d" % (tableName, total))
# 计数
startIndex = 0
sum = 0
# 后期记录表中的记录
if record.findKeyExist('migrate'):
# 存在该记录, 取出处理的数据量
handleNum = record.getHandledNum('migrate')
if handleNum != total:
startIndex = handleNum
sum = handleNum
print("%s startIndex= %d, sum = %d" % (tableName, startIndex, sum))
# 读取数据
while startIndex < total:
limit = everyCount
if startIndex + limit > total:
limit = total - startIndex
# 获取数据
count, data = mongoDb.getDataFromDocument(tableName, startIndex, limit)
#print("dataLen=%d" % count)
# 插入数据到mongodb
# 每次处理的数据量
everyCount = 10000
# 存储处理数据历史记录的文件地址
recordPath = 'update.db'
if __name__ == '__main__':
mongoDb = MongoClient(mongoConn)
record = Record(recordPath)
name_list = []
# 后期记录表中的记录
if record.findKeyExist("record"):
# 存在该记录, 取出处理的数据量
name_list.extend(record.getHandledNum("record"))
collection_names = mongoDb.getDocument("tb_vehicle_status_his")
print("all collections: {}".format(collection_names))
last_list = []
for name in collection_names:
if name in collection_names:
if name.startswith('tb_vehicle_status_his_20_06_'
) and name != "tb_vehicle_status_his_20_06_12":
print("更新表格：", name)
mongoDb.updateTime(name)
name_list.append(name)
record.updateHandledNum("record", name_list)
from record.record import Record
if __name__ == '__main__':
r = Record('test.db')
print(r.findKeyExist('tb_vehicle_status_his_20_05_16'))
r.updateHandledNum('tb_vehicle_status_his_20_05_16', 10)
print(r.getHandledNum('tb_vehicle_status_his_20_05_16'))
print(r.findKeyExist('tb_vehicle_status_his_20_05_16'))
r.updateHandledNum('tb_vehicle_status_his_20_05_16', 20)
print(r.getHandledNum('tb_vehicle_status_his_20_05_16'))
# 获取tableName
tableNameList = sqlDb.getAllTableName('tb_vehicle_status_his_')
for tableName in tableNameList:
print("开始迁移表：%s" % tableName)
total = sqlDb.getAllRowCount(tableName)
print("%s数据总量：%d" % (tableName, total))
# 计数
startIndex = 0
sum = 0
# 后期记录表中的记录
if record.findKeyExist(tableName.replace('_', '')):
# 存在该记录, 取出处理的数据量
handleNum = record.getHandledNum(tableName.replace('_', ''))
if handleNum == total:
# 全部数据已处理
continue
else:
startIndex = handleNum
sum = handleNum
print("%s startIndex= %d, sum = %d" % (tableName, startIndex, sum))
while startIndex < total:
limit = everyCount
if startIndex + limit > total:
limit = total - startIndex
# 获取数据
count, data = sqlDb.getDataFromTable(tableName, startIndex, limit)