def delete_tables(self, delete_tables_list):
if isinstance(delete_tables_list, bool):
delete_tables_list = [self._table_folder + "*"]
elif not isinstance(delete_tables_list, (list, tuple)):
delete_tables_list = [delete_tables_list]
redis = RedisDB()
for delete_tab in delete_tables_list:
if delete_tab == "*":
delete_tab = self._table_folder + "*"
tables = redis.getkeys(delete_tab)
for table in tables:
log.info("正在删除表 %s" % table)
redis.clear(table)
class ExportData(object):
def __init__(self):
self._redisdb = RedisDB()
self._to_db = MysqlDB()
def export(self, from_table, to_table, auto_update=False, batch_count=100):
"""
@summary:
用于从redis的item中导出数据到关系型数据库，如mysql/oracle
from_table与to_table表结构必须一致
---------
@param from_table:
@param to_table:
@param auto_update: 当数据存在时是否自动更新 默认否
---------
@result:
"""
total_count = 0
while True:
datas = []
try:
datas = self._redisdb.sget(from_table,
count=batch_count,
is_pop=False)
if not datas:
log.info("""
\r%s -> %s 共导出 %s 条数据""" %
(from_table, to_table, total_count))
break
json_datas = [eval(data) for data in datas]
sql, json_datas = tools.make_batch_sql(to_table, json_datas,
auto_update)
if self._to_db.add_batch(sql, json_datas):
total_count += len(json_datas)
self._redisdb.srem(from_table, datas)
except Exception as e:
log.exception(e)
log.error(datas)
def export_all(
self,
tables,
auto_update=False,
batch_count=100,
every_table_per_export_callback=None,
):
"""
@summary: 导出所有item
---------
@param tables: 如qidian 则导出起点下面所有的items
数据库中的表格式必须有规律 如导出 qidian:comment:s_qidian_book_comment_dynamic_item 对应导入 qidian_book_comment_dynamic
@param auto_update: 是否自动更新
@param batch_count: 每批次导出的数量
@every_table_per_export_callback: 导出前的回调函数, 用来修改特定表的参数 to_table, auto_update, batch_count
如：
def every_table_per_export_callback(to_table, auto_update, batch_count):
if to_table == 'xxx':
auto_update = True
return to_table, auto_update, batch_count
---------
@result:
"""
tables = (self._redisdb.getkeys(tables + "*_item")
if not isinstance(tables, list) else tables)
if not tables:
log.info("无表数据")
for table in tables:
from_table = table
to_table = tools.get_info(str(from_table),
":s_(.*?)_item",
fetch_one=True)
if callable(every_table_per_export_callback):
to_table, auto_update, batch_count = every_table_per_export_callback(
to_table, auto_update, batch_count)
log.info("""
\r正在导出 %s -> %s""" % (from_table, to_table))
self.export(from_table, to_table, auto_update, batch_count)
def export_items(self, tab_item, items_data):
"""
@summary:
---------
@param tab_item: redis中items的表名
@param items_data: [item.to_dict] 数据
---------
@result:
"""
to_table = tools.get_info(tab_item, ":s_(.*?)_item", fetch_one=True)
sql, datas = tools.make_batch_sql(to_table, items_data)
add_count = self._to_db.add_batch(sql, datas)
datas_size = len(datas)
if add_count is None:
log.error("导出数据到表 %s 失败" % (to_table))
else:
log.info("共导出 %s 条数据 到 %s, 重复 %s 条" %
(datas_size, to_table, datas_size - add_count))
return add_count != None
def update_items(self, tab_item, items_data, update_keys=()):
"""
@summary:
---------
@param tab_item: redis中items的表名
@param items_data: [item.to_dict] 数据
@param update_keys: 更新的字段
---------
@result:
"""
to_table = tools.get_info(tab_item, ":s_(.*?)_item", fetch_one=True)
sql, datas = tools.make_batch_sql(
to_table,
items_data,
update_columns=update_keys or list(items_data[0].keys()),
)
update_count = self._to_db.add_batch(sql, datas)
if update_count is None:
log.error("更新表 %s 数据失败" % (to_table))
else:
msg = "共更新 %s 条数据 到 %s" % (update_count // 2, to_table)
if update_keys:
msg += " 更新字段为 {}".format(update_keys)
log.info(msg)
return update_count != None