def session_commit(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception as e:
storage.error('db operation error，here are details{}'.format(e))
storage.warning('transaction rollbacks')
db_session.rollback()
def session_commit(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception as e:
storage.error('db operation error，here are details{}'.format(e))
storage.warning('transaction rollbacks')
db_session.rollback()
def add_search_cont(search_list):
save_sql = (
'insert into weibo_search (mk_primary,mid,murl,create_time,praise_count,repost_count,comment_count,'
'content,device,user_id,username,uheadimage,user_home,keyword) values(:mk_primary, :mid, '
':murl, :create_time, :praise_count,:repost_count, :comment_count, :content, :device, '
':user_id, :username,:uheadimage, :user_home, :keyword)')
with db_connect.db_execute() as conn:
for search_cont in search_list:
search_info = {
'mk_primary': search_cont.mk_primary,
'mid': search_cont.mid,
'murl': search_cont.murl,
'create_time': search_cont.create_time,
'praise_count': search_cont.praise_count,
'repost_count': search_cont.repost_count,
'comment_count': search_cont.comment_count,
'content': search_cont.content,
'device': search_cont.device,
'user_id': search_cont.user_id,
'username': search_cont.username,
'uheadimage': search_cont.uheadimage,
'user_home': search_cont.user_home,
'keyword': search_cont.keyword
}
try:
db_connect.db_dml_parms(conn, save_sql, search_info)
except Exception as why:
storage.error('插入出错,具体原因为:{why}, 插入数据是{info}'.format(
why=why, info=search_info.__dict__))
async def save_data(self,item):
try:
await self.db.infoq_details.update_one(
{'uuid': item.get("uuid")},
{'$set': item},
upsert=True)
except Exception as e:
storage.error(f'数据插入出错:{e.args},此时的item是{item}')
async def change_status(self,
condition,
col="discogs_seed_data",
status_code=1):
# status_code 0:初始,1:开始下载，2下载完了
try:
item = {}
item["status"] = status_code
# storage.info(f"修改状态,此时的数据是:{item}")
await self.mb[col].update_one(condition, {'$set': item})
except Exception as e:
storage.error(f"修改状态出错:{e.args}此时的数据是:{item}")
async def change_status(self,uuid,item,status_code=0):
"""
status_code 0:初始 1:开始下载 2:下载完成
"""
try:
item['status'] = status_code
await self.db.infoq_seed.update_one({'uuid': uuid}, {'$set': item}, upsert=True)
except Exception as e:
if 'immutable' in e.args[0]:
await self.db.infoq_seed.delete_one({'_id':item['_id']})
storage.info(f'数据重复删除:{e.args},此时的数据是:{item}')
else:
storage.error(f'修改状态出错:{e.args},此时的数据是:{item}')
def db_execute():
con = None
try:
con = get_engine()
except Exception as e:
storage.error('连接数据库错误，具体信息是{e}'.format(e=e))
try:
yield con
except Exception as e:
storage.error('操作数据库出错，具体信息是{e},\n堆栈信息是{detail}'.format(
e=e, detail=repr(traceback.format_stack())))
def save_data(self, items, col="infoq_seed"):
if isinstance(items, list):
for item in items:
try:
self.db[col].update_one({'uuid': item.get("uuid")},
{'$set': item},
upsert=True)
except Exception as e:
storage.error(f"数据插入出错:{e.args},此时的item是:{item}")
else:
try:
self.db[col].update_one({'uuid': items.get("uuid")},
{'$set': items},
upsert=True)
except Exception as e:
storage.error(f"数据插入出错:{e.args},此时的item是:{item}")
async def save_data(self, items, col="discogs_index_data", key="obj_id"):
# storage.info(f"此时的items:{items}")
if isinstance(items, list):
for item in items:
try:
item[key] = item[key]
await self.mb[col].update_one({key: item.get(key)},
{'$set': item},
upsert=True)
except Exception as e:
storage.error(f"数据插入出错:{e.args}此时的item是:{item}")
elif isinstance(items, dict):
try:
items[key] = items[key]
await self.mb[col].update_one({key: items.get(key)},
{'$set': items},
upsert=True)
except Exception as e:
storage.error(f"数据插入出错:{e.args}此时的item是:{items}")
def session_commit(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception as e:
storage.error('数据库操作失败，具体信息是{}'.format(e))
db_session.rollback()
def save(sos):
ins_count = 0
insert_sql = (
'insert into weibo_spread_other (user_id,user_screenname,user_province,user_city,user_location,'
'user_description,user_url,user_profileimageurl,user_gender,user_followerscount,user_friendscount,'
'user_statusescount,user_createdat,user_verifiedtype,user_verifiedreason,status_createdat,'
'status_mid,status_source,status_repostscount,status_commentscount,upper_user_id,'
'original_status_id,status_url) '
" values (:user_id,:user_screenname,:user_province,:user_city,:user_location,"
":user_description,:user_url,:user_profileimageurl,:user_gender,:user_followerscount,"
":user_friendscount,:user_statusescount,:user_createdat,:user_verifiedtype,:user_verifiedreason,"
":status_createdat,:status_mid,:status_source,:status_repostscount,:status_commentscount,"
":upper_user_id,:original_status_id,:status_url)")
with db_connect.db_execute() as conn:
for item in sos:
if item.verify_type == '':
item.verify_type = 0
try:
args = {
'user_id':
item.id,
'user_url':
item.blog_url,
'user_profileimageurl':
item.headimg_url,
'user_screenname':
item.screen_name.encode('gbk', 'ignore').decode('gbk'),
'user_province':
item.province.encode('gbk', 'ignore').decode('gbk'),
'user_city':
item.city.encode('gbk', 'ignore').decode('gbk'),
'user_location':
item.location.encode('gbk', 'ignore').decode('gbk'),
'user_description':
item.description.encode('gbk', 'ignore').decode('gbk'),
'user_gender':
item.gender.encode('gbk', 'ignore').decode('gbk'),
'user_verifiedreason':
item.verify_info.encode('gbk', 'ignore').decode('gbk'),
'status_source':
item.device.encode('gbk', 'ignore').decode('gbk'),
'user_followerscount':
int(item.followers_count),
'user_friendscount':
int(item.friends_count),
'user_statusescount':
int(item.status_count),
'status_repostscount':
int(item.reposts_count),
'status_commentscount':
int(item.comments_count),
'user_verifiedtype':
item.verify_type,
'user_createdat':
item.register_time,
'status_createdat':
item.status_post_time,
'status_mid':
item.mid,
'upper_user_id':
item.upper_user_id,
'original_status_id':
item.original_status_id,
'status_url':
item.status_url,
}
db_connect.db_dml_parms(conn, insert_sql, args)
except Exception as why:
storage.error(item.__dict__)
storage.error(why)
else:
ins_count += 1
storage.info('一共插入了{ins}条数据'.format(ins=ins_count))
