def session_commit(*args, **kwargs): try: return func(*args, **kwargs) except Exception as e: storage.error('db operation error,here are details{}'.format(e)) storage.warning('transaction rollbacks') db_session.rollback()
def add_search_cont(search_list): save_sql = ( 'insert into weibo_search (mk_primary,mid,murl,create_time,praise_count,repost_count,comment_count,' 'content,device,user_id,username,uheadimage,user_home,keyword) values(:mk_primary, :mid, ' ':murl, :create_time, :praise_count,:repost_count, :comment_count, :content, :device, ' ':user_id, :username,:uheadimage, :user_home, :keyword)') with db_connect.db_execute() as conn: for search_cont in search_list: search_info = { 'mk_primary': search_cont.mk_primary, 'mid': search_cont.mid, 'murl': search_cont.murl, 'create_time': search_cont.create_time, 'praise_count': search_cont.praise_count, 'repost_count': search_cont.repost_count, 'comment_count': search_cont.comment_count, 'content': search_cont.content, 'device': search_cont.device, 'user_id': search_cont.user_id, 'username': search_cont.username, 'uheadimage': search_cont.uheadimage, 'user_home': search_cont.user_home, 'keyword': search_cont.keyword } try: db_connect.db_dml_parms(conn, save_sql, search_info) except Exception as why: storage.error('插入出错,具体原因为:{why}, 插入数据是{info}'.format( why=why, info=search_info.__dict__))
async def save_data(self,item): try: await self.db.infoq_details.update_one( {'uuid': item.get("uuid")}, {'$set': item}, upsert=True) except Exception as e: storage.error(f'数据插入出错:{e.args},此时的item是{item}')
async def change_status(self, condition, col="discogs_seed_data", status_code=1): # status_code 0:初始,1:开始下载,2下载完了 try: item = {} item["status"] = status_code # storage.info(f"修改状态,此时的数据是:{item}") await self.mb[col].update_one(condition, {'$set': item}) except Exception as e: storage.error(f"修改状态出错:{e.args}此时的数据是:{item}")
async def change_status(self,uuid,item,status_code=0): """ status_code 0:初始 1:开始下载 2:下载完成 """ try: item['status'] = status_code await self.db.infoq_seed.update_one({'uuid': uuid}, {'$set': item}, upsert=True) except Exception as e: if 'immutable' in e.args[0]: await self.db.infoq_seed.delete_one({'_id':item['_id']}) storage.info(f'数据重复删除:{e.args},此时的数据是:{item}') else: storage.error(f'修改状态出错:{e.args},此时的数据是:{item}')
def db_execute(): con = None try: con = get_engine() except Exception as e: storage.error('连接数据库错误,具体信息是{e}'.format(e=e)) try: yield con except Exception as e: storage.error('操作数据库出错,具体信息是{e},\n堆栈信息是{detail}'.format( e=e, detail=repr(traceback.format_stack())))
def save_data(self, items, col="infoq_seed"): if isinstance(items, list): for item in items: try: self.db[col].update_one({'uuid': item.get("uuid")}, {'$set': item}, upsert=True) except Exception as e: storage.error(f"数据插入出错:{e.args},此时的item是:{item}") else: try: self.db[col].update_one({'uuid': items.get("uuid")}, {'$set': items}, upsert=True) except Exception as e: storage.error(f"数据插入出错:{e.args},此时的item是:{item}")
async def save_data(self, items, col="discogs_index_data", key="obj_id"): # storage.info(f"此时的items:{items}") if isinstance(items, list): for item in items: try: item[key] = item[key] await self.mb[col].update_one({key: item.get(key)}, {'$set': item}, upsert=True) except Exception as e: storage.error(f"数据插入出错:{e.args}此时的item是:{item}") elif isinstance(items, dict): try: items[key] = items[key] await self.mb[col].update_one({key: items.get(key)}, {'$set': items}, upsert=True) except Exception as e: storage.error(f"数据插入出错:{e.args}此时的item是:{items}")
def session_commit(*args, **kwargs): try: return func(*args, **kwargs) except Exception as e: storage.error('数据库操作失败,具体信息是{}'.format(e)) db_session.rollback()
def save(sos): ins_count = 0 insert_sql = ( 'insert into weibo_spread_other (user_id,user_screenname,user_province,user_city,user_location,' 'user_description,user_url,user_profileimageurl,user_gender,user_followerscount,user_friendscount,' 'user_statusescount,user_createdat,user_verifiedtype,user_verifiedreason,status_createdat,' 'status_mid,status_source,status_repostscount,status_commentscount,upper_user_id,' 'original_status_id,status_url) ' " values (:user_id,:user_screenname,:user_province,:user_city,:user_location," ":user_description,:user_url,:user_profileimageurl,:user_gender,:user_followerscount," ":user_friendscount,:user_statusescount,:user_createdat,:user_verifiedtype,:user_verifiedreason," ":status_createdat,:status_mid,:status_source,:status_repostscount,:status_commentscount," ":upper_user_id,:original_status_id,:status_url)") with db_connect.db_execute() as conn: for item in sos: if item.verify_type == '': item.verify_type = 0 try: args = { 'user_id': item.id, 'user_url': item.blog_url, 'user_profileimageurl': item.headimg_url, 'user_screenname': item.screen_name.encode('gbk', 'ignore').decode('gbk'), 'user_province': item.province.encode('gbk', 'ignore').decode('gbk'), 'user_city': item.city.encode('gbk', 'ignore').decode('gbk'), 'user_location': item.location.encode('gbk', 'ignore').decode('gbk'), 'user_description': item.description.encode('gbk', 'ignore').decode('gbk'), 'user_gender': item.gender.encode('gbk', 'ignore').decode('gbk'), 'user_verifiedreason': item.verify_info.encode('gbk', 'ignore').decode('gbk'), 'status_source': item.device.encode('gbk', 'ignore').decode('gbk'), 'user_followerscount': int(item.followers_count), 'user_friendscount': int(item.friends_count), 'user_statusescount': int(item.status_count), 'status_repostscount': int(item.reposts_count), 'status_commentscount': int(item.comments_count), 'user_verifiedtype': item.verify_type, 'user_createdat': item.register_time, 'status_createdat': item.status_post_time, 'status_mid': item.mid, 'upper_user_id': item.upper_user_id, 'original_status_id': item.original_status_id, 'status_url': item.status_url, } db_connect.db_dml_parms(conn, insert_sql, args) except Exception as why: storage.error(item.__dict__) storage.error(why) else: ins_count += 1 storage.info('一共插入了{ins}条数据'.format(ins=ins_count))