def processing_data(j: str, get_data_time: datetime): obj: AV = selfusepy.parse_json(j, AV()) log.info("[Saving] top avs data: %s" % get_data_time.isoformat()) session = DBSession() for i, item in enumerate(obj.onlineList): avInfoDO = AVInfoDO(item) avStatDO = AVStatDO(item, i + 1, get_data_time) exist: AVInfoDO = session.query(AVInfoDO).filter( AVInfoDO.aid == avInfoDO.aid).first() """ 存在则只添加关于av的statistic """ try: if not exist: session.add(avInfoDO) session.add(avStatDO) log.info('[INSERT] aid: %s' % avInfoDO.aid) else: session.add(avStatDO) log.info('[UPDATE] av statistics, aid: %s' % avInfoDO.aid) session.commit() except BaseException as e: session.rollback() raise e else: log.info("[Update or Insert] success") session.close() log.info('[DONE] save top AVs')
def destruct_danmaku(cid: int, danmakus: List[CustomTag]): danmakuMap: MutableMapping[int, DanmakuDO] = {} relationMap: MutableMapping[int, DanmakuRealationDO] = {} print('[FORMER] cid: %s, danmakus: %s' % (cid, danmakus.__len__())) for danmaku in danmakus: # 弹幕出现时间,模式,字体大小,颜色,发送时间戳,弹幕池,用户Hash,数据库ID obj: DanmakuDO = DanmakuDO() obj.content = danmaku.content l: list = danmaku.tag_content.split(',') obj.danmaku_epoch = float(l[0]) obj.mode = int(l[1]) obj.font_size = int(l[2]) obj.font_color = int(l[3]) obj.send_time = datetime.fromtimestamp(int(l[4]), timezone(timedelta(hours=8))) obj.danmaku_pool = int(l[5]) obj.user_hash = int(l[6], 16) # 暂不直接从数据库获取crc32数据, 太慢了 # value = crc32.get_value(l[6]) # if value[0] > 0: # obj.user_id = value[1] obj.id = int(l[7]) relation: DanmakuRealationDO = DanmakuRealationDO() relation.cid = cid relation.danmaku_id = obj.id danmakuMap[obj.id] = obj relationMap[relation.danmaku_id] = relation session = DBSession() try: removeExist(cid, danmakuMap, relationMap) if danmakuMap.__len__() == relationMap.__len__( ) and relationMap.__len__() == 0: print('cid: %s, has saved all danmaku' % cid) return session.bulk_save_objects( danmakuMap.values() if danmakuMap.values().__len__() > 0 else None) session.bulk_save_objects(relationMap.values( ) if relationMap.values().__len__() > 0 else None) session.commit() except BaseException as e: session.rollback() print(e) print('cid: %s, has error. ' % cid) else: print('cid: %s, Saved into DB.' % cid) finally: session.close() print('[SAVED] danmakuMap.len: %s' % danmakuMap.__len__()) print('[SAVED] relationMap.len: %s' % relationMap.__len__()) danmakuMap.clear() relationMap.clear()
async def save_cid_aid_relation(cid_aid: MutableMapping[int, int], cid_info: MutableMapping[int, AvDanmakuCid]): """ 保存av与cid的关系 """ if cid_aid.keys().__len__() < 1: return objs: List[AVCidsDO] = [] sql: str = 'select cid from av_cids where cid in (%s)' % ','.join( '%s' % item for item in cid_aid.keys()) cids: ResultProxy = await execute_sql(sql) exist_cids: Set[int] = set() for item in cids.fetchall(): """ 保存已经存在的关系 """ exist_cids.add(int(item[0])) if not exist_cids.__len__() == cid_aid.__len__(): session = DBSession() for cid, aid in cid_aid.items(): if exist_cids.__contains__(cid): continue obj: AVCidsDO = AVCidsDO() obj.cid = cid obj.aid = aid objs.append(obj) for cid in exist_cids: cid_info.pop(cid, None) try: if cid_info.values().__len__() > 0: for item in cid_info.values(): await execute_sql( "update av_cids set page = %s, page_name = '%s' where cid = %s;" % (item.page, item.pagename, item.cid)) session.bulk_save_objects(objs) session.commit() except BaseException as e: session.rollback() raise e else: log.info('[Saved] av-cid relation. len: %s' % objs.__len__()) finally: session.close() else: log.info('All av-cid relation exist')
def save_danmaku_to_db(q: Queue, danmakuMap: MutableMapping[int, DanmakuDO], relationMap: MutableMapping[int, DanmakuRealationDO], cid_danmakuIdSet: MutableMapping[int, Set[int]]): session = DBSession() try: remove_db_exist_ids(danmakuMap, relationMap, cid_danmakuIdSet.keys()) print('[After Removed DB ids] danmaku len: %s, relation len: %s' % (danmakuMap.__len__(), relationMap.__len__())) if danmakuMap.__len__() != relationMap.__len__(): raise Exception("danmaku's len is not eq relation's len") if danmakuMap.values(): session.bulk_save_objects(danmakuMap.values()) if relationMap.values(): session.bulk_save_objects(relationMap.values()) session.commit() except BaseException: session.rollback() name = multiprocessing.current_process().name _map: MutableMapping[str, str] = {name: traceback.format_exc()} q.put(_map) print('Oops: ', name) else: print('Save to DB success, len: %s' % danmakuMap.__len__()) for cid, value in cid_danmakuIdSet.items(): try: red.sadd(cid, *value) except BaseException: traceback.print_exc() print('[ERROR] redis. cid: %s' % cid) print('[DONE] save danmaku ids to redis') finally: session.close() del danmakuMap del relationMap gc.collect()