def processing_data(j: str, get_data_time: datetime):
obj: AV = selfusepy.parse_json(j, AV())
log.info("[Saving] top avs data: %s" % get_data_time.isoformat())
session = DBSession()
for i, item in enumerate(obj.onlineList):
avInfoDO = AVInfoDO(item)
avStatDO = AVStatDO(item, i + 1, get_data_time)
exist: AVInfoDO = session.query(AVInfoDO).filter(
AVInfoDO.aid == avInfoDO.aid).first()
"""
存在则只添加关于av的statistic
"""
try:
if not exist:
session.add(avInfoDO)
session.add(avStatDO)
log.info('[INSERT] aid: %s' % avInfoDO.aid)
else:
session.add(avStatDO)
log.info('[UPDATE] av statistics, aid: %s' % avInfoDO.aid)
session.commit()
except BaseException as e:
session.rollback()
raise e
else:
log.info("[Update or Insert] success")
session.close()
log.info('[DONE] save top AVs')
def destruct_danmaku(cid: int, danmakus: List[CustomTag]):
danmakuMap: MutableMapping[int, DanmakuDO] = {}
relationMap: MutableMapping[int, DanmakuRealationDO] = {}
print('[FORMER] cid: %s, danmakus: %s' % (cid, danmakus.__len__()))
for danmaku in danmakus:
# 弹幕出现时间,模式,字体大小,颜色,发送时间戳,弹幕池,用户Hash,数据库ID
obj: DanmakuDO = DanmakuDO()
obj.content = danmaku.content
l: list = danmaku.tag_content.split(',')
obj.danmaku_epoch = float(l[0])
obj.mode = int(l[1])
obj.font_size = int(l[2])
obj.font_color = int(l[3])
obj.send_time = datetime.fromtimestamp(int(l[4]),
timezone(timedelta(hours=8)))
obj.danmaku_pool = int(l[5])
obj.user_hash = int(l[6], 16)
# 暂不直接从数据库获取crc32数据, 太慢了
# value = crc32.get_value(l[6])
# if value[0] > 0:
# obj.user_id = value[1]
obj.id = int(l[7])
relation: DanmakuRealationDO = DanmakuRealationDO()
relation.cid = cid
relation.danmaku_id = obj.id
danmakuMap[obj.id] = obj
relationMap[relation.danmaku_id] = relation
session = DBSession()
try:
removeExist(cid, danmakuMap, relationMap)
if danmakuMap.__len__() == relationMap.__len__(
) and relationMap.__len__() == 0:
print('cid: %s, has saved all danmaku' % cid)
return
session.bulk_save_objects(
danmakuMap.values() if danmakuMap.values().__len__() > 0 else None)
session.bulk_save_objects(relationMap.values(
) if relationMap.values().__len__() > 0 else None)
session.commit()
except BaseException as e:
session.rollback()
print(e)
print('cid: %s, has error. ' % cid)
else:
print('cid: %s, Saved into DB.' % cid)
finally:
session.close()
print('[SAVED] danmakuMap.len: %s' % danmakuMap.__len__())
print('[SAVED] relationMap.len: %s' % relationMap.__len__())
danmakuMap.clear()
relationMap.clear()
async def save_cid_aid_relation(cid_aid: MutableMapping[int, int],
cid_info: MutableMapping[int, AvDanmakuCid]):
"""
保存av与cid的关系
"""
if cid_aid.keys().__len__() < 1:
return
objs: List[AVCidsDO] = []
sql: str = 'select cid from av_cids where cid in (%s)' % ','.join(
'%s' % item for item in cid_aid.keys())
cids: ResultProxy = await execute_sql(sql)
exist_cids: Set[int] = set()
for item in cids.fetchall():
"""
保存已经存在的关系
"""
exist_cids.add(int(item[0]))
if not exist_cids.__len__() == cid_aid.__len__():
session = DBSession()
for cid, aid in cid_aid.items():
if exist_cids.__contains__(cid):
continue
obj: AVCidsDO = AVCidsDO()
obj.cid = cid
obj.aid = aid
objs.append(obj)
for cid in exist_cids:
cid_info.pop(cid, None)
try:
if cid_info.values().__len__() > 0:
for item in cid_info.values():
await execute_sql(
"update av_cids set page = %s, page_name = '%s' where cid = %s;"
% (item.page, item.pagename, item.cid))
session.bulk_save_objects(objs)
session.commit()
except BaseException as e:
session.rollback()
raise e
else:
log.info('[Saved] av-cid relation. len: %s' % objs.__len__())
finally:
session.close()
else:
log.info('All av-cid relation exist')
def save_danmaku_to_db(q: Queue, danmakuMap: MutableMapping[int, DanmakuDO],
relationMap: MutableMapping[int, DanmakuRealationDO],
cid_danmakuIdSet: MutableMapping[int, Set[int]]):
session = DBSession()
try:
remove_db_exist_ids(danmakuMap, relationMap, cid_danmakuIdSet.keys())
print('[After Removed DB ids] danmaku len: %s, relation len: %s' %
(danmakuMap.__len__(), relationMap.__len__()))
if danmakuMap.__len__() != relationMap.__len__():
raise Exception("danmaku's len is not eq relation's len")
if danmakuMap.values():
session.bulk_save_objects(danmakuMap.values())
if relationMap.values():
session.bulk_save_objects(relationMap.values())
session.commit()
except BaseException:
session.rollback()
name = multiprocessing.current_process().name
_map: MutableMapping[str, str] = {name: traceback.format_exc()}
q.put(_map)
print('Oops: ', name)
else:
print('Save to DB success, len: %s' % danmakuMap.__len__())
for cid, value in cid_danmakuIdSet.items():
try:
red.sadd(cid, *value)
except BaseException:
traceback.print_exc()
print('[ERROR] redis. cid: %s' % cid)
print('[DONE] save danmaku ids to redis')
finally:
session.close()
del danmakuMap
del relationMap
gc.collect()