def viewTaghistory(vid, language) :
all_items = db.tag_history.aggregate([
{'$match':{ 'vid': ObjectId(vid)}},
{'$lookup': {'from': 'users', 'localField': 'user', 'foreignField': '_id', 'as': 'user_obj'}},
{'$project': {'user_obj._id': 1, 'user_obj.profile.username': 1, 'user_obj.profile.image': 1, 'tags': 1, 'del': 1, 'add': 1, 'time': 1}},
{'$sort': {"time": -1}}
])
all_items = list(all_items)
for item in all_items :
item['tags'], _, _ = tagdb.translate_tag_ids_to_user_language(item['tags'], language)
item['del'], _, _ = tagdb.translate_tag_ids_to_user_language(item['del'], language)
item['add'], _, _ = tagdb.translate_tag_ids_to_user_language(item['add'], language)
return all_items
def getCommonTags(user_language, videos, max_count = 20) : if len(videos) <= 0 : return [] all_tags = list(itertools.chain(*[vid['tags'] for vid in videos])) tag_map = Counter(all_tags).most_common(n = max_count) tag_ids = [item[0] for item in tag_map] return tagdb.translate_tag_ids_to_user_language(tag_ids, user_language)[0]
def queryBlacklist(user, language) : if 'blacklist' in user['settings'] : if isinstance(user['settings']['blacklist'], list) : return tagdb.translate_tag_ids_to_user_language(user['settings']['blacklist'], language)[0] else : return 'default' else : return 'default'
def getAuthorRecord(tag, language):
tag_obj = tagdb._tag(tag)
if not 'author' in tag_obj:
raise UserError('RECORD_NOT_FOUND')
author_obj = db.authors.find_one({'_id': tag_obj['author']})
assert author_obj
author_obj['common_tags'] = tagdb.translate_tag_ids_to_user_language(
author_obj['common_tagids'], language)
return author_obj
def getPopularTags(user_language, max_count = 20) :
try :
assert isinstance(max_count, int) and max_count <= 100 and max_count > 0
response = get_page(TAG_TRACKER_ADDRESS + "/get?count=%d" % max_count)
json_obj = json.loads(response)
tag_ids = [int(i) for i in json_obj['tags']]
tags, _, _ = tagdb.translate_tag_ids_to_user_language(tag_ids, user_language, id_data_map = json_obj['pops'])
return [i[0] for i in tags], {i[0]: i[1] for i in tags}
except :
return []
def getPopularTags(user_language, max_count=20):
try:
assert isinstance(max_count,
int) and max_count <= 100 and max_count > 0
response = get_page(TAG_TRACKER_ADDRESS + "/get?count=%d" % max_count)
json_obj = json.loads(response)
tag_ids = [int(i) for i in json_obj['tags']]
return tagdb.translate_tag_ids_to_user_language(
tag_ids, user_language)[0]
except:
return []
def getRelatedTagsFixedMainTags(user_language, tags, exclude=[], max_count=10):
exclude_tag_ids = tagdb.filter_and_translate_tags(exclude)
exclude_tags, _, _ = tagdb.translate_tag_ids_to_user_language(
exclude_tag_ids, 'CHS')
all_tags = remove_stop_words(
[
'东方MMD',
'剧情MMD',
'舞蹈MMD',
'东方3D',
'游戏',
'东方FTG',
'东方STG',
'游戏宣传', # exclude
'音乐游戏',
'mugen',
'Minecraft',
'实况',
'攻略',
'跑团',
'音乐',
'东方Arrange',
'东方风Arrange',
'东方PV',
'XFD',
'MV',
'演奏',
'东方手书',
'漫画',
'动画',
'有配音',
'Walfas',
'MAD',
'AMV',
'音MAD',
'鬼畜',
'东方杂谈',
'东方科普',
'东方考据',
'访谈',
'电台',
'排行',
'线下活动',
'Cosplay',
'绘画过程',
'模因', # exclude
#'Shitpost', # exclude
'手工艺',
'VTuber',
'图集',
'主标签完成'
],
exclude_tags)
return [{k: 1} for k in all_tags]
def getPlaylist(pid, lang):
ret = playlist_db.retrive_item(pid)
if not ret:
raise UserError('PLAYLIST_NOT_EXIST')
if 'tags' in ret:
ret['tags_translated'], ret['tags_category'], ret[
'tags_tags'] = tagdb.translate_tag_ids_to_user_language(
ret['tags'], lang)
else:
ret['tags_translated'], ret['tags_category'], ret[
'tags_tags'] = [], {}, {}
return ret
def inferTagsFromVideo(utags, title, desc, user_language):
log(obj={
'title': title,
'desc': desc,
'utags': utags,
'lang': user_language
})
utags = [u.lower() for u in utags]
utags.append(title)
utags.append(desc)
all_text = ' 3e7dT2ibT7dM '.join(utags)
tagids = inferTagidsFromText(all_text)
return db.translate_tag_ids_to_user_language(tagids, user_language)[0]
def listVideoQuery(user, query_str, offset, limit, order = 'latest', user_language = 'CHS', hide_placeholder = True, qtype = 'tag', additional_constraint = '', human_readable_tag = False):
log(obj = {'q': query_str, 'offset': offset, 'limit': limit, 'order': order, 'lang': user_language})
if order not in ['latest', 'oldest', 'video_latest', 'video_oldest', 'last_modified'] :
raise UserError('INCORRECT_ORDER')
query_obj, tags = db.compile_query(query_str, qtype)
query_obj_extra, _ = db.compile_query(additional_constraint, 'tag')
log(obj = {'query': dumps(query_obj)})
default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')]
if user and 'settings' in user :
if user['settings']['blacklist'] == 'default' :
query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
else :
query_obj = {'$and': [query_obj, {'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]}
else :
query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
updateTagSearch(tags)
exStats1 = None
exStats2 = None
try :
result = db.retrive_items(query_obj)
exStats1 = result.explain()
if order == 'latest':
result = result.sort([("meta.created_at", -1)])
elif order == 'oldest':
result = result.sort([("meta.created_at", 1)])
elif order == 'video_latest':
result = result.sort([("item.upload_time", -1)])
elif order == 'video_oldest':
result = result.sort([("item.upload_time", 1)])
elif order == 'last_modified':
result = result.sort([("meta.modified_at", -1)])
ret = result.skip(offset).limit(limit)
exStats2 = ret.explain()
count = ret.count()
videos = [item for item in ret]
videos = filterVideoList(videos, user)
for i in range(len(videos)) :
videos[i]['tags'] = list(filter(lambda x: x < 0x80000000, videos[i]['tags']))
if human_readable_tag :
videos[i]['tags_readable'] = db.translate_tag_ids_to_user_language(videos[i]['tags'], user_language)[0]
if hide_placeholder :
videos = _filterPlaceholder(videos)
except pymongo.errors.OperationFailure as ex:
if '$not' in str(ex) :
raise UserError('FAILED_NOT_OP')
else :
log(level = 'ERR', obj = {'ex': str(ex)})
raise UserError('FAILED_UNKNOWN')
return videos, *getCommonTags(user_language, videos), count, query_obj, exStats1, exStats2
def listVideo(offset, limit, user, order = 'latest', user_language = 'CHS', hide_placeholder = True, additional_constraint = '', human_readable_tag = False):
if order not in ['latest', 'oldest', 'video_latest', 'video_oldest', 'last_modified'] :
raise UserError('INCORRECT_ORDER')
default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')]
query_obj_extra, _ = db.compile_query(additional_constraint, 'tag')
query_obj = {}
empty_query = True
if user and 'settings' in user :
if user['settings']['blacklist'] == 'default' :
empty_query = False
query_obj = {'$and': [{'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
else :
if user['settings']['blacklist'] or query_obj_extra :
empty_query = False
query_obj = {'$and': [{'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]}
else :
empty_query = False
query_obj = {'$and': [{'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
if empty_query :
query_obj = {}
exStats1 = None
exStats2 = None
result = db.retrive_items(query_obj)
exStats1 = result.explain()
if order == 'latest':
result = result.sort([("meta.created_at", -1)])
elif order == 'oldest':
result = result.sort([("meta.created_at", 1)])
elif order == 'video_latest':
result = result.sort([("item.upload_time", -1)])
elif order == 'video_oldest':
result = result.sort([("item.upload_time", 1)])
elif order == 'last_modified':
result = result.sort([("meta.modified_at", -1)])
videos = result.skip(offset).limit(limit)
exStats2 = videos.explain()
video_count = videos.count()
videos = [i for i in videos]
videos = filterVideoList(videos, user)
for i in range(len(videos)) :
videos[i]['tags'] = list(filter(lambda x: x < 0x80000000, videos[i]['tags']))
if human_readable_tag :
videos[i]['tags_readable'] = db.translate_tag_ids_to_user_language(videos[i]['tags'], user_language)[0]
if hide_placeholder :
videos = _filterPlaceholder(videos)
tags, pops, pop_tagid_map = getPopularTags(user_language)
return videos, video_count, tags, pops, query_obj, pop_tagid_map, exStats1, exStats2
def inferTagsFromVideo(utags,
title,
desc,
user_language,
video_url: str = '',
user_urls: [str] = []):
log(
obj={
'title': title,
'desc': desc,
'utags': utags,
'lang': user_language,
'video_url': video_url,
'user_urls': user_urls
})
video_url = video_url.strip()
tagids = []
if video_url:
obj, cleanURL = dispatch(video_url)
if obj is not None:
uid = obj.unique_id(obj, cleanURL)
vid_item = db.retrive_item({'item.unique_id': uid})
if vid_item is not None:
tagids = list(
filter(lambda x: x < 0x80000000, vid_item['tags']))
if not tagids:
utags = [u.lower() for u in utags]
utags.append(title)
utags.append(desc)
all_text = ' 3e7dT2ibT7dM '.join(utags)
tagids = inferTagidsFromText(all_text)
matched_author_records, matched_author_tags = matchUserSpace(user_urls)
matched_common_ids = itertools.chain.from_iterable(
[x['common_tagids'] for x in matched_author_records])
tagids = list(
set(tagids) | set([x['id'] for x in matched_author_tags])
| set(matched_common_ids))
return db.translate_tag_ids_to_user_language(tagids, user_language)[0]
def listYourVideo(uid, offset, limit, user, order = 'latest', human_readable_tag = False, user_language = 'CHS'):
if order not in ['latest', 'oldest', 'video_latest', 'video_oldest', 'last_modified'] :
raise UserError('INCORRECT_ORDER')
result = db.retrive_items({'meta.created_by': ObjectId(uid)})
if order == 'latest':
result = result.sort([("meta.created_at", -1)])
elif order == 'oldest':
result = result.sort([("meta.created_at", 1)])
elif order == 'video_latest':
result = result.sort([("item.upload_time", -1)])
elif order == 'video_oldest':
result = result.sort([("item.upload_time", 1)])
elif order == 'last_modified':
result = result.sort([("meta.modified_at", -1)])
videos = result.skip(offset).limit(limit)
video_count = videos.count()
videos = [i for i in videos]
videos = filterVideoList(videos, user)
for i in range(len(videos)) :
videos[i]['tags'] = list(filter(lambda x: x < 0x80000000, videos[i]['tags']))
if human_readable_tag :
videos[i]['tags_readable'] = db.translate_tag_ids_to_user_language(videos[i]['tags'], user_language)[0]
return videos, video_count
def listVideoRandimzied(user, limit, query_str = '', user_language = 'CHS', qtype = 'tag', additional_constraint = '', human_readable_tag = False) :
query_obj, _ = db.compile_query(query_str, qtype)
query_obj_extra, _ = db.compile_query(additional_constraint, 'tag')
log(obj = {'query': dumps(query_obj)})
default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')]
if user and 'settings' in user :
if user['settings']['blacklist'] == 'default' :
query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
else :
query_obj = {'$and': [query_obj, {'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]}
else :
query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
videos = list(db.aggregate([
{'$match': query_obj},
{'$sample': {'size': limit * 2}}
]))
videos = filterVideoList(videos, user)
for i in range(len(videos)) :
videos[i]['tags'] = list(filter(lambda x: x < 0x80000000, videos[i]['tags']))
if human_readable_tag :
videos[i]['tags_readable'] = db.translate_tag_ids_to_user_language(videos[i]['tags'], user_language)[0]
videos = _filterPlaceholder(videos)
videos = videos[: limit]
return videos, *getCommonTags(user_language, videos)
def listCommonTags(user, pid, language):
return tagdb.translate_tag_ids_to_user_language(
listCommonTagIDs(pid, user), language)[0]
def getDefaultBlacklist(language):
return tagdb.translate_tag_ids_to_user_language(
[int(i) for i in Config.DEFAULT_BLACKLIST_POPULAR_TAG.split(',')],
language)[0]
def viewRawTagHistory(offset, limit, language):
all_items = db.tag_history.aggregate([
{
'$sort': {
"time": -1
}
},
{
'$skip': offset
},
{
'$limit': limit
},
{
'$lookup': {
'from': 'users',
'localField': 'user',
'foreignField': '_id',
'as': 'user_obj'
}
},
{
'$project': {
'vid': 1,
'user_obj._id': 1,
'user_obj.profile.username': 1,
'user_obj.profile.image': 1,
'tags': 1,
'del': 1,
'add': 1,
'time': 1
}
},
{
'$lookup': {
'from': 'videos',
'localField': 'vid',
'foreignField': '_id',
'as': 'video_obj'
}
},
{
'$project': {
'vid': 1,
'user_obj._id': 1,
'user_obj.profile.username': 1,
'user_obj.profile.image': 1,
'tags': 1,
'del': 1,
'add': 1,
'time': 1,
'video_obj.item': 1
}
},
])
all_items = list(all_items)
for item in all_items:
item['tags'], _, _ = tagdb.translate_tag_ids_to_user_language(
item['tags'], language)
item['del'], _, _ = tagdb.translate_tag_ids_to_user_language(
item['del'], language)
item['add'], _, _ = tagdb.translate_tag_ids_to_user_language(
item['add'], language)
return all_items
def inferTagsFromUtags(utags, user_language):
log(obj={'utags': utags, 'lang': user_language})
tagids = inferTagidsFromUtags(utags)
return db.translate_tag_ids_to_user_language(tagids, user_language)[0]
def listSubscriptionTags(user, language = 'CHS') :
ret = list(db.subs.find({'meta.created_by': makeUserMeta(user), 'tagid': {'$exists': True}}))
return tagdb.translate_tag_ids_to_user_language([x['tagid'] for x in ret], language)[0]
