def listYourPlaylists(user,
uid,
offset=0,
limit=10000,
query_str='',
additional_constraint='',
order='last_modified'):
if order not in ['latest', 'oldest', 'last_modified']:
raise UserError('INCORRECT_ORDER')
if isObjectAgnosticOperationPermitted('viewPrivatePlaylist', user):
auth_obj = {}
else:
auth_obj = {'item.private': False}
query_obj, _ = tagdb.compile_query(query_str, 'text')
query_obj_extra, _ = tagdb.compile_query(additional_constraint, 'tag')
query = {
'$and': [{
'meta.created_by': ObjectId(uid)
}, query_obj, query_obj_extra, auth_obj]
}
result = playlist_db.retrive_items(query)
if order == 'last_modified':
result = result.sort([("meta.modified_at", -1)])
if order == 'latest':
result = result.sort([("meta.created_at", -1)])
if order == 'oldest':
result = result.sort([("meta.created_at", 1)])
result = result.skip(offset).limit(limit)
count = result.count()
result = [i for i in result]
for i in range(len(result)):
result[i]['tags'] = list(
filter(lambda x: x < 0x80000000, result[i]['tags']))
return result, count
def listMyPlaylists(user,
offset=0,
limit=10000,
query_str='',
additional_constraint='',
order='last_modified'):
if order not in ['latest', 'oldest', 'last_modified']:
raise UserError('INCORRECT_ORDER')
query_obj, _ = tagdb.compile_query(query_str, 'text')
query_obj_extra, _ = tagdb.compile_query(additional_constraint, 'tag')
query = {
'$and': [{
'meta.created_by': ObjectId(user['_id'])
}, query_obj, query_obj_extra]
}
result = playlist_db.retrive_items(query)
if order == 'last_modified':
result = result.sort([("meta.modified_at", -1)])
if order == 'latest':
result = result.sort([("meta.created_at", -1)])
if order == 'oldest':
result = result.sort([("meta.created_at", 1)])
result = result.skip(offset).limit(limit)
total_count = result.count()
result = [i for i in result]
for i in range(len(result)):
result[i]['tags'] = list(
filter(lambda x: x < 0x80000000, result[i]['tags']))
return result, total_count
def listVideoQuery(user, query_str, offset, limit, order = 'latest', user_language = 'CHS', hide_placeholder = True, qtype = 'tag', additional_constraint = '', human_readable_tag = False):
log(obj = {'q': query_str, 'offset': offset, 'limit': limit, 'order': order, 'lang': user_language})
if order not in ['latest', 'oldest', 'video_latest', 'video_oldest', 'last_modified'] :
raise UserError('INCORRECT_ORDER')
query_obj, tags = db.compile_query(query_str, qtype)
query_obj_extra, _ = db.compile_query(additional_constraint, 'tag')
log(obj = {'query': dumps(query_obj)})
default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')]
if user and 'settings' in user :
if user['settings']['blacklist'] == 'default' :
query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
else :
query_obj = {'$and': [query_obj, {'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]}
else :
query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
updateTagSearch(tags)
exStats1 = None
exStats2 = None
try :
result = db.retrive_items(query_obj)
exStats1 = result.explain()
if order == 'latest':
result = result.sort([("meta.created_at", -1)])
elif order == 'oldest':
result = result.sort([("meta.created_at", 1)])
elif order == 'video_latest':
result = result.sort([("item.upload_time", -1)])
elif order == 'video_oldest':
result = result.sort([("item.upload_time", 1)])
elif order == 'last_modified':
result = result.sort([("meta.modified_at", -1)])
ret = result.skip(offset).limit(limit)
exStats2 = ret.explain()
count = ret.count()
videos = [item for item in ret]
videos = filterVideoList(videos, user)
for i in range(len(videos)) :
videos[i]['tags'] = list(filter(lambda x: x < 0x80000000, videos[i]['tags']))
if human_readable_tag :
videos[i]['tags_readable'] = db.translate_tag_ids_to_user_language(videos[i]['tags'], user_language)[0]
if hide_placeholder :
videos = _filterPlaceholder(videos)
except pymongo.errors.OperationFailure as ex:
if '$not' in str(ex) :
raise UserError('FAILED_NOT_OP')
else :
log(level = 'ERR', obj = {'ex': str(ex)})
raise UserError('FAILED_UNKNOWN')
return videos, *getCommonTags(user_language, videos), count, query_obj, exStats1, exStats2
def listSubscriptedItems(user,
page_idx,
page_size,
user_language,
hide_placeholder=True,
order='latest_video'):
subs = list(db.subs.find({'meta.created_by': makeUserMeta(user)}))
q = [tagdb.compile_query(q['qs'], q['qt']) for q in subs]
query_obj = {'$or': []}
for qi, _ in q:
query_obj['$or'].append(qi)
for i in range(len(q)):
subs[i]['obj'] = q[i][0]
subs[i]['obj_tags'] = q[i][1]
default_blacklist_tagids = [
int(i) for i in Config.DEFAULT_BLACKLIST.split(',')
]
if user and 'settings' in user:
if user['settings']['blacklist'] == 'default':
query_obj = {
'$and':
[query_obj, {
'tags': {
'$nin': default_blacklist_tagids
}
}]
}
else:
query_obj = {
'$and':
[query_obj, {
'tags': {
'$nin': user['settings']['blacklist']
}
}]
}
elif user is None:
query_obj = {
'$and': [query_obj, {
'tags': {
'$nin': default_blacklist_tagids
}
}]
}
result = tagdb.retrive_items(query_obj)
if order == 'latest':
result = result.sort([("meta.created_at", -1)])
if order == 'oldest':
result = result.sort([("meta.created_at", 1)])
if order == 'video_latest':
result = result.sort([("item.upload_time", -1)])
if order == 'video_oldest':
result = result.sort([("item.upload_time", 1)])
ret = result.skip(page_idx * page_size).limit(page_size)
count = ret.count()
videos = [item for item in ret]
videos = filterVideoList(videos, user)
if hide_placeholder:
videos = _filterPlaceholder(videos)
return videos, subs, getCommonTags(user_language, videos), count
def editVideoTagsQuery(query, query_type, tags_to_add, tags_to_remove, user):
if query_type not in ['tag', 'text']:
raise UserError('INCORRECT_QUERY_TYPE')
filterOperation('batchVideoTagEdit', user)
query_obj, _ = tagdb.compile_query(query)
log(obj={'query': dumps(query_obj)})
tagids_to_add = tagdb.filter_and_translate_tags(tags_to_add)
tagids_to_remove = tagdb.filter_and_translate_tags(tags_to_remove)
try:
count = 0
with MongoTransaction(client) as s_read, MongoTransaction(
client) as s_write:
result_cursor = tagdb.retrive_items(query_obj, session=s_read())
batch = _batchedRead(result_cursor)
while batch:
item_ids = [item['_id'] for item in batch]
tagdb.update_many_items_tags_pull(item_ids,
tagids_to_remove,
makeUserMeta(user),
session=s_write())
tagdb.update_many_items_tags_merge(item_ids,
tagids_to_add,
makeUserMeta(user),
session=s_write())
count += len(batch)
batch = _batchedRead(result_cursor)
s_write.mark_succeed()
return count
except pymongo.errors.OperationFailure as ex:
if '$not' in str(ex):
raise UserError('FAILED_NOT_OP')
else:
log(level='ERR', obj={'ex': str(ex)})
raise UserError('FAILED_UNKNOWN')
def updateSubScription(user, sub_id, query_str : str, qtype : str = '', name = '') :
obj = db.subs.find_one({'_id': ObjectId(sub_id)})
if obj is None :
raise UserError('SUB_NOT_EXIST')
# TODO: add duplicated query check
tagdb.compile_query(query_str, qtype)
if not name :
name = obj['name']
if not qtype :
qtype = obj['qt']
db.subs.update_one({'_id': ObjectId(sub_id)}, {'$set': {
'qs': query_str,
'qt': qtype,
'name': name,
'meta.modified_by': makeUserMeta(user),
'meta.modified_at': datetime.now()
}})
def listSubscriptedItems(user, offset, limit, user_language, hide_placeholder = True, order = 'video_latest', visibleSubs = [''], additional_constraint = '') :
subs = list(db.subs.find({'meta.created_by': makeUserMeta(user)}))
q = [(tagdb.compile_query(q['qs'], q['qt']), str(q['_id'])) for q in subs]
query_obj = {'$or': []}
if '' in visibleSubs :
for (qi, _), _ in q :
query_obj['$or'].append(qi)
else :
for (qi, _), qid in q :
if qid in visibleSubs :
query_obj['$or'].append(qi)
for i in range(len(q)) :
(qobj, qtags), _ = q[i]
subs[i]['obj'] = qobj
subs[i]['obj_tags'] = qtags
if not query_obj['$or'] :
return [], subs, [], 0
default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')]
query_obj_extra, _ = tagdb.compile_query(additional_constraint, 'tag')
if user and 'settings' in user :
if user['settings']['blacklist'] == 'default' :
query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
else :
query_obj = {'$and': [query_obj, {'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]}
elif user is None :
query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
result = tagdb.retrive_items(query_obj)
if order == 'latest':
result = result.sort([("meta.created_at", -1)])
elif order == 'oldest':
result = result.sort([("meta.created_at", 1)])
elif order == 'video_latest':
result = result.sort([("item.upload_time", -1)])
elif order == 'video_oldest':
result = result.sort([("item.upload_time", 1)])
elif order == 'last_modified':
result = result.sort([("meta.modified_at", -1)])
ret = result.skip(offset).limit(limit)
count = ret.count()
videos = [item for item in ret]
videos = filterVideoList(videos, user)
if hide_placeholder :
videos = _filterPlaceholder(videos)
return videos, subs, *getCommonTags(user_language, videos), count
def listVideo(page_idx,
page_size,
user,
order='latest',
user_language='CHS',
hide_placeholder=True,
additional_constraint=''):
if order not in ['latest', 'oldest', 'video_latest', 'video_oldest']:
raise UserError('INCORRECT_ORDER')
default_blacklist_tagids = [
int(i) for i in Config.DEFAULT_BLACKLIST.split(',')
]
query_obj_extra, _ = db.compile_query(additional_constraint, 'tag')
query_obj = {}
if user and 'settings' in user:
if user['settings']['blacklist'] == 'default':
query_obj = {
'$and': [{
'tags': {
'$nin': default_blacklist_tagids
}
}, query_obj_extra]
}
else:
query_obj = {
'$and': [{
'tags': {
'$nin': user['settings']['blacklist']
}
}, query_obj_extra]
}
else:
query_obj = {
'$and': [{
'tags': {
'$nin': default_blacklist_tagids
}
}, query_obj_extra]
}
result = db.retrive_items(query_obj)
if order == 'latest':
result = result.sort([("meta.created_at", -1)])
if order == 'oldest':
result = result.sort([("meta.created_at", 1)])
if order == 'video_latest':
result = result.sort([("item.upload_time", -1)])
if order == 'video_oldest':
result = result.sort([("item.upload_time", 1)])
videos = result.skip(page_idx * page_size).limit(page_size)
video_count = videos.count()
videos = [i for i in videos]
videos = filterVideoList(videos, user)
if hide_placeholder:
videos = _filterPlaceholder(videos)
tags, pops = getPopularTags(user_language)
return videos, video_count, tags, pops
def addSubscription(user, query_str : str, qtype = 'tag', name = '') :
query_str = query_str.strip()
if not query_str :
raise UserError('EMPTY_QUERY')
# TODO: add duplicated query check
qobj, qtags = tagdb.compile_query(query_str, qtype)
if len(qtags) == 1 and 'tags' in qobj and isinstance(qobj['tags'], int) and qobj['tags'] < 0x80000000:
subid = db.subs.insert_one({'qs': query_str.strip(), 'qt': qtype, 'name': name, 'tagid': qobj['tags'], 'meta': makeUserMetaObject(user)}).inserted_id
else :
subid = db.subs.insert_one({'qs': query_str, 'qt': qtype, 'name': name, 'meta': makeUserMetaObject(user)}).inserted_id
return str(subid)
def listMyPlaylistsAgainstSingleVideo(user,
vid,
offset=0,
limit=10000,
query_str='',
additional_constraint='',
order='last_modified'):
if order not in ['latest', 'oldest', 'last_modified']:
raise UserError('INCORRECT_ORDER')
query_obj, _ = tagdb.compile_query(query_str, 'text')
query_obj_extra, _ = tagdb.compile_query(additional_constraint, 'tag')
query = {
'$and': [{
'meta.created_by': ObjectId(user['_id'])
}, query_obj, query_obj_extra]
}
result = playlist_db.retrive_items(query)
if order == 'last_modified':
result = result.sort([("meta.modified_at", -1)])
if order == 'latest':
result = result.sort([("meta.created_at", -1)])
if order == 'oldest':
result = result.sort([("meta.created_at", 1)])
result = result.skip(offset).limit(limit)
count = result.count()
result = [i for i in result]
for i in range(len(result)):
result[i]['tags'] = list(
filter(lambda x: x < 0x80000000, result[i]['tags']))
result_dict = {str(x['_id']): x for x in result}
pids = [i['_id'] for i in result]
video_items = db.playlist_items.find({
'pid': {
'$in': pids
},
'vid': ObjectId(vid)
})
for item in video_items:
result_dict[str(item['pid'])]['exist'] = True
return result_dict.values(), count
def listSubscriptedItemsRandomized(user, limit, user_language, visibleSubs = [''], additional_constraint = '') :
subs = list(db.subs.find({'meta.created_by': makeUserMeta(user)}))
q = [(tagdb.compile_query(q['qs'], q['qt']), str(q['_id'])) for q in subs]
query_obj = {'$or': []}
if '' in visibleSubs :
for (qi, _), _ in q :
query_obj['$or'].append(qi)
else :
for (qi, _), qid in q :
if qid in visibleSubs :
query_obj['$or'].append(qi)
for i in range(len(q)) :
(qobj, qtags), _ = q[i]
subs[i]['obj'] = qobj
subs[i]['obj_tags'] = qtags
if not query_obj['$or'] :
return [], subs, []
default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')]
query_obj_extra, _ = tagdb.compile_query(additional_constraint, 'tag')
if user and 'settings' in user :
if user['settings']['blacklist'] == 'default' :
query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
else :
query_obj = {'$and': [query_obj, {'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]}
elif user is None :
query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
videos = list(tagdb.aggregate([
{'$match': query_obj},
{'$sample': {'size': limit * 2}}
]))
videos = filterVideoList(videos, user)
for i in range(len(videos)) :
videos[i]['tags'] = list(filter(lambda x: x < 0x80000000, videos[i]['tags']))
videos = _filterPlaceholder(videos)
videos = videos[: limit]
return videos, subs, *getCommonTags(user_language, videos)
def listVideoRandimzied(user, limit, query_str = '', user_language = 'CHS', qtype = 'tag', additional_constraint = '', human_readable_tag = False) :
query_obj, _ = db.compile_query(query_str, qtype)
query_obj_extra, _ = db.compile_query(additional_constraint, 'tag')
log(obj = {'query': dumps(query_obj)})
default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')]
if user and 'settings' in user :
if user['settings']['blacklist'] == 'default' :
query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
else :
query_obj = {'$and': [query_obj, {'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]}
else :
query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
videos = list(db.aggregate([
{'$match': query_obj},
{'$sample': {'size': limit * 2}}
]))
videos = filterVideoList(videos, user)
for i in range(len(videos)) :
videos[i]['tags'] = list(filter(lambda x: x < 0x80000000, videos[i]['tags']))
if human_readable_tag :
videos[i]['tags_readable'] = db.translate_tag_ids_to_user_language(videos[i]['tags'], user_language)[0]
videos = _filterPlaceholder(videos)
videos = videos[: limit]
return videos, *getCommonTags(user_language, videos)
def listVideo(offset, limit, user, order = 'latest', user_language = 'CHS', hide_placeholder = True, additional_constraint = '', human_readable_tag = False):
if order not in ['latest', 'oldest', 'video_latest', 'video_oldest', 'last_modified'] :
raise UserError('INCORRECT_ORDER')
default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')]
query_obj_extra, _ = db.compile_query(additional_constraint, 'tag')
query_obj = {}
empty_query = True
if user and 'settings' in user :
if user['settings']['blacklist'] == 'default' :
empty_query = False
query_obj = {'$and': [{'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
else :
if user['settings']['blacklist'] or query_obj_extra :
empty_query = False
query_obj = {'$and': [{'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]}
else :
empty_query = False
query_obj = {'$and': [{'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
if empty_query :
query_obj = {}
exStats1 = None
exStats2 = None
result = db.retrive_items(query_obj)
exStats1 = result.explain()
if order == 'latest':
result = result.sort([("meta.created_at", -1)])
elif order == 'oldest':
result = result.sort([("meta.created_at", 1)])
elif order == 'video_latest':
result = result.sort([("item.upload_time", -1)])
elif order == 'video_oldest':
result = result.sort([("item.upload_time", 1)])
elif order == 'last_modified':
result = result.sort([("meta.modified_at", -1)])
videos = result.skip(offset).limit(limit)
exStats2 = videos.explain()
video_count = videos.count()
videos = [i for i in videos]
videos = filterVideoList(videos, user)
for i in range(len(videos)) :
videos[i]['tags'] = list(filter(lambda x: x < 0x80000000, videos[i]['tags']))
if human_readable_tag :
videos[i]['tags_readable'] = db.translate_tag_ids_to_user_language(videos[i]['tags'], user_language)[0]
if hide_placeholder :
videos = _filterPlaceholder(videos)
tags, pops, pop_tagid_map = getPopularTags(user_language)
return videos, video_count, tags, pops, query_obj, pop_tagid_map, exStats1, exStats2
def listVideoQuery(user,
query_str,
page_idx,
page_size,
order='latest',
user_language='CHS',
hide_placeholder=True):
log(
obj={
'q': query_str,
'page': page_idx,
'page_size': page_size,
'order': order,
'lang': user_language
})
if order not in ['latest', 'oldest', 'video_latest', 'video_oldest']:
raise UserError('INCORRECT_ORDER')
query_obj, tags = db.compile_query(query_str)
log(obj={'query': dumps(query_obj)})
default_blacklist_tagids = [
int(i) for i in Config.DEFAULT_BLACKLIST.split(',')
]
if user and 'settings' in user:
if user['settings']['blacklist'] == 'default':
query_obj = {
'$and':
[query_obj, {
'tags': {
'$nin': default_blacklist_tagids
}
}]
}
else:
query_obj = {
'$and':
[query_obj, {
'tags': {
'$nin': user['settings']['blacklist']
}
}]
}
elif user is None:
query_obj = {
'$and': [query_obj, {
'tags': {
'$nin': default_blacklist_tagids
}
}]
}
updateTagSearch(tags)
try:
result = db.retrive_items(query_obj)
if order == 'latest':
result = result.sort([("meta.created_at", -1)])
if order == 'oldest':
result = result.sort([("meta.created_at", 1)])
if order == 'video_latest':
result = result.sort([("item.upload_time", -1)])
if order == 'video_oldest':
result = result.sort([("item.upload_time", 1)])
ret = result.skip(page_idx * page_size).limit(page_size)
count = ret.count()
videos = [item for item in ret]
videos = filterVideoList(videos, user)
if hide_placeholder:
videos = _filterPlaceholder(videos)
except pymongo.errors.OperationFailure as ex:
if '$not' in str(ex):
raise UserError('FAILED_NOT_OP')
else:
log(level='ERR', obj={'ex': str(ex)})
raise UserError('FAILED_UNKNOWN')
return videos, getCommonTags(user_language, videos), count
def listPlaylists(user,
offset,
limit,
query_str='',
order='latest',
qtype='tag',
additional_constraint=''):
sort_obj = {"meta.created_at": 1}
if order == 'latest':
sort_obj = {"meta.created_at": -1}
if order == 'last_modified':
sort_obj = {"meta.modified_at": -1}
if order == 'oldest':
sort_obj = {"meta.created_at": 1}
if order == 'views':
sort_obj = {"item.views": 1}
if isObjectAgnosticOperationPermitted('viewPrivatePlaylist', user):
auth_obj = {}
else:
auth_obj = {
'$or': [{
'meta.created_by': user['_id'] if user else ''
}, {
'item.private': False,
'item.videos': {
'$gt': 1
}
}]
}
query_obj, _ = tagdb.compile_query(query_str, qtype)
query_obj_extra, _ = tagdb.compile_query(additional_constraint, 'tag')
query = {'$and': [query_obj, query_obj_extra, auth_obj]}
ans_obj = playlist_db.aggregate([{
"$match": query
}, {
"$lookup": {
"from": "users",
"localField": "meta.created_by",
"foreignField": "_id",
"as": "user_detail"
}
}, {
"$unwind": {
"path": "$user_detail"
}
}, {
"$project": {
"user_detail.crypto": 0,
"user_detail.access_control": 0,
"user_detail.meta": 0,
"user_detail.profile.email": 0,
"user_detail.settings": 0
}
}, {
'$facet': {
'result': [{
'$sort': sort_obj
}, {
'$skip': offset
}, {
'$limit': limit
}],
'count': [{
'$count': 'count'
}]
}
}])
ans_obj = [i for i in ans_obj][0]
if ans_obj['result']:
result = [i for i in ans_obj['result']]
for i in range(len(result)):
result[i]['tags'] = list(
filter(lambda x: x < 0x80000000, result[i]['tags']))
return result, ans_obj['count'][0]['count']
else:
return [], 0
