def listYourPlaylists(user, uid, offset=0, limit=10000, query_str='', additional_constraint='', order='last_modified'): if order not in ['latest', 'oldest', 'last_modified']: raise UserError('INCORRECT_ORDER') if isObjectAgnosticOperationPermitted('viewPrivatePlaylist', user): auth_obj = {} else: auth_obj = {'item.private': False} query_obj, _ = tagdb.compile_query(query_str, 'text') query_obj_extra, _ = tagdb.compile_query(additional_constraint, 'tag') query = { '$and': [{ 'meta.created_by': ObjectId(uid) }, query_obj, query_obj_extra, auth_obj] } result = playlist_db.retrive_items(query) if order == 'last_modified': result = result.sort([("meta.modified_at", -1)]) if order == 'latest': result = result.sort([("meta.created_at", -1)]) if order == 'oldest': result = result.sort([("meta.created_at", 1)]) result = result.skip(offset).limit(limit) count = result.count() result = [i for i in result] for i in range(len(result)): result[i]['tags'] = list( filter(lambda x: x < 0x80000000, result[i]['tags'])) return result, count
def listMyPlaylists(user, offset=0, limit=10000, query_str='', additional_constraint='', order='last_modified'): if order not in ['latest', 'oldest', 'last_modified']: raise UserError('INCORRECT_ORDER') query_obj, _ = tagdb.compile_query(query_str, 'text') query_obj_extra, _ = tagdb.compile_query(additional_constraint, 'tag') query = { '$and': [{ 'meta.created_by': ObjectId(user['_id']) }, query_obj, query_obj_extra] } result = playlist_db.retrive_items(query) if order == 'last_modified': result = result.sort([("meta.modified_at", -1)]) if order == 'latest': result = result.sort([("meta.created_at", -1)]) if order == 'oldest': result = result.sort([("meta.created_at", 1)]) result = result.skip(offset).limit(limit) total_count = result.count() result = [i for i in result] for i in range(len(result)): result[i]['tags'] = list( filter(lambda x: x < 0x80000000, result[i]['tags'])) return result, total_count
def listVideoQuery(user, query_str, offset, limit, order = 'latest', user_language = 'CHS', hide_placeholder = True, qtype = 'tag', additional_constraint = '', human_readable_tag = False): log(obj = {'q': query_str, 'offset': offset, 'limit': limit, 'order': order, 'lang': user_language}) if order not in ['latest', 'oldest', 'video_latest', 'video_oldest', 'last_modified'] : raise UserError('INCORRECT_ORDER') query_obj, tags = db.compile_query(query_str, qtype) query_obj_extra, _ = db.compile_query(additional_constraint, 'tag') log(obj = {'query': dumps(query_obj)}) default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')] if user and 'settings' in user : if user['settings']['blacklist'] == 'default' : query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]} else : query_obj = {'$and': [query_obj, {'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]} else : query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]} updateTagSearch(tags) exStats1 = None exStats2 = None try : result = db.retrive_items(query_obj) exStats1 = result.explain() if order == 'latest': result = result.sort([("meta.created_at", -1)]) elif order == 'oldest': result = result.sort([("meta.created_at", 1)]) elif order == 'video_latest': result = result.sort([("item.upload_time", -1)]) elif order == 'video_oldest': result = result.sort([("item.upload_time", 1)]) elif order == 'last_modified': result = result.sort([("meta.modified_at", -1)]) ret = result.skip(offset).limit(limit) exStats2 = ret.explain() count = ret.count() videos = [item for item in ret] videos = filterVideoList(videos, user) for i in range(len(videos)) : videos[i]['tags'] = list(filter(lambda x: x < 0x80000000, videos[i]['tags'])) if human_readable_tag : videos[i]['tags_readable'] = db.translate_tag_ids_to_user_language(videos[i]['tags'], user_language)[0] if hide_placeholder : videos = _filterPlaceholder(videos) except pymongo.errors.OperationFailure as ex: if '$not' in str(ex) : raise UserError('FAILED_NOT_OP') else : log(level = 'ERR', obj = {'ex': str(ex)}) raise UserError('FAILED_UNKNOWN') return videos, *getCommonTags(user_language, videos), count, query_obj, exStats1, exStats2
def listSubscriptedItems(user, page_idx, page_size, user_language, hide_placeholder=True, order='latest_video'): subs = list(db.subs.find({'meta.created_by': makeUserMeta(user)})) q = [tagdb.compile_query(q['qs'], q['qt']) for q in subs] query_obj = {'$or': []} for qi, _ in q: query_obj['$or'].append(qi) for i in range(len(q)): subs[i]['obj'] = q[i][0] subs[i]['obj_tags'] = q[i][1] default_blacklist_tagids = [ int(i) for i in Config.DEFAULT_BLACKLIST.split(',') ] if user and 'settings' in user: if user['settings']['blacklist'] == 'default': query_obj = { '$and': [query_obj, { 'tags': { '$nin': default_blacklist_tagids } }] } else: query_obj = { '$and': [query_obj, { 'tags': { '$nin': user['settings']['blacklist'] } }] } elif user is None: query_obj = { '$and': [query_obj, { 'tags': { '$nin': default_blacklist_tagids } }] } result = tagdb.retrive_items(query_obj) if order == 'latest': result = result.sort([("meta.created_at", -1)]) if order == 'oldest': result = result.sort([("meta.created_at", 1)]) if order == 'video_latest': result = result.sort([("item.upload_time", -1)]) if order == 'video_oldest': result = result.sort([("item.upload_time", 1)]) ret = result.skip(page_idx * page_size).limit(page_size) count = ret.count() videos = [item for item in ret] videos = filterVideoList(videos, user) if hide_placeholder: videos = _filterPlaceholder(videos) return videos, subs, getCommonTags(user_language, videos), count
def editVideoTagsQuery(query, query_type, tags_to_add, tags_to_remove, user): if query_type not in ['tag', 'text']: raise UserError('INCORRECT_QUERY_TYPE') filterOperation('batchVideoTagEdit', user) query_obj, _ = tagdb.compile_query(query) log(obj={'query': dumps(query_obj)}) tagids_to_add = tagdb.filter_and_translate_tags(tags_to_add) tagids_to_remove = tagdb.filter_and_translate_tags(tags_to_remove) try: count = 0 with MongoTransaction(client) as s_read, MongoTransaction( client) as s_write: result_cursor = tagdb.retrive_items(query_obj, session=s_read()) batch = _batchedRead(result_cursor) while batch: item_ids = [item['_id'] for item in batch] tagdb.update_many_items_tags_pull(item_ids, tagids_to_remove, makeUserMeta(user), session=s_write()) tagdb.update_many_items_tags_merge(item_ids, tagids_to_add, makeUserMeta(user), session=s_write()) count += len(batch) batch = _batchedRead(result_cursor) s_write.mark_succeed() return count except pymongo.errors.OperationFailure as ex: if '$not' in str(ex): raise UserError('FAILED_NOT_OP') else: log(level='ERR', obj={'ex': str(ex)}) raise UserError('FAILED_UNKNOWN')
def updateSubScription(user, sub_id, query_str : str, qtype : str = '', name = '') : obj = db.subs.find_one({'_id': ObjectId(sub_id)}) if obj is None : raise UserError('SUB_NOT_EXIST') # TODO: add duplicated query check tagdb.compile_query(query_str, qtype) if not name : name = obj['name'] if not qtype : qtype = obj['qt'] db.subs.update_one({'_id': ObjectId(sub_id)}, {'$set': { 'qs': query_str, 'qt': qtype, 'name': name, 'meta.modified_by': makeUserMeta(user), 'meta.modified_at': datetime.now() }})
def listSubscriptedItems(user, offset, limit, user_language, hide_placeholder = True, order = 'video_latest', visibleSubs = [''], additional_constraint = '') : subs = list(db.subs.find({'meta.created_by': makeUserMeta(user)})) q = [(tagdb.compile_query(q['qs'], q['qt']), str(q['_id'])) for q in subs] query_obj = {'$or': []} if '' in visibleSubs : for (qi, _), _ in q : query_obj['$or'].append(qi) else : for (qi, _), qid in q : if qid in visibleSubs : query_obj['$or'].append(qi) for i in range(len(q)) : (qobj, qtags), _ = q[i] subs[i]['obj'] = qobj subs[i]['obj_tags'] = qtags if not query_obj['$or'] : return [], subs, [], 0 default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')] query_obj_extra, _ = tagdb.compile_query(additional_constraint, 'tag') if user and 'settings' in user : if user['settings']['blacklist'] == 'default' : query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]} else : query_obj = {'$and': [query_obj, {'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]} elif user is None : query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]} result = tagdb.retrive_items(query_obj) if order == 'latest': result = result.sort([("meta.created_at", -1)]) elif order == 'oldest': result = result.sort([("meta.created_at", 1)]) elif order == 'video_latest': result = result.sort([("item.upload_time", -1)]) elif order == 'video_oldest': result = result.sort([("item.upload_time", 1)]) elif order == 'last_modified': result = result.sort([("meta.modified_at", -1)]) ret = result.skip(offset).limit(limit) count = ret.count() videos = [item for item in ret] videos = filterVideoList(videos, user) if hide_placeholder : videos = _filterPlaceholder(videos) return videos, subs, *getCommonTags(user_language, videos), count
def listVideo(page_idx, page_size, user, order='latest', user_language='CHS', hide_placeholder=True, additional_constraint=''): if order not in ['latest', 'oldest', 'video_latest', 'video_oldest']: raise UserError('INCORRECT_ORDER') default_blacklist_tagids = [ int(i) for i in Config.DEFAULT_BLACKLIST.split(',') ] query_obj_extra, _ = db.compile_query(additional_constraint, 'tag') query_obj = {} if user and 'settings' in user: if user['settings']['blacklist'] == 'default': query_obj = { '$and': [{ 'tags': { '$nin': default_blacklist_tagids } }, query_obj_extra] } else: query_obj = { '$and': [{ 'tags': { '$nin': user['settings']['blacklist'] } }, query_obj_extra] } else: query_obj = { '$and': [{ 'tags': { '$nin': default_blacklist_tagids } }, query_obj_extra] } result = db.retrive_items(query_obj) if order == 'latest': result = result.sort([("meta.created_at", -1)]) if order == 'oldest': result = result.sort([("meta.created_at", 1)]) if order == 'video_latest': result = result.sort([("item.upload_time", -1)]) if order == 'video_oldest': result = result.sort([("item.upload_time", 1)]) videos = result.skip(page_idx * page_size).limit(page_size) video_count = videos.count() videos = [i for i in videos] videos = filterVideoList(videos, user) if hide_placeholder: videos = _filterPlaceholder(videos) tags, pops = getPopularTags(user_language) return videos, video_count, tags, pops
def addSubscription(user, query_str : str, qtype = 'tag', name = '') : query_str = query_str.strip() if not query_str : raise UserError('EMPTY_QUERY') # TODO: add duplicated query check qobj, qtags = tagdb.compile_query(query_str, qtype) if len(qtags) == 1 and 'tags' in qobj and isinstance(qobj['tags'], int) and qobj['tags'] < 0x80000000: subid = db.subs.insert_one({'qs': query_str.strip(), 'qt': qtype, 'name': name, 'tagid': qobj['tags'], 'meta': makeUserMetaObject(user)}).inserted_id else : subid = db.subs.insert_one({'qs': query_str, 'qt': qtype, 'name': name, 'meta': makeUserMetaObject(user)}).inserted_id return str(subid)
def listMyPlaylistsAgainstSingleVideo(user, vid, offset=0, limit=10000, query_str='', additional_constraint='', order='last_modified'): if order not in ['latest', 'oldest', 'last_modified']: raise UserError('INCORRECT_ORDER') query_obj, _ = tagdb.compile_query(query_str, 'text') query_obj_extra, _ = tagdb.compile_query(additional_constraint, 'tag') query = { '$and': [{ 'meta.created_by': ObjectId(user['_id']) }, query_obj, query_obj_extra] } result = playlist_db.retrive_items(query) if order == 'last_modified': result = result.sort([("meta.modified_at", -1)]) if order == 'latest': result = result.sort([("meta.created_at", -1)]) if order == 'oldest': result = result.sort([("meta.created_at", 1)]) result = result.skip(offset).limit(limit) count = result.count() result = [i for i in result] for i in range(len(result)): result[i]['tags'] = list( filter(lambda x: x < 0x80000000, result[i]['tags'])) result_dict = {str(x['_id']): x for x in result} pids = [i['_id'] for i in result] video_items = db.playlist_items.find({ 'pid': { '$in': pids }, 'vid': ObjectId(vid) }) for item in video_items: result_dict[str(item['pid'])]['exist'] = True return result_dict.values(), count
def listSubscriptedItemsRandomized(user, limit, user_language, visibleSubs = [''], additional_constraint = '') : subs = list(db.subs.find({'meta.created_by': makeUserMeta(user)})) q = [(tagdb.compile_query(q['qs'], q['qt']), str(q['_id'])) for q in subs] query_obj = {'$or': []} if '' in visibleSubs : for (qi, _), _ in q : query_obj['$or'].append(qi) else : for (qi, _), qid in q : if qid in visibleSubs : query_obj['$or'].append(qi) for i in range(len(q)) : (qobj, qtags), _ = q[i] subs[i]['obj'] = qobj subs[i]['obj_tags'] = qtags if not query_obj['$or'] : return [], subs, [] default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')] query_obj_extra, _ = tagdb.compile_query(additional_constraint, 'tag') if user and 'settings' in user : if user['settings']['blacklist'] == 'default' : query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]} else : query_obj = {'$and': [query_obj, {'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]} elif user is None : query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]} videos = list(tagdb.aggregate([ {'$match': query_obj}, {'$sample': {'size': limit * 2}} ])) videos = filterVideoList(videos, user) for i in range(len(videos)) : videos[i]['tags'] = list(filter(lambda x: x < 0x80000000, videos[i]['tags'])) videos = _filterPlaceholder(videos) videos = videos[: limit] return videos, subs, *getCommonTags(user_language, videos)
def listVideoRandimzied(user, limit, query_str = '', user_language = 'CHS', qtype = 'tag', additional_constraint = '', human_readable_tag = False) : query_obj, _ = db.compile_query(query_str, qtype) query_obj_extra, _ = db.compile_query(additional_constraint, 'tag') log(obj = {'query': dumps(query_obj)}) default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')] if user and 'settings' in user : if user['settings']['blacklist'] == 'default' : query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]} else : query_obj = {'$and': [query_obj, {'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]} else : query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]} videos = list(db.aggregate([ {'$match': query_obj}, {'$sample': {'size': limit * 2}} ])) videos = filterVideoList(videos, user) for i in range(len(videos)) : videos[i]['tags'] = list(filter(lambda x: x < 0x80000000, videos[i]['tags'])) if human_readable_tag : videos[i]['tags_readable'] = db.translate_tag_ids_to_user_language(videos[i]['tags'], user_language)[0] videos = _filterPlaceholder(videos) videos = videos[: limit] return videos, *getCommonTags(user_language, videos)
def listVideo(offset, limit, user, order = 'latest', user_language = 'CHS', hide_placeholder = True, additional_constraint = '', human_readable_tag = False): if order not in ['latest', 'oldest', 'video_latest', 'video_oldest', 'last_modified'] : raise UserError('INCORRECT_ORDER') default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')] query_obj_extra, _ = db.compile_query(additional_constraint, 'tag') query_obj = {} empty_query = True if user and 'settings' in user : if user['settings']['blacklist'] == 'default' : empty_query = False query_obj = {'$and': [{'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]} else : if user['settings']['blacklist'] or query_obj_extra : empty_query = False query_obj = {'$and': [{'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]} else : empty_query = False query_obj = {'$and': [{'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]} if empty_query : query_obj = {} exStats1 = None exStats2 = None result = db.retrive_items(query_obj) exStats1 = result.explain() if order == 'latest': result = result.sort([("meta.created_at", -1)]) elif order == 'oldest': result = result.sort([("meta.created_at", 1)]) elif order == 'video_latest': result = result.sort([("item.upload_time", -1)]) elif order == 'video_oldest': result = result.sort([("item.upload_time", 1)]) elif order == 'last_modified': result = result.sort([("meta.modified_at", -1)]) videos = result.skip(offset).limit(limit) exStats2 = videos.explain() video_count = videos.count() videos = [i for i in videos] videos = filterVideoList(videos, user) for i in range(len(videos)) : videos[i]['tags'] = list(filter(lambda x: x < 0x80000000, videos[i]['tags'])) if human_readable_tag : videos[i]['tags_readable'] = db.translate_tag_ids_to_user_language(videos[i]['tags'], user_language)[0] if hide_placeholder : videos = _filterPlaceholder(videos) tags, pops, pop_tagid_map = getPopularTags(user_language) return videos, video_count, tags, pops, query_obj, pop_tagid_map, exStats1, exStats2
def listVideoQuery(user, query_str, page_idx, page_size, order='latest', user_language='CHS', hide_placeholder=True): log( obj={ 'q': query_str, 'page': page_idx, 'page_size': page_size, 'order': order, 'lang': user_language }) if order not in ['latest', 'oldest', 'video_latest', 'video_oldest']: raise UserError('INCORRECT_ORDER') query_obj, tags = db.compile_query(query_str) log(obj={'query': dumps(query_obj)}) default_blacklist_tagids = [ int(i) for i in Config.DEFAULT_BLACKLIST.split(',') ] if user and 'settings' in user: if user['settings']['blacklist'] == 'default': query_obj = { '$and': [query_obj, { 'tags': { '$nin': default_blacklist_tagids } }] } else: query_obj = { '$and': [query_obj, { 'tags': { '$nin': user['settings']['blacklist'] } }] } elif user is None: query_obj = { '$and': [query_obj, { 'tags': { '$nin': default_blacklist_tagids } }] } updateTagSearch(tags) try: result = db.retrive_items(query_obj) if order == 'latest': result = result.sort([("meta.created_at", -1)]) if order == 'oldest': result = result.sort([("meta.created_at", 1)]) if order == 'video_latest': result = result.sort([("item.upload_time", -1)]) if order == 'video_oldest': result = result.sort([("item.upload_time", 1)]) ret = result.skip(page_idx * page_size).limit(page_size) count = ret.count() videos = [item for item in ret] videos = filterVideoList(videos, user) if hide_placeholder: videos = _filterPlaceholder(videos) except pymongo.errors.OperationFailure as ex: if '$not' in str(ex): raise UserError('FAILED_NOT_OP') else: log(level='ERR', obj={'ex': str(ex)}) raise UserError('FAILED_UNKNOWN') return videos, getCommonTags(user_language, videos), count
def listPlaylists(user, offset, limit, query_str='', order='latest', qtype='tag', additional_constraint=''): sort_obj = {"meta.created_at": 1} if order == 'latest': sort_obj = {"meta.created_at": -1} if order == 'last_modified': sort_obj = {"meta.modified_at": -1} if order == 'oldest': sort_obj = {"meta.created_at": 1} if order == 'views': sort_obj = {"item.views": 1} if isObjectAgnosticOperationPermitted('viewPrivatePlaylist', user): auth_obj = {} else: auth_obj = { '$or': [{ 'meta.created_by': user['_id'] if user else '' }, { 'item.private': False, 'item.videos': { '$gt': 1 } }] } query_obj, _ = tagdb.compile_query(query_str, qtype) query_obj_extra, _ = tagdb.compile_query(additional_constraint, 'tag') query = {'$and': [query_obj, query_obj_extra, auth_obj]} ans_obj = playlist_db.aggregate([{ "$match": query }, { "$lookup": { "from": "users", "localField": "meta.created_by", "foreignField": "_id", "as": "user_detail" } }, { "$unwind": { "path": "$user_detail" } }, { "$project": { "user_detail.crypto": 0, "user_detail.access_control": 0, "user_detail.meta": 0, "user_detail.profile.email": 0, "user_detail.settings": 0 } }, { '$facet': { 'result': [{ '$sort': sort_obj }, { '$skip': offset }, { '$limit': limit }], 'count': [{ '$count': 'count' }] } }]) ans_obj = [i for i in ans_obj][0] if ans_obj['result']: result = [i for i in ans_obj['result']] for i in range(len(result)): result[i]['tags'] = list( filter(lambda x: x < 0x80000000, result[i]['tags'])) return result, ans_obj['count'][0]['count'] else: return [], 0