def listSubscriptedItems(user, page_idx, page_size, user_language, hide_placeholder=True, order='latest_video'): subs = list(db.subs.find({'meta.created_by': makeUserMeta(user)})) q = [tagdb.compile_query(q['qs'], q['qt']) for q in subs] query_obj = {'$or': []} for qi, _ in q: query_obj['$or'].append(qi) for i in range(len(q)): subs[i]['obj'] = q[i][0] subs[i]['obj_tags'] = q[i][1] default_blacklist_tagids = [ int(i) for i in Config.DEFAULT_BLACKLIST.split(',') ] if user and 'settings' in user: if user['settings']['blacklist'] == 'default': query_obj = { '$and': [query_obj, { 'tags': { '$nin': default_blacklist_tagids } }] } else: query_obj = { '$and': [query_obj, { 'tags': { '$nin': user['settings']['blacklist'] } }] } elif user is None: query_obj = { '$and': [query_obj, { 'tags': { '$nin': default_blacklist_tagids } }] } result = tagdb.retrive_items(query_obj) if order == 'latest': result = result.sort([("meta.created_at", -1)]) if order == 'oldest': result = result.sort([("meta.created_at", 1)]) if order == 'video_latest': result = result.sort([("item.upload_time", -1)]) if order == 'video_oldest': result = result.sort([("item.upload_time", 1)]) ret = result.skip(page_idx * page_size).limit(page_size) count = ret.count() videos = [item for item in ret] videos = filterVideoList(videos, user) if hide_placeholder: videos = _filterPlaceholder(videos) return videos, subs, getCommonTags(user_language, videos), count
def queryAndProcessQueuingRequests(user, max_videos: int, worker_id: str): filterOperation('subtitleocr_queryAndProcessQueuingRequests', user) # step 1: max_videos > 0 and max_videos <= 100 if max_videos <= 0 or max_videos > Subtitles.MAX_WORKER_JOBS: raise UserError('TOO_MANY_JOBS') with redis_lock.Lock(rdb, "mmdocr_global_lock"), MongoTransaction(client) as s: # step 2: get top k oldest requests ret = list( db.subtitle_ocr.find({ "status": "Queuing" }, session=s()).sort([("meta.modified_at", 1) ]).limit(max_videos)) # FIFO ret_vids = [i['vid'] for i in ret] ret_ids = [i['_id'] for i in ret] # step 3: retrive video URLs video_items = tagdb.retrive_items({"_id": { "$in": ret_vids }}, session=s()) video_urls = [{ "url": i["item"]["url"], "unique_id": i["item"]["unique_id"] } for i in video_items] # step 4: mark reserved db.subtitle_ocr.update_many({"_id": { "$in": ret_ids }}, {"$set": { "status": "Reserved", "worker_id": worker_id }}, session=s()) s.mark_succeed() # step 5: return return video_urls
def editVideoTagsQuery(query, query_type, tags_to_add, tags_to_remove, user): if query_type not in ['tag', 'text']: raise UserError('INCORRECT_QUERY_TYPE') filterOperation('batchVideoTagEdit', user) query_obj, _ = tagdb.compile_query(query) log(obj={'query': dumps(query_obj)}) tagids_to_add = tagdb.filter_and_translate_tags(tags_to_add) tagids_to_remove = tagdb.filter_and_translate_tags(tags_to_remove) try: count = 0 with MongoTransaction(client) as s_read, MongoTransaction( client) as s_write: result_cursor = tagdb.retrive_items(query_obj, session=s_read()) batch = _batchedRead(result_cursor) while batch: item_ids = [item['_id'] for item in batch] tagdb.update_many_items_tags_pull(item_ids, tagids_to_remove, makeUserMeta(user), session=s_write()) tagdb.update_many_items_tags_merge(item_ids, tagids_to_add, makeUserMeta(user), session=s_write()) count += len(batch) batch = _batchedRead(result_cursor) s_write.mark_succeed() return count except pymongo.errors.OperationFailure as ex: if '$not' in str(ex): raise UserError('FAILED_NOT_OP') else: log(level='ERR', obj={'ex': str(ex)}) raise UserError('FAILED_UNKNOWN')
def listVideo(page_idx, page_size, user, order='latest', user_language='CHS', hide_placeholder=True): if order not in ['latest', 'oldest', 'video_latest', 'video_oldest']: raise UserError('INCORRECT_ORDER') default_blacklist_tagids = [ int(i) for i in Config.DEFAULT_BLACKLIST.split(',') ] query_obj = {} if user and 'settings' in user: if user['settings']['blacklist'] == 'default': query_obj = {'tags': {'$nin': default_blacklist_tagids}} else: query_obj = {'tags': {'$nin': user['settings']['blacklist']}} elif user is None: query_obj = {} result = db.retrive_items(query_obj) if order == 'latest': result = result.sort([("meta.created_at", -1)]) if order == 'oldest': result = result.sort([("meta.created_at", 1)]) if order == 'video_latest': result = result.sort([("item.upload_time", -1)]) if order == 'video_oldest': result = result.sort([("item.upload_time", 1)]) videos = result.skip(page_idx * page_size).limit(page_size) video_count = videos.count() videos = [i for i in videos] videos = filterVideoList(videos, user) if hide_placeholder: videos = _filterPlaceholder(videos) return videos, video_count, getPopularTags(user_language)
def listVideoQuery(user, query_str, offset, limit, order = 'latest', user_language = 'CHS', hide_placeholder = True, qtype = 'tag', additional_constraint = '', human_readable_tag = False): log(obj = {'q': query_str, 'offset': offset, 'limit': limit, 'order': order, 'lang': user_language}) if order not in ['latest', 'oldest', 'video_latest', 'video_oldest', 'last_modified'] : raise UserError('INCORRECT_ORDER') query_obj, tags = db.compile_query(query_str, qtype) query_obj_extra, _ = db.compile_query(additional_constraint, 'tag') log(obj = {'query': dumps(query_obj)}) default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')] if user and 'settings' in user : if user['settings']['blacklist'] == 'default' : query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]} else : query_obj = {'$and': [query_obj, {'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]} else : query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]} updateTagSearch(tags) exStats1 = None exStats2 = None try : result = db.retrive_items(query_obj) exStats1 = result.explain() if order == 'latest': result = result.sort([("meta.created_at", -1)]) elif order == 'oldest': result = result.sort([("meta.created_at", 1)]) elif order == 'video_latest': result = result.sort([("item.upload_time", -1)]) elif order == 'video_oldest': result = result.sort([("item.upload_time", 1)]) elif order == 'last_modified': result = result.sort([("meta.modified_at", -1)]) ret = result.skip(offset).limit(limit) exStats2 = ret.explain() count = ret.count() videos = [item for item in ret] videos = filterVideoList(videos, user) for i in range(len(videos)) : videos[i]['tags'] = list(filter(lambda x: x < 0x80000000, videos[i]['tags'])) if human_readable_tag : videos[i]['tags_readable'] = db.translate_tag_ids_to_user_language(videos[i]['tags'], user_language)[0] if hide_placeholder : videos = _filterPlaceholder(videos) except pymongo.errors.OperationFailure as ex: if '$not' in str(ex) : raise UserError('FAILED_NOT_OP') else : log(level = 'ERR', obj = {'ex': str(ex)}) raise UserError('FAILED_UNKNOWN') return videos, *getCommonTags(user_language, videos), count, query_obj, exStats1, exStats2
def listVideo(offset, limit, user, order = 'latest', user_language = 'CHS', hide_placeholder = True, additional_constraint = '', human_readable_tag = False): if order not in ['latest', 'oldest', 'video_latest', 'video_oldest', 'last_modified'] : raise UserError('INCORRECT_ORDER') default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')] query_obj_extra, _ = db.compile_query(additional_constraint, 'tag') query_obj = {} empty_query = True if user and 'settings' in user : if user['settings']['blacklist'] == 'default' : empty_query = False query_obj = {'$and': [{'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]} else : if user['settings']['blacklist'] or query_obj_extra : empty_query = False query_obj = {'$and': [{'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]} else : empty_query = False query_obj = {'$and': [{'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]} if empty_query : query_obj = {} exStats1 = None exStats2 = None result = db.retrive_items(query_obj) exStats1 = result.explain() if order == 'latest': result = result.sort([("meta.created_at", -1)]) elif order == 'oldest': result = result.sort([("meta.created_at", 1)]) elif order == 'video_latest': result = result.sort([("item.upload_time", -1)]) elif order == 'video_oldest': result = result.sort([("item.upload_time", 1)]) elif order == 'last_modified': result = result.sort([("meta.modified_at", -1)]) videos = result.skip(offset).limit(limit) exStats2 = videos.explain() video_count = videos.count() videos = [i for i in videos] videos = filterVideoList(videos, user) for i in range(len(videos)) : videos[i]['tags'] = list(filter(lambda x: x < 0x80000000, videos[i]['tags'])) if human_readable_tag : videos[i]['tags_readable'] = db.translate_tag_ids_to_user_language(videos[i]['tags'], user_language)[0] if hide_placeholder : videos = _filterPlaceholder(videos) tags, pops, pop_tagid_map = getPopularTags(user_language) return videos, video_count, tags, pops, query_obj, pop_tagid_map, exStats1, exStats2
def listYourVideo(uid, page_idx, page_size, user, order='latest'): if order not in ['latest', 'oldest', 'video_latest', 'video_oldest']: raise UserError('INCORRECT_ORDER') result = db.retrive_items({'meta.created_by': ObjectId(uid)}) if order == 'latest': result = result.sort([("meta.created_at", -1)]) if order == 'oldest': result = result.sort([("meta.created_at", 1)]) if order == 'video_latest': result = result.sort([("item.upload_time", -1)]) if order == 'video_oldest': result = result.sort([("item.upload_time", 1)]) videos = result.skip(page_idx * page_size).limit(page_size) video_count = videos.count() videos = [i for i in videos] videos = filterVideoList(videos, user) return videos, video_count
def listSubscriptedItems(user, offset, limit, user_language, hide_placeholder = True, order = 'video_latest', visibleSubs = [''], additional_constraint = '') : subs = list(db.subs.find({'meta.created_by': makeUserMeta(user)})) q = [(tagdb.compile_query(q['qs'], q['qt']), str(q['_id'])) for q in subs] query_obj = {'$or': []} if '' in visibleSubs : for (qi, _), _ in q : query_obj['$or'].append(qi) else : for (qi, _), qid in q : if qid in visibleSubs : query_obj['$or'].append(qi) for i in range(len(q)) : (qobj, qtags), _ = q[i] subs[i]['obj'] = qobj subs[i]['obj_tags'] = qtags if not query_obj['$or'] : return [], subs, [], 0 default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')] query_obj_extra, _ = tagdb.compile_query(additional_constraint, 'tag') if user and 'settings' in user : if user['settings']['blacklist'] == 'default' : query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]} else : query_obj = {'$and': [query_obj, {'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]} elif user is None : query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]} result = tagdb.retrive_items(query_obj) if order == 'latest': result = result.sort([("meta.created_at", -1)]) elif order == 'oldest': result = result.sort([("meta.created_at", 1)]) elif order == 'video_latest': result = result.sort([("item.upload_time", -1)]) elif order == 'video_oldest': result = result.sort([("item.upload_time", 1)]) elif order == 'last_modified': result = result.sort([("meta.modified_at", -1)]) ret = result.skip(offset).limit(limit) count = ret.count() videos = [item for item in ret] videos = filterVideoList(videos, user) if hide_placeholder : videos = _filterPlaceholder(videos) return videos, subs, *getCommonTags(user_language, videos), count
def listYourVideo(uid, offset, limit, user, order = 'latest', human_readable_tag = False, user_language = 'CHS'): if order not in ['latest', 'oldest', 'video_latest', 'video_oldest', 'last_modified'] : raise UserError('INCORRECT_ORDER') result = db.retrive_items({'meta.created_by': ObjectId(uid)}) if order == 'latest': result = result.sort([("meta.created_at", -1)]) elif order == 'oldest': result = result.sort([("meta.created_at", 1)]) elif order == 'video_latest': result = result.sort([("item.upload_time", -1)]) elif order == 'video_oldest': result = result.sort([("item.upload_time", 1)]) elif order == 'last_modified': result = result.sort([("meta.modified_at", -1)]) videos = result.skip(offset).limit(limit) video_count = videos.count() videos = [i for i in videos] videos = filterVideoList(videos, user) for i in range(len(videos)) : videos[i]['tags'] = list(filter(lambda x: x < 0x80000000, videos[i]['tags'])) if human_readable_tag : videos[i]['tags_readable'] = db.translate_tag_ids_to_user_language(videos[i]['tags'], user_language)[0] return videos, video_count
def listVideoQuery(user, query_str, page_idx, page_size, order='latest', user_language='CHS', hide_placeholder=True): log( obj={ 'q': query_str, 'page': page_idx, 'page_size': page_size, 'order': order, 'lang': user_language }) if order not in ['latest', 'oldest', 'video_latest', 'video_oldest']: raise UserError('INCORRECT_ORDER') query_obj, tags = db.compile_query(query_str) log(obj={'query': dumps(query_obj)}) default_blacklist_tagids = [ int(i) for i in Config.DEFAULT_BLACKLIST.split(',') ] if user and 'settings' in user: if user['settings']['blacklist'] == 'default': query_obj = { '$and': [query_obj, { 'tags': { '$nin': default_blacklist_tagids } }] } else: query_obj = { '$and': [query_obj, { 'tags': { '$nin': user['settings']['blacklist'] } }] } elif user is None: query_obj = { '$and': [query_obj, { 'tags': { '$nin': default_blacklist_tagids } }] } updateTagSearch(tags) try: result = db.retrive_items(query_obj) if order == 'latest': result = result.sort([("meta.created_at", -1)]) if order == 'oldest': result = result.sort([("meta.created_at", 1)]) if order == 'video_latest': result = result.sort([("item.upload_time", -1)]) if order == 'video_oldest': result = result.sort([("item.upload_time", 1)]) ret = result.skip(page_idx * page_size).limit(page_size) count = ret.count() videos = [item for item in ret] videos = filterVideoList(videos, user) if hide_placeholder: videos = _filterPlaceholder(videos) except pymongo.errors.OperationFailure as ex: if '$not' in str(ex): raise UserError('FAILED_NOT_OP') else: log(level='ERR', obj={'ex': str(ex)}) raise UserError('FAILED_UNKNOWN') return videos, getCommonTags(user_language, videos), count
from db import tagdb from services.playlist import listPlaylistsForVideoNoAuth for item in tagdb.retrive_items({}).batch_size(100): try: listPlaylistsForVideoNoAuth(item['_id']) except: print(item['_id'], item['item']['url'])