Esempio n. 1
0
def listSubscriptedItems(user,
                         page_idx,
                         page_size,
                         user_language,
                         hide_placeholder=True,
                         order='latest_video'):
    subs = list(db.subs.find({'meta.created_by': makeUserMeta(user)}))
    q = [tagdb.compile_query(q['qs'], q['qt']) for q in subs]
    query_obj = {'$or': []}
    for qi, _ in q:
        query_obj['$or'].append(qi)
    for i in range(len(q)):
        subs[i]['obj'] = q[i][0]
        subs[i]['obj_tags'] = q[i][1]
    default_blacklist_tagids = [
        int(i) for i in Config.DEFAULT_BLACKLIST.split(',')
    ]
    if user and 'settings' in user:
        if user['settings']['blacklist'] == 'default':
            query_obj = {
                '$and':
                [query_obj, {
                    'tags': {
                        '$nin': default_blacklist_tagids
                    }
                }]
            }
        else:
            query_obj = {
                '$and':
                [query_obj, {
                    'tags': {
                        '$nin': user['settings']['blacklist']
                    }
                }]
            }
    elif user is None:
        query_obj = {
            '$and': [query_obj, {
                'tags': {
                    '$nin': default_blacklist_tagids
                }
            }]
        }
    result = tagdb.retrive_items(query_obj)
    if order == 'latest':
        result = result.sort([("meta.created_at", -1)])
    if order == 'oldest':
        result = result.sort([("meta.created_at", 1)])
    if order == 'video_latest':
        result = result.sort([("item.upload_time", -1)])
    if order == 'video_oldest':
        result = result.sort([("item.upload_time", 1)])
    ret = result.skip(page_idx * page_size).limit(page_size)
    count = ret.count()
    videos = [item for item in ret]
    videos = filterVideoList(videos, user)
    if hide_placeholder:
        videos = _filterPlaceholder(videos)
    return videos, subs, getCommonTags(user_language, videos), count
Esempio n. 2
0
def queryAndProcessQueuingRequests(user, max_videos: int, worker_id: str):
    filterOperation('subtitleocr_queryAndProcessQueuingRequests', user)
    # step 1: max_videos > 0 and max_videos <= 100
    if max_videos <= 0 or max_videos > Subtitles.MAX_WORKER_JOBS:
        raise UserError('TOO_MANY_JOBS')
    with redis_lock.Lock(rdb,
                         "mmdocr_global_lock"), MongoTransaction(client) as s:
        # step 2: get top k oldest requests
        ret = list(
            db.subtitle_ocr.find({
                "status": "Queuing"
            }, session=s()).sort([("meta.modified_at", 1)
                                  ]).limit(max_videos))  # FIFO
        ret_vids = [i['vid'] for i in ret]
        ret_ids = [i['_id'] for i in ret]
        # step 3: retrive video URLs
        video_items = tagdb.retrive_items({"_id": {
            "$in": ret_vids
        }},
                                          session=s())
        video_urls = [{
            "url": i["item"]["url"],
            "unique_id": i["item"]["unique_id"]
        } for i in video_items]
        # step 4: mark reserved
        db.subtitle_ocr.update_many({"_id": {
            "$in": ret_ids
        }}, {"$set": {
            "status": "Reserved",
            "worker_id": worker_id
        }},
                                    session=s())
        s.mark_succeed()
        # step 5: return
        return video_urls
Esempio n. 3
0
def editVideoTagsQuery(query, query_type, tags_to_add, tags_to_remove, user):
    if query_type not in ['tag', 'text']:
        raise UserError('INCORRECT_QUERY_TYPE')
    filterOperation('batchVideoTagEdit', user)
    query_obj, _ = tagdb.compile_query(query)
    log(obj={'query': dumps(query_obj)})
    tagids_to_add = tagdb.filter_and_translate_tags(tags_to_add)
    tagids_to_remove = tagdb.filter_and_translate_tags(tags_to_remove)
    try:
        count = 0
        with MongoTransaction(client) as s_read, MongoTransaction(
                client) as s_write:
            result_cursor = tagdb.retrive_items(query_obj, session=s_read())
            batch = _batchedRead(result_cursor)
            while batch:
                item_ids = [item['_id'] for item in batch]
                tagdb.update_many_items_tags_pull(item_ids,
                                                  tagids_to_remove,
                                                  makeUserMeta(user),
                                                  session=s_write())
                tagdb.update_many_items_tags_merge(item_ids,
                                                   tagids_to_add,
                                                   makeUserMeta(user),
                                                   session=s_write())
                count += len(batch)
                batch = _batchedRead(result_cursor)
            s_write.mark_succeed()
        return count
    except pymongo.errors.OperationFailure as ex:
        if '$not' in str(ex):
            raise UserError('FAILED_NOT_OP')
        else:
            log(level='ERR', obj={'ex': str(ex)})
            raise UserError('FAILED_UNKNOWN')
Esempio n. 4
0
def listVideo(page_idx,
              page_size,
              user,
              order='latest',
              user_language='CHS',
              hide_placeholder=True):
    if order not in ['latest', 'oldest', 'video_latest', 'video_oldest']:
        raise UserError('INCORRECT_ORDER')
    default_blacklist_tagids = [
        int(i) for i in Config.DEFAULT_BLACKLIST.split(',')
    ]
    query_obj = {}
    if user and 'settings' in user:
        if user['settings']['blacklist'] == 'default':
            query_obj = {'tags': {'$nin': default_blacklist_tagids}}
        else:
            query_obj = {'tags': {'$nin': user['settings']['blacklist']}}
    elif user is None:
        query_obj = {}
    result = db.retrive_items(query_obj)
    if order == 'latest':
        result = result.sort([("meta.created_at", -1)])
    if order == 'oldest':
        result = result.sort([("meta.created_at", 1)])
    if order == 'video_latest':
        result = result.sort([("item.upload_time", -1)])
    if order == 'video_oldest':
        result = result.sort([("item.upload_time", 1)])
    videos = result.skip(page_idx * page_size).limit(page_size)
    video_count = videos.count()
    videos = [i for i in videos]
    videos = filterVideoList(videos, user)
    if hide_placeholder:
        videos = _filterPlaceholder(videos)
    return videos, video_count, getPopularTags(user_language)
Esempio n. 5
0
def listVideoQuery(user, query_str, offset, limit, order = 'latest', user_language = 'CHS', hide_placeholder = True, qtype = 'tag', additional_constraint = '', human_readable_tag = False):
	log(obj = {'q': query_str, 'offset': offset, 'limit': limit, 'order': order, 'lang': user_language})
	if order not in ['latest', 'oldest', 'video_latest', 'video_oldest', 'last_modified'] :
		raise UserError('INCORRECT_ORDER')
	query_obj, tags = db.compile_query(query_str, qtype)
	query_obj_extra, _ = db.compile_query(additional_constraint, 'tag')
	log(obj = {'query': dumps(query_obj)})
	default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')]
	if user and 'settings' in user :
		if user['settings']['blacklist'] == 'default' :
			query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
		else :
			query_obj = {'$and': [query_obj, {'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]}
	else :
		query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
	updateTagSearch(tags)
	exStats1 = None
	exStats2 = None
	try :
		result = db.retrive_items(query_obj)
		exStats1 = result.explain()
		if order == 'latest':
			result = result.sort([("meta.created_at", -1)])
		elif order == 'oldest':
			result = result.sort([("meta.created_at", 1)])
		elif order == 'video_latest':
			result = result.sort([("item.upload_time", -1)])
		elif order == 'video_oldest':
			result = result.sort([("item.upload_time", 1)])
		elif order == 'last_modified':
			result = result.sort([("meta.modified_at", -1)])
		ret = result.skip(offset).limit(limit)
		exStats2 = ret.explain()
		count = ret.count()
		videos = [item for item in ret]
		videos = filterVideoList(videos, user)
		for i in range(len(videos)) :
			videos[i]['tags'] = list(filter(lambda x: x < 0x80000000, videos[i]['tags']))
			if human_readable_tag :
				videos[i]['tags_readable'] = db.translate_tag_ids_to_user_language(videos[i]['tags'], user_language)[0]
		if hide_placeholder :
			videos = _filterPlaceholder(videos)
	except pymongo.errors.OperationFailure as ex:
		if '$not' in str(ex) :
			raise UserError('FAILED_NOT_OP')
		else :
			log(level = 'ERR', obj = {'ex': str(ex)})
			raise UserError('FAILED_UNKNOWN')
	return videos, *getCommonTags(user_language, videos), count, query_obj, exStats1, exStats2
Esempio n. 6
0
def listVideo(offset, limit, user, order = 'latest', user_language = 'CHS', hide_placeholder = True, additional_constraint = '', human_readable_tag = False):
	if order not in ['latest', 'oldest', 'video_latest', 'video_oldest', 'last_modified'] :
		raise UserError('INCORRECT_ORDER')
	default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')]
	query_obj_extra, _ = db.compile_query(additional_constraint, 'tag')
	query_obj = {}
	empty_query = True
	if user and 'settings' in user :
		if user['settings']['blacklist'] == 'default' :
			empty_query = False
			query_obj = {'$and': [{'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
		else :
			if user['settings']['blacklist'] or query_obj_extra :
				empty_query = False
			query_obj = {'$and': [{'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]}
	else :
		empty_query = False
		query_obj = {'$and': [{'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
	if empty_query :
		query_obj = {}
	exStats1 = None
	exStats2 = None
	result = db.retrive_items(query_obj)
	exStats1 = result.explain()
	if order == 'latest':
		result = result.sort([("meta.created_at", -1)])
	elif order == 'oldest':
		result = result.sort([("meta.created_at", 1)])
	elif order == 'video_latest':
		result = result.sort([("item.upload_time", -1)])
	elif order == 'video_oldest':
		result = result.sort([("item.upload_time", 1)])
	elif order == 'last_modified':
		result = result.sort([("meta.modified_at", -1)])
	videos = result.skip(offset).limit(limit)
	exStats2 = videos.explain()
	video_count = videos.count()
	videos = [i for i in videos]
	videos = filterVideoList(videos, user)
	for i in range(len(videos)) :
		videos[i]['tags'] = list(filter(lambda x: x < 0x80000000, videos[i]['tags']))
		if human_readable_tag :
			videos[i]['tags_readable'] = db.translate_tag_ids_to_user_language(videos[i]['tags'], user_language)[0]
	if hide_placeholder :
		videos = _filterPlaceholder(videos)
	tags, pops, pop_tagid_map = getPopularTags(user_language)
	return videos, video_count, tags, pops, query_obj, pop_tagid_map, exStats1, exStats2
Esempio n. 7
0
def listYourVideo(uid, page_idx, page_size, user, order='latest'):
    if order not in ['latest', 'oldest', 'video_latest', 'video_oldest']:
        raise UserError('INCORRECT_ORDER')
    result = db.retrive_items({'meta.created_by': ObjectId(uid)})
    if order == 'latest':
        result = result.sort([("meta.created_at", -1)])
    if order == 'oldest':
        result = result.sort([("meta.created_at", 1)])
    if order == 'video_latest':
        result = result.sort([("item.upload_time", -1)])
    if order == 'video_oldest':
        result = result.sort([("item.upload_time", 1)])
    videos = result.skip(page_idx * page_size).limit(page_size)
    video_count = videos.count()
    videos = [i for i in videos]
    videos = filterVideoList(videos, user)
    return videos, video_count
Esempio n. 8
0
def listSubscriptedItems(user, offset, limit, user_language, hide_placeholder = True, order = 'video_latest', visibleSubs = [''], additional_constraint = '') :
	subs = list(db.subs.find({'meta.created_by': makeUserMeta(user)}))
	q = [(tagdb.compile_query(q['qs'], q['qt']), str(q['_id'])) for q in subs]
	query_obj = {'$or': []}
	if '' in visibleSubs :
		for (qi, _), _ in q :
			query_obj['$or'].append(qi)
	else :
		for (qi, _), qid in q :
			if qid in visibleSubs :
				query_obj['$or'].append(qi)
	for i in range(len(q)) :
		(qobj, qtags), _ = q[i]
		subs[i]['obj'] = qobj
		subs[i]['obj_tags'] = qtags
	if not query_obj['$or'] :
		return [], subs, [], 0
	default_blacklist_tagids = [int(i) for i in Config.DEFAULT_BLACKLIST.split(',')]
	query_obj_extra, _ = tagdb.compile_query(additional_constraint, 'tag')
	if user and 'settings' in user :
		if user['settings']['blacklist'] == 'default' :
			query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
		else :
			query_obj = {'$and': [query_obj, {'tags': {'$nin': user['settings']['blacklist']}}, query_obj_extra]}
	elif user is None :
		query_obj = {'$and': [query_obj, {'tags': {'$nin': default_blacklist_tagids}}, query_obj_extra]}
	result = tagdb.retrive_items(query_obj)
	if order == 'latest':
		result = result.sort([("meta.created_at", -1)])
	elif order == 'oldest':
		result = result.sort([("meta.created_at", 1)])
	elif order == 'video_latest':
		result = result.sort([("item.upload_time", -1)])
	elif order == 'video_oldest':
		result = result.sort([("item.upload_time", 1)])
	elif order == 'last_modified':
		result = result.sort([("meta.modified_at", -1)])
	ret = result.skip(offset).limit(limit)
	count = ret.count()
	videos = [item for item in ret]
	videos = filterVideoList(videos, user)
	if hide_placeholder :
		videos = _filterPlaceholder(videos)
	return videos, subs, *getCommonTags(user_language, videos), count
Esempio n. 9
0
def listYourVideo(uid, offset, limit, user, order = 'latest', human_readable_tag = False, user_language = 'CHS'):
	if order not in ['latest', 'oldest', 'video_latest', 'video_oldest', 'last_modified'] :
		raise UserError('INCORRECT_ORDER')
	result = db.retrive_items({'meta.created_by': ObjectId(uid)})
	if order == 'latest':
		result = result.sort([("meta.created_at", -1)])
	elif order == 'oldest':
		result = result.sort([("meta.created_at", 1)])
	elif order == 'video_latest':
		result = result.sort([("item.upload_time", -1)])
	elif order == 'video_oldest':
		result = result.sort([("item.upload_time", 1)])
	elif order == 'last_modified':
		result = result.sort([("meta.modified_at", -1)])
	videos = result.skip(offset).limit(limit)
	video_count = videos.count()
	videos = [i for i in videos]
	videos = filterVideoList(videos, user)
	for i in range(len(videos)) :
		videos[i]['tags'] = list(filter(lambda x: x < 0x80000000, videos[i]['tags']))
		if human_readable_tag :
			videos[i]['tags_readable'] = db.translate_tag_ids_to_user_language(videos[i]['tags'], user_language)[0]
	return videos, video_count
Esempio n. 10
0
def listVideoQuery(user,
                   query_str,
                   page_idx,
                   page_size,
                   order='latest',
                   user_language='CHS',
                   hide_placeholder=True):
    log(
        obj={
            'q': query_str,
            'page': page_idx,
            'page_size': page_size,
            'order': order,
            'lang': user_language
        })
    if order not in ['latest', 'oldest', 'video_latest', 'video_oldest']:
        raise UserError('INCORRECT_ORDER')
    query_obj, tags = db.compile_query(query_str)
    log(obj={'query': dumps(query_obj)})
    default_blacklist_tagids = [
        int(i) for i in Config.DEFAULT_BLACKLIST.split(',')
    ]
    if user and 'settings' in user:
        if user['settings']['blacklist'] == 'default':
            query_obj = {
                '$and':
                [query_obj, {
                    'tags': {
                        '$nin': default_blacklist_tagids
                    }
                }]
            }
        else:
            query_obj = {
                '$and':
                [query_obj, {
                    'tags': {
                        '$nin': user['settings']['blacklist']
                    }
                }]
            }
    elif user is None:
        query_obj = {
            '$and': [query_obj, {
                'tags': {
                    '$nin': default_blacklist_tagids
                }
            }]
        }
    updateTagSearch(tags)
    try:
        result = db.retrive_items(query_obj)
        if order == 'latest':
            result = result.sort([("meta.created_at", -1)])
        if order == 'oldest':
            result = result.sort([("meta.created_at", 1)])
        if order == 'video_latest':
            result = result.sort([("item.upload_time", -1)])
        if order == 'video_oldest':
            result = result.sort([("item.upload_time", 1)])
        ret = result.skip(page_idx * page_size).limit(page_size)
        count = ret.count()
        videos = [item for item in ret]
        videos = filterVideoList(videos, user)
        if hide_placeholder:
            videos = _filterPlaceholder(videos)
    except pymongo.errors.OperationFailure as ex:
        if '$not' in str(ex):
            raise UserError('FAILED_NOT_OP')
        else:
            log(level='ERR', obj={'ex': str(ex)})
            raise UserError('FAILED_UNKNOWN')
    return videos, getCommonTags(user_language, videos), count
Esempio n. 11
0
from db import tagdb
from services.playlist import listPlaylistsForVideoNoAuth

for item in tagdb.retrive_items({}).batch_size(100):
    try:
        listPlaylistsForVideoNoAuth(item['_id'])
    except:
        print(item['_id'], item['item']['url'])