def insertIntoPlaylist(pid, vid, rank, user) : log(obj = {'pid': pid, 'vid': vid, 'rank': rank}) with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s : playlist = db.playlists.find_one({'_id': ObjectId(pid)}, session = s()) if playlist is None : raise UserError('PLAYLIST_NOT_EXIST') filterOperation('editPlaylist', user, playlist) if tagdb.retrive_item({'_id': ObjectId(vid)}, session = s()) is None : raise UserError('VIDEO_NOT_EXIST') if playlist["videos"] > PlaylistConfig.MAX_VIDEO_PER_PLAYLIST : raise UserError('VIDEO_LIMIT_EXCEEDED') conflicting_item = db.playlist_items.find_one({'pid': ObjectId(pid), 'vid': ObjectId(vid)}, session = s()) if conflicting_item is not None : editPlaylist_MoveLockFree(pid, conflicting_item, rank, session = s()) db.playlists.update_one({'_id': ObjectId(pid)}, {'$set': { 'meta.modified_by': makeUserMeta(user), 'meta.modified_at': datetime.now()}}, session = s()) s.mark_succeed() return if rank < 0 : raise UserError('OUT_OF_RANGE') if rank > playlist['videos'] : rank = int(playlist['videos']) playlists = tagdb.retrive_item({'_id': ObjectId(vid)}, session = s())['item']['series'] playlists.append(ObjectId(pid)) playlists = list(set(playlists)) tagdb.update_item_query(ObjectId(vid), {'$set': {'item.series': playlists}}, makeUserMeta(user), session = s()) db.playlists.update_one({"_id": ObjectId(pid)}, {"$inc": {"videos": int(1)}}, session = s()) db.playlist_items.update_many({'pid': ObjectId(pid), 'rank': {'$gte': rank}}, {'$inc': {'rank': int(1)}}, session = s()) db.playlist_items.insert_one({"pid": ObjectId(pid), "vid": ObjectId(vid), "rank": int(rank), "meta": makeUserMeta(user)}, session = s()) db.playlists.update_one({'_id': ObjectId(pid)}, {'$set': { 'meta.modified_by': makeUserMeta(user), 'meta.modified_at': datetime.now()}}, session = s()) s.mark_succeed()
def insertIntoPlaylistLockFree(pid, vid, rank, user, session) : log(obj = {'pid': pid, 'vid': vid, 'rank': rank}) if tagdb.retrive_item({'_id': ObjectId(vid)}, session = session) is None : #raise UserError('VIDEO_NOT_EXIST') return False conflicting_item = db.playlist_items.find_one({'pid': ObjectId(pid), 'vid': ObjectId(vid)}, session = session) if conflicting_item is not None : editPlaylist_MoveLockFree(pid, conflicting_item, rank, session = session) db.playlists.update_one({'_id': ObjectId(pid)}, {'$set': { 'meta.modified_by': makeUserMeta(user), 'meta.modified_at': datetime.now()}}, session = session) if conflicting_item['rank'] >= rank : return True else : return False playlists = tagdb.retrive_item({'_id': ObjectId(vid)}, session = session)['item']['series'] playlists.append(ObjectId(pid)) playlists = list(set(playlists)) tagdb.update_item_query(ObjectId(vid), {'$set': {'item.series': playlists}}, makeUserMeta(user), session = session) db.playlists.update_one({"_id": ObjectId(pid)}, {"$inc": {"videos": int(1)}}, session = session) db.playlist_items.update_many({'pid': ObjectId(pid), 'rank': {'$gte': rank}}, {'$inc': {'rank': int(1)}}, session = session) db.playlist_items.insert_one({"pid": ObjectId(pid), "vid": ObjectId(vid), "rank": int(rank), "meta": makeUserMeta(user)}, session = session) db.playlists.update_one({'_id': ObjectId(pid)}, {'$set': { 'meta.modified_by': makeUserMeta(user), 'meta.modified_at': datetime.now()}}, session = session) return True
def addVideoToPlaylist(pid, vid, user): log(obj={'pid': pid, 'vid': vid}) with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s: playlist = playlist_db.retrive_item(pid) if playlist is None: raise UserError('PLAYLIST_NOT_EXIST') filterOperation('editPlaylist', user, playlist) if tagdb.retrive_item({'_id': ObjectId(vid)}, session=s()) is None: raise UserError('VIDEO_NOT_EXIST') if playlist['item']["videos"] > PlaylistConfig.MAX_VIDEO_PER_PLAYLIST: raise UserError('VIDEO_LIMIT_EXCEEDED') conflicting_item = db.playlist_items.find_one( { 'pid': ObjectId(pid), 'vid': ObjectId(vid) }, session=s()) if conflicting_item is not None: editPlaylist_MoveLockFree(pid, conflicting_item, int(playlist['item']["videos"]), session=s()) playlist_db.update_item_query(playlist, {}, user=makeUserMeta(user), session=s()) s.mark_succeed() return playlists = tagdb.retrive_item({'_id': ObjectId(vid)}, session=s())['item']['series'] playlists.append(ObjectId(pid)) playlists = list(set(playlists)) tagdb.update_item_query(ObjectId(vid), {'$set': { 'item.series': playlists }}, user=makeUserMeta(user), session=s()) db.playlist_items.insert_one( { "pid": ObjectId(pid), "vid": ObjectId(vid), "rank": int(playlist['item']["videos"]), "meta": makeUserMeta(user) }, session=s()) playlist_db.update_item_query(playlist, {"$inc": { "item.videos": int(1) }}, user=makeUserMeta(user), session=s()) s.mark_succeed()
def _getAllCopies(vid, session, use_unique_id=False): if not vid: return [] if use_unique_id: this_video = tagdb.retrive_item({"item.unique_id": vid}, session=session) else: this_video = tagdb.retrive_item({"_id": ObjectId(vid)}, session=session) if this_video is None: return [] copies = this_video['item']['copies'] # add self copies.append(ObjectId(this_video['_id'])) # use set to remove duplicated items return list(set(copies))
def postVideoNoMerge(user, url, tags, copy, pid, rank, repost_type): log(obj={'url': url, 'tags': tags, 'copy': copy, 'pid': pid, 'rank': rank}) filterOperation('postVideo', user) tags = [tag.strip() for tag in tags] if not url: raise UserError('EMPTY_URL') if len(url) > VideoConfig.MAX_URL_LENGTH: raise UserError('URL_TOO_LONG') if len(tags) > VideoConfig.MAX_TAGS_PER_VIDEO: raise UserError('TAGS_LIMIT_EXCEEDED') obj, cleanURL = dispatch(url) if obj is None: log(level='WARN', obj={'url': url}) raise UserError('UNSUPPORTED_WEBSITE') if not cleanURL: raise UserError('EMPTY_URL') uid = obj.unique_id(obj, cleanURL) vid_item = tagdb.retrive_item({'item.unique_id': uid}) if vid_item is None: tags = filterTags(tags) log(obj={'url': cleanURL}) task_id = postTask( _createJsonForPosting(cleanURL, tags, copy, pid, rank, [], repost_type, user, use_autotag=True)) return task_id else: return 'no-suck-task'
def createPlaylistFromCopies(pid, site, user): if site not in ["youtube", "bilibili", "nicovideo", "twitter", "acfun"]: raise UserError("UNSUPPORTED_SITE") videos, _, playlist_obj = listAllPlaylistVideosOrdered(pid, user) new_pid = createPlaylist('english', playlist_obj['title']['english'] + ' - %s' % site, playlist_obj['desc']['english'], playlist_obj['cover'], user, playlist_obj['private']) with redis_lock.Lock(rdb, 'editLink'), redis_lock.Lock( rdb, "playlistEdit:" + str(new_pid)), MongoTransaction(client) as s: rank = 0 for video in videos: copies = video['item']['item']['copies'] for cp in copies: item = tagdb.retrive_item(cp, session=s()) if item['_id'] != video['vid'] and item['item']['site'] == site: addVideoToPlaylistLockFree(new_pid, item['_id'], user, rank, session=s()) rank += 1 break s.mark_succeed() return new_pid
def getVideosByURLs(urls): ret = [] for url in urls: obj, cleanURL = dispatch(url) if obj is None: ret.append({ 'url': url, 'exist': False, 'reason': 'UNSUPPORTED_WEBSITE' }) continue if not cleanURL: ret.append({'url': url, 'exist': False, 'reason': 'EMPTY_URL'}) continue uid = obj.unique_id(obj, cleanURL) obj = db.retrive_item({'item.unique_id': uid}) if obj: ret.append({'url': url, 'exist': True, 'id': obj['_id']}) else: ret.append({ 'url': url, 'exist': False, 'reason': 'VIDEO_NOT_EXIST' }) return ret
def requestSubtitleOCR(user, vid: ObjectId): # step 1: verify user and video filterOperation('requestSubtitleOCR', user) video_item = tagdb.retrive_item(vid) if video_item is None: raise UserError('VIDEO_NOT_FOUND') # step 2: check if request exists with redis_lock.Lock( rdb, "videoEdit:" + video_item['item']['unique_id']), redis_lock.Lock( rdb, "mmdocr_global_lock"), MongoTransaction(client) as s: ocr_record = db.subtitle_ocr.find_one({"vid": vid}, session=s()) if ocr_record is None: # create new record record = { "vid": vid, "status": "Queuing", "version": 0, # version is set in postSubtitleOCRResult "worker_id": "", # worker_id is set in queryAndProcessQueuingRequests "meta": makeUserMetaObject(user) } db.subtitle_ocr.insert_one(record, session=s()) else: status = ocr_record['status'] record_id = ocr_record['_id'] record_version = ocr_record['version'] mmdocr_version = int(Config.MMDOCR_VERSION) if status in ['NoRecord', 'RecordOutOfDate', 'Error']: assert status != "NoRecord" db.subtitle_ocr.update_one({"_id": record_id}, { "$set": { "status": "Queuing", "meta.modified_at": datetime.utcnow(), "meta.modified_by": ObjectId(user['_id']) } }, session=s()) pass elif status == 'RecordExists': if record_version < mmdocr_version: # newer version of mmdocr exists db.subtitle_ocr.update_one({"_id": record_id}, { "$set": { "status": "Queuing", "meta.modified_at": datetime.utcnow(), "meta.modified_by": ObjectId(user['_id']) } }, session=s()) pass else: raise UserError('RECORD_ALREADY_EXISTS') else: raise UserError('VIDEO_BEING_PROCESSED') s.mark_succeed()
def filterSingleVideo(vid, user, raise_error = True) : item = tagdb.retrive_item(vid) if item is None : return None if _is_authorised(item, user) : return item else : if raise_error : raise UserError('ITEM_NOT_FOUND') else : return None
def getVideoByURL(url): obj, cleanURL = dispatch(url) if obj is None: raise UserError('UNSUPPORTED_WEBSITE') if not cleanURL: raise UserError('EMPTY_URL') uid = obj.unique_id(obj, cleanURL) obj = db.retrive_item({'item.unique_id': uid}) if obj: return obj raise UserError('VIDEO_NOT_EXIST')
def _removeThisCopy(dst_vid, this_vid, user, session): if this_vid is None: return dst_video = tagdb.retrive_item({"_id": ObjectId(dst_vid)}, session) if dst_video is None: return dst_copies = dst_video['item']['copies'] dst_copies = list(set(dst_copies) - set([ObjectId(this_vid)])) tagdb.update_item_query(ObjectId(dst_vid), {"$set": { "item.copies": dst_copies }}, user, session)
def querySubtitleOCRStatus(vid: ObjectId): # step 1: verify video video_item = tagdb.retrive_item(vid) if video_item is None: raise UserError('VIDEO_NOT_FOUND') with redis_lock.Lock(rdb, "mmdocr_global_lock"): # step 2: query and return ocr_record = db.subtitle_ocr.find_one({"vid": vid}) if ocr_record is None: return "NoRecord" else: return ocr_record['status']
def addVideoToPlaylistLockFree(pid, vid, user, rank, session): log(obj={'pid': pid, 'vid': vid}) if tagdb.retrive_item({'_id': ObjectId(vid)}, session=session) is None: #raise UserError('VIDEO_NOT_EXIST') return False conflicting_item = db.playlist_items.find_one( { 'pid': ObjectId(pid), 'vid': ObjectId(vid) }, session=session) if conflicting_item is not None: editPlaylist_MoveLockFree(pid, conflicting_item, rank, session=session) playlist_db.update_item_query(pid, {}, user=makeUserMeta(user), session=session) return False playlists = tagdb.retrive_item({'_id': ObjectId(vid)}, session=session)['item']['series'] playlists.append(ObjectId(pid)) playlists = list(set(playlists)) tagdb.update_item_query(ObjectId(vid), {'$set': { 'item.series': playlists }}, user=makeUserMeta(user), session=session) db.playlist_items.insert_one( { "pid": ObjectId(pid), "vid": ObjectId(vid), "rank": int(rank), "meta": makeUserMeta(user) }, session=session) playlist_db.update_item_query(pid, {"$inc": { "item.videos": int(1) }}, user=makeUserMeta(user), session=session) return True
def listPlaylistsForVideo(user, vid) : video = tagdb.retrive_item({'_id': ObjectId(vid)}) if video is None : raise UserError('VIDEO_NOT_EXIST') if isObjectAgnosticOperationPermitted('viewPrivatePlaylist', user) : auth_obj = {} else : auth_obj = {'$or': [{'playlist.meta.created_by': user['_id'] if user else ''}, {'playlist.private': False, 'playlist.videos': {'$gt': 1}}]} result = db.playlist_items.aggregate([ { '$match': { '$and': [ { 'pid': { '$in': video['item']['series'] } }, { 'vid': video['_id'] } ] } }, { '$lookup': { 'from': 'playlists', 'localField': 'pid', 'foreignField': '_id', 'as': 'playlist' } }, { '$unwind': { 'path': '$playlist' } }, { '$match': auth_obj } ]) ans = [] for obj in result : playlist_obj = obj['playlist'] playlist_obj['prev'] = '' playlist_obj['next'] = '' rank = obj['rank'] if rank > 0 : playlist_obj['prev'] = str(db.playlist_items.find_one({'pid': playlist_obj['_id'], 'rank': int(rank - 1)})['vid']) if rank + 1 < playlist_obj['videos'] : playlist_obj['next'] = str(db.playlist_items.find_one({'pid': playlist_obj['_id'], 'rank': int(rank + 1)})['vid']) ans.append(playlist_obj) return ans
def postSubtitle(user, vid: ObjectId, language: str, subformat: str, content: str): if language not in VALID_LANGUAGES: raise UserError('INVALID_LANGUAGE') subformat = subformat.lower() if subformat not in VALID_SUBTITLE_FORMAT: raise UserError('INVALID_SUBTITLE_FORMAT') video_item = tagdb.retrive_item(vid) if video_item is None: raise UserError('VIDEO_NOT_FOUND') try: size = len(content.encode('utf-8')) except: size = -1 filterOperation('postSubtitle', user) with redis_lock.Lock( rdb, "videoEdit:" + video_item['item']['unique_id']), MongoTransaction(client) as s: existing_item = db.subtitles.find_one( { 'vid': vid, 'lang': language, 'format': subformat, 'meta.created_by': makeUserMeta(user) }, session=s()) if existing_item is None: subid = db.subtitles.insert_one( { 'vid': vid, 'lang': language, 'format': subformat, 'content': content, 'size': size, 'deleted': False, 'autogen': False, 'version': 0, 'meta': makeUserMetaObject(user) }, session=s()).inserted_id else: db.subtitles.update_one({'_id': existing_item['_id']}, { '$set': { 'content': content, 'size': size, 'meta.modified_at': datetime.utcnow() } }, session=s()) subid = existing_item['_id'] s.mark_succeed() return ObjectId(subid)
def postSubtitleOCRResult(user, unique_id: str, content: str, subformat: str, version: int, worker_id: str): # step 1: verify and post filterOperation('subtitleocr_postSubtitleOCRResult', user) subformat = subformat.lower() if subformat not in VALID_SUBTITLE_FORMAT: raise UserError('INVALID_SUBTITLE_FORMAT') video_item = tagdb.retrive_item({"item.unique_id": unique_id}) if video_item is None: raise UserError('VIDEO_NOT_FOUND') try: size = len(content.encode('utf-8')) except: size = -1 with redis_lock.Lock( rdb, "videoEdit:" + video_item['item']['unique_id']), redis_lock.Lock( rdb, "mmdocr_global_lock"), MongoTransaction(client) as s: # delete old versions db.subtitles.delete_many({ 'vid': video_item['_id'], 'autogen': True }, session=s()) subid = db.subtitles.insert_one( { 'vid': video_item['_id'], 'lang': 'UNKNOWN', 'format': subformat, 'content': content, 'size': size, 'deleted': False, 'version': version, 'autogen': True, 'meta': makeUserMetaObject(None) }, session=s()).inserted_id # step 2: update subtitle_ocr db.subtitle_ocr.update_one({"vid": video_item['_id']}, { "$set": { "status": "RecordExists", "version": version, "worker_id": worker_id } }, session=s()) s.mark_succeed() return subid
def _getAllCopies(vid_or_obj, session=None): if not vid_or_obj: return [] this_video = None if isinstance(vid_or_obj, str) or isinstance(vid_or_obj, ObjectId): this_video = tagdb.retrive_item({"_id": ObjectId(vid_or_obj)}, session=session) else: this_video = vid_or_obj if this_video is None: return [] copies = this_video['item']['copies'] # add self copies.append(this_video['_id']) # use set to remove duplicated items return list(set(copies))
def listPlaylistsForVideoNoAuth(vid): video = tagdb.retrive_item({'_id': ObjectId(vid)}) if video is None: raise UserError('VIDEO_NOT_EXIST') result = db.playlist_items.aggregate([{ '$match': { '$and': [{ 'pid': { '$in': video['item']['series'] } }, { 'vid': video['_id'] }] } }, { '$lookup': { 'from': 'playlist_metas', 'localField': 'pid', 'foreignField': '_id', 'as': 'playlist' } }, { '$unwind': { 'path': '$playlist' } }]) ans = [] for obj in result: playlist_obj = obj['playlist'] playlist_obj['prev'] = '' playlist_obj['next'] = '' rank = obj['rank'] if rank > 0: playlist_obj['prev'] = str( db.playlist_items.find_one({ 'pid': playlist_obj['_id'], 'rank': int(rank - 1) })['vid']) if rank + 1 < playlist_obj['item']['videos']: playlist_obj['next'] = str( db.playlist_items.find_one({ 'pid': playlist_obj['_id'], 'rank': int(rank + 1) })['vid']) ans.append(playlist_obj) return ans
def _addThiscopy(dst_vid, this_vid, user, session): if this_vid is None: return dst_video = tagdb.retrive_item({"_id": ObjectId(dst_vid)}, session=session) if dst_video is None: return dst_copies = dst_video['item']['copies'] if isinstance(this_vid, list): dst_copies.extend(this_vid) else: dst_copies.append(ObjectId(this_vid)) dst_copies = list(set(dst_copies) - set([ObjectId(dst_vid)])) tagdb.update_item_query(ObjectId(dst_vid), {"$set": { "item.copies": dst_copies }}, user=user, session=session)
def syncTags(dst, src, user): if dst == src: raise UserError('SAME_VIDEO') filterOperation('syncTags', user, (dst, src)) src_item, src_tags, _, _ = tagdb.retrive_item_with_tag_category_map( src, 'CHS') log(obj={'src_tags': src_tags, 'src_id': src}) dst_item = tagdb.retrive_item(dst) if dst_item is None: raise UserError('ITEM_NOT_EXIST') with redis_lock.Lock( rdb, "videoEdit:" + dst_item["item"]["unique_id"]), MongoTransaction(client) as s: tagdb.update_item_tags_merge(ObjectId(dst), src_tags, makeUserMeta(user), session=s()) s.mark_succeed()
def setVideoRepostType(vid, repost_type, user): filterOperation('setVideoRepostType', user) if repost_type not in [ 'official', 'official_repost', 'authorized_translation', 'authorized_repost', 'translation', 'repost' ]: raise UserError('INCORRECT_REPOST_TYPE') video_obj = tagdb.retrive_item(vid) if video_obj is None: raise UserError('VIDEO_NOT_FOUND') lock_id = "videoEdit:" + video_obj['item']['unique_id'] with redis_lock.Lock(rdb, lock_id), MongoTransaction(client) as s: tagdb.update_item_query(video_obj, {'$set': { 'item.repost_type': repost_type }}, session=s()) s.mark_succeed()
def editPlaylist_Move(pid, vid, to_rank, user) : log(obj = {'pid': pid, 'vid': vid, 'to_rank': to_rank}) with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s : playlist = db.playlists.find_one({'_id': ObjectId(pid)}, session = s()) if playlist is None : raise UserError('PLAYLIST_NOT_EXIST') filterOperation('editPlaylist', user, playlist) if tagdb.retrive_item({'_id': ObjectId(vid)}, session = s()) is None : raise UserError('VIDEO_NOT_EXIST') if to_rank < 0 : raise UserError('OUT_OF_RANGE') if to_rank > playlist['videos'] : to_rank = int(playlist['videos']) editPlaylist_MoveLockFree(pid, vid, to_rank, session = s()) db.playlists.update_one({'_id': ObjectId(pid)}, {'$set': { 'meta.modified_by': makeUserMeta(user), 'meta.modified_at': datetime.now()}}, session = s()) s.mark_succeed()
def broadcastTags(src, user): filterOperation('broadcastTags', user, src) _, src_tags, _, _ = tagdb.retrive_item_with_tag_category_map(src, 'CHS') log(obj={'src_tags': src_tags, 'src_id': src}) with redis_lock.Lock(rdb, "editLink"), MongoTransaction(client) as s: copies = _getAllCopies(src, session=s()) log(obj={'clique': copies}) for copy in copies: if copy != ObjectId(src): # prevent self updating copy_obj = tagdb.retrive_item({"_id": ObjectId(copy)}, session=s()) assert copy_obj is not None, 'copy_obj %s not exist' % copy with redis_lock.Lock( rdb, "videoEdit:" + copy_obj["item"]["unique_id"]): tagdb.update_item_tags_merge(ObjectId(copy), src_tags, makeUserMeta(user), session=s()) s.mark_succeed()
def editPlaylist_Move(pid, vid, to_rank, user): log(obj={'pid': pid, 'vid': vid, 'to_rank': to_rank}) with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s: playlist = playlist_db.retrive_item(pid, session=s()) if playlist is None: raise UserError('PLAYLIST_NOT_EXIST') filterOperation('editPlaylist', user, playlist) if tagdb.retrive_item({'_id': ObjectId(vid)}, session=s()) is None: raise UserError('VIDEO_NOT_EXIST') if to_rank < 0: raise UserError('OUT_OF_RANGE') if to_rank > playlist['item']['videos']: to_rank = int(playlist['item']['videos']) editPlaylist_MoveLockFree(pid, vid, to_rank, session=s()) playlist_db.update_item_query(playlist, {}, user=makeUserMeta(user), session=s()) s.mark_succeed()
def refreshVideoDetail(vid, user): log(obj={'vid': vid}) filterOperation('refreshVideoDetail', user, vid) filterSingleVideo(vid, user) item = tagdb.retrive_item(vid) if item is None: raise UserError('ITEM_NOT_EXIST') json_str = dumps({ 'url': item['item']['url'], 'tags': [], 'dst_copy': '', 'dst_playlist': '', 'dst_rank': -1, 'other_copies': [], 'user': user, 'playlist_ordered': None, 'update_video_detail': True, 'event_id': getEventID() }) postTask(json_str)
def inferTagsFromVideo(utags, title, desc, user_language, video_url: str = '', user_urls: [str] = []): log( obj={ 'title': title, 'desc': desc, 'utags': utags, 'lang': user_language, 'video_url': video_url, 'user_urls': user_urls }) video_url = video_url.strip() tagids = [] if video_url: obj, cleanURL = dispatch(video_url) if obj is not None: uid = obj.unique_id(obj, cleanURL) vid_item = db.retrive_item({'item.unique_id': uid}) if vid_item is not None: tagids = list( filter(lambda x: x < 0x80000000, vid_item['tags'])) if not tagids: utags = [u.lower() for u in utags] utags.append(title) utags.append(desc) all_text = ' 3e7dT2ibT7dM '.join(utags) tagids = inferTagidsFromText(all_text) matched_author_records, matched_author_tags = matchUserSpace(user_urls) matched_common_ids = itertools.chain.from_iterable( [x['common_tagids'] for x in matched_author_records]) tagids = list( set(tagids) | set([x['id'] for x in matched_author_tags]) | set(matched_common_ids)) return db.translate_tag_ids_to_user_language(tagids, user_language)[0]
def updateRequestStatus(user, video_status_map, worker_id: str): # map of unique_id to status filterOperation('subtitleocr_updateRequestStatus', user) with redis_lock.Lock(rdb, "mmdocr_global_lock"), MongoTransaction(client) as s: for unqiue_id, status in video_status_map.items(): if status not in [ 'PendingDownload', 'Downloading', 'PendingProcess', 'Processing', 'Error', 'Queuing' ]: continue # step 1: verify videos video_item = tagdb.retrive_item({"item.unique_id": unqiue_id}, session=s()) if video_item is not None: # step 2: update db.subtitle_ocr.update_one( {"vid": video_item['_id']}, {"$set": { "status": status, "worker_id": worker_id }}, session=s()) s.mark_succeed()
def getVideoDetailNoFilter(vid): return db.retrive_item(vid)
async def postVideoAsync(url, tags, dst_copy, dst_playlist, dst_rank, other_copies, repost_type, playlist_ordered, user, update_video_detail, event_id, field_override=None, use_autotag=False): parsed = None try: dst_playlist = str(dst_playlist) dst_rank = -1 if dst_rank is None else dst_rank #tags = tagdb.filter_and_translate_tags(tags) parsed, url = dispatch(url) except: pass if parsed is None: log_e(event_id, user, 'dispatcher', 'ERR', { 'msg': 'PARSE_FAILED', 'url': url }) await _playlist_reorder_helper.post_video_failed( url, dst_playlist, playlist_ordered, dst_rank, user, event_id) return "PARSE_FAILED", {} unique_id = await parsed.unique_id_async( self=parsed, link=url) # empty unique_id for b23.tv posts, f**k bilibli if not unique_id: ret = await parsed.get_metadata_async(parsed, url, update_video_detail) unique_id = ret['data']['unique_id'] log_e( event_id, user, 'scraper', 'MSG', { 'url': url, 'dst_copy': dst_copy, 'other_copies': other_copies, 'dst_playlist': dst_playlist }) setEventOp('scraper') try: lock_id = "videoEdit:" + unique_id async with RedisLockAsync(rdb, lock_id): unique, conflicting_item = verifyUniqueness(unique_id) if unique or update_video_detail: async with _download_sem: ret = await parsed.get_metadata_async( parsed, url, update_video_detail) if repost_type: ret['data']['repost_type'] = repost_type else: ret['data']['repost_type'] = 'unknown' if ret["status"] == 'FAILED': log_e(event_id, user, 'downloader', 'WARN', { 'msg': 'FETCH_FAILED', 'ret': ret }) await _playlist_reorder_helper.post_video_failed( unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) return "FETCH_FAILED", ret else: unique_id = ret['data']['unique_id'] else: # build ret ret = makeResponseSuccess({ 'thumbnailURL': conflicting_item['item']['thumbnail_url'], 'title': conflicting_item['item']['title'], 'desc': conflicting_item['item']['desc'], 'site': conflicting_item['item']['site'], 'uploadDate': conflicting_item['item']['upload_time'], "unique_id": conflicting_item['item']['unique_id'], "utags": conflicting_item['item']['utags'] }) for k, v in conflicting_item['item'].items(): ret['data'][k] = v if 'part_name' in conflicting_item['item']: ret['part_name'] = conflicting_item['item']['part_name'] if 'repost_type' in conflicting_item[ 'item'] and conflicting_item['item']['repost_type']: ret['data']['repost_type'] = repost_type else: ret['data']['repost_type'] = 'unknown' tagdb.update_item_query( conflicting_item, {'$set': { 'item.repost_type': repost_type }}, user=makeUserMeta(user)) #if hasattr(parsed, 'LOCAL_CRAWLER') : # url = ret["data"]["url"] #else : # url = clear_url(url) use_override = False if field_override and '__condition' in field_override: condition = field_override['__condition'] del field_override['__condition'] if condition == 'any': use_override = True elif condition == 'placeholder' and 'placeholder' in ret[ "data"] and ret["data"]['placeholder']: use_override = True if use_override: for key in field_override: ret['data'][key] = field_override[key] playlists = [] #playlist_lock = None if dst_playlist: #playlist_lock = RedisLockAsync(rdb, "playlistEdit:" + str(dst_playlist)) #playlist_lock.acquire() if playlist_db.retrive_item(dst_playlist) is not None: playlists = [ObjectId(dst_playlist)] if not unique: log_e(event_id, user, 'scraper', level='MSG', obj={ 'msg': 'ALREADY_EXIST', 'unique_id': ret["data"]["unique_id"] }) """ Update existing video """ if update_video_detail: log_e(event_id, user, 'scraper', level='MSG', obj='Updating video detail') with MongoTransaction(client) as s: old_item = tagdb.retrive_item(conflicting_item['_id'], session=s())['item'] if old_item['thumbnail_url'] and old_item[ 'cover_image']: # old thumbnail exists, no need to download again new_detail = await _make_video_data_update( ret["data"], url, user, event_id) else: # old thumbnail does not exists, add to dict new_detail = await _make_video_data_update( ret["data"], url, user, event_id, ret["data"]["thumbnailURL"]) for key in new_detail.keys(): old_item[key] = new_detail[ key] # overwrite or add new field setEventUserAndID(user, event_id) tagdb.update_item_query(conflicting_item['_id'], {'$set': { 'item': old_item }}, ['title', 'desc'], user=makeUserMeta(user), session=s()) s.mark_succeed() return 'SUCCEED', conflicting_item['_id'] # this video already exist in the database # if the operation is to add a link to other copies and not adding self if (dst_copy and dst_copy != conflicting_item['_id']) or other_copies: log_e(event_id, user, 'scraper', level='MSG', obj='Adding to to copies') async with RedisLockAsync( rdb, 'editLink'), MongoTransaction(client) as s: log_e(event_id, user, level='MSG', obj='Adding to to copies, lock acquired') # find all copies of video dst_copy, self included all_copies = _getAllCopies(dst_copy, session=s()) # find all videos linked to source video all_copies += _getAllCopies(conflicting_item['_id'], session=s()) # add videos from other copies for uid in other_copies: all_copies += _getAllCopies(uid, session=s(), use_unique_id=True) # remove duplicated items all_copies = list(set(all_copies)) # add this video to all other copies found if len(all_copies) <= VideoConfig.MAX_COPIES: for dst_vid in all_copies: setEventUserAndID(user, event_id) _addThiscopy(dst_vid, all_copies, makeUserMeta(user), session=s()) log_e(event_id, user, 'scraper', level='MSG', obj='Successfully added to copies') s.mark_succeed() else: #if playlist_lock : # playlist_lock.release() log_e(event_id, user, 'scraper', level='MSG', obj='Too many copies') await _playlist_reorder_helper.post_video_failed( unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) return "TOO_MANY_COPIES", {} # if the operation is adding this video to playlist if dst_playlist: log_e(event_id, user, 'scraper', level='MSG', obj={ 'msg': 'Adding to playlist at position', 'rank': dst_rank }) if playlist_ordered: await _playlist_reorder_helper.post_video_succeed( conflicting_item['_id'], unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) else: setEventUserAndID(user, event_id) if dst_rank == -1: addVideoToPlaylist(dst_playlist, conflicting_item['_id'], user) else: insertIntoPlaylist(dst_playlist, conflicting_item['_id'], dst_rank, user) # merge tags async with MongoTransaction(client) as s: log_e(event_id, user, 'scraper', level='MSG', obj='Merging tags') setEventUserAndID(user, event_id) tagdb.update_item_tags_merge(conflicting_item['_id'], tags, makeUserMeta(user), session=s(), remove_tagids=[354]) s.mark_succeed() #if playlist_lock : # playlist_lock.release() #return "VIDEO_ALREADY_EXIST", conflicting_item['_id'] return "SUCCEED", conflicting_item['_id'] else: # expand dst_copy to all copies linked to dst_copy if dst_copy or other_copies: log_e(event_id, user, 'scraper', level='MSG', obj='Adding to to copies') async with RedisLockAsync( rdb, 'editLink'), MongoTransaction(client) as s: log_e(event_id, user, 'scraper', level='MSG', obj='Adding to to copies, lock acquired') all_copies = _getAllCopies(dst_copy, session=s()) # add videos from other copies for uid in other_copies: all_copies += _getAllCopies(uid, session=s(), use_unique_id=True) video_data = await _make_video_data( ret["data"], all_copies, playlists, url, user, event_id) setEventUserAndID(user, event_id) new_item_id = tagdb.add_item(tags, video_data, 3, ['title', 'desc'], makeUserMeta(user), session=s()) all_copies.append(ObjectId(new_item_id)) # remove duplicated items all_copies = list(set(all_copies)) if len(all_copies) <= VideoConfig.MAX_COPIES: for dst_vid in all_copies: setEventUserAndID(user, event_id) _addThiscopy(dst_vid, all_copies, makeUserMeta(user), session=s()) log_e(event_id, user, 'scraper', level='MSG', obj='Successfully added to copies') s.mark_succeed() else: #if playlist_lock : # playlist_lock.release() log_e(event_id, user, 'scraper', level='MSG', obj='Too many copies') await _playlist_reorder_helper.post_video_failed( unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) return "TOO_MANY_COPIES", {} else: async with MongoTransaction(client) as s: video_data = await _make_video_data( ret["data"], [], playlists, url, user, event_id) setEventUserAndID(user, event_id) if use_autotag: tags.extend( inferTagsFromVideo( video_data['utags'], video_data['title'], video_data['desc'], 'CHS', video_data['url'], video_data['user_space_urls'])) new_item_id = tagdb.add_item(tags, video_data, 3, ['title', 'desc'], makeUserMeta(user), session=s()) log_e(event_id, user, 'scraper', level='MSG', obj={ 'msg': 'New video added to database', 'vid': new_item_id }) s.mark_succeed() # if the operation is adding this video to playlist if dst_playlist: log_e(event_id, user, 'scraper', level='MSG', obj={ 'msg': 'Adding to playlist at position', 'rank': dst_rank }) if playlist_ordered: await _playlist_reorder_helper.post_video_succeed( new_item_id, unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) else: setEventUserAndID(user, event_id) if dst_rank == -1: addVideoToPlaylist(dst_playlist, new_item_id, user) else: insertIntoPlaylist(dst_playlist, new_item_id, dst_rank, user) #if playlist_lock : # playlist_lock.release() log_e(event_id, user, 'scraper', level='MSG', obj='Done') return 'SUCCEED', new_item_id except UserError as ue: await _playlist_reorder_helper.post_video_failed( unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) log_e(event_id, user, 'scraper', level='WARN', obj={'ue': str(ue)}) return ue.msg, {"aux": ue.aux, "traceback": traceback.format_exc()} except Exception as ex: await _playlist_reorder_helper.post_video_failed( unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) log_e(event_id, user, 'scraper', level='ERR', obj={'ex': str(ex)}) try: problematic_lock = RedisLockAsync(rdb, 'editLink') problematic_lock.reset() except: pass return "UNKNOWN", '\n'.join( [repr(traceback.format_exc()), repr(traceback.extract_stack())])
def verifyUniqueness(postingId): if not postingId: return True, None val = tagdb.retrive_item({"item.unique_id": postingId}) return val is None, val