def getPlaylistRatingAggregate(pid: ObjectId): playlist_obj = playlist_db.retrive_item(pid) if playlist_obj is None: raise UserError('PLAYLIST_NOT_EXIST') if 'total_rating' in playlist_obj: return playlist_obj['total_rating'], playlist_obj['total_rating_user'] raise UserError('NOT_RATED')
def updatePlaylistCoverFromFile(pid, user, file_key): log(obj={'pid': pid, 'file_key': file_key}) photo_file = None if file_key.startswith("upload-image-"): filename = rdb.get(file_key) if filename: photo_file = filename.decode('ascii') if photo_file is None: raise UserError('NO_COVER') with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s: list_obj = playlist_db.retrive_item(pid) log(obj={'playlist': list_obj}) if list_obj is None: raise UserError('PLAYLIST_NOT_EXIST') filterOperation('editPlaylist', user, list_obj) playlist_db.update_item_query(list_obj, {'$set': { "item.cover": photo_file }}, user=makeUserMeta(user), session=s()) s.mark_succeed()
def listAllPlaylistVideosOrdered(pid, user): playlist = playlist_db.retrive_item(pid) if playlist is None: raise UserError('PLAYLIST_NOT_EXIST') if playlist['item']['private']: filterOperation('viewPrivatePlaylist', user, playlist) ans_obj = db.playlist_items.aggregate([{ '$match': { "pid": ObjectId(pid) } }, { '$lookup': { 'from': 'videos', 'localField': "vid", 'foreignField': "_id", 'as': 'item' } }, { '$unwind': { 'path': '$item' } }, { '$sort': { 'rank': 1 } }]) return [i for i in ans_obj], playlist['item']['videos'], playlist
def addToPlaylist(user, pid: ObjectId, text: str, use_bleach=True): filterOperation('postComment', user) playlist_obj = playlist_db.retrive_item(pid) if playlist_obj is None: raise UserError('PLAYLIST_NOT_EXIST') with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)): if 'comment_thread' in playlist_obj: cid = addComment(user, playlist_obj['comment_thread'], text, use_bleach=use_bleach) return playlist_obj['comment_thread'], cid else: with MongoTransaction(client) as s: tid = createThread('playlist', playlist_obj['_id'], playlist_obj['meta']['created_by'], session=s()) playlist_db.update_item_query( playlist_obj, {'$set': { 'comment_thread': tid }}, session=s()) s.mark_succeed() cid = addComment(user, tid, text) return tid, cid
def addPlaylistsToFolder(user, path, playlists) : _verifyPath(path) with redis_lock.Lock(rdb, f"folderEdit:{str(makeUserMeta(user))}:{path}"), MongoTransaction(client) as s : folder_obj = _findFolder(user, path) filterOperation('addPlaylistsToFolder', user, folder_obj) for pid in playlists : playlist = playlist_db.retrive_item(pid, session = s()) if playlist is None : continue # skip non-exist playlist if playlist['item']['private'] and not filterOperation('viewPrivatePlaylist', user, playlist, raise_exception = False) : continue # skip other's private playlist playlist_path = path + "\\" + str(playlist['_id']) + "\\/" if _findFolder(user, playlist_path, raise_exception = False) : continue # skip duplicated playlist playlist_obj = { 'user': makeUserMeta(user), 'leaf': True, 'playlist': playlist['_id'], 'name': None, 'path': playlist_path, 'privateView': folder_obj['privateView'], 'privateEdit': folder_obj['privateEdit'], 'meta': makeUserMetaObject(user) } db.playlist_folders.insert_one(playlist_obj, session = s()) db.playlist_folders.update_one({'_id': folder_obj['_id']}, {'$set': { 'meta.modified_by': makeUserMeta(user), 'meta.modified_at': datetime.now() }}, session = s()) s.mark_succeed()
def updateCommonTags(pid, tags, user): log(obj={'pid': pid, 'tags': tags}) with MongoTransaction(client) as s: playlist_obj = playlist_db.retrive_item(pid) if playlist_obj is None: raise UserError('PLAYLIST_NOT_EXIST') filterOperation('editPlaylist', user, playlist_obj) # user is editing video tags, not the playlist itself, no need to lock playlist tags = tagdb.filter_and_translate_tags(tags, session=s()) old_tags = listCommonTagIDs(pid, user) log(obj={'old_tags': old_tags}) old_tags_set = set(old_tags) new_tags_set = set(tags) tags_added = list((old_tags_set ^ new_tags_set) - old_tags_set) tags_to_remove = list((old_tags_set ^ new_tags_set) - new_tags_set) if len(tags_added) - len( tags_to_remove) > PlaylistConfig.MAX_COMMON_TAGS: raise UserError('TOO_MANY_TAGS') all_video_ids, _ = listAllPlaylistVideosUnordered(pid) if tags_to_remove: tagdb.update_many_items_tags_pull(all_video_ids, tags_to_remove, makeUserMeta(user), session=s()) if tags_added: tagdb.update_many_items_tags_merge(all_video_ids, tags_added, makeUserMeta(user), session=s()) s.mark_succeed()
def listAllPlaylistVideosUnordered(pid): playlist = playlist_db.retrive_item(pid) if playlist is None: raise UserError('PLAYLIST_NOT_EXIST') ans_obj = db.playlist_items.find({"pid": ObjectId(pid)}) return [ObjectId(item['vid']) for item in ans_obj], playlist['item']['videos']
def ratePlaylist(user, pid: ObjectId, stars: int): stars = max(min(int(stars), 10), 1) playlist_obj = playlist_db.retrive_item(pid) if playlist_obj is None: raise UserError('PLAYLIST_NOT_EXIST') with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s: rating_obj = db.playlist_ratings.find_one( { 'pid': pid, 'uid': ObjectId(user['_id']) }, session=s()) user_rated = 0 if rating_obj: user_rated = 1 db.playlist_ratings.update_one( { 'pid': pid, 'uid': ObjectId(user['_id']) }, {'$set': { 'v': int(stars) }}, session=s()) else: user_rated = 0 db.playlist_ratings.insert_one( { 'pid': pid, 'uid': ObjectId(user['_id']), 'v': int(stars) }, session=s()) if 'total_rating' in playlist_obj: if rating_obj: playlist_db.update_item_query(playlist_obj, { '$inc': { 'total_rating': int(stars - rating_obj['v']), 'total_rating_user': int(1 - user_rated) } }, session=s()) else: playlist_db.update_item_query(playlist_obj, { '$inc': { 'total_rating': int(stars), 'total_rating_user': int(1 - user_rated) } }, session=s()) else: playlist_db.update_item_query(playlist_obj, { '$set': { 'total_rating': int(stars), 'total_rating_user': int(1) } }, session=s()) s.mark_succeed()
def isOwner(pid_or_obj, user): if isinstance(pid_or_obj, str): obj = playlist_db.retrive_item(pid_or_obj) else: obj = pid_or_obj creator = str(obj['meta']['created_by']) user_id = str(user['_id']) return creator == user_id or ( 'editPlaylist' in user['access_control']['allowed_ops'] ) or user['access_control']['status'] == 'admin'
def extendPlaylistFromExistingPlaylist(pid, url, user, use_autotag=False): log(obj={'url': url, 'pid': pid}) filterOperation('editPlaylist', user) if not playlist_db.retrive_item(pid): raise UserError('PLAYLIST_NOT_EXIST') cralwer, cleanURL = dispatch_playlist(url) if not cralwer: raise UserError('UNSUPPORTED_PLAYLIST_URL') task_id = _postPlaylistTask(cleanURL, pid, use_autotag, user, extend=True) return task_id
def getPlaylist(pid, lang): ret = playlist_db.retrive_item(pid) if not ret: raise UserError('PLAYLIST_NOT_EXIST') if 'tags' in ret: ret['tags_translated'], ret['tags_category'], ret[ 'tags_tags'] = tagdb.translate_tag_ids_to_user_language( ret['tags'], lang) else: ret['tags_translated'], ret['tags_category'], ret[ 'tags_tags'] = [], {}, {} return ret
def addVideoToPlaylist(pid, vid, user): log(obj={'pid': pid, 'vid': vid}) with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s: playlist = playlist_db.retrive_item(pid) if playlist is None: raise UserError('PLAYLIST_NOT_EXIST') filterOperation('editPlaylist', user, playlist) if tagdb.retrive_item({'_id': ObjectId(vid)}, session=s()) is None: raise UserError('VIDEO_NOT_EXIST') if playlist['item']["videos"] > PlaylistConfig.MAX_VIDEO_PER_PLAYLIST: raise UserError('VIDEO_LIMIT_EXCEEDED') conflicting_item = db.playlist_items.find_one( { 'pid': ObjectId(pid), 'vid': ObjectId(vid) }, session=s()) if conflicting_item is not None: editPlaylist_MoveLockFree(pid, conflicting_item, int(playlist['item']["videos"]), session=s()) playlist_db.update_item_query(playlist, {}, user=makeUserMeta(user), session=s()) s.mark_succeed() return playlists = tagdb.retrive_item({'_id': ObjectId(vid)}, session=s())['item']['series'] playlists.append(ObjectId(pid)) playlists = list(set(playlists)) tagdb.update_item_query(ObjectId(vid), {'$set': { 'item.series': playlists }}, user=makeUserMeta(user), session=s()) db.playlist_items.insert_one( { "pid": ObjectId(pid), "vid": ObjectId(vid), "rank": int(playlist['item']["videos"]), "meta": makeUserMeta(user) }, session=s()) playlist_db.update_item_query(playlist, {"$inc": { "item.videos": int(1) }}, user=makeUserMeta(user), session=s()) s.mark_succeed()
def updatePlaylistInfo(pid, title, desc, cover, user, private=False, privateEdit=True): log( obj={ 'title': title, 'desc': desc, 'cover': cover, 'private': private, 'privateEdit': privateEdit }) if len(title) > PlaylistConfig.MAX_TITLE_LENGTH: raise UserError('TITLE_TOO_LONG') if len(desc) > PlaylistConfig.MAX_DESC_LENGTH: raise UserError('DESC_TOO_LONG') if cover and len(cover) > PlaylistConfig.MAX_COVER_URL_LENGTH: raise UserError('URL_TOO_LONG') if not title: raise UserError('EMPTY_TITLE') if not desc: raise UserError('EMPTY_DESC') with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s: list_obj = playlist_db.retrive_item(pid) log(obj={'playlist': list_obj}) if list_obj is None: raise UserError('PLAYLIST_NOT_EXIST') filterOperation('editPlaylist', user, list_obj) if cover: playlist_db.update_item_query(list_obj, {'$set': { "item.cover": cover }}, user=makeUserMeta(user), session=s()) playlist_db.update_item_query( list_obj, { '$set': { "item.title": title, "item.desc": desc, "item.private": private, "item.privateEdit": privateEdit } }, fields_to_index=['item.title', 'item.desc'], user=makeUserMeta(user), session=s()) s.mark_succeed()
def updatePlaylistCover(pid, cover, user): log(obj={'pid': pid, 'cover': cover}) with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s: list_obj = playlist_db.retrive_item(pid) log(obj={'playlist': list_obj}) if list_obj is None: raise UserError('PLAYLIST_NOT_EXIST') filterOperation('editPlaylist', user, list_obj) playlist_db.update_item_query(list_obj, {'$set': { "item.cover": cover }}, user=makeUserMeta(user), session=s()) s.mark_succeed()
def removePlaylist(pid, user): log(obj={'pid': pid}) with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s: list_obj = playlist_db.retrive_item(pid) log(obj={'playlist': list_obj}) if list_obj is None: raise UserError('PLAYLIST_NOT_EXIST') filterOperation('removePlaylist', user, list_obj) all_items = db.playlist_items.find({"pid": ObjectId(pid)}, session=s()) log(obj={'items': [i for i in all_items]}) db.playlist_items.delete_many({"pid": ObjectId(pid)}, session=s()) playlist_db.remove_item(pid, user, session=s()) from services.playlistFolder import deletePlaylist deletePlaylist(pid, session=s()) s.mark_succeed()
def editPlaylist_MoveUp(pid, vid, offset, limit, user): log(obj={'pid': pid, 'vid': vid}) with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s: playlist = playlist_db.retrive_item(pid, session=s()) if playlist is None: raise UserError('PLAYLIST_NOT_EXIST') filterOperation('editPlaylist', user, playlist) if playlist['item']["videos"] > 0: entry = db.playlist_items.find_one( { "pid": ObjectId(pid), "vid": ObjectId(vid) }, session=s()) if entry is None: s.mark_failover() raise UserError('VIDEO_NOT_EXIST_OR_NOT_IN_PLAYLIST') if entry['rank'] <= 0: return None exchange_entry = db.playlist_items.find_one( { "pid": ObjectId(pid), "rank": int(entry['rank'] - 1) }, session=s()) db.playlist_items.update_one( {'_id': entry['_id']}, {'$set': { 'rank': int(entry['rank'] - 1) }}, session=s()) db.playlist_items.update_one( {'_id': exchange_entry['_id']}, {'$set': { 'rank': int(entry['rank']) }}, session=s()) playlist_db.update_item_query(playlist, {}, user=makeUserMeta(user), session=s()) #video_page, video_count = listPlaylistVideos(pid, page - 1, page_size, user) s.mark_succeed() #return {'videos': video_page, 'video_count': video_count, 'page': page} else: raise UserError('EMPTY_PLAYLIST')
def updatePlaylistTags(pid, new_tags, user): log(obj={'pid': pid, 'new_tags': new_tags}) with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s: list_obj = playlist_db.retrive_item(pid) log( obj={ 'playlist': list_obj, 'old_tags': list_obj['tags'] if 'tags' in list_obj else [] }) if list_obj is None: raise UserError('PLAYLIST_NOT_EXIST') #filterOperation('editPlaylist', user, list_obj) playlist_db.update_item_tags(list_obj, new_tags, makeUserMeta(user), session=s()) s.mark_succeed()
def removeVideoFromPlaylist(pid, vid, offset, limit, user): log(obj={'pid': pid, 'vid': vid}) with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s: playlist = playlist_db.retrive_item(pid, session=s()) if playlist is None: raise UserError('PLAYLIST_NOT_EXIST') filterOperation('editPlaylist', user, playlist) if playlist['item']["videos"] > 0: entry = db.playlist_items.find_one( { "pid": ObjectId(pid), "vid": ObjectId(vid) }, session=s()) if entry is None: raise UserError('VIDEO_NOT_EXIST_OR_NOT_IN_PLAYLIST') db.playlist_items.update_many( { 'pid': ObjectId(pid), 'rank': { '$gt': entry['rank'] } }, {'$inc': { 'rank': int(-1) }}, session=s()) db.playlist_items.delete_one({'_id': entry['_id']}, session=s()) playlist_db.update_item_query(playlist, {"$inc": { "item.videos": int(-1) }}, user=makeUserMeta(user), session=s()) else: raise UserError('EMPTY_PLAYLIST') """ video_page, video_count = listPlaylistVideos(pid, page - 1, page_size, user) if len(video_page) == 0 and page > 1 and video_count > 0 : # in case deleting video results in current page becomes empty, show the previous page video_page, video_count = listPlaylistVideos(pid, page - 2, page_size, user) s.mark_succeed() return {'videos': video_page, 'video_count': video_count, 'page': page - 1} """ s.mark_succeed()
def editPlaylist_Move(pid, vid, to_rank, user): log(obj={'pid': pid, 'vid': vid, 'to_rank': to_rank}) with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s: playlist = playlist_db.retrive_item(pid, session=s()) if playlist is None: raise UserError('PLAYLIST_NOT_EXIST') filterOperation('editPlaylist', user, playlist) if tagdb.retrive_item({'_id': ObjectId(vid)}, session=s()) is None: raise UserError('VIDEO_NOT_EXIST') if to_rank < 0: raise UserError('OUT_OF_RANGE') if to_rank > playlist['item']['videos']: to_rank = int(playlist['item']['videos']) editPlaylist_MoveLockFree(pid, vid, to_rank, session=s()) playlist_db.update_item_query(playlist, {}, user=makeUserMeta(user), session=s()) s.mark_succeed()
def inversePlaylistOrder(pid, user): log(obj={'pid': pid}) with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s: playlist = playlist_db.retrive_item(pid, session=s()) if playlist is None: raise UserError('PLAYLIST_NOT_EXIST') filterOperation('editPlaylist', user, playlist) db.playlist_items.update_many({"pid": playlist["_id"]}, {'$bit': { 'rank': { 'xor': int(-1) } }}) db.playlist_items.update_many( {"pid": playlist["_id"]}, {'$inc': { 'rank': int(playlist['item']['videos']) }}) s.mark_succeed()
def updatePlaylistCoverVID(pid, vid, offset, limit, user): log(obj={'pid': pid, 'vid': vid}) with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s: list_obj = playlist_db.retrive_item(pid) log(obj={'playlist': list_obj}) if list_obj is None: raise UserError('PLAYLIST_NOT_EXIST') filterOperation('editPlaylist', user, list_obj) video_obj = filterSingleVideo(vid, user) if video_obj is None: raise UserError('VIDEO_NOT_EXIST') cover = video_obj['item']['cover_image'] playlist_db.update_item_query(list_obj, {'$set': { "item.cover": cover }}, user=makeUserMeta(user), session=s()) #video_page, video_count = listPlaylistVideos(pid, page - 1, page_size, user) s.mark_succeed()
def listPlaylistVideosWithAuthorizationInfo(pid, offset, limit, user): playlist = playlist_db.retrive_item(pid) if playlist is None: raise UserError('PLAYLIST_NOT_EXIST') if playlist['item']['private']: filterOperation('viewPrivatePlaylist', user, playlist) ans_obj = db.playlist_items.aggregate([{ '$match': { "pid": ObjectId(pid) } }, { '$lookup': { 'from': 'videos', 'localField': "vid", 'foreignField': "_id", 'as': 'item' } }, { '$unwind': { 'path': '$item' } }, { '$sort': { 'rank': 1 } }, { '$skip': offset, }, { '$limit': limit }]) ret = [] for obj in ans_obj: ret_obj = obj['item'] ret_obj['rank'] = obj['rank'] ret.append(ret_obj) ret = filterVideoList(ret, user) return ret, playlist['item']['videos'], isAuthorisedToEdit( playlist, user), isOwner(playlist, user)
async def postVideoAsync(url, tags, dst_copy, dst_playlist, dst_rank, other_copies, repost_type, playlist_ordered, user, update_video_detail, event_id, field_override=None, use_autotag=False): parsed = None try: dst_playlist = str(dst_playlist) dst_rank = -1 if dst_rank is None else dst_rank #tags = tagdb.filter_and_translate_tags(tags) parsed, url = dispatch(url) except: pass if parsed is None: log_e(event_id, user, 'dispatcher', 'ERR', { 'msg': 'PARSE_FAILED', 'url': url }) await _playlist_reorder_helper.post_video_failed( url, dst_playlist, playlist_ordered, dst_rank, user, event_id) return "PARSE_FAILED", {} unique_id = await parsed.unique_id_async( self=parsed, link=url) # empty unique_id for b23.tv posts, f**k bilibli if not unique_id: ret = await parsed.get_metadata_async(parsed, url, update_video_detail) unique_id = ret['data']['unique_id'] log_e( event_id, user, 'scraper', 'MSG', { 'url': url, 'dst_copy': dst_copy, 'other_copies': other_copies, 'dst_playlist': dst_playlist }) setEventOp('scraper') try: lock_id = "videoEdit:" + unique_id async with RedisLockAsync(rdb, lock_id): unique, conflicting_item = verifyUniqueness(unique_id) if unique or update_video_detail: async with _download_sem: ret = await parsed.get_metadata_async( parsed, url, update_video_detail) if repost_type: ret['data']['repost_type'] = repost_type else: ret['data']['repost_type'] = 'unknown' if ret["status"] == 'FAILED': log_e(event_id, user, 'downloader', 'WARN', { 'msg': 'FETCH_FAILED', 'ret': ret }) await _playlist_reorder_helper.post_video_failed( unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) return "FETCH_FAILED", ret else: unique_id = ret['data']['unique_id'] else: # build ret ret = makeResponseSuccess({ 'thumbnailURL': conflicting_item['item']['thumbnail_url'], 'title': conflicting_item['item']['title'], 'desc': conflicting_item['item']['desc'], 'site': conflicting_item['item']['site'], 'uploadDate': conflicting_item['item']['upload_time'], "unique_id": conflicting_item['item']['unique_id'], "utags": conflicting_item['item']['utags'] }) for k, v in conflicting_item['item'].items(): ret['data'][k] = v if 'part_name' in conflicting_item['item']: ret['part_name'] = conflicting_item['item']['part_name'] if 'repost_type' in conflicting_item[ 'item'] and conflicting_item['item']['repost_type']: ret['data']['repost_type'] = repost_type else: ret['data']['repost_type'] = 'unknown' tagdb.update_item_query( conflicting_item, {'$set': { 'item.repost_type': repost_type }}, user=makeUserMeta(user)) #if hasattr(parsed, 'LOCAL_CRAWLER') : # url = ret["data"]["url"] #else : # url = clear_url(url) use_override = False if field_override and '__condition' in field_override: condition = field_override['__condition'] del field_override['__condition'] if condition == 'any': use_override = True elif condition == 'placeholder' and 'placeholder' in ret[ "data"] and ret["data"]['placeholder']: use_override = True if use_override: for key in field_override: ret['data'][key] = field_override[key] playlists = [] #playlist_lock = None if dst_playlist: #playlist_lock = RedisLockAsync(rdb, "playlistEdit:" + str(dst_playlist)) #playlist_lock.acquire() if playlist_db.retrive_item(dst_playlist) is not None: playlists = [ObjectId(dst_playlist)] if not unique: log_e(event_id, user, 'scraper', level='MSG', obj={ 'msg': 'ALREADY_EXIST', 'unique_id': ret["data"]["unique_id"] }) """ Update existing video """ if update_video_detail: log_e(event_id, user, 'scraper', level='MSG', obj='Updating video detail') with MongoTransaction(client) as s: old_item = tagdb.retrive_item(conflicting_item['_id'], session=s())['item'] if old_item['thumbnail_url'] and old_item[ 'cover_image']: # old thumbnail exists, no need to download again new_detail = await _make_video_data_update( ret["data"], url, user, event_id) else: # old thumbnail does not exists, add to dict new_detail = await _make_video_data_update( ret["data"], url, user, event_id, ret["data"]["thumbnailURL"]) for key in new_detail.keys(): old_item[key] = new_detail[ key] # overwrite or add new field setEventUserAndID(user, event_id) tagdb.update_item_query(conflicting_item['_id'], {'$set': { 'item': old_item }}, ['title', 'desc'], user=makeUserMeta(user), session=s()) s.mark_succeed() return 'SUCCEED', conflicting_item['_id'] # this video already exist in the database # if the operation is to add a link to other copies and not adding self if (dst_copy and dst_copy != conflicting_item['_id']) or other_copies: log_e(event_id, user, 'scraper', level='MSG', obj='Adding to to copies') async with RedisLockAsync( rdb, 'editLink'), MongoTransaction(client) as s: log_e(event_id, user, level='MSG', obj='Adding to to copies, lock acquired') # find all copies of video dst_copy, self included all_copies = _getAllCopies(dst_copy, session=s()) # find all videos linked to source video all_copies += _getAllCopies(conflicting_item['_id'], session=s()) # add videos from other copies for uid in other_copies: all_copies += _getAllCopies(uid, session=s(), use_unique_id=True) # remove duplicated items all_copies = list(set(all_copies)) # add this video to all other copies found if len(all_copies) <= VideoConfig.MAX_COPIES: for dst_vid in all_copies: setEventUserAndID(user, event_id) _addThiscopy(dst_vid, all_copies, makeUserMeta(user), session=s()) log_e(event_id, user, 'scraper', level='MSG', obj='Successfully added to copies') s.mark_succeed() else: #if playlist_lock : # playlist_lock.release() log_e(event_id, user, 'scraper', level='MSG', obj='Too many copies') await _playlist_reorder_helper.post_video_failed( unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) return "TOO_MANY_COPIES", {} # if the operation is adding this video to playlist if dst_playlist: log_e(event_id, user, 'scraper', level='MSG', obj={ 'msg': 'Adding to playlist at position', 'rank': dst_rank }) if playlist_ordered: await _playlist_reorder_helper.post_video_succeed( conflicting_item['_id'], unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) else: setEventUserAndID(user, event_id) if dst_rank == -1: addVideoToPlaylist(dst_playlist, conflicting_item['_id'], user) else: insertIntoPlaylist(dst_playlist, conflicting_item['_id'], dst_rank, user) # merge tags async with MongoTransaction(client) as s: log_e(event_id, user, 'scraper', level='MSG', obj='Merging tags') setEventUserAndID(user, event_id) tagdb.update_item_tags_merge(conflicting_item['_id'], tags, makeUserMeta(user), session=s(), remove_tagids=[354]) s.mark_succeed() #if playlist_lock : # playlist_lock.release() #return "VIDEO_ALREADY_EXIST", conflicting_item['_id'] return "SUCCEED", conflicting_item['_id'] else: # expand dst_copy to all copies linked to dst_copy if dst_copy or other_copies: log_e(event_id, user, 'scraper', level='MSG', obj='Adding to to copies') async with RedisLockAsync( rdb, 'editLink'), MongoTransaction(client) as s: log_e(event_id, user, 'scraper', level='MSG', obj='Adding to to copies, lock acquired') all_copies = _getAllCopies(dst_copy, session=s()) # add videos from other copies for uid in other_copies: all_copies += _getAllCopies(uid, session=s(), use_unique_id=True) video_data = await _make_video_data( ret["data"], all_copies, playlists, url, user, event_id) setEventUserAndID(user, event_id) new_item_id = tagdb.add_item(tags, video_data, 3, ['title', 'desc'], makeUserMeta(user), session=s()) all_copies.append(ObjectId(new_item_id)) # remove duplicated items all_copies = list(set(all_copies)) if len(all_copies) <= VideoConfig.MAX_COPIES: for dst_vid in all_copies: setEventUserAndID(user, event_id) _addThiscopy(dst_vid, all_copies, makeUserMeta(user), session=s()) log_e(event_id, user, 'scraper', level='MSG', obj='Successfully added to copies') s.mark_succeed() else: #if playlist_lock : # playlist_lock.release() log_e(event_id, user, 'scraper', level='MSG', obj='Too many copies') await _playlist_reorder_helper.post_video_failed( unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) return "TOO_MANY_COPIES", {} else: async with MongoTransaction(client) as s: video_data = await _make_video_data( ret["data"], [], playlists, url, user, event_id) setEventUserAndID(user, event_id) if use_autotag: tags.extend( inferTagsFromVideo( video_data['utags'], video_data['title'], video_data['desc'], 'CHS', video_data['url'], video_data['user_space_urls'])) new_item_id = tagdb.add_item(tags, video_data, 3, ['title', 'desc'], makeUserMeta(user), session=s()) log_e(event_id, user, 'scraper', level='MSG', obj={ 'msg': 'New video added to database', 'vid': new_item_id }) s.mark_succeed() # if the operation is adding this video to playlist if dst_playlist: log_e(event_id, user, 'scraper', level='MSG', obj={ 'msg': 'Adding to playlist at position', 'rank': dst_rank }) if playlist_ordered: await _playlist_reorder_helper.post_video_succeed( new_item_id, unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) else: setEventUserAndID(user, event_id) if dst_rank == -1: addVideoToPlaylist(dst_playlist, new_item_id, user) else: insertIntoPlaylist(dst_playlist, new_item_id, dst_rank, user) #if playlist_lock : # playlist_lock.release() log_e(event_id, user, 'scraper', level='MSG', obj='Done') return 'SUCCEED', new_item_id except UserError as ue: await _playlist_reorder_helper.post_video_failed( unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) log_e(event_id, user, 'scraper', level='WARN', obj={'ue': str(ue)}) return ue.msg, {"aux": ue.aux, "traceback": traceback.format_exc()} except Exception as ex: await _playlist_reorder_helper.post_video_failed( unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) log_e(event_id, user, 'scraper', level='ERR', obj={'ex': str(ex)}) try: problematic_lock = RedisLockAsync(rdb, 'editLink') problematic_lock.reset() except: pass return "UNKNOWN", '\n'.join( [repr(traceback.format_exc()), repr(traceback.extract_stack())])
async def _add_to_playlist(self, dst_playlist, event_id, user_global): if self.playlist_map[dst_playlist]: dst_rank = self.playlist_map[dst_playlist]['rank'] playlist_ordered = self.playlist_map[dst_playlist]['all'] try: # fast method async with RedisLockAsync( rdb, "playlistEdit:" + dst_playlist), MongoTransaction(client) as s: cur_rank = 0 playlist = playlist_db.retrive_item(dst_playlist, session=s()) if playlist is None: raise UserError('PLAYLIST_NOT_EXIST') if playlist["item"]["videos"] + len( self.playlist_map[dst_playlist] ['succeed']) > PlaylistConfig.MAX_VIDEO_PER_PLAYLIST: raise UserError('VIDEO_LIMIT_EXCEEDED') playlist_videos = playlist["item"]['videos'] for unique_id in playlist_ordered: if unique_id in self.playlist_map[dst_playlist][ 'succeed']: (video_id, _, user) = self.playlist_map[ dst_playlist]['succeed'][unique_id] if dst_rank == -1: if filterOperation('editPlaylist', user, playlist, False): if addVideoToPlaylistLockFree( dst_playlist, video_id, user, playlist_videos, session=s()): playlist_videos += 1 else: if filterOperation('editPlaylist', user, playlist, False): if insertIntoPlaylistLockFree(dst_playlist, video_id, dst_rank + cur_rank, user, session=s()): cur_rank += 1 s.mark_succeed() except UserError as ue: # UserError, rereaise to upper level log_e(event_id, user_global, '_add_to_playlist', 'ERR', { 'ex': str(ex), 'tb': traceback.format_exc() }) del self.playlist_map[dst_playlist] rdb.set(f'playlist-batch-post-event-{dst_playlist}', b'done') raise ue except Exception as ex: # if anything goes wrong, fallback to slow method log_e(event_id, user_global, '_add_to_playlist', 'ERR', { 'ex': str(ex), 'tb': traceback.format_exc() }) cur_rank = 0 for unique_id in playlist_ordered: if unique_id in self.playlist_map[dst_playlist]['succeed']: (video_id, _, user) = self.playlist_map[dst_playlist][ 'succeed'][unique_id] # ignore error, add next video try: if dst_rank == -1: addVideoToPlaylist(dst_playlist, video_id, user) else: insertIntoPlaylist(dst_playlist, video_id, dst_rank + cur_rank, user) cur_rank += 1 except: pass log_e( event_id, user_global, '_add_to_playlist', 'MSG', { 'succedd': len(self.playlist_map[dst_playlist]['succeed']), 'all': len(self.playlist_map[dst_playlist]['all']), 'pid': dst_playlist }) del self.playlist_map[dst_playlist] rdb.set(f'playlist-batch-post-event-{dst_playlist}', b'done')
def addComment(user, thread_id: ObjectId, text: str, notification_type: str = 'comment_reply', use_bleach=True): # user can add comments filterOperation('postComment', user) if use_bleach: text = bleach.clean(text, tags=[], attributes=[], styles=[]) l = len(text) if l > Comments.MAX_COMMENT_LENGTH_LONG: raise UserError('COMMENT_TOO_LONG') elif l > Comments.MAX_COMMENT_LENGTH_REGULAR and not filterOperation( 'postLongComment', user, raise_exception=False): raise UserError('COMMENT_TOO_LONG') thread_obj = db.comment_threads.find_one({'_id': thread_id}) if thread_obj is None: raise UserError('THREAD_NOT_EXIST') with redis_lock.Lock(rdb, "thread:" + str(thread_id)), MongoTransaction(client) as s: cid = str( db.comment_items.insert_one( { 'thread': thread_id, 'content': text, 'hidden': False, 'deleted': False, 'pinned': False, 'upvotes': 0, 'downvotes': 0, 'meta': makeUserMetaObject(user) }, session=s()).inserted_id) db.comment_threads.update_one({'_id': thread_id}, {'$inc': { 'count': int(1) }}, session=s()) note_obj = { "cid": ObjectId(cid), "replied_by": makeUserMeta(user), "content": text[:Comments.NOTIFICATION_CONTENT_LENGTH] } # =========================================================== if 'obj_type' in thread_obj and 'obj_id' in thread_obj: note_obj['replied_type'] = thread_obj['obj_type'] note_obj['replied_obj'] = thread_obj['obj_id'] else: obj = db.videos.find_one({'comment_thread': thread_id}, session=s()) if obj: note_obj['replied_type'] = 'video' note_obj['replied_obj'] = obj['_id'] db.comment_threads.update_one( {'_id': thread_id}, {'$set': { 'obj_type': 'video', 'obj_id': obj['_id'] }}, session=s()) else: obj = playlist_db.retrive_item({'comment_thread': thread_id}, session=s()) if obj: note_obj['replied_type'] = 'playlist' note_obj['replied_obj'] = obj['_id'] db.comment_threads.update_one({'_id': thread_id}, { '$set': { 'obj_type': 'playlist', 'obj_id': obj['_id'] } }, session=s()) else: obj = db.users.find_one({'comment_thread': thread_id}, session=s()) if obj: note_obj['replied_type'] = 'user' note_obj['replied_obj'] = obj['_id'] db.comment_threads.update_one({'_id': thread_id}, { '$set': { 'obj_type': 'user', 'obj_id': obj['_id'] } }, session=s()) else: log(level='ERR', obj={ 'msg': 'orphan thread found!!', 'thread_id': thread_id, 'thread_obj': thread_obj }) raise UserError('UNKNOWN_ERROR') # =========================================================== if notification_type and thread_obj['owner'] != user[ '_id']: # empty means do not notify user createNotification(notification_type, thread_obj['owner'], session=s(), other=note_obj) if note_obj[ 'replied_type'] == 'forum': # forum comment, set modified_at date db.forum_threads.update_one({'_id': note_obj['replied_obj']}, { '$set': { 'meta.modified_at': datetime.now(), 'meta.modified_by': user['_id'] } }, session=s()) s.mark_succeed() return cid
def addReply(user, reply_to: ObjectId, text: str, notification_type: str = 'comment_reply', use_bleach=True): # user can add comments """ reply_to: comment id """ filterOperation('postComment', user) if use_bleach: text = bleach.clean(text, tags=[], attributes=[], styles=[]) l = len(text) if l > Comments.MAX_COMMENT_LENGTH_LONG: raise UserError('COMMENT_TOO_LONG') elif l > Comments.MAX_COMMENT_LENGTH_REGULAR and not filterOperation( 'postLongComment', user, raise_exception=False): raise UserError('COMMENT_TOO_LONG') parent_obj = db.comment_items.find_one({'_id': reply_to}) if parent_obj is None: raise UserError('PARENT_NOT_EXIST') with MongoTransaction(client) as s: if 'thread' in parent_obj: # reply to primary comment cid = str( db.comment_items.insert_one( { 'parent': reply_to, 'content': text, 'hidden': False, 'deleted': False, 'pinned': False, 'upvotes': 0, 'downvotes': 0, 'meta': makeUserMetaObject(user) }, session=s()).inserted_id) thread_obj = db.comment_threads.find_one( {'_id': parent_obj['thread']}, session=s()) if thread_obj is None: log(level='ERR', obj={ 'msg': 'orphan comment found!!', 'cid': parent_obj['_id'], 'obj': parent_obj }) raise UserError('UNKNOWN_ERROR') thread_id = thread_obj['_id'] else: # reply to secondary comment cid = str( db.comment_items.insert_one( { 'parent': parent_obj['parent'], 'reply_to': reply_to, 'content': text, 'hidden': False, 'deleted': False, 'pinned': False, 'upvotes': 0, 'downvotes': 0, 'meta': makeUserMetaObject(user) }, session=s()).inserted_id) parent_parent_obj = db.comment_items.find_one( {'_id': parent_obj['parent']}, session=s()) if parent_parent_obj is None: log(level='ERR', obj={ 'msg': 'orphan comment found!!', 'cid': parent_obj['_id'], 'obj': parent_obj }) raise UserError('UNKNOWN_ERROR') thread_obj = db.comment_threads.find_one( {'_id': parent_parent_obj['thread']}, session=s()) if thread_obj is None: log(level='ERR', obj={ 'msg': 'orphan comment found!!', 'cid': parent_parent_obj['_id'], 'obj': parent_parent_obj }) raise UserError('UNKNOWN_ERROR') thread_id = thread_obj['_id'] note_obj = { "cid": ObjectId(cid), "replied_by": makeUserMeta(user), "content": text[:Comments.NOTIFICATION_CONTENT_LENGTH] } # =========================================================== if 'obj_type' in thread_obj and 'obj_id' in thread_obj: note_obj['replied_type'] = thread_obj['obj_type'] note_obj['replied_obj'] = thread_obj['obj_id'] else: obj = db.videos.find_one({'comment_thread': thread_id}, session=s()) if obj: note_obj['replied_type'] = 'video' note_obj['replied_obj'] = obj['_id'] db.comment_threads.update_one( {'_id': thread_id}, {'$set': { 'obj_type': 'video', 'obj_id': obj['_id'] }}, session=s()) else: obj = playlist_db.retrive_item({'comment_thread': thread_id}, session=s()) if obj: note_obj['replied_type'] = 'playlist' note_obj['replied_obj'] = obj['_id'] db.comment_threads.update_one({'_id': thread_id}, { '$set': { 'obj_type': 'playlist', 'obj_id': obj['_id'] } }, session=s()) else: obj = db.users.find_one({'comment_thread': thread_id}, session=s()) if obj: note_obj['replied_type'] = 'user' note_obj['replied_obj'] = obj['_id'] db.comment_threads.update_one({'_id': thread_id}, { '$set': { 'obj_type': 'user', 'obj_id': obj['_id'] } }, session=s()) else: log(level='ERR', obj={ 'msg': 'orphan thread found!!', 'thread_id': thread_id, 'thread_obj': thread_obj }) raise UserError('UNKNOWN_ERROR') # =========================================================== if notification_type and parent_obj['meta']['created_by'] != user[ '_id']: # empty means do not notify user and we don't want to notify users who reply to themselves createNotification(notification_type, parent_obj['meta']['created_by'], session=s(), other=note_obj) if note_obj[ 'replied_type'] == 'forum': # forum reply, set modified_at date db.forum_threads.update_one({'_id': note_obj['replied_obj']}, { '$set': { 'meta.modified_at': datetime.now(), 'meta.modified_by': user['_id'] } }, session=s()) s.mark_succeed()
def listCommonTagIDs(pid, user): playlist = playlist_db.retrive_item(pid) if playlist is None: raise UserError('PLAYLIST_NOT_EXIST') if playlist['item']['private']: filterOperation('viewPrivatePlaylist', user, playlist) result = db.playlist_items.aggregate([{ "$match": { "pid": ObjectId(pid) } }, { "$lookup": { "from": 'videos', "localField": "vid", "foreignField": "_id", "as": "video" } }, { "$project": { "video.tags": 1 } }, { "$unwind": { "path": "$video" } }, { "$project": { "tags": { '$filter': { 'input': '$video.tags', 'as': 'tag', 'cond': { '$lt': ['$$tag', 0x80000000] } } } } }, { "$group": { "_id": 0, "tags": { "$push": "$tags" }, "initialTags": { "$first": "$tags" } } }, { "$project": { "commonTags": { "$reduce": { "input": "$tags", "initialValue": "$initialTags", "in": { "$setIntersection": ["$$value", "$$this"] } } } } }]) ret = [i for i in result] if ret: return ret[0]['commonTags'] else: return []