def postThreadToForum(user, forum_id: ObjectId, title: str, text: str, use_bleach=True): # create a thread and post a comment filterOperation('postThreadToForum', user) if db.forum_metas.find_one({'_id': forum_id}) is None: raise UserError('FORUM_NOT_EXIST') if len(title) > Forums.MAX_TITLE_LENGTH: raise UserError('TITLE_TOO_LONG') l = len(text) if l > Comments.MAX_COMMENT_LENGTH_LONG: raise UserError('COMMENT_TOO_LONG') elif l > Comments.MAX_COMMENT_LENGTH_REGULAR and not filterOperation( 'postLongComment', user, raise_exception=False): raise UserError('COMMENT_TOO_LONG') if use_bleach: text = bleach.clean(text, tags=[], attributes=[], styles=[]) with MongoTransaction(client) as s: thread_id = createThread('forum', None, user['_id'], session=s()) ftid = ObjectId( db.forum_threads.insert_one( { 'forum_id': forum_id, 'title': title, 'tid': thread_id, 'hidden': False, 'deleted': False, 'pinned': False, 'meta': makeUserMetaObject(user) }, session=s()).inserted_id) db.comment_items.insert_one( { 'thread': thread_id, 'content': text, 'hidden': False, 'deleted': False, 'pinned': False, 'upvotes': 0, 'downvotes': 0, 'meta': makeUserMetaObject(user) }, session=s()) db.comment_threads.update_one({'_id': thread_id}, { '$inc': { 'count': int(1) }, '$set': { 'obj_id': ftid } }, session=s()) s.mark_succeed() return str(ftid)
def createPlaylist(language, title, desc, cover, user, private=False): log(obj={'title': title, 'desc': desc, 'cover': cover, 'private': private}) filterOperation('createPlaylist', user) if len(title) > PlaylistConfig.MAX_TITLE_LENGTH: raise UserError('TITLE_TOO_LONG') if len(desc) > PlaylistConfig.MAX_DESC_LENGTH: raise UserError('DESC_TOO_LONG') if len(cover) > PlaylistConfig.MAX_COVER_URL_LENGTH: raise UserError('URL_TOO_LONG') if not cover: cover = 'default-cover.jpg' if not title: raise UserError('EMPTY_TITLE') if not desc: raise UserError('EMPTY_DESC') obj = { "title": { language: title }, "desc": { language: desc }, "private": private, "views": int(0), "videos": int(0), "cover": cover, "meta": makeUserMetaObject(user) } pid = db.playlists.insert_one(obj) log(obj={'pid': pid.inserted_id}) return str(pid.inserted_id)
def addPlaylistsToFolder(user, path, playlists) : _verifyPath(path) with redis_lock.Lock(rdb, f"folderEdit:{str(makeUserMeta(user))}:{path}"), MongoTransaction(client) as s : folder_obj = _findFolder(user, path) filterOperation('addPlaylistsToFolder', user, folder_obj) for pid in playlists : playlist = playlist_db.retrive_item(pid, session = s()) if playlist is None : continue # skip non-exist playlist if playlist['item']['private'] and not filterOperation('viewPrivatePlaylist', user, playlist, raise_exception = False) : continue # skip other's private playlist playlist_path = path + "\\" + str(playlist['_id']) + "\\/" if _findFolder(user, playlist_path, raise_exception = False) : continue # skip duplicated playlist playlist_obj = { 'user': makeUserMeta(user), 'leaf': True, 'playlist': playlist['_id'], 'name': None, 'path': playlist_path, 'privateView': folder_obj['privateView'], 'privateEdit': folder_obj['privateEdit'], 'meta': makeUserMetaObject(user) } db.playlist_folders.insert_one(playlist_obj, session = s()) db.playlist_folders.update_one({'_id': folder_obj['_id']}, {'$set': { 'meta.modified_by': makeUserMeta(user), 'meta.modified_at': datetime.now() }}, session = s()) s.mark_succeed()
def createFolder(user, root, name, privateView=False, privateEdit=True): filterOperation('createFolder', user, root) _verifyPath(root) _verifyFolderName(name) with redis_lock.Lock(rdb, f"folderEdit:{str(makeUserMeta(user))}:{root}"): _findFolder(user, root) fullpath = root + name + "/" with redis_lock.Lock( rdb, f"folderEdit:{str(makeUserMeta(user))}:{fullpath}" ), MongoTransaction(client) as s: obj = _findFolder(user, fullpath, raise_exception=False) if obj: raise UserError('FOLDER_ALREADY_EXIST') folder_obj = { 'user': makeUserMeta(user), 'leaf': False, 'playlist': None, 'name': name, 'path': fullpath, 'privateView': privateView, 'privateEdit': privateEdit, 'meta': makeUserMetaObject(user) } db.playlist_folders.insert_one(folder_obj, session=s()) s.mark_succeed()
def requestSubtitleOCR(user, vid: ObjectId): # step 1: verify user and video filterOperation('requestSubtitleOCR', user) video_item = tagdb.retrive_item(vid) if video_item is None: raise UserError('VIDEO_NOT_FOUND') # step 2: check if request exists with redis_lock.Lock( rdb, "videoEdit:" + video_item['item']['unique_id']), redis_lock.Lock( rdb, "mmdocr_global_lock"), MongoTransaction(client) as s: ocr_record = db.subtitle_ocr.find_one({"vid": vid}, session=s()) if ocr_record is None: # create new record record = { "vid": vid, "status": "Queuing", "version": 0, # version is set in postSubtitleOCRResult "worker_id": "", # worker_id is set in queryAndProcessQueuingRequests "meta": makeUserMetaObject(user) } db.subtitle_ocr.insert_one(record, session=s()) else: status = ocr_record['status'] record_id = ocr_record['_id'] record_version = ocr_record['version'] mmdocr_version = int(Config.MMDOCR_VERSION) if status in ['NoRecord', 'RecordOutOfDate', 'Error']: assert status != "NoRecord" db.subtitle_ocr.update_one({"_id": record_id}, { "$set": { "status": "Queuing", "meta.modified_at": datetime.utcnow(), "meta.modified_by": ObjectId(user['_id']) } }, session=s()) pass elif status == 'RecordExists': if record_version < mmdocr_version: # newer version of mmdocr exists db.subtitle_ocr.update_one({"_id": record_id}, { "$set": { "status": "Queuing", "meta.modified_at": datetime.utcnow(), "meta.modified_by": ObjectId(user['_id']) } }, session=s()) pass else: raise UserError('RECORD_ALREADY_EXISTS') else: raise UserError('VIDEO_BEING_PROCESSED') s.mark_succeed()
def addSubscription(user, query_str : str, qtype = 'tag', name = '') : query_str = query_str.strip() if not query_str : raise UserError('EMPTY_QUERY') # TODO: add duplicated query check qobj, qtags = tagdb.compile_query(query_str, qtype) if len(qtags) == 1 and 'tags' in qobj and isinstance(qobj['tags'], int) and qobj['tags'] < 0x80000000: subid = db.subs.insert_one({'qs': query_str.strip(), 'qt': qtype, 'name': name, 'tagid': qobj['tags'], 'meta': makeUserMetaObject(user)}).inserted_id else : subid = db.subs.insert_one({'qs': query_str, 'qt': qtype, 'name': name, 'meta': makeUserMetaObject(user)}).inserted_id return str(subid)
def postSubtitle(user, vid: ObjectId, language: str, subformat: str, content: str): if language not in VALID_LANGUAGES: raise UserError('INVALID_LANGUAGE') subformat = subformat.lower() if subformat not in VALID_SUBTITLE_FORMAT: raise UserError('INVALID_SUBTITLE_FORMAT') video_item = tagdb.retrive_item(vid) if video_item is None: raise UserError('VIDEO_NOT_FOUND') try: size = len(content.encode('utf-8')) except: size = -1 filterOperation('postSubtitle', user) with redis_lock.Lock( rdb, "videoEdit:" + video_item['item']['unique_id']), MongoTransaction(client) as s: existing_item = db.subtitles.find_one( { 'vid': vid, 'lang': language, 'format': subformat, 'meta.created_by': makeUserMeta(user) }, session=s()) if existing_item is None: subid = db.subtitles.insert_one( { 'vid': vid, 'lang': language, 'format': subformat, 'content': content, 'size': size, 'deleted': False, 'autogen': False, 'version': 0, 'meta': makeUserMetaObject(user) }, session=s()).inserted_id else: db.subtitles.update_one({'_id': existing_item['_id']}, { '$set': { 'content': content, 'size': size, 'meta.modified_at': datetime.utcnow() } }, session=s()) subid = existing_item['_id'] s.mark_succeed() return ObjectId(subid)
def postSubtitleOCRResult(user, unique_id: str, content: str, subformat: str, version: int, worker_id: str): # step 1: verify and post filterOperation('subtitleocr_postSubtitleOCRResult', user) subformat = subformat.lower() if subformat not in VALID_SUBTITLE_FORMAT: raise UserError('INVALID_SUBTITLE_FORMAT') video_item = tagdb.retrive_item({"item.unique_id": unique_id}) if video_item is None: raise UserError('VIDEO_NOT_FOUND') try: size = len(content.encode('utf-8')) except: size = -1 with redis_lock.Lock( rdb, "videoEdit:" + video_item['item']['unique_id']), redis_lock.Lock( rdb, "mmdocr_global_lock"), MongoTransaction(client) as s: # delete old versions db.subtitles.delete_many({ 'vid': video_item['_id'], 'autogen': True }, session=s()) subid = db.subtitles.insert_one( { 'vid': video_item['_id'], 'lang': 'UNKNOWN', 'format': subformat, 'content': content, 'size': size, 'deleted': False, 'version': version, 'autogen': True, 'meta': makeUserMetaObject(None) }, session=s()).inserted_id # step 2: update subtitle_ocr db.subtitle_ocr.update_one({"vid": video_item['_id']}, { "$set": { "status": "RecordExists", "version": version, "worker_id": worker_id } }, session=s()) s.mark_succeed() return subid
def addComment(user, thread_id: ObjectId, text: str, notification_type: str = 'comment_reply', use_bleach=True): # user can add comments filterOperation('postComment', user) if use_bleach: text = bleach.clean(text, tags=[], attributes=[], styles=[]) l = len(text) if l > Comments.MAX_COMMENT_LENGTH_LONG: raise UserError('COMMENT_TOO_LONG') elif l > Comments.MAX_COMMENT_LENGTH_REGULAR and not filterOperation( 'postLongComment', user, raise_exception=False): raise UserError('COMMENT_TOO_LONG') thread_obj = db.comment_threads.find_one({'_id': thread_id}) if thread_obj is None: raise UserError('THREAD_NOT_EXIST') with redis_lock.Lock(rdb, "thread:" + str(thread_id)), MongoTransaction(client) as s: cid = str( db.comment_items.insert_one( { 'thread': thread_id, 'content': text, 'hidden': False, 'deleted': False, 'pinned': False, 'upvotes': 0, 'downvotes': 0, 'meta': makeUserMetaObject(user) }, session=s()).inserted_id) db.comment_threads.update_one({'_id': thread_id}, {'$inc': { 'count': int(1) }}, session=s()) note_obj = { "cid": ObjectId(cid), "replied_by": makeUserMeta(user), "content": text[:Comments.NOTIFICATION_CONTENT_LENGTH] } # =========================================================== if 'obj_type' in thread_obj and 'obj_id' in thread_obj: note_obj['replied_type'] = thread_obj['obj_type'] note_obj['replied_obj'] = thread_obj['obj_id'] else: obj = db.videos.find_one({'comment_thread': thread_id}, session=s()) if obj: note_obj['replied_type'] = 'video' note_obj['replied_obj'] = obj['_id'] db.comment_threads.update_one( {'_id': thread_id}, {'$set': { 'obj_type': 'video', 'obj_id': obj['_id'] }}, session=s()) else: obj = playlist_db.retrive_item({'comment_thread': thread_id}, session=s()) if obj: note_obj['replied_type'] = 'playlist' note_obj['replied_obj'] = obj['_id'] db.comment_threads.update_one({'_id': thread_id}, { '$set': { 'obj_type': 'playlist', 'obj_id': obj['_id'] } }, session=s()) else: obj = db.users.find_one({'comment_thread': thread_id}, session=s()) if obj: note_obj['replied_type'] = 'user' note_obj['replied_obj'] = obj['_id'] db.comment_threads.update_one({'_id': thread_id}, { '$set': { 'obj_type': 'user', 'obj_id': obj['_id'] } }, session=s()) else: log(level='ERR', obj={ 'msg': 'orphan thread found!!', 'thread_id': thread_id, 'thread_obj': thread_obj }) raise UserError('UNKNOWN_ERROR') # =========================================================== if notification_type and thread_obj['owner'] != user[ '_id']: # empty means do not notify user createNotification(notification_type, thread_obj['owner'], session=s(), other=note_obj) if note_obj[ 'replied_type'] == 'forum': # forum comment, set modified_at date db.forum_threads.update_one({'_id': note_obj['replied_obj']}, { '$set': { 'meta.modified_at': datetime.now(), 'meta.modified_by': user['_id'] } }, session=s()) s.mark_succeed() return cid
def addReply(user, reply_to: ObjectId, text: str, notification_type: str = 'comment_reply', use_bleach=True): # user can add comments """ reply_to: comment id """ filterOperation('postComment', user) if use_bleach: text = bleach.clean(text, tags=[], attributes=[], styles=[]) l = len(text) if l > Comments.MAX_COMMENT_LENGTH_LONG: raise UserError('COMMENT_TOO_LONG') elif l > Comments.MAX_COMMENT_LENGTH_REGULAR and not filterOperation( 'postLongComment', user, raise_exception=False): raise UserError('COMMENT_TOO_LONG') parent_obj = db.comment_items.find_one({'_id': reply_to}) if parent_obj is None: raise UserError('PARENT_NOT_EXIST') with MongoTransaction(client) as s: if 'thread' in parent_obj: # reply to primary comment cid = str( db.comment_items.insert_one( { 'parent': reply_to, 'content': text, 'hidden': False, 'deleted': False, 'pinned': False, 'upvotes': 0, 'downvotes': 0, 'meta': makeUserMetaObject(user) }, session=s()).inserted_id) thread_obj = db.comment_threads.find_one( {'_id': parent_obj['thread']}, session=s()) if thread_obj is None: log(level='ERR', obj={ 'msg': 'orphan comment found!!', 'cid': parent_obj['_id'], 'obj': parent_obj }) raise UserError('UNKNOWN_ERROR') thread_id = thread_obj['_id'] else: # reply to secondary comment cid = str( db.comment_items.insert_one( { 'parent': parent_obj['parent'], 'reply_to': reply_to, 'content': text, 'hidden': False, 'deleted': False, 'pinned': False, 'upvotes': 0, 'downvotes': 0, 'meta': makeUserMetaObject(user) }, session=s()).inserted_id) parent_parent_obj = db.comment_items.find_one( {'_id': parent_obj['parent']}, session=s()) if parent_parent_obj is None: log(level='ERR', obj={ 'msg': 'orphan comment found!!', 'cid': parent_obj['_id'], 'obj': parent_obj }) raise UserError('UNKNOWN_ERROR') thread_obj = db.comment_threads.find_one( {'_id': parent_parent_obj['thread']}, session=s()) if thread_obj is None: log(level='ERR', obj={ 'msg': 'orphan comment found!!', 'cid': parent_parent_obj['_id'], 'obj': parent_parent_obj }) raise UserError('UNKNOWN_ERROR') thread_id = thread_obj['_id'] note_obj = { "cid": ObjectId(cid), "replied_by": makeUserMeta(user), "content": text[:Comments.NOTIFICATION_CONTENT_LENGTH] } # =========================================================== if 'obj_type' in thread_obj and 'obj_id' in thread_obj: note_obj['replied_type'] = thread_obj['obj_type'] note_obj['replied_obj'] = thread_obj['obj_id'] else: obj = db.videos.find_one({'comment_thread': thread_id}, session=s()) if obj: note_obj['replied_type'] = 'video' note_obj['replied_obj'] = obj['_id'] db.comment_threads.update_one( {'_id': thread_id}, {'$set': { 'obj_type': 'video', 'obj_id': obj['_id'] }}, session=s()) else: obj = playlist_db.retrive_item({'comment_thread': thread_id}, session=s()) if obj: note_obj['replied_type'] = 'playlist' note_obj['replied_obj'] = obj['_id'] db.comment_threads.update_one({'_id': thread_id}, { '$set': { 'obj_type': 'playlist', 'obj_id': obj['_id'] } }, session=s()) else: obj = db.users.find_one({'comment_thread': thread_id}, session=s()) if obj: note_obj['replied_type'] = 'user' note_obj['replied_obj'] = obj['_id'] db.comment_threads.update_one({'_id': thread_id}, { '$set': { 'obj_type': 'user', 'obj_id': obj['_id'] } }, session=s()) else: log(level='ERR', obj={ 'msg': 'orphan thread found!!', 'thread_id': thread_id, 'thread_obj': thread_obj }) raise UserError('UNKNOWN_ERROR') # =========================================================== if notification_type and parent_obj['meta']['created_by'] != user[ '_id']: # empty means do not notify user and we don't want to notify users who reply to themselves createNotification(notification_type, parent_obj['meta']['created_by'], session=s(), other=note_obj) if note_obj[ 'replied_type'] == 'forum': # forum reply, set modified_at date db.forum_threads.update_one({'_id': note_obj['replied_obj']}, { '$set': { 'meta.modified_at': datetime.now(), 'meta.modified_by': user['_id'] } }, session=s()) s.mark_succeed()
def addComment(user, thread_id: ObjectId, text: str): # user can add comments # TODO notify user being replied to filterOperation('postComment', user) l = len(text) if l > Comments.MAX_COMMENT_LENGTH_LONG: raise UserError('COMMENT_TOO_LONG') elif l > Comments.MAX_COMMENT_LENGTH_REGULAR and not filterOperation( 'postLongComment', user, raise_exception=False): raise UserError('COMMENT_TOO_LONG') thread_obj = db.comment_threads.find_one({'_id': thread_id}) if thread_obj is None: raise UserError('THREAD_NOT_EXIST') with redis_lock.Lock(rdb, "thread:" + str(thread_id)), MongoTransaction(client) as s: cid = str( db.comment_items.insert_one( { 'thread': thread_id, 'content': text, 'hidden': False, 'deleted': False, 'upvotes': 0, 'downvotes': 0, 'meta': makeUserMetaObject(user) }, session=s()).inserted_id) db.comment_threads.update_one({'_id': thread_id}, {'$inc': { 'count': int(1) }}, session=s()) note_obj = { "cid": ObjectId(cid), "replied_by": makeUserMeta(user), "content": text[:Comments.NOTIFICATION_CONTENT_LENGTH] } # =========================================================== if 'obj_type' in thread_obj and 'obj_id' in thread_obj: note_obj['replied_type'] = thread_obj['obj_type'] note_obj['replied_obj'] = thread_obj['obj_id'] else: obj = db.items.find_one({'comment_thread': thread_id}, session=s()) if obj: note_obj['replied_type'] = 'video' note_obj['replied_obj'] = obj['_id'] db.comment_threads.update_one( {'_id': thread_id}, {'$set': { 'obj_type': 'video', 'obj_id': obj['_id'] }}, session=s()) else: obj = db.playlists.find_one({'comment_thread': thread_id}, session=s()) if obj: note_obj['replied_type'] = 'playlist' note_obj['replied_obj'] = obj['_id'] db.comment_threads.update_one({'_id': thread_id}, { '$set': { 'obj_type': 'playlist', 'obj_id': obj['_id'] } }, session=s()) else: obj = db.users.find_one({'comment_thread': thread_id}, session=s()) if obj: note_obj['replied_type'] = 'user' note_obj['replied_obj'] = obj['_id'] db.comment_threads.update_one({'_id': thread_id}, { '$set': { 'obj_type': 'user', 'obj_id': obj['_id'] } }, session=s()) else: log(level='ERR', obj={ 'msg': 'orphan thread found!!', 'thread_id': thread_id, 'thread_obj': thread_obj }) raise UserError('UNKNOWN_ERROR') # =========================================================== createNotification('comment_reply', thread_obj['owner'], session=s(), other=note_obj) s.mark_succeed() return cid
def addReply(user, reply_to: ObjectId, text: str): # user can add comments """ reply_to: comment id """ filterOperation('postComment', user) # TODO notify user being replied to l = len(text) if l > Comments.MAX_COMMENT_LENGTH_LONG: raise UserError('COMMENT_TOO_LONG') elif l > Comments.MAX_COMMENT_LENGTH_REGULAR and not filterOperation( 'postLongComment', user, raise_exception=False): raise UserError('COMMENT_TOO_LONG') parent_obj = db.comment_items.find_one({'_id': reply_to}) if parent_obj is None: raise UserError('PARENT_NOT_EXIST') with MongoTransaction(client) as s: if 'thread' in parent_obj: # reply to primary comment cid = str( db.comment_items.insert_one( { 'parent': reply_to, 'content': text, 'hidden': False, 'deleted': False, 'upvotes': 0, 'downvotes': 0, 'meta': makeUserMetaObject(user) }, session=s()).inserted_id) thread_obj = db.comment_threads.find_one( {'_id': parent_obj['thread']}, session=s()) if thread_obj is None: log(level='ERR', obj={ 'msg': 'orphan comment found!!', 'cid': parent_obj['_id'], 'obj': parent_obj }) raise UserError('UNKNOWN_ERROR') thread_id = thread_obj['_id'] else: # reply to secondary comment cid = str( db.comment_items.insert_one( { 'parent': parent_obj['parent'], 'reply_to': reply_to, 'content': text, 'hidden': False, 'deleted': False, 'upvotes': 0, 'downvotes': 0, 'meta': makeUserMetaObject(user) }, session=s()).inserted_id) parent_parent_obj = db.comment_items.find_one( {'_id': parent_obj['parent']}, session=s()) if parent_parent_obj is None: log(level='ERR', obj={ 'msg': 'orphan comment found!!', 'cid': parent_obj['_id'], 'obj': parent_obj }) raise UserError('UNKNOWN_ERROR') thread_obj = db.comment_threads.find_one( {'_id': parent_parent_obj['thread']}, session=s()) if thread_obj is None: log(level='ERR', obj={ 'msg': 'orphan comment found!!', 'cid': parent_parent_obj['_id'], 'obj': parent_parent_obj }) raise UserError('UNKNOWN_ERROR') thread_id = thread_obj['_id'] note_obj = { "cid": ObjectId(cid), "replied_by": makeUserMeta(user), "content": text[:Comments.NOTIFICATION_CONTENT_LENGTH] } # =========================================================== if 'obj_type' in thread_obj and 'obj_id' in thread_obj: note_obj['replied_type'] = thread_obj['obj_type'] note_obj['replied_obj'] = thread_obj['obj_id'] else: obj = db.items.find_one({'comment_thread': thread_id}, session=s()) if obj: note_obj['replied_type'] = 'video' note_obj['replied_obj'] = obj['_id'] db.comment_threads.update_one( {'_id': thread_id}, {'$set': { 'obj_type': 'video', 'obj_id': obj['_id'] }}, session=s()) else: obj = db.playlists.find_one({'comment_thread': thread_id}, session=s()) if obj: note_obj['replied_type'] = 'playlist' note_obj['replied_obj'] = obj['_id'] db.comment_threads.update_one({'_id': thread_id}, { '$set': { 'obj_type': 'playlist', 'obj_id': obj['_id'] } }, session=s()) else: obj = db.users.find_one({'comment_thread': thread_id}, session=s()) if obj: note_obj['replied_type'] = 'user' note_obj['replied_obj'] = obj['_id'] db.comment_threads.update_one({'_id': thread_id}, { '$set': { 'obj_type': 'user', 'obj_id': obj['_id'] } }, session=s()) else: log(level='ERR', obj={ 'msg': 'orphan thread found!!', 'thread_id': thread_id, 'thread_obj': thread_obj }) raise UserError('UNKNOWN_ERROR') # =========================================================== createNotification('comment_reply', parent_obj['meta']['created_by'], session=s(), other=note_obj) s.mark_succeed()