def update_item(self, item_id, item, user = '', session = None):
item = self.db.items.find_one({'_id': ObjectId(item_id)}, session = session)
if item is None:
raise UserError('ITEM_NOT_EXIST')
self.db.items.update_one({'_id': ObjectId(item_id)}, {'$set': {'item': item, 'meta.modified_by': user, 'meta.modified_at': datetime.now()}}, session = session)
def add_or_rename_tag(self, tag_name, new_tag_name, language, user = '', session = None) :
self._check_language(language)
tag_obj = self._tag(tag_name, session = session)
new_tag_alias_obj = self.db.tag_alias.find_one({'tag': new_tag_name}, session = session)
if new_tag_alias_obj is not None and new_tag_alias_obj['dst'] != tag_obj['_id'] :
raise UserError('TAG_ALREADY_EXIST')
if new_tag_alias_obj is None :
if not isinstance(tag_name, int) :
rc, lang_referenced = self._get_tag_name_reference_count(tag_name, tag_obj)
assert rc > 0
# if it is only referenced once AND it is exactly referenced by the given language
# then it is a unique rename operation, we have to delete the old name
if rc == 1 and lang_referenced == language :
self.db.tag_alias.update_one({'tag': tag_name}, {
'$set': {
'tag': new_tag_name,
'meta.modified_by': user, 'meta.modified_at': datetime.now()
}
}, session = session)
self.aci.DeleteWord(tag_name)
self.aci.AddWord([(tag_obj['id'], new_tag_name, language)])
else :
self.db.tag_alias.insert_one({
'tag': new_tag_name,
'dst': tag_obj['_id'],
'meta': {'created_by': user, 'created_at': datetime.now(), 'modified_by': user, 'modified_at': datetime.now()}
}, session = session)
self.aci.AddWord([(tag_obj['id'], new_tag_name, language)])
else :
self.db.tag_alias.insert_one({
'tag': new_tag_name,
'dst': tag_obj['_id'],
'meta': {'created_by': user, 'created_at': datetime.now(), 'modified_by': user, 'modified_at': datetime.now()}
}, session = session)
self.aci.AddWord([(tag_obj['id'], new_tag_name, language)])
else :
# since tag_alias already exists for new_tag_name, no need to insert new tag_alias
# but we need to consider whether or not to delete the old one
if not isinstance(tag_name, int) :
rc, lang_referenced = self._get_tag_name_reference_count(tag_name, tag_obj)
assert rc > 0
if lang_referenced is None :
# we have an alias with the same name
raise UserError('TAG_ALREADY_EXIST')
# delete ONLY IF it is referenced only once AND it is exactly referenced by the given language
if rc == 1 and lang_referenced == language and tag_name != new_tag_name :
self.db.tag_alias.delete_one({'tag': tag_name}, session = session)
self.aci.DeleteWord(tag_name)
else :
rc, lang_referenced = self._get_tag_name_reference_count(new_tag_name, tag_obj)
assert rc > 0
if lang_referenced is None :
# we have an alias with the same name
raise UserError('TAG_ALREADY_EXIST')
# add or update tag specified by language
self.db.tags.update_one({'_id': tag_obj['_id']}, {
'$set': {
f'languages.{language}': new_tag_name,
'meta.modified_by': user, 'meta.modified_at': datetime.now()
}
}, session = session)
def verify_tags(self, tags, session = None) :
found_tags = self.db.tag_alias.find({'tag': {'$in': tags}}, session = session)
tm = [tag['tag'] for tag in found_tags]
for tag in tags :
if tag not in tm :
raise UserError('TAG_NOT_EXIST', tag)
def login(username, password, challenge, login_session_id):
log(
obj={
'username': username,
'challenge': challenge,
'login_session_id': login_session_id
})
if len(username) > UserConfig.MAX_USERNAME_LENGTH:
raise UserError('USERNAME_TOO_LONG')
if len(username) < UserConfig.MIN_USERNAME_LENGTH:
raise UserError('USERNAME_TOO_SHORT')
if len(password) > UserConfig.MAX_PASSWORD_LENGTH:
raise UserError('PASSWORD_TOO_LONG')
if len(password) < UserConfig.MIN_PASSWORD_LENGTH:
raise UserError('PASSWORD_TOO_SHORT')
if verify_session(login_session_id, 'LOGIN'):
user_obj = db.users.find_one({'profile.username': username})
if not user_obj:
log(level='SEC', obj={'msg': 'USER_NOT_EXIST'})
raise UserError('INCORRECT_LOGIN')
if not verify_password_PBKDF2(password, user_obj['crypto']['salt1'],
user_obj['crypto']['password_hashed']):
log(level='SEC', obj={'msg': 'WRONG_PASSWORD'})
raise UserError('INCORRECT_LOGIN')
user_id = str(user_obj['_id'])
redis_user_key_lookup_key = f"user-{user_id}"
redis_user_key = rdb.get(redis_user_key_lookup_key)
logged_in = False
if redis_user_key:
# user already logged in on some other machines
redis_user_obj_json_str = rdb.get(redis_user_key)
if redis_user_obj_json_str:
logged_in = True
# reset expire time
rdb.set(redis_user_key,
redis_user_obj_json_str,
ex=UserConfig.LOGIN_EXPIRE_TIME)
rdb.set(redis_user_key_lookup_key,
redis_user_key,
ex=UserConfig.LOGIN_EXPIRE_TIME)
if logged_in:
return redis_user_key, user_obj['profile']
common_user_obj = {
'_id': user_obj['_id'],
'profile': {
'username': user_obj['profile']['username'],
'image': user_obj['profile']['image'],
'desc': user_obj['profile']['desc'],
'email': user_obj['profile']['email']
},
'access_control': user_obj['access_control'],
'settings': user_obj['settings']
}
redis_user_value = dumps(common_user_obj)
redis_user_key = binascii.hexlify(bytearray(random_bytes(16))).decode()
redis_user_key_lookup_key = f"user-{user_obj['_id']}"
rdb.set(redis_user_key,
redis_user_value,
ex=UserConfig.LOGIN_EXPIRE_TIME)
rdb.set(redis_user_key_lookup_key,
redis_user_key,
ex=UserConfig.LOGIN_EXPIRE_TIME)
log(obj={'redis_user_key': redis_user_key, 'user': common_user_obj})
return redis_user_key, common_user_obj['profile']
raise UserError('INCORRECT_SESSION')
def ajax_notes_send_dm(rd, user, data):
filterOperation('sendDM', user)
if len(data.content) > 65536 :
raise UserError('CONTENT_TOO_LONG')
createDirectMessage(user['_id'], ObjectId(data.dst_user), other = {'content': data.content})
def update_tag_group(self, group_name, new_tags, user = '', session = None):
g_obj = self.db.groups.find_one({'name': group_name}, session = session)
if g_obj is None:
raise UserError('GROUP_NOT_EXIST')
self.db.groups.update_one({'name': group_name}, {'$set': {'tags': new_tags, 'meta.modified_by': user, 'meta.modified_at': datetime.now()}}, session = session)
def _check_category(self, category, session) :
cat = self.db.cats.find_one({'name': category}, session = session)
if cat is None:
raise UserError('CATEGORY_NOT_EXIST')
return cat
def addVideoToPlaylist(pid, vid, user):
log(obj={'pid': pid, 'vid': vid})
with redis_lock.Lock(rdb, "playlistEdit:" +
str(pid)), MongoTransaction(client) as s:
playlist = db.playlists.find_one({'_id': ObjectId(pid)}, session=s())
if playlist is None:
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('addVideoToPlaylist', user, playlist)
if tagdb.retrive_item({'_id': ObjectId(vid)}, session=s()) is None:
raise UserError('VIDEO_NOT_EXIST')
if playlist["videos"] > PlaylistConfig.MAX_VIDEO_PER_PLAYLIST:
raise UserError('VIDEO_LIMIT_EXCEEDED')
conflicting_item = db.playlist_items.find_one(
{
'pid': ObjectId(pid),
'vid': ObjectId(vid)
}, session=s())
if conflicting_item is not None:
editPlaylist_MoveLockFree(pid,
conflicting_item,
int(playlist["videos"]),
session=s())
db.playlists.update_one({'_id': ObjectId(pid)}, {
'$set': {
'meta.modified_by': makeUserMeta(user),
'meta.modified_at': datetime.now()
}
},
session=s())
s.mark_succeed()
return
playlists = tagdb.retrive_item({'_id': ObjectId(vid)},
session=s())['item']['series']
playlists.append(ObjectId(pid))
playlists = list(set(playlists))
tagdb.update_item_query(ObjectId(vid),
{'$set': {
'item.series': playlists
}},
makeUserMeta(user),
session=s())
db.playlist_items.insert_one(
{
"pid": ObjectId(pid),
"vid": ObjectId(vid),
"rank": int(playlist["videos"]),
"meta": makeUserMeta(user)
},
session=s())
db.playlists.update_one({"_id": ObjectId(pid)},
{"$inc": {
"videos": int(1)
}},
session=s())
db.playlists.update_one({'_id': ObjectId(pid)}, {
'$set': {
'meta.modified_by': makeUserMeta(user),
'meta.modified_at': datetime.now()
}
},
session=s())
s.mark_succeed()
def listAllPlaylistVideosUnordered(pid):
playlist = db.playlists.find_one({'_id': ObjectId(pid)})
if playlist is None:
raise UserError('PLAYLIST_NOT_EXIST')
ans_obj = db.playlist_items.find({"pid": ObjectId(pid)})
return [ObjectId(item['vid']) for item in ans_obj], playlist['videos']
def _verifyFolderName(name):
if '/' in name or '\\' in name or '*' in name:
raise UserError('INVALID_PATH')
return True
def getPlaylist(pid):
ret = db.playlists.find_one({'_id': ObjectId(pid)})
if not ret:
raise UserError('PLAYLIST_NOT_EXIST')
return ret
def _verifyPath(path):
if path:
if path[0] == '/' and path[-1] == '/':
return True
raise UserError('INVALID_PATH')
def listFolder(viewing_user, user, path):
if user == 'me':
raise UserError('INCORRECT_USER')
_verifyPath(path)
folder_obj = _findFolder(user, path)
if isinstance(user, dict):
user = ObjectId(user['_id'])
elif isinstance(user, str):
user = ObjectId(user)
if folder_obj['privateView']:
filterOperation('listFolder', viewing_user, folder_obj)
path_escaped = re.escape(path)
query_regex = f'^{path_escaped}[^\\/]*\\/$'
ret = db.playlist_folders.aggregate([{
'$match': {
'user': user,
'path': {
'$regex': query_regex
}
}
}, {
'$lookup': {
'from': 'playlists',
'localField': 'playlist',
'foreignField': '_id',
'as': 'playlist_object'
}
}, {
'$unwind': {
'path': '$playlist_object',
'preserveNullAndEmptyArrays': True
}
}, {
'$sort': {
'path': 1
}
}])
items = [i for i in ret]
ans = []
for item in items:
assert not (('playlist_object' in item) ^ item['leaf'])
if item['privateView'] and (
viewing_user is None or
(str(user) != str(viewing_user['_id'])
and viewing_user['access_control']['status'] != 'admin')):
continue
if 'playlist_object' in item: # playlist item (leaf)
if item['playlist_object'] is None: # leaf playlist does not exist, do not display
# TODO: maybe just display it is gone, not deleting it
with MongoTransaction(client) as s:
db.playlist_folders.delete_one(
{
'user': user,
'path': item['path']
}, session=s())
s.mark_succeed()
elif item['playlist_object']['private']: # playlist is private
if viewing_user is not None and filterOperation(
'viewPrivatePlaylist',
viewing_user,
item['playlist_object'],
raise_exception=False):
ans.append(item)
else:
ans.append(item)
else: # subfolder item
ans.append(item)
return {'cur': folder_obj, 'children': ans}
def renameFolder(user, path, new_name):
_verifyPath(path)
_verifyFolderName(new_name)
if path == "/":
raise UserError('INVALID_PATH')
with redis_lock.Lock(
rdb,
f"folderEdit:{str(makeUserMeta(user))}:{path}"), MongoTransaction(
client) as s:
folder_obj = _findFolder(user, path)
filterOperation('renameFolder', user, folder_obj)
parent_path, cur_folder = _parentPath(path)
if '\\' in cur_folder:
raise UserError('INVALID_PATH')
if db.playlist_folders.find_one({
'user': makeUserMeta(user),
'path': parent_path + new_name + '/'
}):
raise UserError('FOLDER_ALREADY_EXIST')
parent_path_escaped = re.escape(parent_path)
cur_folder_esacped = re.escape(cur_folder)
query_regex = f'^{parent_path_escaped}{cur_folder_esacped}\\/.*'
replace_regex = re.compile(
f'^({parent_path_escaped})({cur_folder_esacped})(\\/.*)')
paths = db.playlist_folders.find(
{
'user': makeUserMeta(user),
'path': {
'$regex': query_regex
}
},
session=s())
db.playlist_folders.update_one(
{
'user': makeUserMeta(user),
'path': {
'$regex': f'^{parent_path_escaped}{cur_folder_esacped}\\/$'
}
}, {'$set': {
'name': new_name
}},
session=s())
new_name_escaped = re.escape(new_name)
for p in paths:
new_path = replace_regex.sub(rf'\g<1>{new_name}\g<3>', p['path'])
db.playlist_folders.update_one({'_id': p['_id']},
{'$set': {
'path': new_path
}},
session=s())
db.playlist_folders.update_one(
{
'user': makeUserMeta(user),
'path': {
'$regex': query_regex
}
}, {
'$set': {
'meta.modified_by': makeUserMeta(user),
'meta.modified_at': datetime.now()
}
},
session=s())
s.mark_succeed()
return parent_path + new_name + '/'
def remove_tag_group(self, group_name, user = '', session = None):
g_obj = self.db.groups.find_one({'name': group_name}, session = session)
if g_obj is None:
raise UserError('GROUP_NOT_EXIST')
self.db.groups.remove({'name': group_name}, session = session)
def listCommonTagIDs(pid, user):
playlist = db.playlists.find_one({'_id': ObjectId(pid)})
if playlist is None:
raise UserError('PLAYLIST_NOT_EXIST')
if playlist['private']:
filterOperation('viewPrivatePlaylist', user, playlist)
result = db.playlist_items.aggregate([{
"$match": {
"pid": ObjectId(pid)
}
}, {
"$lookup": {
"from": "items",
"localField": "vid",
"foreignField": "_id",
"as": "video"
}
}, {
"$project": {
"video.tags": 1
}
}, {
"$unwind": {
"path": "$video"
}
}, {
"$project": {
"tags": {
'$filter': {
'input': '$video.tags',
'as': 'tag',
'cond': {
'$lt': ['$$tag', 0x80000000]
}
}
}
}
}, {
"$group": {
"_id": 0,
"tags": {
"$push": "$tags"
},
"initialTags": {
"$first": "$tags"
}
}
}, {
"$project": {
"commonTags": {
"$reduce": {
"input": "$tags",
"initialValue": "$initialTags",
"in": {
"$setIntersection": ["$$value", "$$this"]
}
}
}
}
}])
ret = [i for i in result]
if ret:
return ret[0]['commonTags']
else:
return []
def list_tag_group(self, group_name, session = None):
g_obj = self.db.groups.find_one({'name': group_name}, session = session)
if g_obj is None:
raise UserError('GROUP_NOT_EXIST')
return g_obj['tags']
def listPlaylistsForVideo(user, vid):
video = tagdb.retrive_item({'_id': ObjectId(vid)})
if video is None:
raise UserError('VIDEO_NOT_EXIST')
if isObjectAgnosticOperationPermitted('viewPrivatePlaylist', user):
auth_obj = {}
else:
auth_obj = {
'$or': [{
'playlist.meta.created_by': user['_id'] if user else ''
}, {
'playlist.private': False,
'playlist.videos': {
'$gt': 1
}
}]
}
result = db.playlist_items.aggregate([{
'$match': {
'$and': [{
'pid': {
'$in': video['item']['series']
}
}, {
'vid': video['_id']
}]
}
}, {
'$lookup': {
'from': 'playlists',
'localField': 'pid',
'foreignField': '_id',
'as': 'playlist'
}
}, {
'$unwind': {
'path': '$playlist'
}
}, {
'$match': auth_obj
}])
ans = []
for obj in result:
playlist_obj = obj['playlist']
playlist_obj['prev'] = ''
playlist_obj['next'] = ''
rank = obj['rank']
if rank > 0:
playlist_obj['prev'] = str(
db.playlist_items.find_one({
'pid': playlist_obj['_id'],
'rank': int(rank - 1)
})['vid'])
if rank + 1 < playlist_obj['videos']:
playlist_obj['next'] = str(
db.playlist_items.find_one({
'pid': playlist_obj['_id'],
'rank': int(rank + 1)
})['vid'])
ans.append(playlist_obj)
return ans
def _check_language(self, language):
if language not in VALID_LANGUAGES :
raise UserError('UNRECOGNIZED_LANGUAGE')
async def _add_to_playlist(self, dst_playlist, event_id, user_global):
if self.playlist_map[dst_playlist]:
dst_rank = self.playlist_map[dst_playlist]['rank']
playlist_ordered = self.playlist_map[dst_playlist]['all']
try:
# fast method
async with RedisLockAsync(
rdb, "playlistEdit:" +
dst_playlist), MongoTransaction(client) as s:
cur_rank = 0
playlist = db.playlists.find_one(
{'_id': ObjectId(dst_playlist)})
if playlist is None:
raise UserError('PLAYLIST_NOT_EXIST')
if playlist["videos"] + len(
self.playlist_map[dst_playlist]
['succeed']) > PlaylistConfig.MAX_VIDEO_PER_PLAYLIST:
raise UserError('VIDEO_LIMIT_EXCEEDED')
playlist_videos = playlist['videos']
for unique_id in playlist_ordered:
if unique_id in self.playlist_map[dst_playlist][
'succeed']:
(video_id, _, user) = self.playlist_map[
dst_playlist]['succeed'][unique_id]
if dst_rank == -1:
if filterOperation('editPlaylist', user,
playlist, False):
if addVideoToPlaylistLockFree(
dst_playlist,
video_id,
user,
playlist_videos,
session=s()):
playlist_videos += 1
else:
if filterOperation('editPlaylist', user,
playlist, False):
if insertIntoPlaylistLockFree(dst_playlist,
video_id,
dst_rank +
cur_rank,
user,
session=s()):
cur_rank += 1
s.mark_succeed()
except UserError as ue:
# UserError, rereaise to upper level
log_e(event_id, user_global, '_add_to_playlist', 'ERR', {
'ex': str(ex),
'tb': traceback.format_exc()
})
del self.playlist_map[dst_playlist]
rdb.set(f'playlist-batch-post-event-{dst_playlist}', b'done')
raise ue
except Exception as ex:
# if anything goes wrong, fallback to slow method
log_e(event_id, user_global, '_add_to_playlist', 'ERR', {
'ex': str(ex),
'tb': traceback.format_exc()
})
cur_rank = 0
for unique_id in playlist_ordered:
if unique_id in self.playlist_map[dst_playlist]['succeed']:
(video_id, _, user) = self.playlist_map[dst_playlist][
'succeed'][unique_id]
# ignore error, add next video
try:
if dst_rank == -1:
addVideoToPlaylist(dst_playlist, video_id,
user)
else:
insertIntoPlaylist(dst_playlist, video_id,
dst_rank + cur_rank, user)
cur_rank += 1
except:
pass
log_e(
event_id, user_global, '_add_to_playlist', 'MSG', {
'succedd': len(self.playlist_map[dst_playlist]['succeed']),
'all': len(self.playlist_map[dst_playlist]['all']),
'pid': dst_playlist
})
del self.playlist_map[dst_playlist]
rdb.set(f'playlist-batch-post-event-{dst_playlist}', b'done')
def signup(username, password, email, challenge, signup_session_id):
log(
obj={
'username': username,
'email': email,
'challenge': challenge,
'signup_session_id': signup_session_id
})
if len(username) > UserConfig.MAX_USERNAME_LENGTH:
raise UserError('USERNAME_TOO_LONG')
if len(username) < UserConfig.MIN_USERNAME_LENGTH:
raise UserError('USERNAME_TOO_SHORT')
if len(password) > UserConfig.MAX_PASSWORD_LENGTH:
raise UserError('PASSWORD_TOO_LONG')
if len(password) < UserConfig.MIN_PASSWORD_LENGTH:
raise UserError('PASSWORD_TOO_SHORT')
if verify_session(signup_session_id, 'SIGNUP'):
if email:
if len(email) > UserConfig.MAX_EMAIL_LENGTH or not re.match(
r"[^@]+@[^@]+\.[^@]+", email):
raise UserError('INCORRECT_EMAIL')
crypto_method, password_hashed, salt1, salt2, master_key_encryptyed = generate_user_crypto_PBKDF2(
password)
with redis_lock.Lock(rdb, 'signup:' + username):
user_obj_find = db.users.find_one({'profile.username': username})
if user_obj_find is not None:
raise UserError('USER_EXIST')
if email:
user_obj_email = db.users.find_one({'profile.email': email})
if user_obj_email is not None:
raise UserError('EMAIL_EXIST')
user_obj = {
'profile': {
'username': username,
'desc': 'Write something here',
'pubkey': '',
'image': 'default',
'email': email
},
'crypto': {
'crypto_method': crypto_method,
'password_hashed': password_hashed,
'salt1': salt1,
'salt2': salt2,
'master_key_encryptyed': master_key_encryptyed
},
'access_control': {
'status': 'normal',
'access_mode': 'blacklist',
'allowed_ops': [],
'denied_ops': []
},
'settings': {
'blacklist': 'default'
},
'meta': {
'created_at': datetime.now()
}
}
uid = db.users.insert_one(user_obj).inserted_id
log(obj={'uid': uid, 'profile': user_obj['profile']})
return uid
raise UserError('INCORRECT_SESSION')
def add_category(self, category, color = '#000', user = '', session = None) :
cat = self.db.cats.find_one({'name': category}, session = session)
if cat is not None:
raise UserError("CATEGORY_ALREADY_EXIST")
self.db.cats.insert_one({'name': category, 'count': 0, 'color': color, 'meta': {'created_by': user, 'created_at': datetime.now()}}, session = session)
def getUserDeniedOps(user_id, user):
filterOperation('getUserDeniedOps', user, user_id)
old_user_obj = db.users.find_one({'_id': ObjectId(user_id)})
if old_user_obj is None:
raise UserError('USER_NOT_EXIST')
return old_user_obj['access_control']['denied_ops']
def insertIntoPlaylist(pid, vid, rank, user):
log(obj={'pid': pid, 'vid': vid, 'rank': rank})
with redis_lock.Lock(rdb, "playlistEdit:" +
str(pid)), MongoTransaction(client) as s:
playlist = playlist_db.retrive_item(pid, session=s())
if playlist is None:
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('editPlaylist', user, playlist)
if tagdb.retrive_item({'_id': ObjectId(vid)}, session=s()) is None:
raise UserError('VIDEO_NOT_EXIST')
if playlist['item']["videos"] > PlaylistConfig.MAX_VIDEO_PER_PLAYLIST:
raise UserError('VIDEO_LIMIT_EXCEEDED')
conflicting_item = db.playlist_items.find_one(
{
'pid': ObjectId(pid),
'vid': ObjectId(vid)
}, session=s())
if conflicting_item is not None:
editPlaylist_MoveLockFree(pid, conflicting_item, rank, session=s())
playlist_db.update_item_query(playlist, {},
user=makeUserMeta(user),
session=s())
s.mark_succeed()
return
if rank < 0:
raise UserError('OUT_OF_RANGE')
if rank > playlist['item']['videos']:
rank = int(playlist['item']['videos'])
playlists = tagdb.retrive_item({'_id': ObjectId(vid)},
session=s())['item']['series']
playlists.append(ObjectId(pid))
playlists = list(set(playlists))
tagdb.update_item_query(ObjectId(vid),
{'$set': {
'item.series': playlists
}},
user=makeUserMeta(user),
session=s())
playlist_db.update_item_query(playlist,
{"$inc": {
"item.videos": int(1)
}},
user=makeUserMeta(user),
session=s())
db.playlist_items.update_many(
{
'pid': ObjectId(pid),
'rank': {
'$gte': rank
}
}, {'$inc': {
'rank': int(1)
}},
session=s())
db.playlist_items.insert_one(
{
"pid": ObjectId(pid),
"vid": ObjectId(vid),
"rank": int(rank),
"meta": makeUserMeta(user)
},
session=s())
s.mark_succeed()