def getPlaylistRatingAggregate(pid: ObjectId):
playlist_obj = playlist_db.retrive_item(pid)
if playlist_obj is None:
raise UserError('PLAYLIST_NOT_EXIST')
if 'total_rating' in playlist_obj:
return playlist_obj['total_rating'], playlist_obj['total_rating_user']
raise UserError('NOT_RATED')
def updatePlaylistCoverFromFile(pid, user, file_key):
log(obj={'pid': pid, 'file_key': file_key})
photo_file = None
if file_key.startswith("upload-image-"):
filename = rdb.get(file_key)
if filename:
photo_file = filename.decode('ascii')
if photo_file is None:
raise UserError('NO_COVER')
with redis_lock.Lock(rdb, "playlistEdit:" +
str(pid)), MongoTransaction(client) as s:
list_obj = playlist_db.retrive_item(pid)
log(obj={'playlist': list_obj})
if list_obj is None:
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('editPlaylist', user, list_obj)
playlist_db.update_item_query(list_obj,
{'$set': {
"item.cover": photo_file
}},
user=makeUserMeta(user),
session=s())
s.mark_succeed()
def listAllPlaylistVideosOrdered(pid, user):
playlist = playlist_db.retrive_item(pid)
if playlist is None:
raise UserError('PLAYLIST_NOT_EXIST')
if playlist['item']['private']:
filterOperation('viewPrivatePlaylist', user, playlist)
ans_obj = db.playlist_items.aggregate([{
'$match': {
"pid": ObjectId(pid)
}
}, {
'$lookup': {
'from': 'videos',
'localField': "vid",
'foreignField': "_id",
'as': 'item'
}
}, {
'$unwind': {
'path': '$item'
}
}, {
'$sort': {
'rank': 1
}
}])
return [i for i in ans_obj], playlist['item']['videos'], playlist
def addToPlaylist(user, pid: ObjectId, text: str, use_bleach=True):
filterOperation('postComment', user)
playlist_obj = playlist_db.retrive_item(pid)
if playlist_obj is None:
raise UserError('PLAYLIST_NOT_EXIST')
with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)):
if 'comment_thread' in playlist_obj:
cid = addComment(user,
playlist_obj['comment_thread'],
text,
use_bleach=use_bleach)
return playlist_obj['comment_thread'], cid
else:
with MongoTransaction(client) as s:
tid = createThread('playlist',
playlist_obj['_id'],
playlist_obj['meta']['created_by'],
session=s())
playlist_db.update_item_query(
playlist_obj, {'$set': {
'comment_thread': tid
}},
session=s())
s.mark_succeed()
cid = addComment(user, tid, text)
return tid, cid
def addPlaylistsToFolder(user, path, playlists) :
_verifyPath(path)
with redis_lock.Lock(rdb, f"folderEdit:{str(makeUserMeta(user))}:{path}"), MongoTransaction(client) as s :
folder_obj = _findFolder(user, path)
filterOperation('addPlaylistsToFolder', user, folder_obj)
for pid in playlists :
playlist = playlist_db.retrive_item(pid, session = s())
if playlist is None :
continue # skip non-exist playlist
if playlist['item']['private'] and not filterOperation('viewPrivatePlaylist', user, playlist, raise_exception = False) :
continue # skip other's private playlist
playlist_path = path + "\\" + str(playlist['_id']) + "\\/"
if _findFolder(user, playlist_path, raise_exception = False) :
continue # skip duplicated playlist
playlist_obj = {
'user': makeUserMeta(user),
'leaf': True,
'playlist': playlist['_id'],
'name': None,
'path': playlist_path,
'privateView': folder_obj['privateView'],
'privateEdit': folder_obj['privateEdit'],
'meta': makeUserMetaObject(user)
}
db.playlist_folders.insert_one(playlist_obj, session = s())
db.playlist_folders.update_one({'_id': folder_obj['_id']}, {'$set': {
'meta.modified_by': makeUserMeta(user),
'meta.modified_at': datetime.now()
}}, session = s())
s.mark_succeed()
def updateCommonTags(pid, tags, user):
log(obj={'pid': pid, 'tags': tags})
with MongoTransaction(client) as s:
playlist_obj = playlist_db.retrive_item(pid)
if playlist_obj is None:
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('editPlaylist', user, playlist_obj)
# user is editing video tags, not the playlist itself, no need to lock playlist
tags = tagdb.filter_and_translate_tags(tags, session=s())
old_tags = listCommonTagIDs(pid, user)
log(obj={'old_tags': old_tags})
old_tags_set = set(old_tags)
new_tags_set = set(tags)
tags_added = list((old_tags_set ^ new_tags_set) - old_tags_set)
tags_to_remove = list((old_tags_set ^ new_tags_set) - new_tags_set)
if len(tags_added) - len(
tags_to_remove) > PlaylistConfig.MAX_COMMON_TAGS:
raise UserError('TOO_MANY_TAGS')
all_video_ids, _ = listAllPlaylistVideosUnordered(pid)
if tags_to_remove:
tagdb.update_many_items_tags_pull(all_video_ids,
tags_to_remove,
makeUserMeta(user),
session=s())
if tags_added:
tagdb.update_many_items_tags_merge(all_video_ids,
tags_added,
makeUserMeta(user),
session=s())
s.mark_succeed()
def listAllPlaylistVideosUnordered(pid):
playlist = playlist_db.retrive_item(pid)
if playlist is None:
raise UserError('PLAYLIST_NOT_EXIST')
ans_obj = db.playlist_items.find({"pid": ObjectId(pid)})
return [ObjectId(item['vid'])
for item in ans_obj], playlist['item']['videos']
def ratePlaylist(user, pid: ObjectId, stars: int):
stars = max(min(int(stars), 10), 1)
playlist_obj = playlist_db.retrive_item(pid)
if playlist_obj is None:
raise UserError('PLAYLIST_NOT_EXIST')
with redis_lock.Lock(rdb, "playlistEdit:" +
str(pid)), MongoTransaction(client) as s:
rating_obj = db.playlist_ratings.find_one(
{
'pid': pid,
'uid': ObjectId(user['_id'])
}, session=s())
user_rated = 0
if rating_obj:
user_rated = 1
db.playlist_ratings.update_one(
{
'pid': pid,
'uid': ObjectId(user['_id'])
}, {'$set': {
'v': int(stars)
}},
session=s())
else:
user_rated = 0
db.playlist_ratings.insert_one(
{
'pid': pid,
'uid': ObjectId(user['_id']),
'v': int(stars)
},
session=s())
if 'total_rating' in playlist_obj:
if rating_obj:
playlist_db.update_item_query(playlist_obj, {
'$inc': {
'total_rating': int(stars - rating_obj['v']),
'total_rating_user': int(1 - user_rated)
}
},
session=s())
else:
playlist_db.update_item_query(playlist_obj, {
'$inc': {
'total_rating': int(stars),
'total_rating_user': int(1 - user_rated)
}
},
session=s())
else:
playlist_db.update_item_query(playlist_obj, {
'$set': {
'total_rating': int(stars),
'total_rating_user': int(1)
}
},
session=s())
s.mark_succeed()
def isOwner(pid_or_obj, user):
if isinstance(pid_or_obj, str):
obj = playlist_db.retrive_item(pid_or_obj)
else:
obj = pid_or_obj
creator = str(obj['meta']['created_by'])
user_id = str(user['_id'])
return creator == user_id or (
'editPlaylist' in user['access_control']['allowed_ops']
) or user['access_control']['status'] == 'admin'
def extendPlaylistFromExistingPlaylist(pid, url, user, use_autotag=False):
log(obj={'url': url, 'pid': pid})
filterOperation('editPlaylist', user)
if not playlist_db.retrive_item(pid):
raise UserError('PLAYLIST_NOT_EXIST')
cralwer, cleanURL = dispatch_playlist(url)
if not cralwer:
raise UserError('UNSUPPORTED_PLAYLIST_URL')
task_id = _postPlaylistTask(cleanURL, pid, use_autotag, user, extend=True)
return task_id
def getPlaylist(pid, lang):
ret = playlist_db.retrive_item(pid)
if not ret:
raise UserError('PLAYLIST_NOT_EXIST')
if 'tags' in ret:
ret['tags_translated'], ret['tags_category'], ret[
'tags_tags'] = tagdb.translate_tag_ids_to_user_language(
ret['tags'], lang)
else:
ret['tags_translated'], ret['tags_category'], ret[
'tags_tags'] = [], {}, {}
return ret
def addVideoToPlaylist(pid, vid, user):
log(obj={'pid': pid, 'vid': vid})
with redis_lock.Lock(rdb, "playlistEdit:" +
str(pid)), MongoTransaction(client) as s:
playlist = playlist_db.retrive_item(pid)
if playlist is None:
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('editPlaylist', user, playlist)
if tagdb.retrive_item({'_id': ObjectId(vid)}, session=s()) is None:
raise UserError('VIDEO_NOT_EXIST')
if playlist['item']["videos"] > PlaylistConfig.MAX_VIDEO_PER_PLAYLIST:
raise UserError('VIDEO_LIMIT_EXCEEDED')
conflicting_item = db.playlist_items.find_one(
{
'pid': ObjectId(pid),
'vid': ObjectId(vid)
}, session=s())
if conflicting_item is not None:
editPlaylist_MoveLockFree(pid,
conflicting_item,
int(playlist['item']["videos"]),
session=s())
playlist_db.update_item_query(playlist, {},
user=makeUserMeta(user),
session=s())
s.mark_succeed()
return
playlists = tagdb.retrive_item({'_id': ObjectId(vid)},
session=s())['item']['series']
playlists.append(ObjectId(pid))
playlists = list(set(playlists))
tagdb.update_item_query(ObjectId(vid),
{'$set': {
'item.series': playlists
}},
user=makeUserMeta(user),
session=s())
db.playlist_items.insert_one(
{
"pid": ObjectId(pid),
"vid": ObjectId(vid),
"rank": int(playlist['item']["videos"]),
"meta": makeUserMeta(user)
},
session=s())
playlist_db.update_item_query(playlist,
{"$inc": {
"item.videos": int(1)
}},
user=makeUserMeta(user),
session=s())
s.mark_succeed()
def updatePlaylistInfo(pid,
title,
desc,
cover,
user,
private=False,
privateEdit=True):
log(
obj={
'title': title,
'desc': desc,
'cover': cover,
'private': private,
'privateEdit': privateEdit
})
if len(title) > PlaylistConfig.MAX_TITLE_LENGTH:
raise UserError('TITLE_TOO_LONG')
if len(desc) > PlaylistConfig.MAX_DESC_LENGTH:
raise UserError('DESC_TOO_LONG')
if cover and len(cover) > PlaylistConfig.MAX_COVER_URL_LENGTH:
raise UserError('URL_TOO_LONG')
if not title:
raise UserError('EMPTY_TITLE')
if not desc:
raise UserError('EMPTY_DESC')
with redis_lock.Lock(rdb, "playlistEdit:" +
str(pid)), MongoTransaction(client) as s:
list_obj = playlist_db.retrive_item(pid)
log(obj={'playlist': list_obj})
if list_obj is None:
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('editPlaylist', user, list_obj)
if cover:
playlist_db.update_item_query(list_obj,
{'$set': {
"item.cover": cover
}},
user=makeUserMeta(user),
session=s())
playlist_db.update_item_query(
list_obj, {
'$set': {
"item.title": title,
"item.desc": desc,
"item.private": private,
"item.privateEdit": privateEdit
}
},
fields_to_index=['item.title', 'item.desc'],
user=makeUserMeta(user),
session=s())
s.mark_succeed()
def updatePlaylistCover(pid, cover, user):
log(obj={'pid': pid, 'cover': cover})
with redis_lock.Lock(rdb, "playlistEdit:" +
str(pid)), MongoTransaction(client) as s:
list_obj = playlist_db.retrive_item(pid)
log(obj={'playlist': list_obj})
if list_obj is None:
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('editPlaylist', user, list_obj)
playlist_db.update_item_query(list_obj,
{'$set': {
"item.cover": cover
}},
user=makeUserMeta(user),
session=s())
s.mark_succeed()
def removePlaylist(pid, user):
log(obj={'pid': pid})
with redis_lock.Lock(rdb, "playlistEdit:" +
str(pid)), MongoTransaction(client) as s:
list_obj = playlist_db.retrive_item(pid)
log(obj={'playlist': list_obj})
if list_obj is None:
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('removePlaylist', user, list_obj)
all_items = db.playlist_items.find({"pid": ObjectId(pid)}, session=s())
log(obj={'items': [i for i in all_items]})
db.playlist_items.delete_many({"pid": ObjectId(pid)}, session=s())
playlist_db.remove_item(pid, user, session=s())
from services.playlistFolder import deletePlaylist
deletePlaylist(pid, session=s())
s.mark_succeed()
def editPlaylist_MoveUp(pid, vid, offset, limit, user):
log(obj={'pid': pid, 'vid': vid})
with redis_lock.Lock(rdb, "playlistEdit:" +
str(pid)), MongoTransaction(client) as s:
playlist = playlist_db.retrive_item(pid, session=s())
if playlist is None:
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('editPlaylist', user, playlist)
if playlist['item']["videos"] > 0:
entry = db.playlist_items.find_one(
{
"pid": ObjectId(pid),
"vid": ObjectId(vid)
}, session=s())
if entry is None:
s.mark_failover()
raise UserError('VIDEO_NOT_EXIST_OR_NOT_IN_PLAYLIST')
if entry['rank'] <= 0:
return None
exchange_entry = db.playlist_items.find_one(
{
"pid": ObjectId(pid),
"rank": int(entry['rank'] - 1)
},
session=s())
db.playlist_items.update_one(
{'_id': entry['_id']},
{'$set': {
'rank': int(entry['rank'] - 1)
}},
session=s())
db.playlist_items.update_one(
{'_id': exchange_entry['_id']},
{'$set': {
'rank': int(entry['rank'])
}},
session=s())
playlist_db.update_item_query(playlist, {},
user=makeUserMeta(user),
session=s())
#video_page, video_count = listPlaylistVideos(pid, page - 1, page_size, user)
s.mark_succeed()
#return {'videos': video_page, 'video_count': video_count, 'page': page}
else:
raise UserError('EMPTY_PLAYLIST')
def updatePlaylistTags(pid, new_tags, user):
log(obj={'pid': pid, 'new_tags': new_tags})
with redis_lock.Lock(rdb, "playlistEdit:" +
str(pid)), MongoTransaction(client) as s:
list_obj = playlist_db.retrive_item(pid)
log(
obj={
'playlist': list_obj,
'old_tags': list_obj['tags'] if 'tags' in list_obj else []
})
if list_obj is None:
raise UserError('PLAYLIST_NOT_EXIST')
#filterOperation('editPlaylist', user, list_obj)
playlist_db.update_item_tags(list_obj,
new_tags,
makeUserMeta(user),
session=s())
s.mark_succeed()
def removeVideoFromPlaylist(pid, vid, offset, limit, user):
log(obj={'pid': pid, 'vid': vid})
with redis_lock.Lock(rdb, "playlistEdit:" +
str(pid)), MongoTransaction(client) as s:
playlist = playlist_db.retrive_item(pid, session=s())
if playlist is None:
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('editPlaylist', user, playlist)
if playlist['item']["videos"] > 0:
entry = db.playlist_items.find_one(
{
"pid": ObjectId(pid),
"vid": ObjectId(vid)
}, session=s())
if entry is None:
raise UserError('VIDEO_NOT_EXIST_OR_NOT_IN_PLAYLIST')
db.playlist_items.update_many(
{
'pid': ObjectId(pid),
'rank': {
'$gt': entry['rank']
}
}, {'$inc': {
'rank': int(-1)
}},
session=s())
db.playlist_items.delete_one({'_id': entry['_id']}, session=s())
playlist_db.update_item_query(playlist,
{"$inc": {
"item.videos": int(-1)
}},
user=makeUserMeta(user),
session=s())
else:
raise UserError('EMPTY_PLAYLIST')
"""
video_page, video_count = listPlaylistVideos(pid, page - 1, page_size, user)
if len(video_page) == 0 and page > 1 and video_count > 0 :
# in case deleting video results in current page becomes empty, show the previous page
video_page, video_count = listPlaylistVideos(pid, page - 2, page_size, user)
s.mark_succeed()
return {'videos': video_page, 'video_count': video_count, 'page': page - 1}
"""
s.mark_succeed()
def editPlaylist_Move(pid, vid, to_rank, user):
log(obj={'pid': pid, 'vid': vid, 'to_rank': to_rank})
with redis_lock.Lock(rdb, "playlistEdit:" +
str(pid)), MongoTransaction(client) as s:
playlist = playlist_db.retrive_item(pid, session=s())
if playlist is None:
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('editPlaylist', user, playlist)
if tagdb.retrive_item({'_id': ObjectId(vid)}, session=s()) is None:
raise UserError('VIDEO_NOT_EXIST')
if to_rank < 0:
raise UserError('OUT_OF_RANGE')
if to_rank > playlist['item']['videos']:
to_rank = int(playlist['item']['videos'])
editPlaylist_MoveLockFree(pid, vid, to_rank, session=s())
playlist_db.update_item_query(playlist, {},
user=makeUserMeta(user),
session=s())
s.mark_succeed()
def inversePlaylistOrder(pid, user):
log(obj={'pid': pid})
with redis_lock.Lock(rdb, "playlistEdit:" +
str(pid)), MongoTransaction(client) as s:
playlist = playlist_db.retrive_item(pid, session=s())
if playlist is None:
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('editPlaylist', user, playlist)
db.playlist_items.update_many({"pid": playlist["_id"]},
{'$bit': {
'rank': {
'xor': int(-1)
}
}})
db.playlist_items.update_many(
{"pid": playlist["_id"]},
{'$inc': {
'rank': int(playlist['item']['videos'])
}})
s.mark_succeed()
def updatePlaylistCoverVID(pid, vid, offset, limit, user):
log(obj={'pid': pid, 'vid': vid})
with redis_lock.Lock(rdb, "playlistEdit:" +
str(pid)), MongoTransaction(client) as s:
list_obj = playlist_db.retrive_item(pid)
log(obj={'playlist': list_obj})
if list_obj is None:
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('editPlaylist', user, list_obj)
video_obj = filterSingleVideo(vid, user)
if video_obj is None:
raise UserError('VIDEO_NOT_EXIST')
cover = video_obj['item']['cover_image']
playlist_db.update_item_query(list_obj,
{'$set': {
"item.cover": cover
}},
user=makeUserMeta(user),
session=s())
#video_page, video_count = listPlaylistVideos(pid, page - 1, page_size, user)
s.mark_succeed()
def listPlaylistVideosWithAuthorizationInfo(pid, offset, limit, user):
playlist = playlist_db.retrive_item(pid)
if playlist is None:
raise UserError('PLAYLIST_NOT_EXIST')
if playlist['item']['private']:
filterOperation('viewPrivatePlaylist', user, playlist)
ans_obj = db.playlist_items.aggregate([{
'$match': {
"pid": ObjectId(pid)
}
}, {
'$lookup': {
'from': 'videos',
'localField': "vid",
'foreignField': "_id",
'as': 'item'
}
}, {
'$unwind': {
'path': '$item'
}
}, {
'$sort': {
'rank': 1
}
}, {
'$skip': offset,
}, {
'$limit': limit
}])
ret = []
for obj in ans_obj:
ret_obj = obj['item']
ret_obj['rank'] = obj['rank']
ret.append(ret_obj)
ret = filterVideoList(ret, user)
return ret, playlist['item']['videos'], isAuthorisedToEdit(
playlist, user), isOwner(playlist, user)
async def postVideoAsync(url,
tags,
dst_copy,
dst_playlist,
dst_rank,
other_copies,
repost_type,
playlist_ordered,
user,
update_video_detail,
event_id,
field_override=None,
use_autotag=False):
parsed = None
try:
dst_playlist = str(dst_playlist)
dst_rank = -1 if dst_rank is None else dst_rank
#tags = tagdb.filter_and_translate_tags(tags)
parsed, url = dispatch(url)
except:
pass
if parsed is None:
log_e(event_id, user, 'dispatcher', 'ERR', {
'msg': 'PARSE_FAILED',
'url': url
})
await _playlist_reorder_helper.post_video_failed(
url, dst_playlist, playlist_ordered, dst_rank, user, event_id)
return "PARSE_FAILED", {}
unique_id = await parsed.unique_id_async(
self=parsed,
link=url) # empty unique_id for b23.tv posts, f**k bilibli
if not unique_id:
ret = await parsed.get_metadata_async(parsed, url, update_video_detail)
unique_id = ret['data']['unique_id']
log_e(
event_id, user, 'scraper', 'MSG', {
'url': url,
'dst_copy': dst_copy,
'other_copies': other_copies,
'dst_playlist': dst_playlist
})
setEventOp('scraper')
try:
lock_id = "videoEdit:" + unique_id
async with RedisLockAsync(rdb, lock_id):
unique, conflicting_item = verifyUniqueness(unique_id)
if unique or update_video_detail:
async with _download_sem:
ret = await parsed.get_metadata_async(
parsed, url, update_video_detail)
if repost_type:
ret['data']['repost_type'] = repost_type
else:
ret['data']['repost_type'] = 'unknown'
if ret["status"] == 'FAILED':
log_e(event_id, user, 'downloader', 'WARN', {
'msg': 'FETCH_FAILED',
'ret': ret
})
await _playlist_reorder_helper.post_video_failed(
unique_id, dst_playlist, playlist_ordered, dst_rank,
user, event_id)
return "FETCH_FAILED", ret
else:
unique_id = ret['data']['unique_id']
else:
# build ret
ret = makeResponseSuccess({
'thumbnailURL':
conflicting_item['item']['thumbnail_url'],
'title':
conflicting_item['item']['title'],
'desc':
conflicting_item['item']['desc'],
'site':
conflicting_item['item']['site'],
'uploadDate':
conflicting_item['item']['upload_time'],
"unique_id":
conflicting_item['item']['unique_id'],
"utags":
conflicting_item['item']['utags']
})
for k, v in conflicting_item['item'].items():
ret['data'][k] = v
if 'part_name' in conflicting_item['item']:
ret['part_name'] = conflicting_item['item']['part_name']
if 'repost_type' in conflicting_item[
'item'] and conflicting_item['item']['repost_type']:
ret['data']['repost_type'] = repost_type
else:
ret['data']['repost_type'] = 'unknown'
tagdb.update_item_query(
conflicting_item,
{'$set': {
'item.repost_type': repost_type
}},
user=makeUserMeta(user))
#if hasattr(parsed, 'LOCAL_CRAWLER') :
# url = ret["data"]["url"]
#else :
# url = clear_url(url)
use_override = False
if field_override and '__condition' in field_override:
condition = field_override['__condition']
del field_override['__condition']
if condition == 'any':
use_override = True
elif condition == 'placeholder' and 'placeholder' in ret[
"data"] and ret["data"]['placeholder']:
use_override = True
if use_override:
for key in field_override:
ret['data'][key] = field_override[key]
playlists = []
#playlist_lock = None
if dst_playlist:
#playlist_lock = RedisLockAsync(rdb, "playlistEdit:" + str(dst_playlist))
#playlist_lock.acquire()
if playlist_db.retrive_item(dst_playlist) is not None:
playlists = [ObjectId(dst_playlist)]
if not unique:
log_e(event_id,
user,
'scraper',
level='MSG',
obj={
'msg': 'ALREADY_EXIST',
'unique_id': ret["data"]["unique_id"]
})
"""
Update existing video
"""
if update_video_detail:
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Updating video detail')
with MongoTransaction(client) as s:
old_item = tagdb.retrive_item(conflicting_item['_id'],
session=s())['item']
if old_item['thumbnail_url'] and old_item[
'cover_image']:
# old thumbnail exists, no need to download again
new_detail = await _make_video_data_update(
ret["data"], url, user, event_id)
else:
# old thumbnail does not exists, add to dict
new_detail = await _make_video_data_update(
ret["data"], url, user, event_id,
ret["data"]["thumbnailURL"])
for key in new_detail.keys():
old_item[key] = new_detail[
key] # overwrite or add new field
setEventUserAndID(user, event_id)
tagdb.update_item_query(conflicting_item['_id'],
{'$set': {
'item': old_item
}}, ['title', 'desc'],
user=makeUserMeta(user),
session=s())
s.mark_succeed()
return 'SUCCEED', conflicting_item['_id']
# this video already exist in the database
# if the operation is to add a link to other copies and not adding self
if (dst_copy and
dst_copy != conflicting_item['_id']) or other_copies:
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Adding to to copies')
async with RedisLockAsync(
rdb, 'editLink'), MongoTransaction(client) as s:
log_e(event_id,
user,
level='MSG',
obj='Adding to to copies, lock acquired')
# find all copies of video dst_copy, self included
all_copies = _getAllCopies(dst_copy, session=s())
# find all videos linked to source video
all_copies += _getAllCopies(conflicting_item['_id'],
session=s())
# add videos from other copies
for uid in other_copies:
all_copies += _getAllCopies(uid,
session=s(),
use_unique_id=True)
# remove duplicated items
all_copies = list(set(all_copies))
# add this video to all other copies found
if len(all_copies) <= VideoConfig.MAX_COPIES:
for dst_vid in all_copies:
setEventUserAndID(user, event_id)
_addThiscopy(dst_vid,
all_copies,
makeUserMeta(user),
session=s())
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Successfully added to copies')
s.mark_succeed()
else:
#if playlist_lock :
# playlist_lock.release()
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Too many copies')
await _playlist_reorder_helper.post_video_failed(
unique_id, dst_playlist, playlist_ordered,
dst_rank, user, event_id)
return "TOO_MANY_COPIES", {}
# if the operation is adding this video to playlist
if dst_playlist:
log_e(event_id,
user,
'scraper',
level='MSG',
obj={
'msg': 'Adding to playlist at position',
'rank': dst_rank
})
if playlist_ordered:
await _playlist_reorder_helper.post_video_succeed(
conflicting_item['_id'], unique_id, dst_playlist,
playlist_ordered, dst_rank, user, event_id)
else:
setEventUserAndID(user, event_id)
if dst_rank == -1:
addVideoToPlaylist(dst_playlist,
conflicting_item['_id'], user)
else:
insertIntoPlaylist(dst_playlist,
conflicting_item['_id'],
dst_rank, user)
# merge tags
async with MongoTransaction(client) as s:
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Merging tags')
setEventUserAndID(user, event_id)
tagdb.update_item_tags_merge(conflicting_item['_id'],
tags,
makeUserMeta(user),
session=s(),
remove_tagids=[354])
s.mark_succeed()
#if playlist_lock :
# playlist_lock.release()
#return "VIDEO_ALREADY_EXIST", conflicting_item['_id']
return "SUCCEED", conflicting_item['_id']
else:
# expand dst_copy to all copies linked to dst_copy
if dst_copy or other_copies:
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Adding to to copies')
async with RedisLockAsync(
rdb, 'editLink'), MongoTransaction(client) as s:
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Adding to to copies, lock acquired')
all_copies = _getAllCopies(dst_copy, session=s())
# add videos from other copies
for uid in other_copies:
all_copies += _getAllCopies(uid,
session=s(),
use_unique_id=True)
video_data = await _make_video_data(
ret["data"], all_copies, playlists, url, user,
event_id)
setEventUserAndID(user, event_id)
new_item_id = tagdb.add_item(tags,
video_data,
3, ['title', 'desc'],
makeUserMeta(user),
session=s())
all_copies.append(ObjectId(new_item_id))
# remove duplicated items
all_copies = list(set(all_copies))
if len(all_copies) <= VideoConfig.MAX_COPIES:
for dst_vid in all_copies:
setEventUserAndID(user, event_id)
_addThiscopy(dst_vid,
all_copies,
makeUserMeta(user),
session=s())
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Successfully added to copies')
s.mark_succeed()
else:
#if playlist_lock :
# playlist_lock.release()
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Too many copies')
await _playlist_reorder_helper.post_video_failed(
unique_id, dst_playlist, playlist_ordered,
dst_rank, user, event_id)
return "TOO_MANY_COPIES", {}
else:
async with MongoTransaction(client) as s:
video_data = await _make_video_data(
ret["data"], [], playlists, url, user, event_id)
setEventUserAndID(user, event_id)
if use_autotag:
tags.extend(
inferTagsFromVideo(
video_data['utags'], video_data['title'],
video_data['desc'], 'CHS',
video_data['url'],
video_data['user_space_urls']))
new_item_id = tagdb.add_item(tags,
video_data,
3, ['title', 'desc'],
makeUserMeta(user),
session=s())
log_e(event_id,
user,
'scraper',
level='MSG',
obj={
'msg': 'New video added to database',
'vid': new_item_id
})
s.mark_succeed()
# if the operation is adding this video to playlist
if dst_playlist:
log_e(event_id,
user,
'scraper',
level='MSG',
obj={
'msg': 'Adding to playlist at position',
'rank': dst_rank
})
if playlist_ordered:
await _playlist_reorder_helper.post_video_succeed(
new_item_id, unique_id, dst_playlist,
playlist_ordered, dst_rank, user, event_id)
else:
setEventUserAndID(user, event_id)
if dst_rank == -1:
addVideoToPlaylist(dst_playlist, new_item_id, user)
else:
insertIntoPlaylist(dst_playlist, new_item_id,
dst_rank, user)
#if playlist_lock :
# playlist_lock.release()
log_e(event_id, user, 'scraper', level='MSG', obj='Done')
return 'SUCCEED', new_item_id
except UserError as ue:
await _playlist_reorder_helper.post_video_failed(
unique_id, dst_playlist, playlist_ordered, dst_rank, user,
event_id)
log_e(event_id, user, 'scraper', level='WARN', obj={'ue': str(ue)})
return ue.msg, {"aux": ue.aux, "traceback": traceback.format_exc()}
except Exception as ex:
await _playlist_reorder_helper.post_video_failed(
unique_id, dst_playlist, playlist_ordered, dst_rank, user,
event_id)
log_e(event_id, user, 'scraper', level='ERR', obj={'ex': str(ex)})
try:
problematic_lock = RedisLockAsync(rdb, 'editLink')
problematic_lock.reset()
except:
pass
return "UNKNOWN", '\n'.join(
[repr(traceback.format_exc()),
repr(traceback.extract_stack())])
async def _add_to_playlist(self, dst_playlist, event_id, user_global):
if self.playlist_map[dst_playlist]:
dst_rank = self.playlist_map[dst_playlist]['rank']
playlist_ordered = self.playlist_map[dst_playlist]['all']
try:
# fast method
async with RedisLockAsync(
rdb, "playlistEdit:" +
dst_playlist), MongoTransaction(client) as s:
cur_rank = 0
playlist = playlist_db.retrive_item(dst_playlist,
session=s())
if playlist is None:
raise UserError('PLAYLIST_NOT_EXIST')
if playlist["item"]["videos"] + len(
self.playlist_map[dst_playlist]
['succeed']) > PlaylistConfig.MAX_VIDEO_PER_PLAYLIST:
raise UserError('VIDEO_LIMIT_EXCEEDED')
playlist_videos = playlist["item"]['videos']
for unique_id in playlist_ordered:
if unique_id in self.playlist_map[dst_playlist][
'succeed']:
(video_id, _, user) = self.playlist_map[
dst_playlist]['succeed'][unique_id]
if dst_rank == -1:
if filterOperation('editPlaylist', user,
playlist, False):
if addVideoToPlaylistLockFree(
dst_playlist,
video_id,
user,
playlist_videos,
session=s()):
playlist_videos += 1
else:
if filterOperation('editPlaylist', user,
playlist, False):
if insertIntoPlaylistLockFree(dst_playlist,
video_id,
dst_rank +
cur_rank,
user,
session=s()):
cur_rank += 1
s.mark_succeed()
except UserError as ue:
# UserError, rereaise to upper level
log_e(event_id, user_global, '_add_to_playlist', 'ERR', {
'ex': str(ex),
'tb': traceback.format_exc()
})
del self.playlist_map[dst_playlist]
rdb.set(f'playlist-batch-post-event-{dst_playlist}', b'done')
raise ue
except Exception as ex:
# if anything goes wrong, fallback to slow method
log_e(event_id, user_global, '_add_to_playlist', 'ERR', {
'ex': str(ex),
'tb': traceback.format_exc()
})
cur_rank = 0
for unique_id in playlist_ordered:
if unique_id in self.playlist_map[dst_playlist]['succeed']:
(video_id, _, user) = self.playlist_map[dst_playlist][
'succeed'][unique_id]
# ignore error, add next video
try:
if dst_rank == -1:
addVideoToPlaylist(dst_playlist, video_id,
user)
else:
insertIntoPlaylist(dst_playlist, video_id,
dst_rank + cur_rank, user)
cur_rank += 1
except:
pass
log_e(
event_id, user_global, '_add_to_playlist', 'MSG', {
'succedd': len(self.playlist_map[dst_playlist]['succeed']),
'all': len(self.playlist_map[dst_playlist]['all']),
'pid': dst_playlist
})
del self.playlist_map[dst_playlist]
rdb.set(f'playlist-batch-post-event-{dst_playlist}', b'done')
def addComment(user,
thread_id: ObjectId,
text: str,
notification_type: str = 'comment_reply',
use_bleach=True): # user can add comments
filterOperation('postComment', user)
if use_bleach:
text = bleach.clean(text, tags=[], attributes=[], styles=[])
l = len(text)
if l > Comments.MAX_COMMENT_LENGTH_LONG:
raise UserError('COMMENT_TOO_LONG')
elif l > Comments.MAX_COMMENT_LENGTH_REGULAR and not filterOperation(
'postLongComment', user, raise_exception=False):
raise UserError('COMMENT_TOO_LONG')
thread_obj = db.comment_threads.find_one({'_id': thread_id})
if thread_obj is None:
raise UserError('THREAD_NOT_EXIST')
with redis_lock.Lock(rdb, "thread:" +
str(thread_id)), MongoTransaction(client) as s:
cid = str(
db.comment_items.insert_one(
{
'thread': thread_id,
'content': text,
'hidden': False,
'deleted': False,
'pinned': False,
'upvotes': 0,
'downvotes': 0,
'meta': makeUserMetaObject(user)
},
session=s()).inserted_id)
db.comment_threads.update_one({'_id': thread_id},
{'$inc': {
'count': int(1)
}},
session=s())
note_obj = {
"cid": ObjectId(cid),
"replied_by": makeUserMeta(user),
"content": text[:Comments.NOTIFICATION_CONTENT_LENGTH]
}
# ===========================================================
if 'obj_type' in thread_obj and 'obj_id' in thread_obj:
note_obj['replied_type'] = thread_obj['obj_type']
note_obj['replied_obj'] = thread_obj['obj_id']
else:
obj = db.videos.find_one({'comment_thread': thread_id},
session=s())
if obj:
note_obj['replied_type'] = 'video'
note_obj['replied_obj'] = obj['_id']
db.comment_threads.update_one(
{'_id': thread_id},
{'$set': {
'obj_type': 'video',
'obj_id': obj['_id']
}},
session=s())
else:
obj = playlist_db.retrive_item({'comment_thread': thread_id},
session=s())
if obj:
note_obj['replied_type'] = 'playlist'
note_obj['replied_obj'] = obj['_id']
db.comment_threads.update_one({'_id': thread_id}, {
'$set': {
'obj_type': 'playlist',
'obj_id': obj['_id']
}
},
session=s())
else:
obj = db.users.find_one({'comment_thread': thread_id},
session=s())
if obj:
note_obj['replied_type'] = 'user'
note_obj['replied_obj'] = obj['_id']
db.comment_threads.update_one({'_id': thread_id}, {
'$set': {
'obj_type': 'user',
'obj_id': obj['_id']
}
},
session=s())
else:
log(level='ERR',
obj={
'msg': 'orphan thread found!!',
'thread_id': thread_id,
'thread_obj': thread_obj
})
raise UserError('UNKNOWN_ERROR')
# ===========================================================
if notification_type and thread_obj['owner'] != user[
'_id']: # empty means do not notify user
createNotification(notification_type,
thread_obj['owner'],
session=s(),
other=note_obj)
if note_obj[
'replied_type'] == 'forum': # forum comment, set modified_at date
db.forum_threads.update_one({'_id': note_obj['replied_obj']}, {
'$set': {
'meta.modified_at': datetime.now(),
'meta.modified_by': user['_id']
}
},
session=s())
s.mark_succeed()
return cid
def addReply(user,
reply_to: ObjectId,
text: str,
notification_type: str = 'comment_reply',
use_bleach=True): # user can add comments
"""
reply_to: comment id
"""
filterOperation('postComment', user)
if use_bleach:
text = bleach.clean(text, tags=[], attributes=[], styles=[])
l = len(text)
if l > Comments.MAX_COMMENT_LENGTH_LONG:
raise UserError('COMMENT_TOO_LONG')
elif l > Comments.MAX_COMMENT_LENGTH_REGULAR and not filterOperation(
'postLongComment', user, raise_exception=False):
raise UserError('COMMENT_TOO_LONG')
parent_obj = db.comment_items.find_one({'_id': reply_to})
if parent_obj is None:
raise UserError('PARENT_NOT_EXIST')
with MongoTransaction(client) as s:
if 'thread' in parent_obj: # reply to primary comment
cid = str(
db.comment_items.insert_one(
{
'parent': reply_to,
'content': text,
'hidden': False,
'deleted': False,
'pinned': False,
'upvotes': 0,
'downvotes': 0,
'meta': makeUserMetaObject(user)
},
session=s()).inserted_id)
thread_obj = db.comment_threads.find_one(
{'_id': parent_obj['thread']}, session=s())
if thread_obj is None:
log(level='ERR',
obj={
'msg': 'orphan comment found!!',
'cid': parent_obj['_id'],
'obj': parent_obj
})
raise UserError('UNKNOWN_ERROR')
thread_id = thread_obj['_id']
else: # reply to secondary comment
cid = str(
db.comment_items.insert_one(
{
'parent': parent_obj['parent'],
'reply_to': reply_to,
'content': text,
'hidden': False,
'deleted': False,
'pinned': False,
'upvotes': 0,
'downvotes': 0,
'meta': makeUserMetaObject(user)
},
session=s()).inserted_id)
parent_parent_obj = db.comment_items.find_one(
{'_id': parent_obj['parent']}, session=s())
if parent_parent_obj is None:
log(level='ERR',
obj={
'msg': 'orphan comment found!!',
'cid': parent_obj['_id'],
'obj': parent_obj
})
raise UserError('UNKNOWN_ERROR')
thread_obj = db.comment_threads.find_one(
{'_id': parent_parent_obj['thread']}, session=s())
if thread_obj is None:
log(level='ERR',
obj={
'msg': 'orphan comment found!!',
'cid': parent_parent_obj['_id'],
'obj': parent_parent_obj
})
raise UserError('UNKNOWN_ERROR')
thread_id = thread_obj['_id']
note_obj = {
"cid": ObjectId(cid),
"replied_by": makeUserMeta(user),
"content": text[:Comments.NOTIFICATION_CONTENT_LENGTH]
}
# ===========================================================
if 'obj_type' in thread_obj and 'obj_id' in thread_obj:
note_obj['replied_type'] = thread_obj['obj_type']
note_obj['replied_obj'] = thread_obj['obj_id']
else:
obj = db.videos.find_one({'comment_thread': thread_id},
session=s())
if obj:
note_obj['replied_type'] = 'video'
note_obj['replied_obj'] = obj['_id']
db.comment_threads.update_one(
{'_id': thread_id},
{'$set': {
'obj_type': 'video',
'obj_id': obj['_id']
}},
session=s())
else:
obj = playlist_db.retrive_item({'comment_thread': thread_id},
session=s())
if obj:
note_obj['replied_type'] = 'playlist'
note_obj['replied_obj'] = obj['_id']
db.comment_threads.update_one({'_id': thread_id}, {
'$set': {
'obj_type': 'playlist',
'obj_id': obj['_id']
}
},
session=s())
else:
obj = db.users.find_one({'comment_thread': thread_id},
session=s())
if obj:
note_obj['replied_type'] = 'user'
note_obj['replied_obj'] = obj['_id']
db.comment_threads.update_one({'_id': thread_id}, {
'$set': {
'obj_type': 'user',
'obj_id': obj['_id']
}
},
session=s())
else:
log(level='ERR',
obj={
'msg': 'orphan thread found!!',
'thread_id': thread_id,
'thread_obj': thread_obj
})
raise UserError('UNKNOWN_ERROR')
# ===========================================================
if notification_type and parent_obj['meta']['created_by'] != user[
'_id']: # empty means do not notify user and we don't want to notify users who reply to themselves
createNotification(notification_type,
parent_obj['meta']['created_by'],
session=s(),
other=note_obj)
if note_obj[
'replied_type'] == 'forum': # forum reply, set modified_at date
db.forum_threads.update_one({'_id': note_obj['replied_obj']}, {
'$set': {
'meta.modified_at': datetime.now(),
'meta.modified_by': user['_id']
}
},
session=s())
s.mark_succeed()
def listCommonTagIDs(pid, user):
playlist = playlist_db.retrive_item(pid)
if playlist is None:
raise UserError('PLAYLIST_NOT_EXIST')
if playlist['item']['private']:
filterOperation('viewPrivatePlaylist', user, playlist)
result = db.playlist_items.aggregate([{
"$match": {
"pid": ObjectId(pid)
}
}, {
"$lookup": {
"from": 'videos',
"localField": "vid",
"foreignField": "_id",
"as": "video"
}
}, {
"$project": {
"video.tags": 1
}
}, {
"$unwind": {
"path": "$video"
}
}, {
"$project": {
"tags": {
'$filter': {
'input': '$video.tags',
'as': 'tag',
'cond': {
'$lt': ['$$tag', 0x80000000]
}
}
}
}
}, {
"$group": {
"_id": 0,
"tags": {
"$push": "$tags"
},
"initialTags": {
"$first": "$tags"
}
}
}, {
"$project": {
"commonTags": {
"$reduce": {
"input": "$tags",
"initialValue": "$initialTags",
"in": {
"$setIntersection": ["$$value", "$$this"]
}
}
}
}
}])
ret = [i for i in result]
if ret:
return ret[0]['commonTags']
else:
return []
