def insertIntoPlaylist(pid, vid, rank, user) :
log(obj = {'pid': pid, 'vid': vid, 'rank': rank})
with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s :
playlist = db.playlists.find_one({'_id': ObjectId(pid)}, session = s())
if playlist is None :
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('editPlaylist', user, playlist)
if tagdb.retrive_item({'_id': ObjectId(vid)}, session = s()) is None :
raise UserError('VIDEO_NOT_EXIST')
if playlist["videos"] > PlaylistConfig.MAX_VIDEO_PER_PLAYLIST :
raise UserError('VIDEO_LIMIT_EXCEEDED')
conflicting_item = db.playlist_items.find_one({'pid': ObjectId(pid), 'vid': ObjectId(vid)}, session = s())
if conflicting_item is not None :
editPlaylist_MoveLockFree(pid, conflicting_item, rank, session = s())
db.playlists.update_one({'_id': ObjectId(pid)}, {'$set': {
'meta.modified_by': makeUserMeta(user),
'meta.modified_at': datetime.now()}}, session = s())
s.mark_succeed()
return
if rank < 0 :
raise UserError('OUT_OF_RANGE')
if rank > playlist['videos'] :
rank = int(playlist['videos'])
playlists = tagdb.retrive_item({'_id': ObjectId(vid)}, session = s())['item']['series']
playlists.append(ObjectId(pid))
playlists = list(set(playlists))
tagdb.update_item_query(ObjectId(vid), {'$set': {'item.series': playlists}}, makeUserMeta(user), session = s())
db.playlists.update_one({"_id": ObjectId(pid)}, {"$inc": {"videos": int(1)}}, session = s())
db.playlist_items.update_many({'pid': ObjectId(pid), 'rank': {'$gte': rank}}, {'$inc': {'rank': int(1)}}, session = s())
db.playlist_items.insert_one({"pid": ObjectId(pid), "vid": ObjectId(vid), "rank": int(rank), "meta": makeUserMeta(user)}, session = s())
db.playlists.update_one({'_id': ObjectId(pid)}, {'$set': {
'meta.modified_by': makeUserMeta(user),
'meta.modified_at': datetime.now()}}, session = s())
s.mark_succeed()
def insertIntoPlaylistLockFree(pid, vid, rank, user, session) :
log(obj = {'pid': pid, 'vid': vid, 'rank': rank})
if tagdb.retrive_item({'_id': ObjectId(vid)}, session = session) is None :
#raise UserError('VIDEO_NOT_EXIST')
return False
conflicting_item = db.playlist_items.find_one({'pid': ObjectId(pid), 'vid': ObjectId(vid)}, session = session)
if conflicting_item is not None :
editPlaylist_MoveLockFree(pid, conflicting_item, rank, session = session)
db.playlists.update_one({'_id': ObjectId(pid)}, {'$set': {
'meta.modified_by': makeUserMeta(user),
'meta.modified_at': datetime.now()}}, session = session)
if conflicting_item['rank'] >= rank :
return True
else :
return False
playlists = tagdb.retrive_item({'_id': ObjectId(vid)}, session = session)['item']['series']
playlists.append(ObjectId(pid))
playlists = list(set(playlists))
tagdb.update_item_query(ObjectId(vid), {'$set': {'item.series': playlists}}, makeUserMeta(user), session = session)
db.playlists.update_one({"_id": ObjectId(pid)}, {"$inc": {"videos": int(1)}}, session = session)
db.playlist_items.update_many({'pid': ObjectId(pid), 'rank': {'$gte': rank}}, {'$inc': {'rank': int(1)}}, session = session)
db.playlist_items.insert_one({"pid": ObjectId(pid), "vid": ObjectId(vid), "rank": int(rank), "meta": makeUserMeta(user)}, session = session)
db.playlists.update_one({'_id': ObjectId(pid)}, {'$set': {
'meta.modified_by': makeUserMeta(user),
'meta.modified_at': datetime.now()}}, session = session)
return True
def addVideoToPlaylist(pid, vid, user):
log(obj={'pid': pid, 'vid': vid})
with redis_lock.Lock(rdb, "playlistEdit:" +
str(pid)), MongoTransaction(client) as s:
playlist = playlist_db.retrive_item(pid)
if playlist is None:
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('editPlaylist', user, playlist)
if tagdb.retrive_item({'_id': ObjectId(vid)}, session=s()) is None:
raise UserError('VIDEO_NOT_EXIST')
if playlist['item']["videos"] > PlaylistConfig.MAX_VIDEO_PER_PLAYLIST:
raise UserError('VIDEO_LIMIT_EXCEEDED')
conflicting_item = db.playlist_items.find_one(
{
'pid': ObjectId(pid),
'vid': ObjectId(vid)
}, session=s())
if conflicting_item is not None:
editPlaylist_MoveLockFree(pid,
conflicting_item,
int(playlist['item']["videos"]),
session=s())
playlist_db.update_item_query(playlist, {},
user=makeUserMeta(user),
session=s())
s.mark_succeed()
return
playlists = tagdb.retrive_item({'_id': ObjectId(vid)},
session=s())['item']['series']
playlists.append(ObjectId(pid))
playlists = list(set(playlists))
tagdb.update_item_query(ObjectId(vid),
{'$set': {
'item.series': playlists
}},
user=makeUserMeta(user),
session=s())
db.playlist_items.insert_one(
{
"pid": ObjectId(pid),
"vid": ObjectId(vid),
"rank": int(playlist['item']["videos"]),
"meta": makeUserMeta(user)
},
session=s())
playlist_db.update_item_query(playlist,
{"$inc": {
"item.videos": int(1)
}},
user=makeUserMeta(user),
session=s())
s.mark_succeed()
def _getAllCopies(vid, session, use_unique_id=False):
if not vid:
return []
if use_unique_id:
this_video = tagdb.retrive_item({"item.unique_id": vid},
session=session)
else:
this_video = tagdb.retrive_item({"_id": ObjectId(vid)},
session=session)
if this_video is None:
return []
copies = this_video['item']['copies']
# add self
copies.append(ObjectId(this_video['_id']))
# use set to remove duplicated items
return list(set(copies))
def postVideoNoMerge(user, url, tags, copy, pid, rank, repost_type):
log(obj={'url': url, 'tags': tags, 'copy': copy, 'pid': pid, 'rank': rank})
filterOperation('postVideo', user)
tags = [tag.strip() for tag in tags]
if not url:
raise UserError('EMPTY_URL')
if len(url) > VideoConfig.MAX_URL_LENGTH:
raise UserError('URL_TOO_LONG')
if len(tags) > VideoConfig.MAX_TAGS_PER_VIDEO:
raise UserError('TAGS_LIMIT_EXCEEDED')
obj, cleanURL = dispatch(url)
if obj is None:
log(level='WARN', obj={'url': url})
raise UserError('UNSUPPORTED_WEBSITE')
if not cleanURL:
raise UserError('EMPTY_URL')
uid = obj.unique_id(obj, cleanURL)
vid_item = tagdb.retrive_item({'item.unique_id': uid})
if vid_item is None:
tags = filterTags(tags)
log(obj={'url': cleanURL})
task_id = postTask(
_createJsonForPosting(cleanURL,
tags,
copy,
pid,
rank, [],
repost_type,
user,
use_autotag=True))
return task_id
else:
return 'no-suck-task'
def createPlaylistFromCopies(pid, site, user):
if site not in ["youtube", "bilibili", "nicovideo", "twitter", "acfun"]:
raise UserError("UNSUPPORTED_SITE")
videos, _, playlist_obj = listAllPlaylistVideosOrdered(pid, user)
new_pid = createPlaylist('english',
playlist_obj['title']['english'] + ' - %s' % site,
playlist_obj['desc']['english'],
playlist_obj['cover'], user,
playlist_obj['private'])
with redis_lock.Lock(rdb, 'editLink'), redis_lock.Lock(
rdb,
"playlistEdit:" + str(new_pid)), MongoTransaction(client) as s:
rank = 0
for video in videos:
copies = video['item']['item']['copies']
for cp in copies:
item = tagdb.retrive_item(cp, session=s())
if item['_id'] != video['vid'] and item['item']['site'] == site:
addVideoToPlaylistLockFree(new_pid,
item['_id'],
user,
rank,
session=s())
rank += 1
break
s.mark_succeed()
return new_pid
def getVideosByURLs(urls):
ret = []
for url in urls:
obj, cleanURL = dispatch(url)
if obj is None:
ret.append({
'url': url,
'exist': False,
'reason': 'UNSUPPORTED_WEBSITE'
})
continue
if not cleanURL:
ret.append({'url': url, 'exist': False, 'reason': 'EMPTY_URL'})
continue
uid = obj.unique_id(obj, cleanURL)
obj = db.retrive_item({'item.unique_id': uid})
if obj:
ret.append({'url': url, 'exist': True, 'id': obj['_id']})
else:
ret.append({
'url': url,
'exist': False,
'reason': 'VIDEO_NOT_EXIST'
})
return ret
def requestSubtitleOCR(user, vid: ObjectId):
# step 1: verify user and video
filterOperation('requestSubtitleOCR', user)
video_item = tagdb.retrive_item(vid)
if video_item is None:
raise UserError('VIDEO_NOT_FOUND')
# step 2: check if request exists
with redis_lock.Lock(
rdb,
"videoEdit:" + video_item['item']['unique_id']), redis_lock.Lock(
rdb, "mmdocr_global_lock"), MongoTransaction(client) as s:
ocr_record = db.subtitle_ocr.find_one({"vid": vid}, session=s())
if ocr_record is None:
# create new record
record = {
"vid": vid,
"status": "Queuing",
"version": 0, # version is set in postSubtitleOCRResult
"worker_id":
"", # worker_id is set in queryAndProcessQueuingRequests
"meta": makeUserMetaObject(user)
}
db.subtitle_ocr.insert_one(record, session=s())
else:
status = ocr_record['status']
record_id = ocr_record['_id']
record_version = ocr_record['version']
mmdocr_version = int(Config.MMDOCR_VERSION)
if status in ['NoRecord', 'RecordOutOfDate', 'Error']:
assert status != "NoRecord"
db.subtitle_ocr.update_one({"_id": record_id}, {
"$set": {
"status": "Queuing",
"meta.modified_at": datetime.utcnow(),
"meta.modified_by": ObjectId(user['_id'])
}
},
session=s())
pass
elif status == 'RecordExists':
if record_version < mmdocr_version:
# newer version of mmdocr exists
db.subtitle_ocr.update_one({"_id": record_id}, {
"$set": {
"status": "Queuing",
"meta.modified_at": datetime.utcnow(),
"meta.modified_by": ObjectId(user['_id'])
}
},
session=s())
pass
else:
raise UserError('RECORD_ALREADY_EXISTS')
else:
raise UserError('VIDEO_BEING_PROCESSED')
s.mark_succeed()
def filterSingleVideo(vid, user, raise_error = True) :
item = tagdb.retrive_item(vid)
if item is None :
return None
if _is_authorised(item, user) :
return item
else :
if raise_error :
raise UserError('ITEM_NOT_FOUND')
else :
return None
def getVideoByURL(url):
obj, cleanURL = dispatch(url)
if obj is None:
raise UserError('UNSUPPORTED_WEBSITE')
if not cleanURL:
raise UserError('EMPTY_URL')
uid = obj.unique_id(obj, cleanURL)
obj = db.retrive_item({'item.unique_id': uid})
if obj:
return obj
raise UserError('VIDEO_NOT_EXIST')
def _removeThisCopy(dst_vid, this_vid, user, session):
if this_vid is None:
return
dst_video = tagdb.retrive_item({"_id": ObjectId(dst_vid)}, session)
if dst_video is None:
return
dst_copies = dst_video['item']['copies']
dst_copies = list(set(dst_copies) - set([ObjectId(this_vid)]))
tagdb.update_item_query(ObjectId(dst_vid),
{"$set": {
"item.copies": dst_copies
}}, user, session)
def querySubtitleOCRStatus(vid: ObjectId):
# step 1: verify video
video_item = tagdb.retrive_item(vid)
if video_item is None:
raise UserError('VIDEO_NOT_FOUND')
with redis_lock.Lock(rdb, "mmdocr_global_lock"):
# step 2: query and return
ocr_record = db.subtitle_ocr.find_one({"vid": vid})
if ocr_record is None:
return "NoRecord"
else:
return ocr_record['status']
def addVideoToPlaylistLockFree(pid, vid, user, rank, session):
log(obj={'pid': pid, 'vid': vid})
if tagdb.retrive_item({'_id': ObjectId(vid)}, session=session) is None:
#raise UserError('VIDEO_NOT_EXIST')
return False
conflicting_item = db.playlist_items.find_one(
{
'pid': ObjectId(pid),
'vid': ObjectId(vid)
}, session=session)
if conflicting_item is not None:
editPlaylist_MoveLockFree(pid, conflicting_item, rank, session=session)
playlist_db.update_item_query(pid, {},
user=makeUserMeta(user),
session=session)
return False
playlists = tagdb.retrive_item({'_id': ObjectId(vid)},
session=session)['item']['series']
playlists.append(ObjectId(pid))
playlists = list(set(playlists))
tagdb.update_item_query(ObjectId(vid),
{'$set': {
'item.series': playlists
}},
user=makeUserMeta(user),
session=session)
db.playlist_items.insert_one(
{
"pid": ObjectId(pid),
"vid": ObjectId(vid),
"rank": int(rank),
"meta": makeUserMeta(user)
},
session=session)
playlist_db.update_item_query(pid, {"$inc": {
"item.videos": int(1)
}},
user=makeUserMeta(user),
session=session)
return True
def listPlaylistsForVideo(user, vid) :
video = tagdb.retrive_item({'_id': ObjectId(vid)})
if video is None :
raise UserError('VIDEO_NOT_EXIST')
if isObjectAgnosticOperationPermitted('viewPrivatePlaylist', user) :
auth_obj = {}
else :
auth_obj = {'$or': [{'playlist.meta.created_by': user['_id'] if user else ''}, {'playlist.private': False, 'playlist.videos': {'$gt': 1}}]}
result = db.playlist_items.aggregate([
{
'$match': {
'$and': [
{
'pid': {
'$in': video['item']['series']
}
},
{
'vid': video['_id']
}
]
}
},
{
'$lookup': {
'from': 'playlists',
'localField': 'pid',
'foreignField': '_id',
'as': 'playlist'
}
},
{
'$unwind': {
'path': '$playlist'
}
},
{
'$match': auth_obj
}
])
ans = []
for obj in result :
playlist_obj = obj['playlist']
playlist_obj['prev'] = ''
playlist_obj['next'] = ''
rank = obj['rank']
if rank > 0 :
playlist_obj['prev'] = str(db.playlist_items.find_one({'pid': playlist_obj['_id'], 'rank': int(rank - 1)})['vid'])
if rank + 1 < playlist_obj['videos'] :
playlist_obj['next'] = str(db.playlist_items.find_one({'pid': playlist_obj['_id'], 'rank': int(rank + 1)})['vid'])
ans.append(playlist_obj)
return ans
def postSubtitle(user, vid: ObjectId, language: str, subformat: str,
content: str):
if language not in VALID_LANGUAGES:
raise UserError('INVALID_LANGUAGE')
subformat = subformat.lower()
if subformat not in VALID_SUBTITLE_FORMAT:
raise UserError('INVALID_SUBTITLE_FORMAT')
video_item = tagdb.retrive_item(vid)
if video_item is None:
raise UserError('VIDEO_NOT_FOUND')
try:
size = len(content.encode('utf-8'))
except:
size = -1
filterOperation('postSubtitle', user)
with redis_lock.Lock(
rdb, "videoEdit:" +
video_item['item']['unique_id']), MongoTransaction(client) as s:
existing_item = db.subtitles.find_one(
{
'vid': vid,
'lang': language,
'format': subformat,
'meta.created_by': makeUserMeta(user)
},
session=s())
if existing_item is None:
subid = db.subtitles.insert_one(
{
'vid': vid,
'lang': language,
'format': subformat,
'content': content,
'size': size,
'deleted': False,
'autogen': False,
'version': 0,
'meta': makeUserMetaObject(user)
},
session=s()).inserted_id
else:
db.subtitles.update_one({'_id': existing_item['_id']}, {
'$set': {
'content': content,
'size': size,
'meta.modified_at': datetime.utcnow()
}
},
session=s())
subid = existing_item['_id']
s.mark_succeed()
return ObjectId(subid)
def postSubtitleOCRResult(user, unique_id: str, content: str, subformat: str,
version: int, worker_id: str):
# step 1: verify and post
filterOperation('subtitleocr_postSubtitleOCRResult', user)
subformat = subformat.lower()
if subformat not in VALID_SUBTITLE_FORMAT:
raise UserError('INVALID_SUBTITLE_FORMAT')
video_item = tagdb.retrive_item({"item.unique_id": unique_id})
if video_item is None:
raise UserError('VIDEO_NOT_FOUND')
try:
size = len(content.encode('utf-8'))
except:
size = -1
with redis_lock.Lock(
rdb,
"videoEdit:" + video_item['item']['unique_id']), redis_lock.Lock(
rdb, "mmdocr_global_lock"), MongoTransaction(client) as s:
# delete old versions
db.subtitles.delete_many({
'vid': video_item['_id'],
'autogen': True
},
session=s())
subid = db.subtitles.insert_one(
{
'vid': video_item['_id'],
'lang': 'UNKNOWN',
'format': subformat,
'content': content,
'size': size,
'deleted': False,
'version': version,
'autogen': True,
'meta': makeUserMetaObject(None)
},
session=s()).inserted_id
# step 2: update subtitle_ocr
db.subtitle_ocr.update_one({"vid": video_item['_id']}, {
"$set": {
"status": "RecordExists",
"version": version,
"worker_id": worker_id
}
},
session=s())
s.mark_succeed()
return subid
def _getAllCopies(vid_or_obj, session=None):
if not vid_or_obj:
return []
this_video = None
if isinstance(vid_or_obj, str) or isinstance(vid_or_obj, ObjectId):
this_video = tagdb.retrive_item({"_id": ObjectId(vid_or_obj)},
session=session)
else:
this_video = vid_or_obj
if this_video is None:
return []
copies = this_video['item']['copies']
# add self
copies.append(this_video['_id'])
# use set to remove duplicated items
return list(set(copies))
def listPlaylistsForVideoNoAuth(vid):
video = tagdb.retrive_item({'_id': ObjectId(vid)})
if video is None:
raise UserError('VIDEO_NOT_EXIST')
result = db.playlist_items.aggregate([{
'$match': {
'$and': [{
'pid': {
'$in': video['item']['series']
}
}, {
'vid': video['_id']
}]
}
}, {
'$lookup': {
'from': 'playlist_metas',
'localField': 'pid',
'foreignField': '_id',
'as': 'playlist'
}
}, {
'$unwind': {
'path': '$playlist'
}
}])
ans = []
for obj in result:
playlist_obj = obj['playlist']
playlist_obj['prev'] = ''
playlist_obj['next'] = ''
rank = obj['rank']
if rank > 0:
playlist_obj['prev'] = str(
db.playlist_items.find_one({
'pid': playlist_obj['_id'],
'rank': int(rank - 1)
})['vid'])
if rank + 1 < playlist_obj['item']['videos']:
playlist_obj['next'] = str(
db.playlist_items.find_one({
'pid': playlist_obj['_id'],
'rank': int(rank + 1)
})['vid'])
ans.append(playlist_obj)
return ans
def _addThiscopy(dst_vid, this_vid, user, session):
if this_vid is None:
return
dst_video = tagdb.retrive_item({"_id": ObjectId(dst_vid)}, session=session)
if dst_video is None:
return
dst_copies = dst_video['item']['copies']
if isinstance(this_vid, list):
dst_copies.extend(this_vid)
else:
dst_copies.append(ObjectId(this_vid))
dst_copies = list(set(dst_copies) - set([ObjectId(dst_vid)]))
tagdb.update_item_query(ObjectId(dst_vid),
{"$set": {
"item.copies": dst_copies
}},
user=user,
session=session)
def syncTags(dst, src, user):
if dst == src:
raise UserError('SAME_VIDEO')
filterOperation('syncTags', user, (dst, src))
src_item, src_tags, _, _ = tagdb.retrive_item_with_tag_category_map(
src, 'CHS')
log(obj={'src_tags': src_tags, 'src_id': src})
dst_item = tagdb.retrive_item(dst)
if dst_item is None:
raise UserError('ITEM_NOT_EXIST')
with redis_lock.Lock(
rdb, "videoEdit:" +
dst_item["item"]["unique_id"]), MongoTransaction(client) as s:
tagdb.update_item_tags_merge(ObjectId(dst),
src_tags,
makeUserMeta(user),
session=s())
s.mark_succeed()
def setVideoRepostType(vid, repost_type, user):
filterOperation('setVideoRepostType', user)
if repost_type not in [
'official', 'official_repost', 'authorized_translation',
'authorized_repost', 'translation', 'repost'
]:
raise UserError('INCORRECT_REPOST_TYPE')
video_obj = tagdb.retrive_item(vid)
if video_obj is None:
raise UserError('VIDEO_NOT_FOUND')
lock_id = "videoEdit:" + video_obj['item']['unique_id']
with redis_lock.Lock(rdb, lock_id), MongoTransaction(client) as s:
tagdb.update_item_query(video_obj,
{'$set': {
'item.repost_type': repost_type
}},
session=s())
s.mark_succeed()
def editPlaylist_Move(pid, vid, to_rank, user) :
log(obj = {'pid': pid, 'vid': vid, 'to_rank': to_rank})
with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s :
playlist = db.playlists.find_one({'_id': ObjectId(pid)}, session = s())
if playlist is None :
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('editPlaylist', user, playlist)
if tagdb.retrive_item({'_id': ObjectId(vid)}, session = s()) is None :
raise UserError('VIDEO_NOT_EXIST')
if to_rank < 0 :
raise UserError('OUT_OF_RANGE')
if to_rank > playlist['videos'] :
to_rank = int(playlist['videos'])
editPlaylist_MoveLockFree(pid, vid, to_rank, session = s())
db.playlists.update_one({'_id': ObjectId(pid)}, {'$set': {
'meta.modified_by': makeUserMeta(user),
'meta.modified_at': datetime.now()}}, session = s())
s.mark_succeed()
def broadcastTags(src, user):
filterOperation('broadcastTags', user, src)
_, src_tags, _, _ = tagdb.retrive_item_with_tag_category_map(src, 'CHS')
log(obj={'src_tags': src_tags, 'src_id': src})
with redis_lock.Lock(rdb, "editLink"), MongoTransaction(client) as s:
copies = _getAllCopies(src, session=s())
log(obj={'clique': copies})
for copy in copies:
if copy != ObjectId(src): # prevent self updating
copy_obj = tagdb.retrive_item({"_id": ObjectId(copy)},
session=s())
assert copy_obj is not None, 'copy_obj %s not exist' % copy
with redis_lock.Lock(
rdb, "videoEdit:" + copy_obj["item"]["unique_id"]):
tagdb.update_item_tags_merge(ObjectId(copy),
src_tags,
makeUserMeta(user),
session=s())
s.mark_succeed()
def editPlaylist_Move(pid, vid, to_rank, user):
log(obj={'pid': pid, 'vid': vid, 'to_rank': to_rank})
with redis_lock.Lock(rdb, "playlistEdit:" +
str(pid)), MongoTransaction(client) as s:
playlist = playlist_db.retrive_item(pid, session=s())
if playlist is None:
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('editPlaylist', user, playlist)
if tagdb.retrive_item({'_id': ObjectId(vid)}, session=s()) is None:
raise UserError('VIDEO_NOT_EXIST')
if to_rank < 0:
raise UserError('OUT_OF_RANGE')
if to_rank > playlist['item']['videos']:
to_rank = int(playlist['item']['videos'])
editPlaylist_MoveLockFree(pid, vid, to_rank, session=s())
playlist_db.update_item_query(playlist, {},
user=makeUserMeta(user),
session=s())
s.mark_succeed()
def refreshVideoDetail(vid, user):
log(obj={'vid': vid})
filterOperation('refreshVideoDetail', user, vid)
filterSingleVideo(vid, user)
item = tagdb.retrive_item(vid)
if item is None:
raise UserError('ITEM_NOT_EXIST')
json_str = dumps({
'url': item['item']['url'],
'tags': [],
'dst_copy': '',
'dst_playlist': '',
'dst_rank': -1,
'other_copies': [],
'user': user,
'playlist_ordered': None,
'update_video_detail': True,
'event_id': getEventID()
})
postTask(json_str)
def inferTagsFromVideo(utags,
title,
desc,
user_language,
video_url: str = '',
user_urls: [str] = []):
log(
obj={
'title': title,
'desc': desc,
'utags': utags,
'lang': user_language,
'video_url': video_url,
'user_urls': user_urls
})
video_url = video_url.strip()
tagids = []
if video_url:
obj, cleanURL = dispatch(video_url)
if obj is not None:
uid = obj.unique_id(obj, cleanURL)
vid_item = db.retrive_item({'item.unique_id': uid})
if vid_item is not None:
tagids = list(
filter(lambda x: x < 0x80000000, vid_item['tags']))
if not tagids:
utags = [u.lower() for u in utags]
utags.append(title)
utags.append(desc)
all_text = ' 3e7dT2ibT7dM '.join(utags)
tagids = inferTagidsFromText(all_text)
matched_author_records, matched_author_tags = matchUserSpace(user_urls)
matched_common_ids = itertools.chain.from_iterable(
[x['common_tagids'] for x in matched_author_records])
tagids = list(
set(tagids) | set([x['id'] for x in matched_author_tags])
| set(matched_common_ids))
return db.translate_tag_ids_to_user_language(tagids, user_language)[0]
def updateRequestStatus(user, video_status_map,
worker_id: str): # map of unique_id to status
filterOperation('subtitleocr_updateRequestStatus', user)
with redis_lock.Lock(rdb,
"mmdocr_global_lock"), MongoTransaction(client) as s:
for unqiue_id, status in video_status_map.items():
if status not in [
'PendingDownload', 'Downloading', 'PendingProcess',
'Processing', 'Error', 'Queuing'
]:
continue
# step 1: verify videos
video_item = tagdb.retrive_item({"item.unique_id": unqiue_id},
session=s())
if video_item is not None:
# step 2: update
db.subtitle_ocr.update_one(
{"vid": video_item['_id']},
{"$set": {
"status": status,
"worker_id": worker_id
}},
session=s())
s.mark_succeed()
def getVideoDetailNoFilter(vid):
return db.retrive_item(vid)
async def postVideoAsync(url,
tags,
dst_copy,
dst_playlist,
dst_rank,
other_copies,
repost_type,
playlist_ordered,
user,
update_video_detail,
event_id,
field_override=None,
use_autotag=False):
parsed = None
try:
dst_playlist = str(dst_playlist)
dst_rank = -1 if dst_rank is None else dst_rank
#tags = tagdb.filter_and_translate_tags(tags)
parsed, url = dispatch(url)
except:
pass
if parsed is None:
log_e(event_id, user, 'dispatcher', 'ERR', {
'msg': 'PARSE_FAILED',
'url': url
})
await _playlist_reorder_helper.post_video_failed(
url, dst_playlist, playlist_ordered, dst_rank, user, event_id)
return "PARSE_FAILED", {}
unique_id = await parsed.unique_id_async(
self=parsed,
link=url) # empty unique_id for b23.tv posts, f**k bilibli
if not unique_id:
ret = await parsed.get_metadata_async(parsed, url, update_video_detail)
unique_id = ret['data']['unique_id']
log_e(
event_id, user, 'scraper', 'MSG', {
'url': url,
'dst_copy': dst_copy,
'other_copies': other_copies,
'dst_playlist': dst_playlist
})
setEventOp('scraper')
try:
lock_id = "videoEdit:" + unique_id
async with RedisLockAsync(rdb, lock_id):
unique, conflicting_item = verifyUniqueness(unique_id)
if unique or update_video_detail:
async with _download_sem:
ret = await parsed.get_metadata_async(
parsed, url, update_video_detail)
if repost_type:
ret['data']['repost_type'] = repost_type
else:
ret['data']['repost_type'] = 'unknown'
if ret["status"] == 'FAILED':
log_e(event_id, user, 'downloader', 'WARN', {
'msg': 'FETCH_FAILED',
'ret': ret
})
await _playlist_reorder_helper.post_video_failed(
unique_id, dst_playlist, playlist_ordered, dst_rank,
user, event_id)
return "FETCH_FAILED", ret
else:
unique_id = ret['data']['unique_id']
else:
# build ret
ret = makeResponseSuccess({
'thumbnailURL':
conflicting_item['item']['thumbnail_url'],
'title':
conflicting_item['item']['title'],
'desc':
conflicting_item['item']['desc'],
'site':
conflicting_item['item']['site'],
'uploadDate':
conflicting_item['item']['upload_time'],
"unique_id":
conflicting_item['item']['unique_id'],
"utags":
conflicting_item['item']['utags']
})
for k, v in conflicting_item['item'].items():
ret['data'][k] = v
if 'part_name' in conflicting_item['item']:
ret['part_name'] = conflicting_item['item']['part_name']
if 'repost_type' in conflicting_item[
'item'] and conflicting_item['item']['repost_type']:
ret['data']['repost_type'] = repost_type
else:
ret['data']['repost_type'] = 'unknown'
tagdb.update_item_query(
conflicting_item,
{'$set': {
'item.repost_type': repost_type
}},
user=makeUserMeta(user))
#if hasattr(parsed, 'LOCAL_CRAWLER') :
# url = ret["data"]["url"]
#else :
# url = clear_url(url)
use_override = False
if field_override and '__condition' in field_override:
condition = field_override['__condition']
del field_override['__condition']
if condition == 'any':
use_override = True
elif condition == 'placeholder' and 'placeholder' in ret[
"data"] and ret["data"]['placeholder']:
use_override = True
if use_override:
for key in field_override:
ret['data'][key] = field_override[key]
playlists = []
#playlist_lock = None
if dst_playlist:
#playlist_lock = RedisLockAsync(rdb, "playlistEdit:" + str(dst_playlist))
#playlist_lock.acquire()
if playlist_db.retrive_item(dst_playlist) is not None:
playlists = [ObjectId(dst_playlist)]
if not unique:
log_e(event_id,
user,
'scraper',
level='MSG',
obj={
'msg': 'ALREADY_EXIST',
'unique_id': ret["data"]["unique_id"]
})
"""
Update existing video
"""
if update_video_detail:
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Updating video detail')
with MongoTransaction(client) as s:
old_item = tagdb.retrive_item(conflicting_item['_id'],
session=s())['item']
if old_item['thumbnail_url'] and old_item[
'cover_image']:
# old thumbnail exists, no need to download again
new_detail = await _make_video_data_update(
ret["data"], url, user, event_id)
else:
# old thumbnail does not exists, add to dict
new_detail = await _make_video_data_update(
ret["data"], url, user, event_id,
ret["data"]["thumbnailURL"])
for key in new_detail.keys():
old_item[key] = new_detail[
key] # overwrite or add new field
setEventUserAndID(user, event_id)
tagdb.update_item_query(conflicting_item['_id'],
{'$set': {
'item': old_item
}}, ['title', 'desc'],
user=makeUserMeta(user),
session=s())
s.mark_succeed()
return 'SUCCEED', conflicting_item['_id']
# this video already exist in the database
# if the operation is to add a link to other copies and not adding self
if (dst_copy and
dst_copy != conflicting_item['_id']) or other_copies:
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Adding to to copies')
async with RedisLockAsync(
rdb, 'editLink'), MongoTransaction(client) as s:
log_e(event_id,
user,
level='MSG',
obj='Adding to to copies, lock acquired')
# find all copies of video dst_copy, self included
all_copies = _getAllCopies(dst_copy, session=s())
# find all videos linked to source video
all_copies += _getAllCopies(conflicting_item['_id'],
session=s())
# add videos from other copies
for uid in other_copies:
all_copies += _getAllCopies(uid,
session=s(),
use_unique_id=True)
# remove duplicated items
all_copies = list(set(all_copies))
# add this video to all other copies found
if len(all_copies) <= VideoConfig.MAX_COPIES:
for dst_vid in all_copies:
setEventUserAndID(user, event_id)
_addThiscopy(dst_vid,
all_copies,
makeUserMeta(user),
session=s())
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Successfully added to copies')
s.mark_succeed()
else:
#if playlist_lock :
# playlist_lock.release()
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Too many copies')
await _playlist_reorder_helper.post_video_failed(
unique_id, dst_playlist, playlist_ordered,
dst_rank, user, event_id)
return "TOO_MANY_COPIES", {}
# if the operation is adding this video to playlist
if dst_playlist:
log_e(event_id,
user,
'scraper',
level='MSG',
obj={
'msg': 'Adding to playlist at position',
'rank': dst_rank
})
if playlist_ordered:
await _playlist_reorder_helper.post_video_succeed(
conflicting_item['_id'], unique_id, dst_playlist,
playlist_ordered, dst_rank, user, event_id)
else:
setEventUserAndID(user, event_id)
if dst_rank == -1:
addVideoToPlaylist(dst_playlist,
conflicting_item['_id'], user)
else:
insertIntoPlaylist(dst_playlist,
conflicting_item['_id'],
dst_rank, user)
# merge tags
async with MongoTransaction(client) as s:
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Merging tags')
setEventUserAndID(user, event_id)
tagdb.update_item_tags_merge(conflicting_item['_id'],
tags,
makeUserMeta(user),
session=s(),
remove_tagids=[354])
s.mark_succeed()
#if playlist_lock :
# playlist_lock.release()
#return "VIDEO_ALREADY_EXIST", conflicting_item['_id']
return "SUCCEED", conflicting_item['_id']
else:
# expand dst_copy to all copies linked to dst_copy
if dst_copy or other_copies:
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Adding to to copies')
async with RedisLockAsync(
rdb, 'editLink'), MongoTransaction(client) as s:
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Adding to to copies, lock acquired')
all_copies = _getAllCopies(dst_copy, session=s())
# add videos from other copies
for uid in other_copies:
all_copies += _getAllCopies(uid,
session=s(),
use_unique_id=True)
video_data = await _make_video_data(
ret["data"], all_copies, playlists, url, user,
event_id)
setEventUserAndID(user, event_id)
new_item_id = tagdb.add_item(tags,
video_data,
3, ['title', 'desc'],
makeUserMeta(user),
session=s())
all_copies.append(ObjectId(new_item_id))
# remove duplicated items
all_copies = list(set(all_copies))
if len(all_copies) <= VideoConfig.MAX_COPIES:
for dst_vid in all_copies:
setEventUserAndID(user, event_id)
_addThiscopy(dst_vid,
all_copies,
makeUserMeta(user),
session=s())
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Successfully added to copies')
s.mark_succeed()
else:
#if playlist_lock :
# playlist_lock.release()
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Too many copies')
await _playlist_reorder_helper.post_video_failed(
unique_id, dst_playlist, playlist_ordered,
dst_rank, user, event_id)
return "TOO_MANY_COPIES", {}
else:
async with MongoTransaction(client) as s:
video_data = await _make_video_data(
ret["data"], [], playlists, url, user, event_id)
setEventUserAndID(user, event_id)
if use_autotag:
tags.extend(
inferTagsFromVideo(
video_data['utags'], video_data['title'],
video_data['desc'], 'CHS',
video_data['url'],
video_data['user_space_urls']))
new_item_id = tagdb.add_item(tags,
video_data,
3, ['title', 'desc'],
makeUserMeta(user),
session=s())
log_e(event_id,
user,
'scraper',
level='MSG',
obj={
'msg': 'New video added to database',
'vid': new_item_id
})
s.mark_succeed()
# if the operation is adding this video to playlist
if dst_playlist:
log_e(event_id,
user,
'scraper',
level='MSG',
obj={
'msg': 'Adding to playlist at position',
'rank': dst_rank
})
if playlist_ordered:
await _playlist_reorder_helper.post_video_succeed(
new_item_id, unique_id, dst_playlist,
playlist_ordered, dst_rank, user, event_id)
else:
setEventUserAndID(user, event_id)
if dst_rank == -1:
addVideoToPlaylist(dst_playlist, new_item_id, user)
else:
insertIntoPlaylist(dst_playlist, new_item_id,
dst_rank, user)
#if playlist_lock :
# playlist_lock.release()
log_e(event_id, user, 'scraper', level='MSG', obj='Done')
return 'SUCCEED', new_item_id
except UserError as ue:
await _playlist_reorder_helper.post_video_failed(
unique_id, dst_playlist, playlist_ordered, dst_rank, user,
event_id)
log_e(event_id, user, 'scraper', level='WARN', obj={'ue': str(ue)})
return ue.msg, {"aux": ue.aux, "traceback": traceback.format_exc()}
except Exception as ex:
await _playlist_reorder_helper.post_video_failed(
unique_id, dst_playlist, playlist_ordered, dst_rank, user,
event_id)
log_e(event_id, user, 'scraper', level='ERR', obj={'ex': str(ex)})
try:
problematic_lock = RedisLockAsync(rdb, 'editLink')
problematic_lock.reset()
except:
pass
return "UNKNOWN", '\n'.join(
[repr(traceback.format_exc()),
repr(traceback.extract_stack())])
def verifyUniqueness(postingId):
if not postingId:
return True, None
val = tagdb.retrive_item({"item.unique_id": postingId})
return val is None, val