def insertIntoPlaylistLockFree(pid, vid, rank, user, session) :
log(obj = {'pid': pid, 'vid': vid, 'rank': rank})
if tagdb.retrive_item({'_id': ObjectId(vid)}, session = session) is None :
#raise UserError('VIDEO_NOT_EXIST')
return False
conflicting_item = db.playlist_items.find_one({'pid': ObjectId(pid), 'vid': ObjectId(vid)}, session = session)
if conflicting_item is not None :
editPlaylist_MoveLockFree(pid, conflicting_item, rank, session = session)
db.playlists.update_one({'_id': ObjectId(pid)}, {'$set': {
'meta.modified_by': makeUserMeta(user),
'meta.modified_at': datetime.now()}}, session = session)
if conflicting_item['rank'] >= rank :
return True
else :
return False
playlists = tagdb.retrive_item({'_id': ObjectId(vid)}, session = session)['item']['series']
playlists.append(ObjectId(pid))
playlists = list(set(playlists))
tagdb.update_item_query(ObjectId(vid), {'$set': {'item.series': playlists}}, makeUserMeta(user), session = session)
db.playlists.update_one({"_id": ObjectId(pid)}, {"$inc": {"videos": int(1)}}, session = session)
db.playlist_items.update_many({'pid': ObjectId(pid), 'rank': {'$gte': rank}}, {'$inc': {'rank': int(1)}}, session = session)
db.playlist_items.insert_one({"pid": ObjectId(pid), "vid": ObjectId(vid), "rank": int(rank), "meta": makeUserMeta(user)}, session = session)
db.playlists.update_one({'_id': ObjectId(pid)}, {'$set': {
'meta.modified_by': makeUserMeta(user),
'meta.modified_at': datetime.now()}}, session = session)
return True
def insertIntoPlaylist(pid, vid, rank, user) :
log(obj = {'pid': pid, 'vid': vid, 'rank': rank})
with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s :
playlist = db.playlists.find_one({'_id': ObjectId(pid)}, session = s())
if playlist is None :
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('editPlaylist', user, playlist)
if tagdb.retrive_item({'_id': ObjectId(vid)}, session = s()) is None :
raise UserError('VIDEO_NOT_EXIST')
if playlist["videos"] > PlaylistConfig.MAX_VIDEO_PER_PLAYLIST :
raise UserError('VIDEO_LIMIT_EXCEEDED')
conflicting_item = db.playlist_items.find_one({'pid': ObjectId(pid), 'vid': ObjectId(vid)}, session = s())
if conflicting_item is not None :
editPlaylist_MoveLockFree(pid, conflicting_item, rank, session = s())
db.playlists.update_one({'_id': ObjectId(pid)}, {'$set': {
'meta.modified_by': makeUserMeta(user),
'meta.modified_at': datetime.now()}}, session = s())
s.mark_succeed()
return
if rank < 0 :
raise UserError('OUT_OF_RANGE')
if rank > playlist['videos'] :
rank = int(playlist['videos'])
playlists = tagdb.retrive_item({'_id': ObjectId(vid)}, session = s())['item']['series']
playlists.append(ObjectId(pid))
playlists = list(set(playlists))
tagdb.update_item_query(ObjectId(vid), {'$set': {'item.series': playlists}}, makeUserMeta(user), session = s())
db.playlists.update_one({"_id": ObjectId(pid)}, {"$inc": {"videos": int(1)}}, session = s())
db.playlist_items.update_many({'pid': ObjectId(pid), 'rank': {'$gte': rank}}, {'$inc': {'rank': int(1)}}, session = s())
db.playlist_items.insert_one({"pid": ObjectId(pid), "vid": ObjectId(vid), "rank": int(rank), "meta": makeUserMeta(user)}, session = s())
db.playlists.update_one({'_id': ObjectId(pid)}, {'$set': {
'meta.modified_by': makeUserMeta(user),
'meta.modified_at': datetime.now()}}, session = s())
s.mark_succeed()
def _removeThisCopy(dst_vid, this_vid, user, session):
if this_vid is None:
return
dst_video = tagdb.retrive_item({"_id": ObjectId(dst_vid)}, session)
if dst_video is None:
return
dst_copies = dst_video['item']['copies']
dst_copies = list(set(dst_copies) - set([ObjectId(this_vid)]))
tagdb.update_item_query(ObjectId(dst_vid),
{"$set": {
"item.copies": dst_copies
}}, user, session)
def addVideoToPlaylist(pid, vid, user):
log(obj={'pid': pid, 'vid': vid})
with redis_lock.Lock(rdb, "playlistEdit:" +
str(pid)), MongoTransaction(client) as s:
playlist = playlist_db.retrive_item(pid)
if playlist is None:
raise UserError('PLAYLIST_NOT_EXIST')
filterOperation('editPlaylist', user, playlist)
if tagdb.retrive_item({'_id': ObjectId(vid)}, session=s()) is None:
raise UserError('VIDEO_NOT_EXIST')
if playlist['item']["videos"] > PlaylistConfig.MAX_VIDEO_PER_PLAYLIST:
raise UserError('VIDEO_LIMIT_EXCEEDED')
conflicting_item = db.playlist_items.find_one(
{
'pid': ObjectId(pid),
'vid': ObjectId(vid)
}, session=s())
if conflicting_item is not None:
editPlaylist_MoveLockFree(pid,
conflicting_item,
int(playlist['item']["videos"]),
session=s())
playlist_db.update_item_query(playlist, {},
user=makeUserMeta(user),
session=s())
s.mark_succeed()
return
playlists = tagdb.retrive_item({'_id': ObjectId(vid)},
session=s())['item']['series']
playlists.append(ObjectId(pid))
playlists = list(set(playlists))
tagdb.update_item_query(ObjectId(vid),
{'$set': {
'item.series': playlists
}},
user=makeUserMeta(user),
session=s())
db.playlist_items.insert_one(
{
"pid": ObjectId(pid),
"vid": ObjectId(vid),
"rank": int(playlist['item']["videos"]),
"meta": makeUserMeta(user)
},
session=s())
playlist_db.update_item_query(playlist,
{"$inc": {
"item.videos": int(1)
}},
user=makeUserMeta(user),
session=s())
s.mark_succeed()
def breakLink(vid, user):
filterOperation('breakLink', user, vid)
with redis_lock.Lock(rdb, 'editLink'), MongoTransaction(client) as s:
nodes = _getAllCopies(vid)
log(obj={'old_clique': nodes, 'node_remove': vid})
if nodes:
for node in nodes:
_removeThisCopy(node, vid, makeUserMeta(user), session=s())
tagdb.update_item_query(ObjectId(vid),
{"$set": {
"item.copies": []
}},
makeUserMeta(user),
session=s())
s.mark_succeed()
def _addThiscopy(dst_vid, this_vid, user, session):
if this_vid is None:
return
dst_video = tagdb.retrive_item({"_id": ObjectId(dst_vid)}, session=session)
if dst_video is None:
return
dst_copies = dst_video['item']['copies']
if isinstance(this_vid, list):
dst_copies.extend(this_vid)
else:
dst_copies.append(ObjectId(this_vid))
dst_copies = list(set(dst_copies) - set([ObjectId(dst_vid)]))
tagdb.update_item_query(ObjectId(dst_vid),
{"$set": {
"item.copies": dst_copies
}},
user=user,
session=session)
def setVideoRepostType(vid, repost_type, user):
filterOperation('setVideoRepostType', user)
if repost_type not in [
'official', 'official_repost', 'authorized_translation',
'authorized_repost', 'translation', 'repost'
]:
raise UserError('INCORRECT_REPOST_TYPE')
video_obj = tagdb.retrive_item(vid)
if video_obj is None:
raise UserError('VIDEO_NOT_FOUND')
lock_id = "videoEdit:" + video_obj['item']['unique_id']
with redis_lock.Lock(rdb, lock_id), MongoTransaction(client) as s:
tagdb.update_item_query(video_obj,
{'$set': {
'item.repost_type': repost_type
}},
session=s())
s.mark_succeed()
def addVideoToPlaylistLockFree(pid, vid, user, rank, session):
log(obj={'pid': pid, 'vid': vid})
if tagdb.retrive_item({'_id': ObjectId(vid)}, session=session) is None:
#raise UserError('VIDEO_NOT_EXIST')
return False
conflicting_item = db.playlist_items.find_one(
{
'pid': ObjectId(pid),
'vid': ObjectId(vid)
}, session=session)
if conflicting_item is not None:
editPlaylist_MoveLockFree(pid, conflicting_item, rank, session=session)
playlist_db.update_item_query(pid, {},
user=makeUserMeta(user),
session=session)
return False
playlists = tagdb.retrive_item({'_id': ObjectId(vid)},
session=session)['item']['series']
playlists.append(ObjectId(pid))
playlists = list(set(playlists))
tagdb.update_item_query(ObjectId(vid),
{'$set': {
'item.series': playlists
}},
user=makeUserMeta(user),
session=session)
db.playlist_items.insert_one(
{
"pid": ObjectId(pid),
"vid": ObjectId(vid),
"rank": int(rank),
"meta": makeUserMeta(user)
},
session=session)
playlist_db.update_item_query(pid, {"$inc": {
"item.videos": int(1)
}},
user=makeUserMeta(user),
session=session)
return True
async def postVideoAsync(url,
tags,
dst_copy,
dst_playlist,
dst_rank,
other_copies,
repost_type,
playlist_ordered,
user,
update_video_detail,
event_id,
field_override=None,
use_autotag=False):
parsed = None
try:
dst_playlist = str(dst_playlist)
dst_rank = -1 if dst_rank is None else dst_rank
#tags = tagdb.filter_and_translate_tags(tags)
parsed, url = dispatch(url)
except:
pass
if parsed is None:
log_e(event_id, user, 'dispatcher', 'ERR', {
'msg': 'PARSE_FAILED',
'url': url
})
await _playlist_reorder_helper.post_video_failed(
url, dst_playlist, playlist_ordered, dst_rank, user, event_id)
return "PARSE_FAILED", {}
unique_id = await parsed.unique_id_async(
self=parsed,
link=url) # empty unique_id for b23.tv posts, f**k bilibli
if not unique_id:
ret = await parsed.get_metadata_async(parsed, url, update_video_detail)
unique_id = ret['data']['unique_id']
log_e(
event_id, user, 'scraper', 'MSG', {
'url': url,
'dst_copy': dst_copy,
'other_copies': other_copies,
'dst_playlist': dst_playlist
})
setEventOp('scraper')
try:
lock_id = "videoEdit:" + unique_id
async with RedisLockAsync(rdb, lock_id):
unique, conflicting_item = verifyUniqueness(unique_id)
if unique or update_video_detail:
async with _download_sem:
ret = await parsed.get_metadata_async(
parsed, url, update_video_detail)
if repost_type:
ret['data']['repost_type'] = repost_type
else:
ret['data']['repost_type'] = 'unknown'
if ret["status"] == 'FAILED':
log_e(event_id, user, 'downloader', 'WARN', {
'msg': 'FETCH_FAILED',
'ret': ret
})
await _playlist_reorder_helper.post_video_failed(
unique_id, dst_playlist, playlist_ordered, dst_rank,
user, event_id)
return "FETCH_FAILED", ret
else:
unique_id = ret['data']['unique_id']
else:
# build ret
ret = makeResponseSuccess({
'thumbnailURL':
conflicting_item['item']['thumbnail_url'],
'title':
conflicting_item['item']['title'],
'desc':
conflicting_item['item']['desc'],
'site':
conflicting_item['item']['site'],
'uploadDate':
conflicting_item['item']['upload_time'],
"unique_id":
conflicting_item['item']['unique_id'],
"utags":
conflicting_item['item']['utags']
})
for k, v in conflicting_item['item'].items():
ret['data'][k] = v
if 'part_name' in conflicting_item['item']:
ret['part_name'] = conflicting_item['item']['part_name']
if 'repost_type' in conflicting_item[
'item'] and conflicting_item['item']['repost_type']:
ret['data']['repost_type'] = repost_type
else:
ret['data']['repost_type'] = 'unknown'
tagdb.update_item_query(
conflicting_item,
{'$set': {
'item.repost_type': repost_type
}},
user=makeUserMeta(user))
#if hasattr(parsed, 'LOCAL_CRAWLER') :
# url = ret["data"]["url"]
#else :
# url = clear_url(url)
use_override = False
if field_override and '__condition' in field_override:
condition = field_override['__condition']
del field_override['__condition']
if condition == 'any':
use_override = True
elif condition == 'placeholder' and 'placeholder' in ret[
"data"] and ret["data"]['placeholder']:
use_override = True
if use_override:
for key in field_override:
ret['data'][key] = field_override[key]
playlists = []
#playlist_lock = None
if dst_playlist:
#playlist_lock = RedisLockAsync(rdb, "playlistEdit:" + str(dst_playlist))
#playlist_lock.acquire()
if playlist_db.retrive_item(dst_playlist) is not None:
playlists = [ObjectId(dst_playlist)]
if not unique:
log_e(event_id,
user,
'scraper',
level='MSG',
obj={
'msg': 'ALREADY_EXIST',
'unique_id': ret["data"]["unique_id"]
})
"""
Update existing video
"""
if update_video_detail:
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Updating video detail')
with MongoTransaction(client) as s:
old_item = tagdb.retrive_item(conflicting_item['_id'],
session=s())['item']
if old_item['thumbnail_url'] and old_item[
'cover_image']:
# old thumbnail exists, no need to download again
new_detail = await _make_video_data_update(
ret["data"], url, user, event_id)
else:
# old thumbnail does not exists, add to dict
new_detail = await _make_video_data_update(
ret["data"], url, user, event_id,
ret["data"]["thumbnailURL"])
for key in new_detail.keys():
old_item[key] = new_detail[
key] # overwrite or add new field
setEventUserAndID(user, event_id)
tagdb.update_item_query(conflicting_item['_id'],
{'$set': {
'item': old_item
}}, ['title', 'desc'],
user=makeUserMeta(user),
session=s())
s.mark_succeed()
return 'SUCCEED', conflicting_item['_id']
# this video already exist in the database
# if the operation is to add a link to other copies and not adding self
if (dst_copy and
dst_copy != conflicting_item['_id']) or other_copies:
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Adding to to copies')
async with RedisLockAsync(
rdb, 'editLink'), MongoTransaction(client) as s:
log_e(event_id,
user,
level='MSG',
obj='Adding to to copies, lock acquired')
# find all copies of video dst_copy, self included
all_copies = _getAllCopies(dst_copy, session=s())
# find all videos linked to source video
all_copies += _getAllCopies(conflicting_item['_id'],
session=s())
# add videos from other copies
for uid in other_copies:
all_copies += _getAllCopies(uid,
session=s(),
use_unique_id=True)
# remove duplicated items
all_copies = list(set(all_copies))
# add this video to all other copies found
if len(all_copies) <= VideoConfig.MAX_COPIES:
for dst_vid in all_copies:
setEventUserAndID(user, event_id)
_addThiscopy(dst_vid,
all_copies,
makeUserMeta(user),
session=s())
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Successfully added to copies')
s.mark_succeed()
else:
#if playlist_lock :
# playlist_lock.release()
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Too many copies')
await _playlist_reorder_helper.post_video_failed(
unique_id, dst_playlist, playlist_ordered,
dst_rank, user, event_id)
return "TOO_MANY_COPIES", {}
# if the operation is adding this video to playlist
if dst_playlist:
log_e(event_id,
user,
'scraper',
level='MSG',
obj={
'msg': 'Adding to playlist at position',
'rank': dst_rank
})
if playlist_ordered:
await _playlist_reorder_helper.post_video_succeed(
conflicting_item['_id'], unique_id, dst_playlist,
playlist_ordered, dst_rank, user, event_id)
else:
setEventUserAndID(user, event_id)
if dst_rank == -1:
addVideoToPlaylist(dst_playlist,
conflicting_item['_id'], user)
else:
insertIntoPlaylist(dst_playlist,
conflicting_item['_id'],
dst_rank, user)
# merge tags
async with MongoTransaction(client) as s:
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Merging tags')
setEventUserAndID(user, event_id)
tagdb.update_item_tags_merge(conflicting_item['_id'],
tags,
makeUserMeta(user),
session=s(),
remove_tagids=[354])
s.mark_succeed()
#if playlist_lock :
# playlist_lock.release()
#return "VIDEO_ALREADY_EXIST", conflicting_item['_id']
return "SUCCEED", conflicting_item['_id']
else:
# expand dst_copy to all copies linked to dst_copy
if dst_copy or other_copies:
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Adding to to copies')
async with RedisLockAsync(
rdb, 'editLink'), MongoTransaction(client) as s:
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Adding to to copies, lock acquired')
all_copies = _getAllCopies(dst_copy, session=s())
# add videos from other copies
for uid in other_copies:
all_copies += _getAllCopies(uid,
session=s(),
use_unique_id=True)
video_data = await _make_video_data(
ret["data"], all_copies, playlists, url, user,
event_id)
setEventUserAndID(user, event_id)
new_item_id = tagdb.add_item(tags,
video_data,
3, ['title', 'desc'],
makeUserMeta(user),
session=s())
all_copies.append(ObjectId(new_item_id))
# remove duplicated items
all_copies = list(set(all_copies))
if len(all_copies) <= VideoConfig.MAX_COPIES:
for dst_vid in all_copies:
setEventUserAndID(user, event_id)
_addThiscopy(dst_vid,
all_copies,
makeUserMeta(user),
session=s())
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Successfully added to copies')
s.mark_succeed()
else:
#if playlist_lock :
# playlist_lock.release()
log_e(event_id,
user,
'scraper',
level='MSG',
obj='Too many copies')
await _playlist_reorder_helper.post_video_failed(
unique_id, dst_playlist, playlist_ordered,
dst_rank, user, event_id)
return "TOO_MANY_COPIES", {}
else:
async with MongoTransaction(client) as s:
video_data = await _make_video_data(
ret["data"], [], playlists, url, user, event_id)
setEventUserAndID(user, event_id)
if use_autotag:
tags.extend(
inferTagsFromVideo(
video_data['utags'], video_data['title'],
video_data['desc'], 'CHS',
video_data['url'],
video_data['user_space_urls']))
new_item_id = tagdb.add_item(tags,
video_data,
3, ['title', 'desc'],
makeUserMeta(user),
session=s())
log_e(event_id,
user,
'scraper',
level='MSG',
obj={
'msg': 'New video added to database',
'vid': new_item_id
})
s.mark_succeed()
# if the operation is adding this video to playlist
if dst_playlist:
log_e(event_id,
user,
'scraper',
level='MSG',
obj={
'msg': 'Adding to playlist at position',
'rank': dst_rank
})
if playlist_ordered:
await _playlist_reorder_helper.post_video_succeed(
new_item_id, unique_id, dst_playlist,
playlist_ordered, dst_rank, user, event_id)
else:
setEventUserAndID(user, event_id)
if dst_rank == -1:
addVideoToPlaylist(dst_playlist, new_item_id, user)
else:
insertIntoPlaylist(dst_playlist, new_item_id,
dst_rank, user)
#if playlist_lock :
# playlist_lock.release()
log_e(event_id, user, 'scraper', level='MSG', obj='Done')
return 'SUCCEED', new_item_id
except UserError as ue:
await _playlist_reorder_helper.post_video_failed(
unique_id, dst_playlist, playlist_ordered, dst_rank, user,
event_id)
log_e(event_id, user, 'scraper', level='WARN', obj={'ue': str(ue)})
return ue.msg, {"aux": ue.aux, "traceback": traceback.format_exc()}
except Exception as ex:
await _playlist_reorder_helper.post_video_failed(
unique_id, dst_playlist, playlist_ordered, dst_rank, user,
event_id)
log_e(event_id, user, 'scraper', level='ERR', obj={'ex': str(ex)})
try:
problematic_lock = RedisLockAsync(rdb, 'editLink')
problematic_lock.reset()
except:
pass
return "UNKNOWN", '\n'.join(
[repr(traceback.format_exc()),
repr(traceback.extract_stack())])
