def insertIntoPlaylistLockFree(pid, vid, rank, user, session) : log(obj = {'pid': pid, 'vid': vid, 'rank': rank}) if tagdb.retrive_item({'_id': ObjectId(vid)}, session = session) is None : #raise UserError('VIDEO_NOT_EXIST') return False conflicting_item = db.playlist_items.find_one({'pid': ObjectId(pid), 'vid': ObjectId(vid)}, session = session) if conflicting_item is not None : editPlaylist_MoveLockFree(pid, conflicting_item, rank, session = session) db.playlists.update_one({'_id': ObjectId(pid)}, {'$set': { 'meta.modified_by': makeUserMeta(user), 'meta.modified_at': datetime.now()}}, session = session) if conflicting_item['rank'] >= rank : return True else : return False playlists = tagdb.retrive_item({'_id': ObjectId(vid)}, session = session)['item']['series'] playlists.append(ObjectId(pid)) playlists = list(set(playlists)) tagdb.update_item_query(ObjectId(vid), {'$set': {'item.series': playlists}}, makeUserMeta(user), session = session) db.playlists.update_one({"_id": ObjectId(pid)}, {"$inc": {"videos": int(1)}}, session = session) db.playlist_items.update_many({'pid': ObjectId(pid), 'rank': {'$gte': rank}}, {'$inc': {'rank': int(1)}}, session = session) db.playlist_items.insert_one({"pid": ObjectId(pid), "vid": ObjectId(vid), "rank": int(rank), "meta": makeUserMeta(user)}, session = session) db.playlists.update_one({'_id': ObjectId(pid)}, {'$set': { 'meta.modified_by': makeUserMeta(user), 'meta.modified_at': datetime.now()}}, session = session) return True
def insertIntoPlaylist(pid, vid, rank, user) : log(obj = {'pid': pid, 'vid': vid, 'rank': rank}) with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s : playlist = db.playlists.find_one({'_id': ObjectId(pid)}, session = s()) if playlist is None : raise UserError('PLAYLIST_NOT_EXIST') filterOperation('editPlaylist', user, playlist) if tagdb.retrive_item({'_id': ObjectId(vid)}, session = s()) is None : raise UserError('VIDEO_NOT_EXIST') if playlist["videos"] > PlaylistConfig.MAX_VIDEO_PER_PLAYLIST : raise UserError('VIDEO_LIMIT_EXCEEDED') conflicting_item = db.playlist_items.find_one({'pid': ObjectId(pid), 'vid': ObjectId(vid)}, session = s()) if conflicting_item is not None : editPlaylist_MoveLockFree(pid, conflicting_item, rank, session = s()) db.playlists.update_one({'_id': ObjectId(pid)}, {'$set': { 'meta.modified_by': makeUserMeta(user), 'meta.modified_at': datetime.now()}}, session = s()) s.mark_succeed() return if rank < 0 : raise UserError('OUT_OF_RANGE') if rank > playlist['videos'] : rank = int(playlist['videos']) playlists = tagdb.retrive_item({'_id': ObjectId(vid)}, session = s())['item']['series'] playlists.append(ObjectId(pid)) playlists = list(set(playlists)) tagdb.update_item_query(ObjectId(vid), {'$set': {'item.series': playlists}}, makeUserMeta(user), session = s()) db.playlists.update_one({"_id": ObjectId(pid)}, {"$inc": {"videos": int(1)}}, session = s()) db.playlist_items.update_many({'pid': ObjectId(pid), 'rank': {'$gte': rank}}, {'$inc': {'rank': int(1)}}, session = s()) db.playlist_items.insert_one({"pid": ObjectId(pid), "vid": ObjectId(vid), "rank": int(rank), "meta": makeUserMeta(user)}, session = s()) db.playlists.update_one({'_id': ObjectId(pid)}, {'$set': { 'meta.modified_by': makeUserMeta(user), 'meta.modified_at': datetime.now()}}, session = s()) s.mark_succeed()
def _removeThisCopy(dst_vid, this_vid, user, session): if this_vid is None: return dst_video = tagdb.retrive_item({"_id": ObjectId(dst_vid)}, session) if dst_video is None: return dst_copies = dst_video['item']['copies'] dst_copies = list(set(dst_copies) - set([ObjectId(this_vid)])) tagdb.update_item_query(ObjectId(dst_vid), {"$set": { "item.copies": dst_copies }}, user, session)
def addVideoToPlaylist(pid, vid, user): log(obj={'pid': pid, 'vid': vid}) with redis_lock.Lock(rdb, "playlistEdit:" + str(pid)), MongoTransaction(client) as s: playlist = playlist_db.retrive_item(pid) if playlist is None: raise UserError('PLAYLIST_NOT_EXIST') filterOperation('editPlaylist', user, playlist) if tagdb.retrive_item({'_id': ObjectId(vid)}, session=s()) is None: raise UserError('VIDEO_NOT_EXIST') if playlist['item']["videos"] > PlaylistConfig.MAX_VIDEO_PER_PLAYLIST: raise UserError('VIDEO_LIMIT_EXCEEDED') conflicting_item = db.playlist_items.find_one( { 'pid': ObjectId(pid), 'vid': ObjectId(vid) }, session=s()) if conflicting_item is not None: editPlaylist_MoveLockFree(pid, conflicting_item, int(playlist['item']["videos"]), session=s()) playlist_db.update_item_query(playlist, {}, user=makeUserMeta(user), session=s()) s.mark_succeed() return playlists = tagdb.retrive_item({'_id': ObjectId(vid)}, session=s())['item']['series'] playlists.append(ObjectId(pid)) playlists = list(set(playlists)) tagdb.update_item_query(ObjectId(vid), {'$set': { 'item.series': playlists }}, user=makeUserMeta(user), session=s()) db.playlist_items.insert_one( { "pid": ObjectId(pid), "vid": ObjectId(vid), "rank": int(playlist['item']["videos"]), "meta": makeUserMeta(user) }, session=s()) playlist_db.update_item_query(playlist, {"$inc": { "item.videos": int(1) }}, user=makeUserMeta(user), session=s()) s.mark_succeed()
def breakLink(vid, user): filterOperation('breakLink', user, vid) with redis_lock.Lock(rdb, 'editLink'), MongoTransaction(client) as s: nodes = _getAllCopies(vid) log(obj={'old_clique': nodes, 'node_remove': vid}) if nodes: for node in nodes: _removeThisCopy(node, vid, makeUserMeta(user), session=s()) tagdb.update_item_query(ObjectId(vid), {"$set": { "item.copies": [] }}, makeUserMeta(user), session=s()) s.mark_succeed()
def _addThiscopy(dst_vid, this_vid, user, session): if this_vid is None: return dst_video = tagdb.retrive_item({"_id": ObjectId(dst_vid)}, session=session) if dst_video is None: return dst_copies = dst_video['item']['copies'] if isinstance(this_vid, list): dst_copies.extend(this_vid) else: dst_copies.append(ObjectId(this_vid)) dst_copies = list(set(dst_copies) - set([ObjectId(dst_vid)])) tagdb.update_item_query(ObjectId(dst_vid), {"$set": { "item.copies": dst_copies }}, user=user, session=session)
def setVideoRepostType(vid, repost_type, user): filterOperation('setVideoRepostType', user) if repost_type not in [ 'official', 'official_repost', 'authorized_translation', 'authorized_repost', 'translation', 'repost' ]: raise UserError('INCORRECT_REPOST_TYPE') video_obj = tagdb.retrive_item(vid) if video_obj is None: raise UserError('VIDEO_NOT_FOUND') lock_id = "videoEdit:" + video_obj['item']['unique_id'] with redis_lock.Lock(rdb, lock_id), MongoTransaction(client) as s: tagdb.update_item_query(video_obj, {'$set': { 'item.repost_type': repost_type }}, session=s()) s.mark_succeed()
def addVideoToPlaylistLockFree(pid, vid, user, rank, session): log(obj={'pid': pid, 'vid': vid}) if tagdb.retrive_item({'_id': ObjectId(vid)}, session=session) is None: #raise UserError('VIDEO_NOT_EXIST') return False conflicting_item = db.playlist_items.find_one( { 'pid': ObjectId(pid), 'vid': ObjectId(vid) }, session=session) if conflicting_item is not None: editPlaylist_MoveLockFree(pid, conflicting_item, rank, session=session) playlist_db.update_item_query(pid, {}, user=makeUserMeta(user), session=session) return False playlists = tagdb.retrive_item({'_id': ObjectId(vid)}, session=session)['item']['series'] playlists.append(ObjectId(pid)) playlists = list(set(playlists)) tagdb.update_item_query(ObjectId(vid), {'$set': { 'item.series': playlists }}, user=makeUserMeta(user), session=session) db.playlist_items.insert_one( { "pid": ObjectId(pid), "vid": ObjectId(vid), "rank": int(rank), "meta": makeUserMeta(user) }, session=session) playlist_db.update_item_query(pid, {"$inc": { "item.videos": int(1) }}, user=makeUserMeta(user), session=session) return True
async def postVideoAsync(url, tags, dst_copy, dst_playlist, dst_rank, other_copies, repost_type, playlist_ordered, user, update_video_detail, event_id, field_override=None, use_autotag=False): parsed = None try: dst_playlist = str(dst_playlist) dst_rank = -1 if dst_rank is None else dst_rank #tags = tagdb.filter_and_translate_tags(tags) parsed, url = dispatch(url) except: pass if parsed is None: log_e(event_id, user, 'dispatcher', 'ERR', { 'msg': 'PARSE_FAILED', 'url': url }) await _playlist_reorder_helper.post_video_failed( url, dst_playlist, playlist_ordered, dst_rank, user, event_id) return "PARSE_FAILED", {} unique_id = await parsed.unique_id_async( self=parsed, link=url) # empty unique_id for b23.tv posts, f**k bilibli if not unique_id: ret = await parsed.get_metadata_async(parsed, url, update_video_detail) unique_id = ret['data']['unique_id'] log_e( event_id, user, 'scraper', 'MSG', { 'url': url, 'dst_copy': dst_copy, 'other_copies': other_copies, 'dst_playlist': dst_playlist }) setEventOp('scraper') try: lock_id = "videoEdit:" + unique_id async with RedisLockAsync(rdb, lock_id): unique, conflicting_item = verifyUniqueness(unique_id) if unique or update_video_detail: async with _download_sem: ret = await parsed.get_metadata_async( parsed, url, update_video_detail) if repost_type: ret['data']['repost_type'] = repost_type else: ret['data']['repost_type'] = 'unknown' if ret["status"] == 'FAILED': log_e(event_id, user, 'downloader', 'WARN', { 'msg': 'FETCH_FAILED', 'ret': ret }) await _playlist_reorder_helper.post_video_failed( unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) return "FETCH_FAILED", ret else: unique_id = ret['data']['unique_id'] else: # build ret ret = makeResponseSuccess({ 'thumbnailURL': conflicting_item['item']['thumbnail_url'], 'title': conflicting_item['item']['title'], 'desc': conflicting_item['item']['desc'], 'site': conflicting_item['item']['site'], 'uploadDate': conflicting_item['item']['upload_time'], "unique_id": conflicting_item['item']['unique_id'], "utags": conflicting_item['item']['utags'] }) for k, v in conflicting_item['item'].items(): ret['data'][k] = v if 'part_name' in conflicting_item['item']: ret['part_name'] = conflicting_item['item']['part_name'] if 'repost_type' in conflicting_item[ 'item'] and conflicting_item['item']['repost_type']: ret['data']['repost_type'] = repost_type else: ret['data']['repost_type'] = 'unknown' tagdb.update_item_query( conflicting_item, {'$set': { 'item.repost_type': repost_type }}, user=makeUserMeta(user)) #if hasattr(parsed, 'LOCAL_CRAWLER') : # url = ret["data"]["url"] #else : # url = clear_url(url) use_override = False if field_override and '__condition' in field_override: condition = field_override['__condition'] del field_override['__condition'] if condition == 'any': use_override = True elif condition == 'placeholder' and 'placeholder' in ret[ "data"] and ret["data"]['placeholder']: use_override = True if use_override: for key in field_override: ret['data'][key] = field_override[key] playlists = [] #playlist_lock = None if dst_playlist: #playlist_lock = RedisLockAsync(rdb, "playlistEdit:" + str(dst_playlist)) #playlist_lock.acquire() if playlist_db.retrive_item(dst_playlist) is not None: playlists = [ObjectId(dst_playlist)] if not unique: log_e(event_id, user, 'scraper', level='MSG', obj={ 'msg': 'ALREADY_EXIST', 'unique_id': ret["data"]["unique_id"] }) """ Update existing video """ if update_video_detail: log_e(event_id, user, 'scraper', level='MSG', obj='Updating video detail') with MongoTransaction(client) as s: old_item = tagdb.retrive_item(conflicting_item['_id'], session=s())['item'] if old_item['thumbnail_url'] and old_item[ 'cover_image']: # old thumbnail exists, no need to download again new_detail = await _make_video_data_update( ret["data"], url, user, event_id) else: # old thumbnail does not exists, add to dict new_detail = await _make_video_data_update( ret["data"], url, user, event_id, ret["data"]["thumbnailURL"]) for key in new_detail.keys(): old_item[key] = new_detail[ key] # overwrite or add new field setEventUserAndID(user, event_id) tagdb.update_item_query(conflicting_item['_id'], {'$set': { 'item': old_item }}, ['title', 'desc'], user=makeUserMeta(user), session=s()) s.mark_succeed() return 'SUCCEED', conflicting_item['_id'] # this video already exist in the database # if the operation is to add a link to other copies and not adding self if (dst_copy and dst_copy != conflicting_item['_id']) or other_copies: log_e(event_id, user, 'scraper', level='MSG', obj='Adding to to copies') async with RedisLockAsync( rdb, 'editLink'), MongoTransaction(client) as s: log_e(event_id, user, level='MSG', obj='Adding to to copies, lock acquired') # find all copies of video dst_copy, self included all_copies = _getAllCopies(dst_copy, session=s()) # find all videos linked to source video all_copies += _getAllCopies(conflicting_item['_id'], session=s()) # add videos from other copies for uid in other_copies: all_copies += _getAllCopies(uid, session=s(), use_unique_id=True) # remove duplicated items all_copies = list(set(all_copies)) # add this video to all other copies found if len(all_copies) <= VideoConfig.MAX_COPIES: for dst_vid in all_copies: setEventUserAndID(user, event_id) _addThiscopy(dst_vid, all_copies, makeUserMeta(user), session=s()) log_e(event_id, user, 'scraper', level='MSG', obj='Successfully added to copies') s.mark_succeed() else: #if playlist_lock : # playlist_lock.release() log_e(event_id, user, 'scraper', level='MSG', obj='Too many copies') await _playlist_reorder_helper.post_video_failed( unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) return "TOO_MANY_COPIES", {} # if the operation is adding this video to playlist if dst_playlist: log_e(event_id, user, 'scraper', level='MSG', obj={ 'msg': 'Adding to playlist at position', 'rank': dst_rank }) if playlist_ordered: await _playlist_reorder_helper.post_video_succeed( conflicting_item['_id'], unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) else: setEventUserAndID(user, event_id) if dst_rank == -1: addVideoToPlaylist(dst_playlist, conflicting_item['_id'], user) else: insertIntoPlaylist(dst_playlist, conflicting_item['_id'], dst_rank, user) # merge tags async with MongoTransaction(client) as s: log_e(event_id, user, 'scraper', level='MSG', obj='Merging tags') setEventUserAndID(user, event_id) tagdb.update_item_tags_merge(conflicting_item['_id'], tags, makeUserMeta(user), session=s(), remove_tagids=[354]) s.mark_succeed() #if playlist_lock : # playlist_lock.release() #return "VIDEO_ALREADY_EXIST", conflicting_item['_id'] return "SUCCEED", conflicting_item['_id'] else: # expand dst_copy to all copies linked to dst_copy if dst_copy or other_copies: log_e(event_id, user, 'scraper', level='MSG', obj='Adding to to copies') async with RedisLockAsync( rdb, 'editLink'), MongoTransaction(client) as s: log_e(event_id, user, 'scraper', level='MSG', obj='Adding to to copies, lock acquired') all_copies = _getAllCopies(dst_copy, session=s()) # add videos from other copies for uid in other_copies: all_copies += _getAllCopies(uid, session=s(), use_unique_id=True) video_data = await _make_video_data( ret["data"], all_copies, playlists, url, user, event_id) setEventUserAndID(user, event_id) new_item_id = tagdb.add_item(tags, video_data, 3, ['title', 'desc'], makeUserMeta(user), session=s()) all_copies.append(ObjectId(new_item_id)) # remove duplicated items all_copies = list(set(all_copies)) if len(all_copies) <= VideoConfig.MAX_COPIES: for dst_vid in all_copies: setEventUserAndID(user, event_id) _addThiscopy(dst_vid, all_copies, makeUserMeta(user), session=s()) log_e(event_id, user, 'scraper', level='MSG', obj='Successfully added to copies') s.mark_succeed() else: #if playlist_lock : # playlist_lock.release() log_e(event_id, user, 'scraper', level='MSG', obj='Too many copies') await _playlist_reorder_helper.post_video_failed( unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) return "TOO_MANY_COPIES", {} else: async with MongoTransaction(client) as s: video_data = await _make_video_data( ret["data"], [], playlists, url, user, event_id) setEventUserAndID(user, event_id) if use_autotag: tags.extend( inferTagsFromVideo( video_data['utags'], video_data['title'], video_data['desc'], 'CHS', video_data['url'], video_data['user_space_urls'])) new_item_id = tagdb.add_item(tags, video_data, 3, ['title', 'desc'], makeUserMeta(user), session=s()) log_e(event_id, user, 'scraper', level='MSG', obj={ 'msg': 'New video added to database', 'vid': new_item_id }) s.mark_succeed() # if the operation is adding this video to playlist if dst_playlist: log_e(event_id, user, 'scraper', level='MSG', obj={ 'msg': 'Adding to playlist at position', 'rank': dst_rank }) if playlist_ordered: await _playlist_reorder_helper.post_video_succeed( new_item_id, unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) else: setEventUserAndID(user, event_id) if dst_rank == -1: addVideoToPlaylist(dst_playlist, new_item_id, user) else: insertIntoPlaylist(dst_playlist, new_item_id, dst_rank, user) #if playlist_lock : # playlist_lock.release() log_e(event_id, user, 'scraper', level='MSG', obj='Done') return 'SUCCEED', new_item_id except UserError as ue: await _playlist_reorder_helper.post_video_failed( unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) log_e(event_id, user, 'scraper', level='WARN', obj={'ue': str(ue)}) return ue.msg, {"aux": ue.aux, "traceback": traceback.format_exc()} except Exception as ex: await _playlist_reorder_helper.post_video_failed( unique_id, dst_playlist, playlist_ordered, dst_rank, user, event_id) log_e(event_id, user, 'scraper', level='ERR', obj={'ex': str(ex)}) try: problematic_lock = RedisLockAsync(rdb, 'editLink') problematic_lock.reset() except: pass return "UNKNOWN", '\n'.join( [repr(traceback.format_exc()), repr(traceback.extract_stack())])