def test_title_transformation(self):
p = VideoHTMLParser(html_string=self.get_resource_contents(k_VIDEO_WITH_ILLEGAL_TITLE))
v = Video(video_id='sm')
v._html = p
params = custom_youtube_dl.get_ydl_options(video=v)
self.assertTrue('-％' in params['outtmpl'], "'-％' should be in 'outtmpl'")
self.assertFalse('%(title)s' in params['outtmpl'], "'%(title)s' should not be in 'outtmpl'")
def add_video(self, session_id, user_id):
"""
:param session_id:
:param user_id:
:return:
"""
LOGGER.info("Adding video to session.")
session = SessionManager.get_instance().get_session(
session_id=session_id)
if session['state'] != SessionStatus.ACTIVE.value:
raise IllegalSessionStateException(
"Can only add videos to active sessions.")
user_videos = Video.objects(session=session['id'],
user_id=user_id).count()
video_name = "video{}".format(user_videos)
session = RumbaSession.objects(id=session_id).first()
video_path = FileSystemService.get_instance().create_video_directory(
session_folder=session['folder_url'],
user_id=user_id,
video_name=video_name)
video = Video(session=session['id'],
user_id=user_id,
name=video_name,
video_path=video_path).save()
LOGGER.info("Video successfully added: [id={}]".format(video))
return str(video['id'])
def test_contains_banned_keywords_positive(self):
html_string = self.get_resource_contents(k_VIDEO_HTML)
v = Video(video_id='fake_id_123')
v._html = VideoHTMLParser(html_string=html_string)
contains = download_thread._title_contains_keywords(video=v,
keywords=['そう'])
self.assertTrue(contains, 'The title contains the banned keyword')
def test_contains_banned_keywords_negative(self):
html_string = self.get_resource_contents(k_VIDEO_HTML)
v = Video(video_id='fake_id_123')
v._html = VideoHTMLParser(html_string=html_string)
contains = download_thread._title_contains_keywords(
video=v, keywords=['keyword1', 'keyword2'])
self.assertFalse(contains,
'The title does not contain the banned keyword')
def add_dasher_splitter_thread(self, d_thread, video_id):
"""
This method adds an already started d_thumbs thread to the list of managed threads in
order to monitor it.
The d_thumbs parameter represents a DasherSplitterThread that has been crearted and started
by an external entity. This method is responsible of adding the thread to the
VideoThreadsRepository and also to update the splitter_status field of the related
vie Database.
:param d_thread: Runnig instance of a DasherSplitterThread.
:param video_id: Id of the video which is being splitted.
:raises:
- NotExistingResource, if there's no video with such id.
- ValueError, if the provided thread is not a valid DasherSplitterThread or if
the video id has a wrong format.
"""
LOGGER.info("Adding dasher splitter thread to VideoThreadsManager: [video_id={}]".format(
video_id))
try:
if d_thread is None or type(d_thread) != DasherSplitterThread:
raise ValueError("parameter is not instance of VideoThreadsManager")
GenericValidator.validate_id(video_id)
video = Video.objects(id=video_id).first()
if video is None:
raise NotExistingResource("No video with such id")
VideoThreadsRepository.get_instance().add_dasher_splitter_thread(d_thread=d_thread,
video_id=video_id)
video.update(set__splitter_status=ProcessStatus.IN_PROGRESS.value)
LOGGER.info(
"DasherSplitterThread sucessfully added to VideoThreadsManager: [video_id={}]".
format(video_id))
except Exception as ex:
LOGGER.exception("Error adding DasherSplitterThread: ")
raise ex
def add_thumbs_thread(self, t_thread, video_id):
"""
This method adds an already started t_thumbs thread to the list of managed threads in
order to monitor it.
The t_thumbs parameter represents a ThumbsCreatorThread that has been created and
started by an external entity. This method is responsible of adding the thread to the
VideoThreadsRepository and also to update the thumbs_status field of the related video
in the Database.
:param t_thumbs: Running instance of a ThumbsCreatorThread.
:param video_id: Id of the video which thumbs are being created by the t_thumbs thread.
"""
LOGGER.info("Adding thumbs creator thread to VideoThreadsManager: [video_id={}]".
format(video_id))
try:
if t_thread is None or type(t_thread) != ThumbsCreatorThread:
raise ValueError("parameter is not instance of ThumbsCreatorThread")
GenericValidator.validate_id(video_id)
video = Video.objects(id=video_id).first()
if video is None:
raise NotExistingResource("No video with such id")
VideoThreadsRepository.get_instance().add_thumbs_thread(t_thread=t_thread,
video_id=video_id)
video.update(set__thumbs_status=ProcessStatus.IN_PROGRESS.value)
LOGGER.info(
"ThumbsCreatorThread sucessfully added to VideoThreadsManager: [video_id={}]".
format(video_id))
except Exception as ex:
LOGGER.exception("Error adding ThumbsCreatorThread: ")
raise ex
def list_all_session_videos(self, session_id):
"""
:param session_id:
:return:
"""
LOGGER.info(
"Listing all session videos. [session_id={}]".format(session_id))
GenericValidator.validate_id(session_id)
session = SessionManager.get_instance().get_session(
session_id=session_id)
videos = Video.objects(session=session['id'])
session_videos = DataTransformer.generate_video_list_view(
session=session, db_videos=videos)
for session_video in session_videos:
try:
session_video['ts'] = VideoEditorHelper.get_initial_ts(
video_id=session_video['video_id'])
except Exception:
LOGGER.warn(
"Could not retrieve timestmap of video. Skipping it...")
session_video['ts'] = -1
ordered_list = sorted(session_videos, key=lambda x: float(x['ts']))
LOGGER.info("Retrieved {} videos for session {}".format(
len(ordered_list), session_id))
return ordered_list
def videos(self):
break_now = False
vids = []
for page in range(1, 99):
if break_now:
break
url = '{}?page={}&sort=m&order=d'.format(self.url, page)
r = requests.get(url)
links = str(r.text).split(
'<li class="item" data-video-item data-video-id="')
if len(links) == 1:
break
for link in links:
line_split = link.split('" data-nicoad-video')
video_id = line_split[0]
if len(video_id) > 20:
continue
if len(line_split) > 1:
rest = line_split[1]
pos = k_MYLIST_PATTERN.search(rest).regs[-1]
mylist_count = int(rest[pos[0]:pos[1]].replace(',', ''))
if mylist_count >= config.global_instance['minimum_mylist']:
vid = Video(video_id=video_id,
mylist_count=mylist_count)
vids.append(vid)
else:
break_now = True
break
return vids
def __init__(self, url, logger):
self.logger = logger
self.nico_object = Video(url=url)
self.type = self.k_VIDEO
for nico_object_type, obj in self.k_TYPE_MAP.items():
if nico_object_type in url:
self.nico_object = obj(url=url)
self.type = nico_object_type
break
def list_user_videos(self, user_id):
LOGGER.info(
"Listing all videos of the user. [user_id={}]".format(user_id))
GenericValidator.validate_id(user_id)
videos = Video.objects(user_id=user_id)
user_videos = DataTransformer.generate_user_video_list(
db_videos=videos)
LOGGER.info("Retrieved {} videos for user {}".format(
len(user_videos), user_id))
return user_videos
def _append_all(self, video_ids):
default_logger.debug('len(video_ids)={}'.format(len(video_ids)))
self._lock.acquire()
existing_video_ids = {}
for qe in self._list:
existing_video_ids[str(qe)] = None
for new_video_id in video_ids:
if new_video_id not in existing_video_ids:
video = Video(video_id=new_video_id)
qe = QueueElement(video=video)
self._list.append(qe)
self._lock.release()
def videos(self):
vids = []
r = requests.get(self.url)
items = list(
filter(lambda line: '<div class="itemData">' in line,
str(r.text).split('\n')))
for item in items:
matches = k_MYLIST_PATTERN.search(item)
video_id_pos = list(matches.regs[0])
video_id_pos[1] = item.index('">', video_id_pos[0])
video = Video(video_id=item[video_id_pos[0]:video_id_pos[1]])
vids.append(video)
return vids
def zip_video_thumbs(self, video_id):
"""
Zip all the thumbs of a specific video, identified by its id.
:param video_id: Id of the video.
:return: Buffer containing the content of the zip file.
"""
LOGGER.info("Getting all thumbs of a video: [id={}]".format(video_id))
video = Video.objects(id=video_id).first()
if video is None:
raise NotExistingResource("There's no video with such id.")
thumbs_path = video['video_path'] + "/thumbs"
zipbuffer = FileSystemService.get_instance().zip_directory(
dir_url=thumbs_path, zip_name=video['name'])
LOGGER.info("Video Thumbs successfully collected in ZIP file")
return zipbuffer
def get_video_first_thumb(self, video_id):
LOGGER.info("Getting first video thumb: [id={}]".format(video_id))
try:
video = Video.objects(id=video_id).first()
if video is None:
raise NotExistingResource("There's no video with such id.")
video_path = video['video_path'] + "/thumbs/out1.jpg"
LOGGER.info(
"First video thumb retrieved: [path={}]".format(video_path))
return video_path
except ValueError as ve:
LOGGER.exception("Error validating video id: ")
raise ve
except Exception as ex:
LOGGER.exception("Error trying to get video thumb: ")
raise ex
def add_video_to_active_session(self, user_id):
"""
:param user_id:
:return:
"""
LOGGER.info("Adding video to active session.")
session = SessionManager.get_instance().get_current_session()
if session is None or session['state'] != SessionStatus.ACTIVE.value:
raise IllegalSessionStateException("There's no active session.")
video_id = self.add_video(session_id=session['id'], user_id=user_id)
video_mongo = Video.objects(id=video_id).first()
video_dict = MongoHelper.to_dict(video_mongo)
video_dict['session'] = str(video_dict['session'])
LOGGER.info("Video successfully added to active session")
return video_dict
def get_initial_ts(video_id):
"""
:param video_id:
:return:
"""
GenericValidator.validate_id(video_id)
video = Video.objects(id=video_id).first()
if video is None:
raise NotExistingResource("No video with such id.")
video_path = video['video_path']
ts_filename = "{}/ts.txt".format(video_path)
file = open(ts_filename, "r")
if not os.path.exists(ts_filename):
raise NotExistingResource(
"Video is still being recorded or it failed.")
ts = file.read().rstrip("\n")
return ts
def list_session_videos(self, session_id, user_id):
"""
:param session_id:
:param user_id:
:return:
"""
LOGGER.info(
"Listing all session videos of the user. [session_id={}, user_id={}]"
.format(session_id, user_id))
GenericValidator.validate_id(session_id)
GenericValidator.validate_id(user_id)
session = SessionManager.get_instance().get_session(
session_id=session_id)
videos = Video.objects(session=session['id'], user_id=user_id)
session_videos = DataTransformer.generate_video_list_view(
session=session, db_videos=videos)
LOGGER.info("Retrieved {} videos for session {} and user {}".format(
len(session_videos), session_id, user_id))
return session_videos
def build_next_video_slice_info(video_info):
"""
:return:
"""
if video_info is None or type(video_info) != dict:
raise ValueError("Expected a dictionary as parameter.")
video = Video.objects(id=video_info['id']).first()
if video is None:
raise NotExistingResource("There's no video with such id.")
video_slice_info = {}
video_slice_info['file'] = "{}/dasher-output/video.mp4".format(
video['video_path'])
video_slice_info['inpoint'] = str(
datetime.timedelta(seconds=video_info['init'])).replace(":", ".")
video_slice_info['outpoint'] = str(
datetime.timedelta(seconds=video_info['end'])).replace(":", ".")
video_slice_info['inpoint'] = video_slice_info['inpoint'][-2:] + ".00"
video_slice_info[
'outpoint'] = video_slice_info['outpoint'][-2:] + ".00"
return video_slice_info
def cut_audio_for_user_video(session_id, video_id, video_init_ts,
video_length):
"""
:param session_id:
:param video_id:
:return:
"""
GenericValidator.validate_id(session_id)
GenericValidator.validate_id(video_id)
session = RumbaSession.objects(id=session_id).first()
if session is None:
raise NotExistingResource("There's no session with such id.")
video = Video.objects(id=video_id).first()
if video is None:
raise NotExistingResource("There's no video with such id.")
audio_path = "{}/audio.wav".format(session['folder_url'])
audio_output = "{}/audio-{}.wav".format(session['folder_url'],
uuid.uuid4().hex)
audio_init_ts = AudioManager.get_instance().get_audio_init_ts(
session_id=session_id)
audio_init_offset = VideoEditorHelper.calculate_audio_init_offset(
audio_init_ts=audio_init_ts, video_init_ts=video_init_ts)
ffmpeg_audio_init_offset = DataTransformer.transform_seconds_to_ffmpeg_offset(
float(audio_init_offset))
audio_end_offset = 12
print("Video_id: {}".format(video_id))
print("Video init ts: {}".format(video_init_ts))
print("Audio init ts: {}".format(audio_init_ts))
print("Audio init offset: {}".format(audio_init_offset))
audio_thread = AudioSplitterThread(
inputFile=audio_path,
outputFile=audio_output,
initial_offset=ffmpeg_audio_init_offset,
end_offset=video_length)
audio_thread.start()
audio_thread.join()
if audio_thread.code != 0:
raise Exception("FFMpeg command failed.")
return audio_output
def stop_and_remove_thumbs_thread(self, video_id):
"""
Stops and removes the ThumbsCreatorThread of a specific video.
This method check if there's a ThumbsCreatorThread running for a specific video, which is
is given as parameter. If the thread is still running, this method stops it and updates
the database information, marking the thread as Failure. If the thread is not running, then
this method checks the return code of the thread to mark the state in the DB as
"Failure" or "Finished" depending on its value.
:param video_id: Id of the video.
:raises:
- ValueError, if the given id os not a valid.
- NotExistingResource, if there's no video with such id.
"""
LOGGER.info("Removing thumbs creator thread from VideoThreadsManager: [video_id={}]".
format(video_id))
try:
GenericValidator.validate_id(video_id)
t_thread = VideoThreadsRepository.get_instance().get_thumbs_thread(video_id=video_id)
video = Video.objects(id=video_id).first()
if video is None:
raise NotExistingResource("No video with such id")
# 1) If the thread is running,we stop it and mark it as failure in the db.
if t_thread['thread'].is_alive():
t_thread['thread'].exit()
video.update(set__thumbs_status=ProcessStatus.FAILURE.value)
# 2) If the thread finished, we check the return code and update the db.
elif t_thread['thread'].code != 0:
video.update(set__thumbs_status=ProcessStatus.FAILURE.value)
else:
video.update(set__thumbs_status=ProcessStatus.FINISHED.value)
# 3) Finally we remove it from the REPOSITORY.
VideoThreadsRepository.get_instance().remove_thumbs_thread(video_id=video_id)
LOGGER.info(
"Sucessfully removed thumbs creator thread from VideoThreadsManager: [video_id={}]".
format(video_id))
except Exception as ex:
LOGGER.error("Error removing ThumbsCreatorThread: ")
raise ex
def stop_video(self, video_id):
"""
:param video_id:
:return:
"""
LOGGER.info("Stopping video: [id={}]".format(video_id))
try:
GenericValidator.validate_id(video_id)
video = Video.objects(id=video_id).first()
if video is None:
raise NotExistingResource("No video with such id")
if video['finished']:
raise IllegalResourceState("The video is already stopped.")
video.update(set__finished=True)
LOGGER.info("Video stopped: [id={}]".format(video_id))
except ValueError as ve:
LOGGER.exception("Error validating video id: ")
raise ve
except Exception as ex:
LOGGER.exception("Error trying to stop video: ")
raise ex
def calculate_audio_end_offset(edit_info, audio_init_offset,
audio_init_ts):
"""
:param session_id:
:param edit_info:
:return:
"""
if edit_info is None or type(edit_info) != list:
raise ValueError("Expected a list as parameter.")
last_video = edit_info[-1]
last_video_info = Video.objects(id=last_video['id']).first()
if last_video_info is None:
raise NotExistingResource(
"The last video of the edition does not exist.")
video_initts = VideoEditorHelper.get_initial_ts(
video_id=str(last_video_info['id']))
end_ts = float(video_initts) + float(last_video['thumb']) + 1.0
offset = float(end_ts) - float(audio_init_ts) - float(
audio_init_offset)
offset = round(offset, 3)
return str(offset)
def merge_user_video(self, video_id):
"""
Given the id of a video recorded by a user, it merges the video identified with such id
with the audio of the session.
A video can only be merged with the session audio if the record proces of the video
is already finished.
:param video_id: Id of the video.
"""
GenericValidator.validate_id(video_id)
video = Video.objects(id=video_id).first()
if video is None:
raise NotExistingResource("There's no video with such id.")
video_init_ts = VideoEditorHelper.get_initial_ts(video_id=video_id)
original_video = "{}/dasher-output/video.mp4".format(
video['video_path'])
t_video_length = VideoLengthThread(video_file=original_video)
t_video_length.start()
t_video_length.join()
if (t_video_length.code != 0):
raise Exception("Error merging audio and video of the user.")
audio_path = AudioManager.cut_audio_for_user_video(
session_id=str(video['session']['id']),
video_id=video_id,
video_init_ts=video_init_ts,
video_length=t_video_length.output)
output_file = "{}/dasher-output/video-mixed.mp4".format(
video['video_path'])
mixer = AudioVideoMixerThread(video_file=original_video,
audio_file=audio_path,
output_file=output_file)
mixer.start()
mixer.join()
if mixer.code != 0:
raise Exception(
"Error merging user video with audio. FFmpeg command failed.")
video.update(set__mixed=True)
video.update(set__mixed_video_path=output_file)
def build_next_video_slice_information(video_id, initial_ts, end_ts):
"""
:param video_info:
:return:
"""
if video_id is None or initial_ts is None or end_ts is None:
raise ValueError("All parameters are mandatory.")
video = Video.objects(id=video_id).first()
if video is None:
raise NotExistingResource("There's no video with such id.")
video_slice_info = {}
video_slice_info['file'] = "{}/dasher-output/video.mp4".format(
video['video_path'])
video_init = float(VideoEditorHelper.get_initial_ts(video_id=video_id))
init_offset = float(initial_ts) - float(video_init)
end_offset = float(end_ts) - float(video_init)
video_slice_info['inpoint'] = str(
datetime.timedelta(seconds=init_offset))
video_slice_info['outpoint'] = str(
datetime.timedelta(seconds=end_offset))
return video_slice_info
def __check_splitter_threads(self):
"""
:return:
"""
d_threads = self.thread_repos.get_all_dasher_splitter_threads()
for d_thread_info in d_threads:
LOGGER.debug(
"Checking DasherSplitter thread: [video_id={}]".format(
d_thread_info['video_id']))
video = Video.objects(id=d_thread_info['video_id']).first()
# If there's no video with such id, we kill the thread.
if video is None:
LOGGER.info(
"Found a DasherSplitter thread without associated video. Removing it."
)
d_thread_info['thread'].exit()
self.thread_repos.remove_dasher_splitter_thread(
d_thread_info['video_id'])
# If the thread is not alive, we check the return code and update the
# thread status into the database.
if not d_thread_info['thread'].is_alive():
LOGGER.info(
"DasherSplitter thread finished: [video_id={}, return_code={}]"
.format(d_thread_info['video_id'],
d_thread_info['thread'].code))
if d_thread_info['thread'].code == 0:
video.update(
set__splitter_status=ProcessStatus.FINISHED.value)
else:
video.update(
set__splitter_status=ProcessStatus.FAILURE.value)
self.thread_repos.remove_dasher_splitter_thread(
d_thread_info['video_id'])
else:
LOGGER.info(
"DasherSplitter thread still in progress: [video_id={}]".
format(d_thread_info['video_id']))
def split_video(self, video_id):
"""
:param video_id:
:return:
"""
LOGGER.info("Splitting video in fragments: [id={}]".format(video_id))
try:
GenericValidator.validate_id(video_id)
video = Video.objects(id=video_id).first()
if video is None:
raise NotExistingResource("No video with such id.")
video_path = video['video_path']
t_splitter = DasherSplitterThread(video_path=video_path)
t_splitter.start()
VideoThreadsManager.get_instance().add_dasher_splitter_thread(
d_thread=t_splitter, video_id=video_id)
except ValueError as ve:
LOGGER.exception("Error validating video path: ")
raise ve
except Exception as ex:
LOGGER.exception("Error trying to split video: ")
raise ex
def create_video_thumbs(self, video_id):
"""
:param video_id:
:return:
"""
LOGGER.info("Creating thumbs for video: [id={}]".format(video_id))
try:
GenericValidator.validate_id(video_id)
video = Video.objects(id=video_id).first()
if video is None:
raise NotExistingResource("No video with such id.")
video_path = video['video_path']
t_thumbs_creator = ThumbsCreatorThread(video_path=video_path)
t_thumbs_creator.start()
VideoThreadsManager.get_instance().add_thumbs_thread(
t_thread=t_thumbs_creator, video_id=video_id)
except ValueError as ve:
LOGGER.exception("Error validating video path: ")
raise ve
except Exception as ex:
LOGGER.exception("Error trying to create thumbs for video: ")
raise ex
def get_mixed_video_path(self, video_id):
"""
:param video_id:
:return:
"""
LOGGER.info("Geeting mixed video path [video_id={}]".format(video_id))
try:
video = Video.objects(id=video_id).first()
if video is None:
raise NotExistingResource("There's no video with such id.")
if not video['mixed'] or not video['mixed_video_path']:
raise IllegalResourceState("The video is no mixed yet.")
LOGGER.info(
"Mixed video path retrieved: [video_id={}, path={}]".format(
video_id, video['mixed_video_path']))
return video['mixed_video_path']
except ValueError as ve:
LOGGER.exception("Error validating video id: ")
raise ve
except Exception as ex:
LOGGER.exception("Error getting mixed video: ")
raise ex