def test_compression_max_width_works(self, high_res_video):
video_file = make_video_file(high_res_video)
video_file.ffmpeg_settings = {'crf': 33, 'max_width': 200}
video_filename = video_file.process_file()
video_path = config.get_storage_path(video_filename)
width, height = get_resolution(video_path)
assert width == 200, 'should be compress to 200 hz resolution'
assert video_file.get_preset() == format_presets.VIDEO_LOW_RES, 'Should have low res preset'
def test_convert_and_resize_ogv_works(self, low_res_ogv_video):
video_file = make_video_file(low_res_ogv_video)
video_file.ffmpeg_settings = {'crf': 33, 'max_height': 200}
video_filename = video_file.process_file()
video_path = config.get_storage_path(video_filename)
width, height = get_resolution(video_path)
assert height == 200, 'should be compress to 200 v resolution'
assert video_file.get_preset() == format_presets.VIDEO_LOW_RES, 'Should have low res preset'
def test_basic_video_processing_low_res(self, low_res_video):
expected_video_filename = '897d83a2e5389d454d37feb574587516.mp4'
video_file = make_video_file(low_res_video)
video_filename = video_file.process_file()
assert video_filename == expected_video_filename, "Video file should have filename {}".format(expected_video_filename)
video_path = config.get_storage_path(video_filename)
assert os.path.isfile(video_path), "Video should be stored at {}".format(video_path)
assert video_file.get_preset() == format_presets.VIDEO_LOW_RES, 'Should have low res preset'
def test_default_compression_works(self, high_res_video):
video_file = make_video_file(high_res_video)
video_file.ffmpeg_settings = {'crf': 33}
video_filename = video_file.process_file()
video_path = config.get_storage_path(video_filename)
width, height = get_resolution(video_path)
assert height == 480, 'should compress to 480 v resolution by defualt'
assert video_file.get_preset() == format_presets.VIDEO_LOW_RES, 'Should have low res preset'
def download_file(self, path, hash=None, default_ext='.{}'.format(file_formats.PNG), preset=None, force_ext=False):
""" download_file: downloads file from path
Args:
path (str): local path or url to file to download
hash (hash): hash to use for filename (optional)
default_ext (str): extension to use if none given (optional)
preset (str): preset to use (optional)
force_ext (bool): force manager to use default extension (optional)
Returns: filename of downloaded file
"""
# Access path
r = self.session.get(path, stream=True)
r.raise_for_status()
# Get extension of file or default if none found
extension = path.split(".")[-1].lower()
if force_ext or extension not in self.all_file_extensions:
extension = default_ext
else:
extension = "." + extension
# Write file to temporary file
with tempfile.TemporaryFile() as tempf:
# If a hash was not provided, generate hash during write process
if hash is None:
hash = hashlib.md5()
for chunk in r:
hash.update(chunk)
tempf.write(chunk)
# Otherwise, just write the file
else:
for chunk in r:
tempf.write(chunk)
# Get file metadata (hashed filename, original filename, size)
hashstring = hash.hexdigest()
original_filename = path.split("/")[-1].split(".")[0]
filename = '{0}{ext}'.format(hashstring, ext=extension)
file_size = tempf.tell()
tempf.seek(0)
# Keep track of downloaded file
self.files += [filename]
self._file_mapping.update({filename : {
'original_filename': original_filename,
'source_url': path,
'size': file_size,
'preset':preset,
}})
# Write file to local storage
with open(config.get_storage_path(filename), 'wb') as destf:
shutil.copyfileobj(tempf, destf)
if self.verbose:
print("\tDownloaded '{0}' to {1}".format(original_filename, filename))
return filename
def test_download_to_storage(video_file, video_filename, html_file,
html_filename, audio_file, audio_filename,
document_file, document_filename, epub_file,
epub_filename, thumbnail_file, thumbnail_filename,
subtitle_file, subtitle_filename):
process_files(video_file, html_file, audio_file, document_file, epub_file,
thumbnail_file, subtitle_file)
video_path = config.get_storage_path(video_filename)
html_path = config.get_storage_path(html_filename)
audio_path = config.get_storage_path(audio_filename)
document_path = config.get_storage_path(document_filename)
epub_path = config.get_storage_path(epub_filename)
thumbnail_path = config.get_storage_path(thumbnail_filename)
subtitle_path = config.get_storage_path(subtitle_filename)
assert os.path.isfile(video_path), "Video should be stored at {}".format(
video_path)
assert os.path.isfile(html_path), "HTML should be stored at {}".format(
html_path)
assert os.path.isfile(audio_path), "Audio should be stored at {}".format(
audio_path)
assert os.path.isfile(
document_path), "PDF document should be stored at {}".format(
document_path)
assert os.path.isfile(
epub_path), "ePub document should be stored at {}".format(epub_path)
assert os.path.isfile(
thumbnail_path), "Thumbnail should be stored at {}".format(
thumbnail_path)
assert os.path.isfile(
subtitle_path), "Subtitle should be stored at {}".format(subtitle_path)
def test_srt2vtt_works(self, srt_subtitles_file,
srt_subtitles_file_strings):
subs_file = make_subtitles_file(srt_subtitles_file)
converted_subs_filename = subs_file.process_file()
subs_path = config.get_storage_path(converted_subs_filename)
assert os.path.exists(subs_path), 'no converted file found'
_, dotext = os.path.splitext(subs_path)
assert dotext == '.vtt', 'wrong extension of converted file'
with open(subs_path, 'r') as converted_file:
converted_contents = converted_file.read()
for string in srt_subtitles_file_strings:
assert string in converted_contents
def upload_files(self, file_list):
""" upload_files: uploads files to server
Args: file_list (str): list of files to upload
Returns: None
"""
counter = 0
for f in file_list:
with open(config.get_storage_path(f), 'rb') as file_obj:
response = requests.post(config.file_upload_url(self.domain), files={'file': file_obj})
response.raise_for_status()
counter += 1
if self.verbose:
print("\tUploaded {0} ({count}/{total}) ".format(f, count=counter, total=len(file_list)))
def test_convertible_substitles_ar_srt():
"""
Basic check that srt --> vtt conversion works.
"""
assert os.path.exists("tests/testcontent/testsubtitles_ar.srt")
subtitle_file = SubtitleFile("tests/testcontent/testsubtitles_ar.srt", language='ar')
filename = subtitle_file.process_file()
assert filename, 'conferted filename must exit'
assert filename.endswith('.vtt'), 'conferted filename must have .vtt extension'
storage_path = config.get_storage_path(filename)
with open(storage_path) as converted_vtt:
filecontents = converted_vtt.read()
check_words = 'لناس على'
assert check_words in filecontents, 'missing check word in converted subs'
def check_has_thumbnail(self, node):
thumbnail_files = [
f for f in node.files if isinstance(f, ThumbnailFile)
or isinstance(f, TiledThumbnailFile)
]
assert len(thumbnail_files) == 1, 'expected single thumbnail'
thumbnail_file = thumbnail_files[0]
thumbnail_filename = thumbnail_file.get_filename()
thumbnail_path = config.get_storage_path(thumbnail_filename)
assert os.path.exists(thumbnail_path)
img = PIL.Image.open(thumbnail_path)
img.verify()
if SHOW_THUMBS:
img = PIL.Image.open(thumbnail_path)
img.show()
def test_convertible_substitles_from_pressurcooker(pressurcooker_test_files):
"""
Try to load all the test files used in pressurecooker as riceccooker `SubtitleFile`s.
All subs have the appropriate extension so no need to specify `subtitlesformat`.
"""
for fixture in pressurcooker_test_files:
localpath = fixture['localpath']
assert os.path.exists(localpath), 'Error mising local test file ' + localpath
subtitle_file = SubtitleFile(localpath, language=fixture['language'])
filename = subtitle_file.process_file()
assert filename, 'conferted filename must exit'
assert filename.endswith('.vtt'), 'conferted filename must have .vtt extension'
storage_path = config.get_storage_path(filename)
with open(storage_path) as converted_vtt:
filecontents = converted_vtt.read()
assert fixture['check_words'] in filecontents, 'missing check_words in converted subs'
def test_convertible_substitles_weirdext_subtitlesformat():
"""
Check that we handle cases when ext cannot be guessed from URL or localpath.
Passing `subtitlesformat` allows chef authors to manually specify subs format.
"""
subs_url = 'https://commons.wikimedia.org/w/api.php?' \
+ 'action=timedtext&title=File%3AA_Is_for_Atom_1953.webm&lang=es&trackformat=srt'
subtitle_file = SubtitleFile(
subs_url,
language='es',
subtitlesformat='srt' # set subtitlesformat when can't inferr ext form url
)
filename = subtitle_file.process_file()
assert filename, 'conferted filename must exit'
assert filename.endswith('.vtt'), 'conferted filename must have .vtt extension'
storage_path = config.get_storage_path(filename)
with open(storage_path) as converted_vtt:
filecontents = converted_vtt.read()
assert 'El total de los protones y neutrones de un átomo' in filecontents, \
'missing check words in converted subs'
def watermark_video(filename):
# Check if we've processed this file before -- is it in the cache?
key = files.generate_key("WATERMARKED",
filename,
settings=WATERMARK_SETTINGS)
if not config.UPDATE and files.FILECACHE.get(key):
return files.FILECACHE.get(key).decode('utf-8')
# Create a temporary filename to write the watermarked video.
tempf = tempfile.NamedTemporaryFile(suffix=".{}".format(file_formats.MP4),
delete=False)
tempf.close()
tempfile_name = tempf.name
# Now watermark it with the Touchable Earth logo!
print("\t--- Watermarking ", filename)
video_clip = mpe.VideoFileClip(config.get_storage_path(filename),
audio=True)
logo = (mpe.ImageClip(WATERMARK_SETTINGS["image"]).set_duration(
video_clip.duration).resize(
height=WATERMARK_SETTINGS["height"]).margin(
right=WATERMARK_SETTINGS["right"],
bottom=WATERMARK_SETTINGS["bottom"],
opacity=0).set_pos(WATERMARK_SETTINGS["position"]))
composite = mpe.CompositeVideoClip([video_clip, logo])
composite.duration = video_clip.duration
composite.write_videofile(tempfile_name, threads=4)
# Now move the watermarked file to Ricecooker storage and hash its name
# so it can be validated.
watermarked_filename = "{}.{}".format(files.get_hash(tempfile_name),
file_formats.MP4)
files.copy_file_to_storage(watermarked_filename, tempfile_name)
os.unlink(tempfile_name)
files.FILECACHE.set(key, bytes(watermarked_filename, "utf-8"))
return watermarked_filename
def overlay_and_watermark_video(filename, youtube_id):
# Check if we've processed this file before -- is it in the cache?
key = files.generate_key("WATERMARKED",
filename,
settings=WATERMARK_SETTINGS)
if not config.UPDATE and files.FILECACHE.get(key):
return files.FILECACHE.get(key).decode('utf-8')
# Create a temporary filename to write the watermarked video.
tempf = tempfile.NamedTemporaryFile(suffix=".{}".format(file_formats.MP4),
delete=False)
tempf.close()
tempfile_name = tempf.name
# Now watermark it with the Touchable Earth logo!
print("\t--- Watermarking and adding overlay ", filename)
# First add the overlay image -- this is the image shown as the first frame
# so that when the video hasn't been played yet, it will show this image
# rather than a black screen (since Touchable Earth's videos start from
# a blank black screen).
# Download the overlay image based on the YouTube ID
overlay_src = 'https://i.ytimg.com/vi_webp/%s/maxresdefault.webp' % youtube_id
print("\t ... grabbing overlay image from %s" % overlay_src)
destination = tempfile.mkdtemp()
overlay_filename = "overlay.webp"
overlay_file = os.path.join(destination, overlay_filename)
_, response = download_file(overlay_src,
destination,
request_fn=sess.get,
filename=overlay_filename)
video_clip = mpe.VideoFileClip(config.get_storage_path(filename),
audio=True)
if response.status_code == 200:
overlay_clip = mpe.ImageClip(overlay_file).set_duration(0.1)
concat_clips = mpe.concatenate_videoclips([overlay_clip, video_clip])
else:
concat_clips = video_clip
print("\t WARNING: Could not download overlay image file from %s" %
overlay_src)
# Now create the watermark logo as a clip ...
logo = (mpe.ImageClip(WATERMARK_SETTINGS["image"]).set_duration(
concat_clips.duration).resize(
height=WATERMARK_SETTINGS["height"]).margin(
right=WATERMARK_SETTINGS["right"],
bottom=WATERMARK_SETTINGS["bottom"],
opacity=0).set_pos(WATERMARK_SETTINGS["position"]))
# And then combine it with the video clip.
composite = mpe.CompositeVideoClip([concat_clips, logo])
composite.duration = concat_clips.duration
composite.write_videofile(tempfile_name, threads=4)
# Now move the watermarked file to Ricecooker storage and hash its name
# so it can be validated.
watermarked_filename = "{}.{}".format(files.get_hash(tempfile_name),
file_formats.MP4)
files.copy_file_to_storage(watermarked_filename, tempfile_name)
os.unlink(tempfile_name)
os.unlink(overlay_file)
files.FILECACHE.set(key, bytes(watermarked_filename, "utf-8"))
return watermarked_filename
