def test_compression_max_width_works(self, high_res_video): video_file = make_video_file(high_res_video) video_file.ffmpeg_settings = {'crf': 33, 'max_width': 200} video_filename = video_file.process_file() video_path = config.get_storage_path(video_filename) width, height = get_resolution(video_path) assert width == 200, 'should be compress to 200 hz resolution' assert video_file.get_preset() == format_presets.VIDEO_LOW_RES, 'Should have low res preset'
def test_convert_and_resize_ogv_works(self, low_res_ogv_video): video_file = make_video_file(low_res_ogv_video) video_file.ffmpeg_settings = {'crf': 33, 'max_height': 200} video_filename = video_file.process_file() video_path = config.get_storage_path(video_filename) width, height = get_resolution(video_path) assert height == 200, 'should be compress to 200 v resolution' assert video_file.get_preset() == format_presets.VIDEO_LOW_RES, 'Should have low res preset'
def test_basic_video_processing_low_res(self, low_res_video): expected_video_filename = '897d83a2e5389d454d37feb574587516.mp4' video_file = make_video_file(low_res_video) video_filename = video_file.process_file() assert video_filename == expected_video_filename, "Video file should have filename {}".format(expected_video_filename) video_path = config.get_storage_path(video_filename) assert os.path.isfile(video_path), "Video should be stored at {}".format(video_path) assert video_file.get_preset() == format_presets.VIDEO_LOW_RES, 'Should have low res preset'
def test_default_compression_works(self, high_res_video): video_file = make_video_file(high_res_video) video_file.ffmpeg_settings = {'crf': 33} video_filename = video_file.process_file() video_path = config.get_storage_path(video_filename) width, height = get_resolution(video_path) assert height == 480, 'should compress to 480 v resolution by defualt' assert video_file.get_preset() == format_presets.VIDEO_LOW_RES, 'Should have low res preset'
def download_file(self, path, hash=None, default_ext='.{}'.format(file_formats.PNG), preset=None, force_ext=False): """ download_file: downloads file from path Args: path (str): local path or url to file to download hash (hash): hash to use for filename (optional) default_ext (str): extension to use if none given (optional) preset (str): preset to use (optional) force_ext (bool): force manager to use default extension (optional) Returns: filename of downloaded file """ # Access path r = self.session.get(path, stream=True) r.raise_for_status() # Get extension of file or default if none found extension = path.split(".")[-1].lower() if force_ext or extension not in self.all_file_extensions: extension = default_ext else: extension = "." + extension # Write file to temporary file with tempfile.TemporaryFile() as tempf: # If a hash was not provided, generate hash during write process if hash is None: hash = hashlib.md5() for chunk in r: hash.update(chunk) tempf.write(chunk) # Otherwise, just write the file else: for chunk in r: tempf.write(chunk) # Get file metadata (hashed filename, original filename, size) hashstring = hash.hexdigest() original_filename = path.split("/")[-1].split(".")[0] filename = '{0}{ext}'.format(hashstring, ext=extension) file_size = tempf.tell() tempf.seek(0) # Keep track of downloaded file self.files += [filename] self._file_mapping.update({filename : { 'original_filename': original_filename, 'source_url': path, 'size': file_size, 'preset':preset, }}) # Write file to local storage with open(config.get_storage_path(filename), 'wb') as destf: shutil.copyfileobj(tempf, destf) if self.verbose: print("\tDownloaded '{0}' to {1}".format(original_filename, filename)) return filename
def test_download_to_storage(video_file, video_filename, html_file, html_filename, audio_file, audio_filename, document_file, document_filename, epub_file, epub_filename, thumbnail_file, thumbnail_filename, subtitle_file, subtitle_filename): process_files(video_file, html_file, audio_file, document_file, epub_file, thumbnail_file, subtitle_file) video_path = config.get_storage_path(video_filename) html_path = config.get_storage_path(html_filename) audio_path = config.get_storage_path(audio_filename) document_path = config.get_storage_path(document_filename) epub_path = config.get_storage_path(epub_filename) thumbnail_path = config.get_storage_path(thumbnail_filename) subtitle_path = config.get_storage_path(subtitle_filename) assert os.path.isfile(video_path), "Video should be stored at {}".format( video_path) assert os.path.isfile(html_path), "HTML should be stored at {}".format( html_path) assert os.path.isfile(audio_path), "Audio should be stored at {}".format( audio_path) assert os.path.isfile( document_path), "PDF document should be stored at {}".format( document_path) assert os.path.isfile( epub_path), "ePub document should be stored at {}".format(epub_path) assert os.path.isfile( thumbnail_path), "Thumbnail should be stored at {}".format( thumbnail_path) assert os.path.isfile( subtitle_path), "Subtitle should be stored at {}".format(subtitle_path)
def test_srt2vtt_works(self, srt_subtitles_file, srt_subtitles_file_strings): subs_file = make_subtitles_file(srt_subtitles_file) converted_subs_filename = subs_file.process_file() subs_path = config.get_storage_path(converted_subs_filename) assert os.path.exists(subs_path), 'no converted file found' _, dotext = os.path.splitext(subs_path) assert dotext == '.vtt', 'wrong extension of converted file' with open(subs_path, 'r') as converted_file: converted_contents = converted_file.read() for string in srt_subtitles_file_strings: assert string in converted_contents
def upload_files(self, file_list): """ upload_files: uploads files to server Args: file_list (str): list of files to upload Returns: None """ counter = 0 for f in file_list: with open(config.get_storage_path(f), 'rb') as file_obj: response = requests.post(config.file_upload_url(self.domain), files={'file': file_obj}) response.raise_for_status() counter += 1 if self.verbose: print("\tUploaded {0} ({count}/{total}) ".format(f, count=counter, total=len(file_list)))
def test_convertible_substitles_ar_srt(): """ Basic check that srt --> vtt conversion works. """ assert os.path.exists("tests/testcontent/testsubtitles_ar.srt") subtitle_file = SubtitleFile("tests/testcontent/testsubtitles_ar.srt", language='ar') filename = subtitle_file.process_file() assert filename, 'conferted filename must exit' assert filename.endswith('.vtt'), 'conferted filename must have .vtt extension' storage_path = config.get_storage_path(filename) with open(storage_path) as converted_vtt: filecontents = converted_vtt.read() check_words = 'لناس على' assert check_words in filecontents, 'missing check word in converted subs'
def check_has_thumbnail(self, node): thumbnail_files = [ f for f in node.files if isinstance(f, ThumbnailFile) or isinstance(f, TiledThumbnailFile) ] assert len(thumbnail_files) == 1, 'expected single thumbnail' thumbnail_file = thumbnail_files[0] thumbnail_filename = thumbnail_file.get_filename() thumbnail_path = config.get_storage_path(thumbnail_filename) assert os.path.exists(thumbnail_path) img = PIL.Image.open(thumbnail_path) img.verify() if SHOW_THUMBS: img = PIL.Image.open(thumbnail_path) img.show()
def test_convertible_substitles_from_pressurcooker(pressurcooker_test_files): """ Try to load all the test files used in pressurecooker as riceccooker `SubtitleFile`s. All subs have the appropriate extension so no need to specify `subtitlesformat`. """ for fixture in pressurcooker_test_files: localpath = fixture['localpath'] assert os.path.exists(localpath), 'Error mising local test file ' + localpath subtitle_file = SubtitleFile(localpath, language=fixture['language']) filename = subtitle_file.process_file() assert filename, 'conferted filename must exit' assert filename.endswith('.vtt'), 'conferted filename must have .vtt extension' storage_path = config.get_storage_path(filename) with open(storage_path) as converted_vtt: filecontents = converted_vtt.read() assert fixture['check_words'] in filecontents, 'missing check_words in converted subs'
def test_convertible_substitles_weirdext_subtitlesformat(): """ Check that we handle cases when ext cannot be guessed from URL or localpath. Passing `subtitlesformat` allows chef authors to manually specify subs format. """ subs_url = 'https://commons.wikimedia.org/w/api.php?' \ + 'action=timedtext&title=File%3AA_Is_for_Atom_1953.webm&lang=es&trackformat=srt' subtitle_file = SubtitleFile( subs_url, language='es', subtitlesformat='srt' # set subtitlesformat when can't inferr ext form url ) filename = subtitle_file.process_file() assert filename, 'conferted filename must exit' assert filename.endswith('.vtt'), 'conferted filename must have .vtt extension' storage_path = config.get_storage_path(filename) with open(storage_path) as converted_vtt: filecontents = converted_vtt.read() assert 'El total de los protones y neutrones de un átomo' in filecontents, \ 'missing check words in converted subs'
def watermark_video(filename): # Check if we've processed this file before -- is it in the cache? key = files.generate_key("WATERMARKED", filename, settings=WATERMARK_SETTINGS) if not config.UPDATE and files.FILECACHE.get(key): return files.FILECACHE.get(key).decode('utf-8') # Create a temporary filename to write the watermarked video. tempf = tempfile.NamedTemporaryFile(suffix=".{}".format(file_formats.MP4), delete=False) tempf.close() tempfile_name = tempf.name # Now watermark it with the Touchable Earth logo! print("\t--- Watermarking ", filename) video_clip = mpe.VideoFileClip(config.get_storage_path(filename), audio=True) logo = (mpe.ImageClip(WATERMARK_SETTINGS["image"]).set_duration( video_clip.duration).resize( height=WATERMARK_SETTINGS["height"]).margin( right=WATERMARK_SETTINGS["right"], bottom=WATERMARK_SETTINGS["bottom"], opacity=0).set_pos(WATERMARK_SETTINGS["position"])) composite = mpe.CompositeVideoClip([video_clip, logo]) composite.duration = video_clip.duration composite.write_videofile(tempfile_name, threads=4) # Now move the watermarked file to Ricecooker storage and hash its name # so it can be validated. watermarked_filename = "{}.{}".format(files.get_hash(tempfile_name), file_formats.MP4) files.copy_file_to_storage(watermarked_filename, tempfile_name) os.unlink(tempfile_name) files.FILECACHE.set(key, bytes(watermarked_filename, "utf-8")) return watermarked_filename
def overlay_and_watermark_video(filename, youtube_id): # Check if we've processed this file before -- is it in the cache? key = files.generate_key("WATERMARKED", filename, settings=WATERMARK_SETTINGS) if not config.UPDATE and files.FILECACHE.get(key): return files.FILECACHE.get(key).decode('utf-8') # Create a temporary filename to write the watermarked video. tempf = tempfile.NamedTemporaryFile(suffix=".{}".format(file_formats.MP4), delete=False) tempf.close() tempfile_name = tempf.name # Now watermark it with the Touchable Earth logo! print("\t--- Watermarking and adding overlay ", filename) # First add the overlay image -- this is the image shown as the first frame # so that when the video hasn't been played yet, it will show this image # rather than a black screen (since Touchable Earth's videos start from # a blank black screen). # Download the overlay image based on the YouTube ID overlay_src = 'https://i.ytimg.com/vi_webp/%s/maxresdefault.webp' % youtube_id print("\t ... grabbing overlay image from %s" % overlay_src) destination = tempfile.mkdtemp() overlay_filename = "overlay.webp" overlay_file = os.path.join(destination, overlay_filename) _, response = download_file(overlay_src, destination, request_fn=sess.get, filename=overlay_filename) video_clip = mpe.VideoFileClip(config.get_storage_path(filename), audio=True) if response.status_code == 200: overlay_clip = mpe.ImageClip(overlay_file).set_duration(0.1) concat_clips = mpe.concatenate_videoclips([overlay_clip, video_clip]) else: concat_clips = video_clip print("\t WARNING: Could not download overlay image file from %s" % overlay_src) # Now create the watermark logo as a clip ... logo = (mpe.ImageClip(WATERMARK_SETTINGS["image"]).set_duration( concat_clips.duration).resize( height=WATERMARK_SETTINGS["height"]).margin( right=WATERMARK_SETTINGS["right"], bottom=WATERMARK_SETTINGS["bottom"], opacity=0).set_pos(WATERMARK_SETTINGS["position"])) # And then combine it with the video clip. composite = mpe.CompositeVideoClip([concat_clips, logo]) composite.duration = concat_clips.duration composite.write_videofile(tempfile_name, threads=4) # Now move the watermarked file to Ricecooker storage and hash its name # so it can be validated. watermarked_filename = "{}.{}".format(files.get_hash(tempfile_name), file_formats.MP4) files.copy_file_to_storage(watermarked_filename, tempfile_name) os.unlink(tempfile_name) os.unlink(overlay_file) files.FILECACHE.set(key, bytes(watermarked_filename, "utf-8")) return watermarked_filename