def _execute_ffmpeg( content: bytes, cli: List[str], program: str = "ffmpeg", ignore_error_if_data: bool = False, get_logs: bool = False, ) -> bytes: mime_type = mime.get_mime_type(content) if mime.is_heif(mime_type): # FFmpeg does not support HEIF. # https://trac.ffmpeg.org/ticket/6521 content = convert_heif_to_png(content) cli = [program, "-loglevel", "32" if get_logs else "24"] + cli proc = Popen(cli, stdout=PIPE, stdin=PIPE, stderr=PIPE) out, err = proc.communicate(input=content) if proc.returncode != 0: args = " ".join(shlex.quote(arg) for arg in cli) logger.warning( f"Failed to execute {program} command (cli={args}, err={err})") if (len(out) > 0 and not ignore_error_if_data) or len(out) == 0: raise errors.ProcessingError("Error while processing media.\n" + err.decode("utf-8")) return err if get_logs else out
def _execute( self, cli: List[str], program: str = 'ffmpeg', ignore_error_if_data: bool = False, get_logs: bool = False) -> bytes: extension = mime.get_extension(mime.get_mime_type(self.content)) assert extension with util.create_temp_file(suffix='.' + extension) as handle: handle.write(self.content) handle.flush() cli = [program, '-loglevel', '32' if get_logs else '24'] + cli cli = [part.format(path=handle.name) for part in cli] proc = subprocess.Popen( cli, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) out, err = proc.communicate(input=self.content) if proc.returncode != 0: logger.warning( 'Failed to execute ffmpeg command (cli=%r, err=%r)', ' '.join(shlex.quote(arg) for arg in cli), err) if ((len(out) > 0 and not ignore_error_if_data) or len(out) == 0): raise errors.ProcessingError( 'Error while processing image.\n' + err.decode('utf-8')) return err if get_logs else out
def download(url: str, use_video_downloader: bool = False) -> bytes: assert url youtube_dl_error = None if use_video_downloader: try: url = _get_youtube_dl_content_url(url) or url except errors.ThirdPartyError as ex: youtube_dl_error = ex request = urllib.request.Request(url) if config.config["user_agent"]: request.add_header("User-Agent", config.config["user_agent"]) request.add_header("Referer", url) content_buffer = b"" length_tally = 0 try: with urllib.request.urlopen(request) as handle: while (chunk := handle.read(_dl_chunk_size)): length_tally += len(chunk) if length_tally > config.config["max_dl_filesize"]: raise DownloadTooLargeError(url) content_buffer += chunk except urllib.error.HTTPError as ex: raise DownloadError(url) from ex if (youtube_dl_error and mime.get_mime_type(content_buffer) == "application/octet-stream"): raise youtube_dl_error return content_buffer
def _execute( self, cli: List[str], program: str = "ffmpeg", ignore_error_if_data: bool = False, get_logs: bool = False, ) -> bytes: extension = mime.get_extension(mime.get_mime_type(self.content)) assert extension with util.create_temp_file(suffix="." + extension) as handle: handle.write(self.content) handle.flush() cli = [program, "-loglevel", "32" if get_logs else "24"] + cli cli = [part.format(path=handle.name) for part in cli] proc = subprocess.Popen( cli, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE, ) out, err = proc.communicate(input=self.content) if proc.returncode != 0: logger.warning( "Failed to execute ffmpeg command (cli=%r, err=%r)", " ".join(shlex.quote(arg) for arg in cli), err, ) if (len(out) > 0 and not ignore_error_if_data) or len(out) == 0: raise errors.ProcessingError( "Error while processing image.\n" + err.decode("utf-8")) return err if get_logs else out
def _execute( self, cli: List[str], program: str = 'ffmpeg', ignore_error_if_data: bool = False) -> bytes: extension = mime.get_extension(mime.get_mime_type(self.content)) assert extension with util.create_temp_file(suffix='.' + extension) as handle: handle.write(self.content) handle.flush() cli = [program, '-loglevel', '24'] + cli cli = [part.format(path=handle.name) for part in cli] proc = subprocess.Popen( cli, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) out, err = proc.communicate(input=self.content) if proc.returncode != 0: logger.warning( 'ffmpeg 명령어 실행 실패 (cli=%r, err=%r)', ' '.join(shlex.quote(arg) for arg in cli), err) if ((len(out) > 0 and not ignore_error_if_data) or len(out) == 0): raise errors.ProcessingError( '이미지 처리 중 오류.\n' + err.decode('utf-8')) return out
def get_default_flags(content: bytes) -> List[str]: assert content ret = [] if mime.is_video(mime.get_mime_type(content)): ret.append(model.Post.FLAG_LOOP) if images.Image(content).check_for_sound(): ret.append(model.Post.FLAG_SOUND) return ret
def update_post_content(post: model.Post, content: Optional[bytes]) -> None: assert post if not content: raise InvalidPostContentError('Post content missing.') update_signature = False post.mime_type = mime.get_mime_type(content) if mime.is_flash(post.mime_type): post.type = model.Post.TYPE_FLASH elif mime.is_image(post.mime_type): update_signature = True if mime.is_animated_gif(content): post.type = model.Post.TYPE_ANIMATION else: post.type = model.Post.TYPE_IMAGE elif mime.is_video(post.mime_type): post.type = model.Post.TYPE_VIDEO else: raise InvalidPostContentError( 'Unhandled file type: %r' % post.mime_type) post.checksum = util.get_sha1(content) other_post = ( db.session .query(model.Post) .filter(model.Post.checksum == post.checksum) .filter(model.Post.post_id != post.post_id) .one_or_none()) if other_post \ and other_post.post_id \ and other_post.post_id != post.post_id: raise PostAlreadyUploadedError(other_post) if update_signature: purge_post_signature(post) post.signature = generate_post_signature(post, content) post.file_size = len(content) try: image = images.Image(content) post.canvas_width = image.width post.canvas_height = image.height except errors.ProcessingError: if not config.config['allow_broken_uploads']: raise InvalidPostContentError( 'Unable to process image metadata') else: post.canvas_width = None post.canvas_height = None if (post.canvas_width is not None and post.canvas_width <= 0) \ or (post.canvas_height is not None and post.canvas_height <= 0): if not config.config['allow_broken_uploads']: raise InvalidPostContentError( 'Invalid image dimensions returned during processing') else: post.canvas_width = None post.canvas_height = None setattr(post, '__content', content)
def test_sound(post: model.Post, content: bytes) -> None: assert post assert content if mime.is_video(mime.get_mime_type(content)): if images.Image(content).check_for_sound(): flags = post.flags if model.Post.FLAG_SOUND not in flags: flags.append(model.Post.FLAG_SOUND) update_post_flags(post, flags)
def test_sound(post: model.Post, content: bytes) -> None: assert post assert content if mime.is_video(mime.get_mime_type(content)): if images.Image(content).check_for_sound(): flags = [x for x in post.flags.split(',') if x] if model.Post.FLAG_SOUND not in flags: flags.append(model.Post.FLAG_SOUND) update_post_flags(post, flags)
def download(url: str, use_video_downloader: bool = False) -> bytes: assert url request = urllib.request.Request(url) if config.config['user_agent']: request.add_header('User-Agent', config.config['user_agent']) request.add_header('Referer', url) try: with urllib.request.urlopen(request) as handle: content = handle.read() except Exception as ex: raise errors.ProcessingError('Error downloading %s (%s)' % (url, ex)) if (use_video_downloader and mime.get_mime_type(content) == 'application/octet-stream'): return _youtube_dl_wrapper(url) return content
def update_post_content(post: model.Post, content: Optional[bytes]) -> None: assert post if not content: raise InvalidPostContentError('Post content missing.') post.mime_type = mime.get_mime_type(content) if mime.is_flash(post.mime_type): post.type = model.Post.TYPE_FLASH elif mime.is_image(post.mime_type): if mime.is_animated_gif(content): post.type = model.Post.TYPE_ANIMATION else: post.type = model.Post.TYPE_IMAGE elif mime.is_video(post.mime_type): post.type = model.Post.TYPE_VIDEO else: raise InvalidPostContentError( 'Unhandled file type: %r' % post.mime_type) post.checksum = util.get_sha1(content) other_post = ( db.session .query(model.Post) .filter(model.Post.checksum == post.checksum) .filter(model.Post.post_id != post.post_id) .one_or_none()) if other_post \ and other_post.post_id \ and other_post.post_id != post.post_id: raise PostAlreadyUploadedError(other_post) post.file_size = len(content) try: image = images.Image(content) post.canvas_width = image.width post.canvas_height = image.height except errors.ProcessingError: post.canvas_width = None post.canvas_height = None if (post.canvas_width is not None and post.canvas_width <= 0) \ or (post.canvas_height is not None and post.canvas_height <= 0): post.canvas_width = None post.canvas_height = None setattr(post, '__content', content)
def update_post_content(post: model.Post, content: Optional[bytes]) -> None: assert post if not content: raise InvalidPostContentError('Post content missing.') post.mime_type = mime.get_mime_type(content) if mime.is_flash(post.mime_type): post.type = model.Post.TYPE_FLASH elif mime.is_image(post.mime_type): if mime.is_animated_gif(content): post.type = model.Post.TYPE_ANIMATION else: post.type = model.Post.TYPE_IMAGE elif mime.is_video(post.mime_type): post.type = model.Post.TYPE_VIDEO else: raise InvalidPostContentError('Unhandled file type: %r' % post.mime_type) post.checksum = util.get_sha1(content) other_post = db.session \ .query(model.Post) \ .filter(model.Post.checksum == post.checksum) \ .filter(model.Post.post_id != post.post_id) \ .one_or_none() if other_post \ and other_post.post_id \ and other_post.post_id != post.post_id: raise PostAlreadyUploadedError(other_post) post.file_size = len(content) try: image = images.Image(content) post.canvas_width = image.width post.canvas_height = image.height except errors.ProcessingError: post.canvas_width = None post.canvas_height = None if (post.canvas_width is not None and post.canvas_width <= 0) \ or (post.canvas_height is not None and post.canvas_height <= 0): post.canvas_width = None post.canvas_height = None setattr(post, '__content', content)
def _execute(self, cli: List[str], program: str = 'ffmpeg') -> bytes: extension = mime.get_extension(mime.get_mime_type(self.content)) assert extension with util.create_temp_file(suffix='.' + extension) as handle: handle.write(self.content) handle.flush() cli = [program, '-loglevel', '24'] + cli cli = [part.format(path=handle.name) for part in cli] proc = subprocess.Popen(cli, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) out, err = proc.communicate(input=self.content) if proc.returncode != 0: logger.warning( 'Failed to execute ffmpeg command (cli=%r, err=%r)', ' '.join(shlex.quote(arg) for arg in cli), err) raise errors.ProcessingError( 'Error while processing image.\n' + err.decode('utf-8')) return out
def _execute(self, cli, program='ffmpeg'): extension = mime.get_extension(mime.get_mime_type(self.content)) assert extension with util.create_temp_file(suffix='.' + extension) as handle: handle.write(self.content) handle.flush() cli = [program, '-loglevel', '24'] + cli cli = [part.format(path=handle.name) for part in cli] proc = subprocess.Popen( cli, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) out, err = proc.communicate(input=self.content) if proc.returncode != 0: logger.warning( 'Failed to execute ffmpeg command (cli=%r, err=%r)', ' '.join(shlex.quote(arg) for arg in cli), err) raise errors.ProcessingError( 'Error while processing image.\n' + err.decode('utf-8')) return out
def update_post_content(post, content): if not content: raise InvalidPostContentError('Post content missing.') post.mime_type = mime.get_mime_type(content) if mime.is_flash(post.mime_type): post.type = db.Post.TYPE_FLASH elif mime.is_image(post.mime_type): if mime.is_animated_gif(content): post.type = db.Post.TYPE_ANIMATION else: post.type = db.Post.TYPE_IMAGE elif mime.is_video(post.mime_type): post.type = db.Post.TYPE_VIDEO else: raise InvalidPostContentError('Unhandled file type: %r' % post.mime_type) post.checksum = util.get_md5(content) other_post = db.session \ .query(db.Post) \ .filter(db.Post.checksum == post.checksum) \ .filter(db.Post.post_id != post.post_id) \ .one_or_none() if other_post: raise PostAlreadyUploadedError( 'Post already uploaded (%d)' % other_post.post_id) post.file_size = len(content) try: image = images.Image(content) post.canvas_width = image.width post.canvas_height = image.height except errors.ProcessingError: post.canvas_width = None post.canvas_height = None files.save(get_post_content_path(post), content) update_post_thumbnail(post, content=None, delete=False)
def update_post_content(post: model.Post, content: Optional[bytes]) -> None: assert post if not content: raise InvalidPostContentError("Post content missing.") update_signature = False post.mime_type = mime.get_mime_type(content) if mime.is_flash(post.mime_type): raise InvalidPostContentError( "Flash animations are deprecated in this build and are slowly " + "being phased out.") elif mime.is_image(post.mime_type): update_signature = True if mime.is_animated_gif(content): post.type = model.Post.TYPE_ANIMATION else: post.type = model.Post.TYPE_IMAGE elif mime.is_video(post.mime_type): post.type = model.Post.TYPE_VIDEO else: raise InvalidPostContentError(f"Unhandled file type: {post.mime_type}") post.checksum = util.get_sha1(content) post.checksum_md5 = util.get_md5(content) other_post = (db.session.query( model.Post).filter(model.Post.checksum == post.checksum).filter( model.Post.post_id != post.post_id).one_or_none()) if (other_post and other_post.post_id and other_post.post_id != post.post_id): raise PostAlreadyUploadedError(other_post) if update_signature: purge_post_signature(post) post.signature = generate_post_signature(post, content) post.file_size = len(content) post.canvas_width = None post.canvas_height = None post.date_taken = None post.camera = None try: if post.type == model.Post.TYPE_IMAGE: media = images.Image(content) elif post.type in (model.Post.TYPE_ANIMATION, model.Post.TYPE_VIDEO): media = images.Video(content) except Exception as ex: logger.exception(ex) if not config.config["allow_broken_uploads"]: raise InvalidPostContentError("Unable to process image metadata") else: if not media.width or not media.height: if not config.config["allow_broken_uploads"]: raise InvalidPostContentError( "Invalid image dimensions returned during processing") post.canvas_width = media.width post.canvas_height = media.height post.date_taken = media.date_taken post.camera = media.camera setattr(post, "__content", content)
def test_get_mime_type_for_empty_file(): assert mime.get_mime_type(b'') == 'application/octet-stream'
def test_get_mime_type_for_empty_file(): assert mime.get_mime_type(b"") == "application/octet-stream"
def test_get_mime_type(read_asset, input_path, expected_mime_type): assert mime.get_mime_type(read_asset(input_path)) == expected_mime_type