def _execute_ffmpeg(
content: bytes,
cli: List[str],
program: str = "ffmpeg",
ignore_error_if_data: bool = False,
get_logs: bool = False,
) -> bytes:
mime_type = mime.get_mime_type(content)
if mime.is_heif(mime_type):
# FFmpeg does not support HEIF.
# https://trac.ffmpeg.org/ticket/6521
content = convert_heif_to_png(content)
cli = [program, "-loglevel", "32" if get_logs else "24"] + cli
proc = Popen(cli, stdout=PIPE, stdin=PIPE, stderr=PIPE)
out, err = proc.communicate(input=content)
if proc.returncode != 0:
args = " ".join(shlex.quote(arg) for arg in cli)
logger.warning(
f"Failed to execute {program} command (cli={args}, err={err})")
if (len(out) > 0 and not ignore_error_if_data) or len(out) == 0:
raise errors.ProcessingError("Error while processing media.\n" +
err.decode("utf-8"))
return err if get_logs else out
def _execute(
self,
cli: List[str],
program: str = 'ffmpeg',
ignore_error_if_data: bool = False,
get_logs: bool = False) -> bytes:
extension = mime.get_extension(mime.get_mime_type(self.content))
assert extension
with util.create_temp_file(suffix='.' + extension) as handle:
handle.write(self.content)
handle.flush()
cli = [program, '-loglevel', '32' if get_logs else '24'] + cli
cli = [part.format(path=handle.name) for part in cli]
proc = subprocess.Popen(
cli,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate(input=self.content)
if proc.returncode != 0:
logger.warning(
'Failed to execute ffmpeg command (cli=%r, err=%r)',
' '.join(shlex.quote(arg) for arg in cli),
err)
if ((len(out) > 0 and not ignore_error_if_data)
or len(out) == 0):
raise errors.ProcessingError(
'Error while processing image.\n'
+ err.decode('utf-8'))
return err if get_logs else out
def download(url: str, use_video_downloader: bool = False) -> bytes:
assert url
youtube_dl_error = None
if use_video_downloader:
try:
url = _get_youtube_dl_content_url(url) or url
except errors.ThirdPartyError as ex:
youtube_dl_error = ex
request = urllib.request.Request(url)
if config.config["user_agent"]:
request.add_header("User-Agent", config.config["user_agent"])
request.add_header("Referer", url)
content_buffer = b""
length_tally = 0
try:
with urllib.request.urlopen(request) as handle:
while (chunk := handle.read(_dl_chunk_size)):
length_tally += len(chunk)
if length_tally > config.config["max_dl_filesize"]:
raise DownloadTooLargeError(url)
content_buffer += chunk
except urllib.error.HTTPError as ex:
raise DownloadError(url) from ex
if (youtube_dl_error and mime.get_mime_type(content_buffer)
== "application/octet-stream"):
raise youtube_dl_error
return content_buffer
def _execute(
self,
cli: List[str],
program: str = "ffmpeg",
ignore_error_if_data: bool = False,
get_logs: bool = False,
) -> bytes:
extension = mime.get_extension(mime.get_mime_type(self.content))
assert extension
with util.create_temp_file(suffix="." + extension) as handle:
handle.write(self.content)
handle.flush()
cli = [program, "-loglevel", "32" if get_logs else "24"] + cli
cli = [part.format(path=handle.name) for part in cli]
proc = subprocess.Popen(
cli,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
)
out, err = proc.communicate(input=self.content)
if proc.returncode != 0:
logger.warning(
"Failed to execute ffmpeg command (cli=%r, err=%r)",
" ".join(shlex.quote(arg) for arg in cli),
err,
)
if (len(out) > 0
and not ignore_error_if_data) or len(out) == 0:
raise errors.ProcessingError(
"Error while processing image.\n" +
err.decode("utf-8"))
return err if get_logs else out
def _execute(
self,
cli: List[str],
program: str = 'ffmpeg',
ignore_error_if_data: bool = False) -> bytes:
extension = mime.get_extension(mime.get_mime_type(self.content))
assert extension
with util.create_temp_file(suffix='.' + extension) as handle:
handle.write(self.content)
handle.flush()
cli = [program, '-loglevel', '24'] + cli
cli = [part.format(path=handle.name) for part in cli]
proc = subprocess.Popen(
cli,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate(input=self.content)
if proc.returncode != 0:
logger.warning(
'ffmpeg 명령어 실행 실패 (cli=%r, err=%r)',
' '.join(shlex.quote(arg) for arg in cli),
err)
if ((len(out) > 0 and not ignore_error_if_data)
or len(out) == 0):
raise errors.ProcessingError(
'이미지 처리 중 오류.\n'
+ err.decode('utf-8'))
return out
def get_default_flags(content: bytes) -> List[str]:
assert content
ret = []
if mime.is_video(mime.get_mime_type(content)):
ret.append(model.Post.FLAG_LOOP)
if images.Image(content).check_for_sound():
ret.append(model.Post.FLAG_SOUND)
return ret
def update_post_content(post: model.Post, content: Optional[bytes]) -> None:
assert post
if not content:
raise InvalidPostContentError('Post content missing.')
update_signature = False
post.mime_type = mime.get_mime_type(content)
if mime.is_flash(post.mime_type):
post.type = model.Post.TYPE_FLASH
elif mime.is_image(post.mime_type):
update_signature = True
if mime.is_animated_gif(content):
post.type = model.Post.TYPE_ANIMATION
else:
post.type = model.Post.TYPE_IMAGE
elif mime.is_video(post.mime_type):
post.type = model.Post.TYPE_VIDEO
else:
raise InvalidPostContentError(
'Unhandled file type: %r' % post.mime_type)
post.checksum = util.get_sha1(content)
other_post = (
db.session
.query(model.Post)
.filter(model.Post.checksum == post.checksum)
.filter(model.Post.post_id != post.post_id)
.one_or_none())
if other_post \
and other_post.post_id \
and other_post.post_id != post.post_id:
raise PostAlreadyUploadedError(other_post)
if update_signature:
purge_post_signature(post)
post.signature = generate_post_signature(post, content)
post.file_size = len(content)
try:
image = images.Image(content)
post.canvas_width = image.width
post.canvas_height = image.height
except errors.ProcessingError:
if not config.config['allow_broken_uploads']:
raise InvalidPostContentError(
'Unable to process image metadata')
else:
post.canvas_width = None
post.canvas_height = None
if (post.canvas_width is not None and post.canvas_width <= 0) \
or (post.canvas_height is not None and post.canvas_height <= 0):
if not config.config['allow_broken_uploads']:
raise InvalidPostContentError(
'Invalid image dimensions returned during processing')
else:
post.canvas_width = None
post.canvas_height = None
setattr(post, '__content', content)
def test_sound(post: model.Post, content: bytes) -> None:
assert post
assert content
if mime.is_video(mime.get_mime_type(content)):
if images.Image(content).check_for_sound():
flags = post.flags
if model.Post.FLAG_SOUND not in flags:
flags.append(model.Post.FLAG_SOUND)
update_post_flags(post, flags)
def test_sound(post: model.Post, content: bytes) -> None:
assert post
assert content
if mime.is_video(mime.get_mime_type(content)):
if images.Image(content).check_for_sound():
flags = [x for x in post.flags.split(',') if x]
if model.Post.FLAG_SOUND not in flags:
flags.append(model.Post.FLAG_SOUND)
update_post_flags(post, flags)
def download(url: str, use_video_downloader: bool = False) -> bytes:
assert url
request = urllib.request.Request(url)
if config.config['user_agent']:
request.add_header('User-Agent', config.config['user_agent'])
request.add_header('Referer', url)
try:
with urllib.request.urlopen(request) as handle:
content = handle.read()
except Exception as ex:
raise errors.ProcessingError('Error downloading %s (%s)' % (url, ex))
if (use_video_downloader
and mime.get_mime_type(content) == 'application/octet-stream'):
return _youtube_dl_wrapper(url)
return content
def update_post_content(post: model.Post, content: Optional[bytes]) -> None:
assert post
if not content:
raise InvalidPostContentError('Post content missing.')
post.mime_type = mime.get_mime_type(content)
if mime.is_flash(post.mime_type):
post.type = model.Post.TYPE_FLASH
elif mime.is_image(post.mime_type):
if mime.is_animated_gif(content):
post.type = model.Post.TYPE_ANIMATION
else:
post.type = model.Post.TYPE_IMAGE
elif mime.is_video(post.mime_type):
post.type = model.Post.TYPE_VIDEO
else:
raise InvalidPostContentError(
'Unhandled file type: %r' % post.mime_type)
post.checksum = util.get_sha1(content)
other_post = (
db.session
.query(model.Post)
.filter(model.Post.checksum == post.checksum)
.filter(model.Post.post_id != post.post_id)
.one_or_none())
if other_post \
and other_post.post_id \
and other_post.post_id != post.post_id:
raise PostAlreadyUploadedError(other_post)
post.file_size = len(content)
try:
image = images.Image(content)
post.canvas_width = image.width
post.canvas_height = image.height
except errors.ProcessingError:
post.canvas_width = None
post.canvas_height = None
if (post.canvas_width is not None and post.canvas_width <= 0) \
or (post.canvas_height is not None and post.canvas_height <= 0):
post.canvas_width = None
post.canvas_height = None
setattr(post, '__content', content)
def update_post_content(post: model.Post, content: Optional[bytes]) -> None:
assert post
if not content:
raise InvalidPostContentError('Post content missing.')
post.mime_type = mime.get_mime_type(content)
if mime.is_flash(post.mime_type):
post.type = model.Post.TYPE_FLASH
elif mime.is_image(post.mime_type):
if mime.is_animated_gif(content):
post.type = model.Post.TYPE_ANIMATION
else:
post.type = model.Post.TYPE_IMAGE
elif mime.is_video(post.mime_type):
post.type = model.Post.TYPE_VIDEO
else:
raise InvalidPostContentError('Unhandled file type: %r' %
post.mime_type)
post.checksum = util.get_sha1(content)
other_post = db.session \
.query(model.Post) \
.filter(model.Post.checksum == post.checksum) \
.filter(model.Post.post_id != post.post_id) \
.one_or_none()
if other_post \
and other_post.post_id \
and other_post.post_id != post.post_id:
raise PostAlreadyUploadedError(other_post)
post.file_size = len(content)
try:
image = images.Image(content)
post.canvas_width = image.width
post.canvas_height = image.height
except errors.ProcessingError:
post.canvas_width = None
post.canvas_height = None
if (post.canvas_width is not None and post.canvas_width <= 0) \
or (post.canvas_height is not None and post.canvas_height <= 0):
post.canvas_width = None
post.canvas_height = None
setattr(post, '__content', content)
def _execute(self, cli: List[str], program: str = 'ffmpeg') -> bytes:
extension = mime.get_extension(mime.get_mime_type(self.content))
assert extension
with util.create_temp_file(suffix='.' + extension) as handle:
handle.write(self.content)
handle.flush()
cli = [program, '-loglevel', '24'] + cli
cli = [part.format(path=handle.name) for part in cli]
proc = subprocess.Popen(cli,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate(input=self.content)
if proc.returncode != 0:
logger.warning(
'Failed to execute ffmpeg command (cli=%r, err=%r)',
' '.join(shlex.quote(arg) for arg in cli), err)
raise errors.ProcessingError(
'Error while processing image.\n' + err.decode('utf-8'))
return out
def _execute(self, cli, program='ffmpeg'):
extension = mime.get_extension(mime.get_mime_type(self.content))
assert extension
with util.create_temp_file(suffix='.' + extension) as handle:
handle.write(self.content)
handle.flush()
cli = [program, '-loglevel', '24'] + cli
cli = [part.format(path=handle.name) for part in cli]
proc = subprocess.Popen(
cli,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate(input=self.content)
if proc.returncode != 0:
logger.warning(
'Failed to execute ffmpeg command (cli=%r, err=%r)',
' '.join(shlex.quote(arg) for arg in cli),
err)
raise errors.ProcessingError(
'Error while processing image.\n' + err.decode('utf-8'))
return out
def update_post_content(post, content):
if not content:
raise InvalidPostContentError('Post content missing.')
post.mime_type = mime.get_mime_type(content)
if mime.is_flash(post.mime_type):
post.type = db.Post.TYPE_FLASH
elif mime.is_image(post.mime_type):
if mime.is_animated_gif(content):
post.type = db.Post.TYPE_ANIMATION
else:
post.type = db.Post.TYPE_IMAGE
elif mime.is_video(post.mime_type):
post.type = db.Post.TYPE_VIDEO
else:
raise InvalidPostContentError('Unhandled file type: %r' % post.mime_type)
post.checksum = util.get_md5(content)
other_post = db.session \
.query(db.Post) \
.filter(db.Post.checksum == post.checksum) \
.filter(db.Post.post_id != post.post_id) \
.one_or_none()
if other_post:
raise PostAlreadyUploadedError(
'Post already uploaded (%d)' % other_post.post_id)
post.file_size = len(content)
try:
image = images.Image(content)
post.canvas_width = image.width
post.canvas_height = image.height
except errors.ProcessingError:
post.canvas_width = None
post.canvas_height = None
files.save(get_post_content_path(post), content)
update_post_thumbnail(post, content=None, delete=False)
def update_post_content(post: model.Post, content: Optional[bytes]) -> None:
assert post
if not content:
raise InvalidPostContentError("Post content missing.")
update_signature = False
post.mime_type = mime.get_mime_type(content)
if mime.is_flash(post.mime_type):
raise InvalidPostContentError(
"Flash animations are deprecated in this build and are slowly " +
"being phased out.")
elif mime.is_image(post.mime_type):
update_signature = True
if mime.is_animated_gif(content):
post.type = model.Post.TYPE_ANIMATION
else:
post.type = model.Post.TYPE_IMAGE
elif mime.is_video(post.mime_type):
post.type = model.Post.TYPE_VIDEO
else:
raise InvalidPostContentError(f"Unhandled file type: {post.mime_type}")
post.checksum = util.get_sha1(content)
post.checksum_md5 = util.get_md5(content)
other_post = (db.session.query(
model.Post).filter(model.Post.checksum == post.checksum).filter(
model.Post.post_id != post.post_id).one_or_none())
if (other_post and other_post.post_id
and other_post.post_id != post.post_id):
raise PostAlreadyUploadedError(other_post)
if update_signature:
purge_post_signature(post)
post.signature = generate_post_signature(post, content)
post.file_size = len(content)
post.canvas_width = None
post.canvas_height = None
post.date_taken = None
post.camera = None
try:
if post.type == model.Post.TYPE_IMAGE:
media = images.Image(content)
elif post.type in (model.Post.TYPE_ANIMATION, model.Post.TYPE_VIDEO):
media = images.Video(content)
except Exception as ex:
logger.exception(ex)
if not config.config["allow_broken_uploads"]:
raise InvalidPostContentError("Unable to process image metadata")
else:
if not media.width or not media.height:
if not config.config["allow_broken_uploads"]:
raise InvalidPostContentError(
"Invalid image dimensions returned during processing")
post.canvas_width = media.width
post.canvas_height = media.height
post.date_taken = media.date_taken
post.camera = media.camera
setattr(post, "__content", content)
def test_get_mime_type_for_empty_file():
assert mime.get_mime_type(b'') == 'application/octet-stream'
def test_get_mime_type_for_empty_file():
assert mime.get_mime_type(b"") == "application/octet-stream"
def test_get_mime_type(read_asset, input_path, expected_mime_type):
assert mime.get_mime_type(read_asset(input_path)) == expected_mime_type
def test_get_mime_type(read_asset, input_path, expected_mime_type):
assert mime.get_mime_type(read_asset(input_path)) == expected_mime_type
