def search_by_image_exact(image_content: bytes) -> Optional[model.Post]:
checksum = util.get_sha1(image_content)
return (
db.session
.query(model.Post)
.filter(model.Post.checksum == checksum)
.one_or_none())
def search_by_image_exact(image_content: bytes) -> Optional[model.Post]:
checksum = util.get_sha1(image_content)
return (
db.session
.query(model.Post)
.filter(model.Post.checksum == checksum)
.one_or_none())
def _youtube_dl_wrapper(url: str) -> bytes:
outpath = os.path.join(
config.config["data_dir"],
"temporary-uploads",
"youtubedl-" + util.get_sha1(url)[0:8] + ".dat",
)
options = {
"ignoreerrors": False,
"format": "best[ext=webm]/best[ext=mp4]/best[ext=flv]",
"logger": logger,
"max_filesize": config.config["max_dl_filesize"],
"max_downloads": 1,
"outtmpl": outpath,
}
try:
with YoutubeDL(options) as ydl:
ydl.extract_info(url, download=True)
with open(outpath, "rb") as f:
return f.read()
except YoutubeDLError as ex:
raise errors.ThirdPartyError("Error downloading video %s (%s)" %
(url, ex))
except FileNotFoundError:
raise errors.ThirdPartyError(
"Error downloading video %s (file could not be saved)" % (url))
def update_post_content(post: model.Post, content: Optional[bytes]) -> None:
assert post
if not content:
raise InvalidPostContentError('Post content missing.')
update_signature = False
post.mime_type = mime.get_mime_type(content)
if mime.is_flash(post.mime_type):
post.type = model.Post.TYPE_FLASH
elif mime.is_image(post.mime_type):
update_signature = True
if mime.is_animated_gif(content):
post.type = model.Post.TYPE_ANIMATION
else:
post.type = model.Post.TYPE_IMAGE
elif mime.is_video(post.mime_type):
post.type = model.Post.TYPE_VIDEO
else:
raise InvalidPostContentError(
'Unhandled file type: %r' % post.mime_type)
post.checksum = util.get_sha1(content)
other_post = (
db.session
.query(model.Post)
.filter(model.Post.checksum == post.checksum)
.filter(model.Post.post_id != post.post_id)
.one_or_none())
if other_post \
and other_post.post_id \
and other_post.post_id != post.post_id:
raise PostAlreadyUploadedError(other_post)
if update_signature:
purge_post_signature(post)
post.signature = generate_post_signature(post, content)
post.file_size = len(content)
try:
image = images.Image(content)
post.canvas_width = image.width
post.canvas_height = image.height
except errors.ProcessingError:
if not config.config['allow_broken_uploads']:
raise InvalidPostContentError(
'Unable to process image metadata')
else:
post.canvas_width = None
post.canvas_height = None
if (post.canvas_width is not None and post.canvas_width <= 0) \
or (post.canvas_height is not None and post.canvas_height <= 0):
if not config.config['allow_broken_uploads']:
raise InvalidPostContentError(
'Invalid image dimensions returned during processing')
else:
post.canvas_width = None
post.canvas_height = None
setattr(post, '__content', content)
def update_post_content(post: model.Post, content: Optional[bytes]) -> None:
assert post
if not content:
raise InvalidPostContentError('Post content missing.')
post.mime_type = mime.get_mime_type(content)
if mime.is_flash(post.mime_type):
post.type = model.Post.TYPE_FLASH
elif mime.is_image(post.mime_type):
if mime.is_animated_gif(content):
post.type = model.Post.TYPE_ANIMATION
else:
post.type = model.Post.TYPE_IMAGE
elif mime.is_video(post.mime_type):
post.type = model.Post.TYPE_VIDEO
else:
raise InvalidPostContentError(
'Unhandled file type: %r' % post.mime_type)
post.checksum = util.get_sha1(content)
other_post = (
db.session
.query(model.Post)
.filter(model.Post.checksum == post.checksum)
.filter(model.Post.post_id != post.post_id)
.one_or_none())
if other_post \
and other_post.post_id \
and other_post.post_id != post.post_id:
raise PostAlreadyUploadedError(other_post)
post.file_size = len(content)
try:
image = images.Image(content)
post.canvas_width = image.width
post.canvas_height = image.height
except errors.ProcessingError:
post.canvas_width = None
post.canvas_height = None
if (post.canvas_width is not None and post.canvas_width <= 0) \
or (post.canvas_height is not None and post.canvas_height <= 0):
post.canvas_width = None
post.canvas_height = None
setattr(post, '__content', content)
def update_post_content(post: model.Post, content: Optional[bytes]) -> None:
assert post
if not content:
raise InvalidPostContentError('Post content missing.')
post.mime_type = mime.get_mime_type(content)
if mime.is_flash(post.mime_type):
post.type = model.Post.TYPE_FLASH
elif mime.is_image(post.mime_type):
if mime.is_animated_gif(content):
post.type = model.Post.TYPE_ANIMATION
else:
post.type = model.Post.TYPE_IMAGE
elif mime.is_video(post.mime_type):
post.type = model.Post.TYPE_VIDEO
else:
raise InvalidPostContentError('Unhandled file type: %r' %
post.mime_type)
post.checksum = util.get_sha1(content)
other_post = db.session \
.query(model.Post) \
.filter(model.Post.checksum == post.checksum) \
.filter(model.Post.post_id != post.post_id) \
.one_or_none()
if other_post \
and other_post.post_id \
and other_post.post_id != post.post_id:
raise PostAlreadyUploadedError(other_post)
post.file_size = len(content)
try:
image = images.Image(content)
post.canvas_width = image.width
post.canvas_height = image.height
except errors.ProcessingError:
post.canvas_width = None
post.canvas_height = None
if (post.canvas_width is not None and post.canvas_width <= 0) \
or (post.canvas_height is not None and post.canvas_height <= 0):
post.canvas_width = None
post.canvas_height = None
setattr(post, '__content', content)
def _youtube_dl_wrapper(url: str) -> bytes:
outpath = os.path.join(config.config['data_dir'], 'temporary-uploads',
'youtubedl-' + util.get_sha1(url)[0:8] + '.dat')
options = {
'ignoreerrors': False,
'format': 'best[ext=webm]/best[ext=mp4]/best[ext=flv]',
'logger': logger,
'max_filesize': config.config['max_dl_filesize'],
'max_downloads': 1,
'outtmpl': outpath,
}
try:
with YoutubeDL(options) as ydl:
ydl.extract_info(url, download=True)
with open(outpath, 'rb') as f:
return f.read()
except YoutubeDLError as ex:
raise errors.ThirdPartyError('Error downloading video %s (%s)' %
(url, ex))
except FileNotFoundError as ex:
raise errors.ThirdPartyError(
'Error downloading video %s (file could not be saved)' % (url))
def test_video_download(url, expected_sha1):
actual_content = net.download(url, use_video_downloader=True)
assert get_sha1(actual_content) == expected_sha1
def search_by_image_exact(image_content):
checksum = util.get_sha1(image_content)
return db.session \
.query(db.Post) \
.filter(db.Post.checksum == checksum) \
.one_or_none()
def save(content: bytes) -> str:
checksum = util.get_sha1(content)
path = _get_path(checksum)
if not files.has(path):
files.save(path, content)
return checksum
def save(content: bytes) -> str:
checksum = util.get_sha1(content)
path = _get_path(checksum)
if not files.has(path):
files.save(path, content)
return checksum
def update_post_content(post: model.Post, content: Optional[bytes]) -> None:
assert post
if not content:
raise InvalidPostContentError("Post content missing.")
update_signature = False
post.mime_type = mime.get_mime_type(content)
if mime.is_flash(post.mime_type):
raise InvalidPostContentError(
"Flash animations are deprecated in this build and are slowly " +
"being phased out.")
elif mime.is_image(post.mime_type):
update_signature = True
if mime.is_animated_gif(content):
post.type = model.Post.TYPE_ANIMATION
else:
post.type = model.Post.TYPE_IMAGE
elif mime.is_video(post.mime_type):
post.type = model.Post.TYPE_VIDEO
else:
raise InvalidPostContentError(f"Unhandled file type: {post.mime_type}")
post.checksum = util.get_sha1(content)
post.checksum_md5 = util.get_md5(content)
other_post = (db.session.query(
model.Post).filter(model.Post.checksum == post.checksum).filter(
model.Post.post_id != post.post_id).one_or_none())
if (other_post and other_post.post_id
and other_post.post_id != post.post_id):
raise PostAlreadyUploadedError(other_post)
if update_signature:
purge_post_signature(post)
post.signature = generate_post_signature(post, content)
post.file_size = len(content)
post.canvas_width = None
post.canvas_height = None
post.date_taken = None
post.camera = None
try:
if post.type == model.Post.TYPE_IMAGE:
media = images.Image(content)
elif post.type in (model.Post.TYPE_ANIMATION, model.Post.TYPE_VIDEO):
media = images.Video(content)
except Exception as ex:
logger.exception(ex)
if not config.config["allow_broken_uploads"]:
raise InvalidPostContentError("Unable to process image metadata")
else:
if not media.width or not media.height:
if not config.config["allow_broken_uploads"]:
raise InvalidPostContentError(
"Invalid image dimensions returned during processing")
post.canvas_width = media.width
post.canvas_height = media.height
post.date_taken = media.date_taken
post.camera = media.camera
setattr(post, "__content", content)