def generate_post_thumbnail(post):
if files.has(get_post_thumbnail_backup_path(post)):
content = files.get(get_post_thumbnail_backup_path(post))
else:
content = files.get(get_post_content_path(post))
try:
image = images.Image(content)
image.resize_fill(
int(config.config['thumbnails']['post_width']),
int(config.config['thumbnails']['post_height']))
files.save(get_post_thumbnail_path(post), image.to_jpeg())
except errors.ProcessingError:
files.save(get_post_thumbnail_path(post), EMPTY_PIXEL)
def generate_post_thumbnail(post: model.Post) -> None:
assert post
if files.has(get_post_thumbnail_backup_path(post)):
content = files.get(get_post_thumbnail_backup_path(post))
else:
content = files.get(get_post_content_path(post))
try:
assert content
image = images.Image(content)
image.resize_fill(
int(config.config["thumbnails"]["post_width"]),
int(config.config["thumbnails"]["post_height"]),
)
files.save(get_post_thumbnail_path(post), image.to_jpeg())
except errors.ProcessingError:
files.save(get_post_thumbnail_path(post), EMPTY_PIXEL)
def update_post_thumbnail(post, content=None, delete=True):
if content is None:
content = files.get(get_post_content_path(post))
if delete:
files.delete(get_post_thumbnail_backup_path(post))
else:
files.save(get_post_thumbnail_backup_path(post), content)
generate_post_thumbnail(post)
def generate_post_thumbnail(post: model.Post) -> None:
assert post
if files.has(get_post_thumbnail_backup_path(post)):
content = files.get(get_post_thumbnail_backup_path(post))
else:
content = files.get(get_post_content_path(post))
try:
assert content
if post.type == model.Post.TYPE_IMAGE:
media = images.Image(content)
elif post.type == model.Post.TYPE_VIDEO:
media = images.Video(content)
thumb = media.to_thumbnail(
int(config.config["thumbnails"]["post_width"]),
int(config.config["thumbnails"]["post_height"]),
)
files.save(get_post_thumbnail_path(post), thumb)
except errors.ProcessingError:
files.save(get_post_thumbnail_path(post), EMPTY_PIXEL)
def update_all_md5_checksums() -> None:
posts_to_hash = (
db.session.query(model.Post).filter(
model.Post.checksum_md5 == None) # noqa: E711
.order_by(model.Post.post_id.asc()).all())
for post in posts_to_hash:
try:
post.checksum_md5 = util.get_md5(
files.get(get_post_content_path(post)))
db.session.commit()
logger.info("Created MD5 - Post %d", post.post_id)
except Exception as ex:
logger.exception(ex)
def update_all_post_signatures() -> None:
posts_to_hash = (db.session.query(
model.Post).filter((model.Post.type == model.Post.TYPE_IMAGE) | (
model.Post.type == model.Post.TYPE_ANIMATION)).filter(
model.Post.signature == None).order_by(
model.Post.post_id.asc()).all())
for post in posts_to_hash:
try:
generate_post_signature(post,
files.get(get_post_content_path(post)))
db.session.commit()
logger.info('Hashed Post %d', post.post_id)
except Exception as ex:
logger.exception(ex)
def populate_reverse_search() -> None:
excluded_post_ids = image_hash.get_all_paths()
post_ids_to_hash = (db.session.query(model.Post.post_id).filter(
(model.Post.type == model.Post.TYPE_IMAGE)
| (model.Post.type == model.Post.TYPE_ANIMATION)).filter(
~model.Post.post_id.in_(excluded_post_ids)).order_by(
model.Post.post_id.asc()).all())
for post_ids_chunk in util.chunks(post_ids_to_hash, 100):
posts_chunk = (db.session.query(model.Post).filter(
model.Post.post_id.in_(post_ids_chunk)).all())
for post in posts_chunk:
content_path = get_post_content_path(post)
if files.has(content_path):
image_hash.add_image(post.post_id, files.get(content_path))
def populate_reverse_search():
excluded_post_ids = image_hash.get_all_paths()
post_ids_to_hash = (db.session
.query(db.Post.post_id)
.filter(
(db.Post.type == db.Post.TYPE_IMAGE) |
(db.Post.type == db.Post.TYPE_ANIMATION))
.filter(~db.Post.post_id.in_(excluded_post_ids))
.order_by(db.Post.post_id.asc())
.all())
for post_ids_chunk in util.chunks(post_ids_to_hash, 100):
posts_chunk = (db.session
.query(db.Post)
.filter(db.Post.post_id.in_(post_ids_chunk))
.all())
for post in posts_chunk:
content_path = get_post_content_path(post)
if files.has(content_path):
image_hash.add_image(post.post_id, files.get(content_path))
def merge_posts(source_post: model.Post, target_post: model.Post,
replace_content: bool) -> None:
assert source_post
assert target_post
if source_post.post_id == target_post.post_id:
raise InvalidPostRelationError('Cannot merge post with itself.')
def merge_tables(table: model.Base,
anti_dup_func: Optional[Callable[[model.Base, model.Base],
bool]],
source_post_id: int, target_post_id: int) -> None:
alias1 = table
alias2 = sa.orm.util.aliased(table)
update_stmt = (sa.sql.expression.update(alias1).where(
alias1.post_id == source_post_id))
if anti_dup_func is not None:
update_stmt = (update_stmt.where(
~sa.exists().where(anti_dup_func(alias1, alias2)).where(
alias2.post_id == target_post_id)))
update_stmt = update_stmt.values(post_id=target_post_id)
db.session.execute(update_stmt)
def merge_tags(source_post_id: int, target_post_id: int) -> None:
merge_tables(model.PostTag,
lambda alias1, alias2: alias1.tag_id == alias2.tag_id,
source_post_id, target_post_id)
def merge_scores(source_post_id: int, target_post_id: int) -> None:
merge_tables(model.PostScore,
lambda alias1, alias2: alias1.user_id == alias2.user_id,
source_post_id, target_post_id)
def merge_favorites(source_post_id: int, target_post_id: int) -> None:
merge_tables(model.PostFavorite,
lambda alias1, alias2: alias1.user_id == alias2.user_id,
source_post_id, target_post_id)
def merge_comments(source_post_id: int, target_post_id: int) -> None:
merge_tables(model.Comment, None, source_post_id, target_post_id)
def merge_relations(source_post_id: int, target_post_id: int) -> None:
alias1 = model.PostRelation
alias2 = sa.orm.util.aliased(model.PostRelation)
update_stmt = (sa.sql.expression.update(alias1).where(
alias1.parent_id == source_post_id).where(
alias1.child_id != target_post_id).where(~sa.exists().where(
alias2.child_id == alias1.child_id).where(
alias2.parent_id == target_post_id)).values(
parent_id=target_post_id))
db.session.execute(update_stmt)
update_stmt = (sa.sql.expression.update(alias1).where(
alias1.child_id == source_post_id).where(
alias1.parent_id != target_post_id).where(~sa.exists().where(
alias2.parent_id == alias1.parent_id).where(
alias2.child_id == target_post_id)).values(
child_id=target_post_id))
db.session.execute(update_stmt)
merge_tags(source_post.post_id, target_post.post_id)
merge_comments(source_post.post_id, target_post.post_id)
merge_scores(source_post.post_id, target_post.post_id)
merge_favorites(source_post.post_id, target_post.post_id)
merge_relations(source_post.post_id, target_post.post_id)
delete(source_post)
db.session.flush()
if replace_content:
content = files.get(get_post_content_path(source_post))
update_post_content(target_post, content)
def merge_posts(source_post, target_post, replace_content):
assert source_post
assert target_post
if source_post.post_id == target_post.post_id:
raise InvalidPostRelationError('Cannot merge post with itself.')
def merge_tables(table, anti_dup_func, source_post_id, target_post_id):
alias1 = table
alias2 = sqlalchemy.orm.util.aliased(table)
update_stmt = (sqlalchemy.sql.expression.update(alias1)
.where(alias1.post_id == source_post_id))
if anti_dup_func is not None:
update_stmt = (update_stmt
.where(~sqlalchemy.exists()
.where(anti_dup_func(alias1, alias2))
.where(alias2.post_id == target_post_id)))
update_stmt = update_stmt.values(post_id=target_post_id)
db.session.execute(update_stmt)
def merge_tags(source_post_id, target_post_id):
merge_tables(
db.PostTag,
lambda alias1, alias2: alias1.tag_id == alias2.tag_id,
source_post_id,
target_post_id)
def merge_scores(source_post_id, target_post_id):
merge_tables(
db.PostScore,
lambda alias1, alias2: alias1.user_id == alias2.user_id,
source_post_id,
target_post_id)
def merge_favorites(source_post_id, target_post_id):
merge_tables(
db.PostFavorite,
lambda alias1, alias2: alias1.user_id == alias2.user_id,
source_post_id,
target_post_id)
def merge_comments(source_post_id, target_post_id):
merge_tables(db.Comment, None, source_post_id, target_post_id)
def merge_relations(source_post_id, target_post_id):
alias1 = db.PostRelation
alias2 = sqlalchemy.orm.util.aliased(db.PostRelation)
update_stmt = (sqlalchemy.sql.expression.update(alias1)
.where(alias1.parent_id == source_post_id)
.where(alias1.child_id != target_post_id)
.where(~sqlalchemy.exists()
.where(alias2.child_id == alias1.child_id)
.where(alias2.parent_id == target_post_id))
.values(parent_id=target_post_id))
db.session.execute(update_stmt)
update_stmt = (sqlalchemy.sql.expression.update(alias1)
.where(alias1.child_id == source_post_id)
.where(alias1.parent_id != target_post_id)
.where(~sqlalchemy.exists()
.where(alias2.parent_id == alias1.parent_id)
.where(alias2.child_id == target_post_id))
.values(child_id=target_post_id))
db.session.execute(update_stmt)
merge_tags(source_post.post_id, target_post.post_id)
merge_comments(source_post.post_id, target_post.post_id)
merge_scores(source_post.post_id, target_post.post_id)
merge_favorites(source_post.post_id, target_post.post_id)
merge_relations(source_post.post_id, target_post.post_id)
delete(source_post)
db.session.flush()
if replace_content:
content = files.get(get_post_content_path(source_post))
update_post_content(target_post, content)
def get(checksum: str) -> Optional[bytes]:
return files.get("temporary-uploads/%s.dat" % checksum)
def merge_posts(
source_post: model.Post,
target_post: model.Post,
replace_content: bool) -> None:
assert source_post
assert target_post
if source_post.post_id == target_post.post_id:
raise InvalidPostRelationError('Cannot merge post with itself.')
def merge_tables(
table: model.Base,
anti_dup_func: Optional[Callable[[model.Base, model.Base], bool]],
source_post_id: int,
target_post_id: int) -> None:
alias1 = table
alias2 = sa.orm.util.aliased(table)
update_stmt = (
sa.sql.expression.update(alias1)
.where(alias1.post_id == source_post_id))
if anti_dup_func is not None:
update_stmt = (
update_stmt
.where(
~sa.exists()
.where(anti_dup_func(alias1, alias2))
.where(alias2.post_id == target_post_id)))
update_stmt = update_stmt.values(post_id=target_post_id)
db.session.execute(update_stmt)
def merge_tags(source_post_id: int, target_post_id: int) -> None:
merge_tables(
model.PostTag,
lambda alias1, alias2: alias1.tag_id == alias2.tag_id,
source_post_id,
target_post_id)
def merge_scores(source_post_id: int, target_post_id: int) -> None:
merge_tables(
model.PostScore,
lambda alias1, alias2: alias1.user_id == alias2.user_id,
source_post_id,
target_post_id)
def merge_favorites(source_post_id: int, target_post_id: int) -> None:
merge_tables(
model.PostFavorite,
lambda alias1, alias2: alias1.user_id == alias2.user_id,
source_post_id,
target_post_id)
def merge_comments(source_post_id: int, target_post_id: int) -> None:
merge_tables(model.Comment, None, source_post_id, target_post_id)
def merge_relations(source_post_id: int, target_post_id: int) -> None:
alias1 = model.PostRelation
alias2 = sa.orm.util.aliased(model.PostRelation)
update_stmt = (
sa.sql.expression.update(alias1)
.where(alias1.parent_id == source_post_id)
.where(alias1.child_id != target_post_id)
.where(
~sa.exists()
.where(alias2.child_id == alias1.child_id)
.where(alias2.parent_id == target_post_id))
.values(parent_id=target_post_id))
db.session.execute(update_stmt)
update_stmt = (
sa.sql.expression.update(alias1)
.where(alias1.child_id == source_post_id)
.where(alias1.parent_id != target_post_id)
.where(
~sa.exists()
.where(alias2.parent_id == alias1.parent_id)
.where(alias2.child_id == target_post_id))
.values(child_id=target_post_id))
db.session.execute(update_stmt)
def transfer_flags(source_post_id: int, target_post_id: int) -> None:
target = get_post_by_id(target_post_id)
source = get_post_by_id(source_post_id)
target.flags = source.flags
merge_tags(source_post.post_id, target_post.post_id)
merge_comments(source_post.post_id, target_post.post_id)
merge_scores(source_post.post_id, target_post.post_id)
merge_favorites(source_post.post_id, target_post.post_id)
merge_relations(source_post.post_id, target_post.post_id)
content = None
if replace_content:
content = files.get(get_post_content_path(source_post))
transfer_flags(source_post.post_id, target_post.post_id)
delete(source_post)
db.session.flush()
if content is not None:
update_post_content(target_post, content)
def get(checksum: str) -> Optional[bytes]:
return files.get('temporary-uploads/%s.dat' % checksum)
def get(checksum):
return files.get("temporary-uploads/%s.dat" % checksum)
