def generate_post_thumbnail(post): if files.has(get_post_thumbnail_backup_path(post)): content = files.get(get_post_thumbnail_backup_path(post)) else: content = files.get(get_post_content_path(post)) try: image = images.Image(content) image.resize_fill( int(config.config['thumbnails']['post_width']), int(config.config['thumbnails']['post_height'])) files.save(get_post_thumbnail_path(post), image.to_jpeg()) except errors.ProcessingError: files.save(get_post_thumbnail_path(post), EMPTY_PIXEL)
def generate_post_thumbnail(post: model.Post) -> None: assert post if files.has(get_post_thumbnail_backup_path(post)): content = files.get(get_post_thumbnail_backup_path(post)) else: content = files.get(get_post_content_path(post)) try: assert content image = images.Image(content) image.resize_fill( int(config.config["thumbnails"]["post_width"]), int(config.config["thumbnails"]["post_height"]), ) files.save(get_post_thumbnail_path(post), image.to_jpeg()) except errors.ProcessingError: files.save(get_post_thumbnail_path(post), EMPTY_PIXEL)
def update_post_thumbnail(post, content=None, delete=True): if content is None: content = files.get(get_post_content_path(post)) if delete: files.delete(get_post_thumbnail_backup_path(post)) else: files.save(get_post_thumbnail_backup_path(post), content) generate_post_thumbnail(post)
def generate_post_thumbnail(post: model.Post) -> None: assert post if files.has(get_post_thumbnail_backup_path(post)): content = files.get(get_post_thumbnail_backup_path(post)) else: content = files.get(get_post_content_path(post)) try: assert content if post.type == model.Post.TYPE_IMAGE: media = images.Image(content) elif post.type == model.Post.TYPE_VIDEO: media = images.Video(content) thumb = media.to_thumbnail( int(config.config["thumbnails"]["post_width"]), int(config.config["thumbnails"]["post_height"]), ) files.save(get_post_thumbnail_path(post), thumb) except errors.ProcessingError: files.save(get_post_thumbnail_path(post), EMPTY_PIXEL)
def update_all_md5_checksums() -> None: posts_to_hash = ( db.session.query(model.Post).filter( model.Post.checksum_md5 == None) # noqa: E711 .order_by(model.Post.post_id.asc()).all()) for post in posts_to_hash: try: post.checksum_md5 = util.get_md5( files.get(get_post_content_path(post))) db.session.commit() logger.info("Created MD5 - Post %d", post.post_id) except Exception as ex: logger.exception(ex)
def update_all_post_signatures() -> None: posts_to_hash = (db.session.query( model.Post).filter((model.Post.type == model.Post.TYPE_IMAGE) | ( model.Post.type == model.Post.TYPE_ANIMATION)).filter( model.Post.signature == None).order_by( model.Post.post_id.asc()).all()) for post in posts_to_hash: try: generate_post_signature(post, files.get(get_post_content_path(post))) db.session.commit() logger.info('Hashed Post %d', post.post_id) except Exception as ex: logger.exception(ex)
def populate_reverse_search() -> None: excluded_post_ids = image_hash.get_all_paths() post_ids_to_hash = (db.session.query(model.Post.post_id).filter( (model.Post.type == model.Post.TYPE_IMAGE) | (model.Post.type == model.Post.TYPE_ANIMATION)).filter( ~model.Post.post_id.in_(excluded_post_ids)).order_by( model.Post.post_id.asc()).all()) for post_ids_chunk in util.chunks(post_ids_to_hash, 100): posts_chunk = (db.session.query(model.Post).filter( model.Post.post_id.in_(post_ids_chunk)).all()) for post in posts_chunk: content_path = get_post_content_path(post) if files.has(content_path): image_hash.add_image(post.post_id, files.get(content_path))
def populate_reverse_search(): excluded_post_ids = image_hash.get_all_paths() post_ids_to_hash = (db.session .query(db.Post.post_id) .filter( (db.Post.type == db.Post.TYPE_IMAGE) | (db.Post.type == db.Post.TYPE_ANIMATION)) .filter(~db.Post.post_id.in_(excluded_post_ids)) .order_by(db.Post.post_id.asc()) .all()) for post_ids_chunk in util.chunks(post_ids_to_hash, 100): posts_chunk = (db.session .query(db.Post) .filter(db.Post.post_id.in_(post_ids_chunk)) .all()) for post in posts_chunk: content_path = get_post_content_path(post) if files.has(content_path): image_hash.add_image(post.post_id, files.get(content_path))
def merge_posts(source_post: model.Post, target_post: model.Post, replace_content: bool) -> None: assert source_post assert target_post if source_post.post_id == target_post.post_id: raise InvalidPostRelationError('Cannot merge post with itself.') def merge_tables(table: model.Base, anti_dup_func: Optional[Callable[[model.Base, model.Base], bool]], source_post_id: int, target_post_id: int) -> None: alias1 = table alias2 = sa.orm.util.aliased(table) update_stmt = (sa.sql.expression.update(alias1).where( alias1.post_id == source_post_id)) if anti_dup_func is not None: update_stmt = (update_stmt.where( ~sa.exists().where(anti_dup_func(alias1, alias2)).where( alias2.post_id == target_post_id))) update_stmt = update_stmt.values(post_id=target_post_id) db.session.execute(update_stmt) def merge_tags(source_post_id: int, target_post_id: int) -> None: merge_tables(model.PostTag, lambda alias1, alias2: alias1.tag_id == alias2.tag_id, source_post_id, target_post_id) def merge_scores(source_post_id: int, target_post_id: int) -> None: merge_tables(model.PostScore, lambda alias1, alias2: alias1.user_id == alias2.user_id, source_post_id, target_post_id) def merge_favorites(source_post_id: int, target_post_id: int) -> None: merge_tables(model.PostFavorite, lambda alias1, alias2: alias1.user_id == alias2.user_id, source_post_id, target_post_id) def merge_comments(source_post_id: int, target_post_id: int) -> None: merge_tables(model.Comment, None, source_post_id, target_post_id) def merge_relations(source_post_id: int, target_post_id: int) -> None: alias1 = model.PostRelation alias2 = sa.orm.util.aliased(model.PostRelation) update_stmt = (sa.sql.expression.update(alias1).where( alias1.parent_id == source_post_id).where( alias1.child_id != target_post_id).where(~sa.exists().where( alias2.child_id == alias1.child_id).where( alias2.parent_id == target_post_id)).values( parent_id=target_post_id)) db.session.execute(update_stmt) update_stmt = (sa.sql.expression.update(alias1).where( alias1.child_id == source_post_id).where( alias1.parent_id != target_post_id).where(~sa.exists().where( alias2.parent_id == alias1.parent_id).where( alias2.child_id == target_post_id)).values( child_id=target_post_id)) db.session.execute(update_stmt) merge_tags(source_post.post_id, target_post.post_id) merge_comments(source_post.post_id, target_post.post_id) merge_scores(source_post.post_id, target_post.post_id) merge_favorites(source_post.post_id, target_post.post_id) merge_relations(source_post.post_id, target_post.post_id) delete(source_post) db.session.flush() if replace_content: content = files.get(get_post_content_path(source_post)) update_post_content(target_post, content)
def merge_posts(source_post, target_post, replace_content): assert source_post assert target_post if source_post.post_id == target_post.post_id: raise InvalidPostRelationError('Cannot merge post with itself.') def merge_tables(table, anti_dup_func, source_post_id, target_post_id): alias1 = table alias2 = sqlalchemy.orm.util.aliased(table) update_stmt = (sqlalchemy.sql.expression.update(alias1) .where(alias1.post_id == source_post_id)) if anti_dup_func is not None: update_stmt = (update_stmt .where(~sqlalchemy.exists() .where(anti_dup_func(alias1, alias2)) .where(alias2.post_id == target_post_id))) update_stmt = update_stmt.values(post_id=target_post_id) db.session.execute(update_stmt) def merge_tags(source_post_id, target_post_id): merge_tables( db.PostTag, lambda alias1, alias2: alias1.tag_id == alias2.tag_id, source_post_id, target_post_id) def merge_scores(source_post_id, target_post_id): merge_tables( db.PostScore, lambda alias1, alias2: alias1.user_id == alias2.user_id, source_post_id, target_post_id) def merge_favorites(source_post_id, target_post_id): merge_tables( db.PostFavorite, lambda alias1, alias2: alias1.user_id == alias2.user_id, source_post_id, target_post_id) def merge_comments(source_post_id, target_post_id): merge_tables(db.Comment, None, source_post_id, target_post_id) def merge_relations(source_post_id, target_post_id): alias1 = db.PostRelation alias2 = sqlalchemy.orm.util.aliased(db.PostRelation) update_stmt = (sqlalchemy.sql.expression.update(alias1) .where(alias1.parent_id == source_post_id) .where(alias1.child_id != target_post_id) .where(~sqlalchemy.exists() .where(alias2.child_id == alias1.child_id) .where(alias2.parent_id == target_post_id)) .values(parent_id=target_post_id)) db.session.execute(update_stmt) update_stmt = (sqlalchemy.sql.expression.update(alias1) .where(alias1.child_id == source_post_id) .where(alias1.parent_id != target_post_id) .where(~sqlalchemy.exists() .where(alias2.parent_id == alias1.parent_id) .where(alias2.child_id == target_post_id)) .values(child_id=target_post_id)) db.session.execute(update_stmt) merge_tags(source_post.post_id, target_post.post_id) merge_comments(source_post.post_id, target_post.post_id) merge_scores(source_post.post_id, target_post.post_id) merge_favorites(source_post.post_id, target_post.post_id) merge_relations(source_post.post_id, target_post.post_id) delete(source_post) db.session.flush() if replace_content: content = files.get(get_post_content_path(source_post)) update_post_content(target_post, content)
def get(checksum: str) -> Optional[bytes]: return files.get("temporary-uploads/%s.dat" % checksum)
def merge_posts( source_post: model.Post, target_post: model.Post, replace_content: bool) -> None: assert source_post assert target_post if source_post.post_id == target_post.post_id: raise InvalidPostRelationError('Cannot merge post with itself.') def merge_tables( table: model.Base, anti_dup_func: Optional[Callable[[model.Base, model.Base], bool]], source_post_id: int, target_post_id: int) -> None: alias1 = table alias2 = sa.orm.util.aliased(table) update_stmt = ( sa.sql.expression.update(alias1) .where(alias1.post_id == source_post_id)) if anti_dup_func is not None: update_stmt = ( update_stmt .where( ~sa.exists() .where(anti_dup_func(alias1, alias2)) .where(alias2.post_id == target_post_id))) update_stmt = update_stmt.values(post_id=target_post_id) db.session.execute(update_stmt) def merge_tags(source_post_id: int, target_post_id: int) -> None: merge_tables( model.PostTag, lambda alias1, alias2: alias1.tag_id == alias2.tag_id, source_post_id, target_post_id) def merge_scores(source_post_id: int, target_post_id: int) -> None: merge_tables( model.PostScore, lambda alias1, alias2: alias1.user_id == alias2.user_id, source_post_id, target_post_id) def merge_favorites(source_post_id: int, target_post_id: int) -> None: merge_tables( model.PostFavorite, lambda alias1, alias2: alias1.user_id == alias2.user_id, source_post_id, target_post_id) def merge_comments(source_post_id: int, target_post_id: int) -> None: merge_tables(model.Comment, None, source_post_id, target_post_id) def merge_relations(source_post_id: int, target_post_id: int) -> None: alias1 = model.PostRelation alias2 = sa.orm.util.aliased(model.PostRelation) update_stmt = ( sa.sql.expression.update(alias1) .where(alias1.parent_id == source_post_id) .where(alias1.child_id != target_post_id) .where( ~sa.exists() .where(alias2.child_id == alias1.child_id) .where(alias2.parent_id == target_post_id)) .values(parent_id=target_post_id)) db.session.execute(update_stmt) update_stmt = ( sa.sql.expression.update(alias1) .where(alias1.child_id == source_post_id) .where(alias1.parent_id != target_post_id) .where( ~sa.exists() .where(alias2.parent_id == alias1.parent_id) .where(alias2.child_id == target_post_id)) .values(child_id=target_post_id)) db.session.execute(update_stmt) def transfer_flags(source_post_id: int, target_post_id: int) -> None: target = get_post_by_id(target_post_id) source = get_post_by_id(source_post_id) target.flags = source.flags merge_tags(source_post.post_id, target_post.post_id) merge_comments(source_post.post_id, target_post.post_id) merge_scores(source_post.post_id, target_post.post_id) merge_favorites(source_post.post_id, target_post.post_id) merge_relations(source_post.post_id, target_post.post_id) content = None if replace_content: content = files.get(get_post_content_path(source_post)) transfer_flags(source_post.post_id, target_post.post_id) delete(source_post) db.session.flush() if content is not None: update_post_content(target_post, content)
def get(checksum: str) -> Optional[bytes]: return files.get('temporary-uploads/%s.dat' % checksum)
def get(checksum): return files.get("temporary-uploads/%s.dat" % checksum)