def show_status() -> ServerInfo:
"""
PIMS Server status.
"""
return ServerInfo(version=__version__,
api_version=__api_version__,
settings=get_settings())
async def startup():
# Check PIMS configuration
try:
settings = get_settings()
logger.info("[green bold]PIMS is starting with config:[/]")
for k, v in settings.dict().items():
logger.info(f"[green]* {k}:[/] [blue]{v}[/]",
extra={"highlight": False})
except ValidationError as e:
logger.error("Impossible to read or parse some PIMS settings:")
logger.error(e)
exit(-1)
# Check optimisation are enabled for external libs
from pydantic import compiled as pydantic_compiled
if not pydantic_compiled:
logger.warning(f"[red]Pydantic is running in non compiled mode.")
from pyvips import API_mode as pyvips_binary
if not pyvips_binary:
logger.warning("[red]Pyvips is running in non binary mode.")
from shapely.speedups import enabled as shapely_speedups
if not shapely_speedups:
logger.warning("[red]Shapely is running without speedups.")
# Caching
if not get_settings().cache_enabled:
logger.warning(f"[orange3]Cache is disabled by configuration.")
else:
try:
await _startup_cache(__version__)
logger.info(f"[green]Cache is ready!")
except ConnectionError:
logger.error(f"[red]Impossible to connect to cache database. "
f"Disabling cache!")
def import_collection(self, collection: Path, prefer_copy: bool = False):
"""Import recursively children of the collection."""
cytomine = None
for listener in self.listeners:
if isinstance(listener, CytomineListener):
cytomine = listener
break
if cytomine:
task = Task.IMPORT_WITH_CYTOMINE
else:
task = Task.IMPORT
imported = list()
format_factory = FormatFactory()
tasks = list()
# Collection children are extracted recursively into collection
# directories, until the directory is an image format (we can thus have
# multi-file formats as directories in a collection).
for child in collection.get_extracted_children(
stop_recursion_cond=format_factory.match
):
self.notify(
ImportEventType.REGISTER_FILE, child, self.upload_path
)
try:
if cytomine:
new_listener = cytomine.new_listener_from_registered_child(child)
args = [
new_listener.auth, str(child), child.name,
new_listener, prefer_copy
]
else:
args = [str(child), child.name, prefer_copy]
tasks.append((task, args))
except Exception as _: # noqa
# Do not propagate error to siblings
# Each importer is independent
pass
def _sequential_imports():
for name, args_ in tasks:
func_from_str(BG_TASK_MAPPING.get(name))(*args_)
if not get_settings().task_queue_enabled:
_sequential_imports()
else:
try:
task_group = group([
signature(CELERY_TASK_MAPPING.get(name), args_)
for name, args_ in tasks
])
# WARNING !
# These tasks are synchronous with respect to the parent task (the archive)
# It is required to update the parent (the archive) status when everything is
# finished. Code should be refactored to use Celery callbacks but it does not
# seem so easy.
# Current implementation may cause deadlock if the worker pool is exhausted,
# while the parent task is waiting for subtasks to finish.
# http://docs.celeryq.org/en/latest/userguide/tasks.html#task-synchronous-subtasks
with allow_join_result():
r = task_group.apply_async()
r.get() # Wait for group to finish
except Exception as e: # noqa
# WARNING !
# Catch too many exceptions such as those related to import logic ?
# TODO: identify Celery exception raised when trying to use it while rabbitmq is
# down
# However, it should not happen in production.
_sequential_imports()
return imported
)
from pims.files.histogram import Histogram
from pims.files.image import Image
from pims.formats import AbstractFormat
from pims.formats.utils.factories import FormatFactory, SpatialReadableFormatFactory
from pims.importer.listeners import (
CytomineListener, ImportEventType, ImportListener,
StdoutListener
)
from pims.processing.histograms.utils import build_histogram_file
from pims.tasks.queue import BG_TASK_MAPPING, CELERY_TASK_MAPPING, Task, func_from_str
from pims.utils.strings import unique_name_generator
log = logging.getLogger("pims.app")
PENDING_PATH = Path(get_settings().pending_path)
FILE_ROOT_PATH = Path(get_settings().root)
class FileErrorProblem(BadRequestException):
pass
class ImageParsingProblem(BadRequestException):
pass
class FormatConversionProblem(BadRequestException):
pass
from pims.api.utils.mimetype import OutputExtension, VISUALISATION_MIMETYPES, get_output_format
from pims.api.utils.models import (AssociatedName, CollectionSize, FormatId,
ImageOutDisplayQueryParams, ZoomOrLevel)
from pims.api.utils.output_parameter import (get_thumb_output_dimensions,
safeguard_output_dimensions)
from pims.api.utils.parameter import filepath_parameter, imagepath_parameter, path2filepath
from pims.api.utils.response import FastJsonResponse, convert_quantity, response_list
from pims.cache import cache_image_response
from pims.config import Settings, get_settings
from pims.files.file import FileRole, FileType, Path
from pims.formats.utils.structures.metadata import MetadataType
from pims.processing.image_response import AssociatedResponse
router = APIRouter()
api_tags = ['Metadata']
cache_associated_ttl = get_settings().cache_ttl_thumb
class SingleFileInfo(BaseModel):
"""
Information about a file
"""
file_type: FileType
filepath: str = Field(
...,
description=
'The file path (filename with path, relative to the server root)',
example='/a/b/c/thefile.png',
)
stem: str = Field(
# * See the License for the specific language governing permissions and
# * limitations under the License.
import logging
from enum import Enum
from importlib import import_module
from typing import Callable
from celery import Celery
from starlette.background import BackgroundTasks
from pims.config import get_settings
logger = logging.getLogger("pims.app")
settings = get_settings()
broker_url = f"{settings.task_queue_user}:{settings.task_queue_password}@{settings.task_queue_url}"
celery_app = Celery("worker",
broker=f"amqp://{broker_url}//",
backend=f"rpc://{broker_url}//")
celery_app.conf.update(task_serializer="pickle",
result_serializer="pickle",
accept_content=["pickle"],
broker_transport_options={
"max_retries": 3,
"interval_start": 0,
"interval_step": 0.2,
"interval_max": 0.5
})
from pims.api.utils.parameter import imagepath_parameter
from pims.api.utils.processing_parameter import (
parse_bitdepth, parse_colormap_ids, parse_filter_ids,
parse_intensity_bounds
)
from pims.cache import cache_image_response
from pims.config import Settings, get_settings
from pims.files.file import Path
from pims.filters import FILTERS
from pims.processing.colormaps import ALL_COLORMAPS
from pims.processing.image_response import ResizedResponse
from pims.utils.iterables import check_array_size_parameters, ensure_list
router = APIRouter()
api_tags = ['Resized']
cache_ttl = get_settings().cache_ttl_resized
@router.get('/image/{filepath:path}/resized{extension:path}', tags=api_tags)
async def show_resized(
request: Request, response: Response,
path: Path = Depends(imagepath_parameter),
extension: OutputExtension = Depends(extension_path_parameter),
output: ImageOutDisplayQueryParams = Depends(),
output2: ImageOutProcessingQueryParams = Depends(),
planes: PlaneSelectionQueryParams = Depends(),
operations: ImageOpsProcessingQueryParams = Depends(),
headers: ImageRequestHeaders = Depends(),
config: Settings = Depends(get_settings)
):
"""
check_zoom_validity,
get_window_output_dimensions,
safeguard_output_dimensions)
from pims.api.utils.parameter import imagepath_parameter
from pims.api.window import _show_window
from pims.cache import cache_image_response
from pims.config import Settings, get_settings
from pims.files.file import Path
from pims.processing.annotations import annotation_crop_affine_matrix, get_annotation_region
from pims.processing.image_response import MaskResponse
from pims.utils.color import WHITE
from pims.utils.iterables import ensure_list
router = APIRouter()
api_tags = ['Annotations']
cache_ttl = get_settings().cache_ttl_window
@router.post('/image/{filepath:path}/annotation/mask{extension:path}',
tags=api_tags)
async def show_mask(
request: Request,
response: Response,
body: AnnotationMaskRequest,
path: Path = Depends(imagepath_parameter),
extension: OutputExtension = Depends(extension_path_parameter),
headers: ImageAnnotationRequestHeaders = Depends(),
config: Settings = Depends(get_settings)):
"""
**`GET with body` - when a GET with URL encoded query parameters is not possible due to URL
size limits, a POST with body content must be used.**
# *
# * http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
from enum import Enum
from typing import Optional
from fastapi import Depends, Header
from pims.config import get_settings
DEFAULT_SAFE_MODE = get_settings().default_image_size_safety_mode
DEFAULT_ANNOTATION_ORIGIN = get_settings().default_annotation_origin
def serialize_header(value, style='simple', explode=False): # noqa
"""
Serialize a header according to https://swagger.io/docs/specification/serialization/.
Parameters
----------
value :
Value to serialize
style : str ('simple')
Serialization style.
explode : bool
Explode the object serialization.
from skimage.exposure import histogram
from pims.api.utils.models import Colorspace, HistogramType
from pims.api.utils.output_parameter import get_thumb_output_dimensions
from pims.config import get_settings
from pims.files.file import Path
from pims.files.histogram import Histogram
from pims.processing.histograms import ZarrHistogramFormat
from pims.processing.histograms.format import (ZHF_ATTR_FORMAT, ZHF_ATTR_TYPE,
ZHF_BOUNDS, ZHF_HIST,
ZHF_PER_CHANNEL, ZHF_PER_IMAGE,
ZHF_PER_PLANE)
from pims.processing.pixels import ImagePixels
from pims.utils.math import max_intensity
MAX_PIXELS_COMPLETE_HISTOGRAM = get_settings().max_pixels_complete_histogram
MAX_LENGTH_COMPLETE_HISTOGRAM = get_settings().max_length_complete_histogram
def argmin_nonzero(arr, axis=-1):
"""
Get the smallest indices of non-zero values along the specified axis.
"""
return np.argmax(arr != 0, axis=axis)
def argmax_nonzero(arr, axis=-1):
"""
Get the biggest indices of non-zero values along the specified axis,
in the reverted array. It gives the index of the last non-zero value in
the array.
from pims.api.utils.parameter import imagepath_parameter
from pims.api.utils.processing_parameter import (
parse_colormap_ids, parse_filter_ids,
parse_intensity_bounds
)
from pims.cache import cache_image_response
from pims.config import Settings, get_settings
from pims.files.file import Path
from pims.filters import FILTERS
from pims.processing.colormaps import ALL_COLORMAPS
from pims.processing.image_response import ThumbnailResponse
from pims.utils.iterables import check_array_size_parameters, ensure_list
router = APIRouter()
api_tags = ['Thumbnails']
cache_ttl = get_settings().cache_ttl_thumb
@router.get('/image/{filepath:path}/thumb{extension:path}', tags=api_tags)
async def show_thumb(
request: Request, response: Response,
path: Path = Depends(imagepath_parameter),
extension: OutputExtension = Depends(extension_path_parameter),
output: ImageOutDisplayQueryParams = Depends(),
planes: PlaneSelectionQueryParams = Depends(),
operations: ImageOpsDisplayQueryParams = Depends(),
use_precomputed: bool = Query(True),
headers: ImageRequestHeaders = Depends(),
config: Settings = Depends(get_settings)
):
"""
from pims.api.utils.parameter import imagepath_parameter
from pims.api.utils.processing_parameter import (parse_colormap_ids,
parse_filter_ids,
parse_intensity_bounds)
from pims.cache import cache_image_response
from pims.config import Settings, get_settings
from pims.files.file import Path
from pims.filters import FILTERS
from pims.processing.colormaps import ALL_COLORMAPS
from pims.processing.image_response import TileResponse, WindowResponse
from pims.utils.iterables import check_array_size_parameters, ensure_list
router = APIRouter()
tile_tags = ['Tiles']
norm_tile_tags = ['Normalized tiles']
cache_ttl = get_settings().cache_ttl_tile
@router.post('/image/{filepath:path}/tile{extension:path}', tags=tile_tags)
async def show_tile_with_body(
request: Request,
response: Response,
body: TileRequest,
path: Path = Depends(imagepath_parameter),
extension: OutputExtension = Depends(extension_path_parameter),
headers: ImageRequestHeaders = Depends(),
config: Settings = Depends(get_settings)):
"""
**`GET with body` - when a GET with URL encoded query parameters is not possible due to URL
size limits, a POST with body content must be used.**
