def extract(self, path: Path, clean: bool = True):
"""
Extract the archive content.
Parameters
----------
path : Path
A non-existing directory path where the archive content is extracted
clean : bool (default: True)
Whether the archive content has to be cleaned from Mac OS hidden
files (.DS_STORE, __MACOSX).
"""
if path.exists() and not path.is_dir():
raise ArchiveError(
f"{self} cannot be extracted in {path} because "
f"it already exists or it is not a directory"
)
try:
shutil.unpack_archive(self.absolute(), path, self._format.name)
except shutil.ReadError as e:
raise ArchiveError(str(e))
if clean:
bad_filenames = ['.DS_STORE', '__MACOSX']
for bad_filename in bad_filenames:
for bad_path in path.rglob(bad_filename):
bad_path.unlink(missing_ok=True)
def test_roles(app, settings, fake_files):
root = Path(settings.root)
for ff in fake_files.values():
name = ff['filepath']
role = ff['role']
path = root / Path(name)
if role == "upload":
assert path.has_upload_role()
assert not path.has_original_role()
assert not path.has_spatial_role()
assert not path.has_spectral_role()
elif role == "original":
assert not path.has_upload_role()
assert path.has_original_role()
assert not path.has_spatial_role()
assert not path.has_spectral_role()
elif role == "visualisation":
assert not path.has_upload_role()
assert not path.has_original_role()
assert path.has_spatial_role()
assert not path.has_spectral_role()
elif role == "spectral":
assert not path.has_upload_role()
assert not path.has_original_role()
assert not path.has_spatial_role()
assert path.has_spectral_role()
else:
assert not path.has_upload_role()
assert not path.has_original_role()
assert not path.has_spatial_role()
assert not path.has_spectral_role()
def mkdir(self, directory: Path):
"""Make a directory (with notifications)"""
try:
directory.mkdir() # TODO: mode
except (FileNotFoundError, FileExistsError, OSError) as e:
self.notify(ImportEventType.FILE_ERROR, directory, exception=e)
raise FileErrorProblem(directory)
def export_upload(
background: BackgroundTasks,
path: Path = Depends(imagepath_parameter),
):
"""
Export the upload representation of an image.
"""
image = path.get_original()
check_representation_existence(image)
upload_file = image.get_upload().resolve()
media_type = image.media_type
if upload_file.is_dir():
# if archive has been deleted
tmp_export = Path(f"/tmp/{unique_name_generator()}")
make_zip_archive(tmp_export, upload_file)
def cleanup(tmp):
tmp.unlink(missing_ok=True)
background.add_task(cleanup, tmp_export)
upload_file = tmp_export
media_type = "application/zip"
return FileResponse(upload_file, media_type=media_type, filename=path.name)
def test_collection(app, settings, fake_files):
root = Path(settings.root)
for ff in fake_files.values():
name = ff['filepath']
is_collection = ff['collection']
path = root / Path(name)
assert path.is_collection() == is_collection
assert path.is_single() == (not is_collection)
def mksymlink(self, path: Path, target: Path):
"""Make a symlink from path to target (with notifications)"""
try:
path.symlink_to(
target,
target_is_directory=target.is_dir()
)
except (FileNotFoundError, FileExistsError, OSError) as e:
self.notify(ImportEventType.FILE_ERROR, path, exception=e)
raise FileErrorProblem(path)
def show_representation(representation: FileRole,
path: Path = Depends(imagepath_parameter)):
"""
Get image representation info
"""
rpr = path.get_representation(representation)
return RepresentationInfo.from_path(rpr)
def show_plane_histogram(
z_slices: conint(ge=0),
timepoints: conint(ge=0),
path: Path = Depends(imagepath_parameter),
channels: Optional[List[conint(ge=0)]] = Query(
None, description="Only return histograms for these channels"
),
):
"""
Get histogram per plane.
"""
in_image = path.get_spatial()
check_representation_existence(in_image)
channels = ensure_list(channels)
z_slices = ensure_list(z_slices)
timepoints = ensure_list(timepoints)
channels = get_channel_indexes(in_image, channels)
z_slices = get_zslice_indexes(in_image, z_slices)
timepoints = get_timepoint_indexes(in_image, timepoints)
hist_info = []
htype = in_image.histogram_type()
for c, z, t in itertools.product(channels, z_slices, timepoints):
mini, maxi = in_image.plane_bounds(c, z, t)
hist_info.append(
PlaneHistogramInfo(
channel=c, z_slice=z, timepoint=t, type=htype,
color=in_image.channels[c].hex_color,
minimum=mini, maximum=maxi
)
)
return response_list(hist_info)
def show_channels_histogram_bounds(
path: Path = Depends(imagepath_parameter),
channels: Optional[List[conint(ge=0)]] = Query(
None, description="Only return histograms for these channels"
),
):
"""
Get histogram bounds per channel where all planes (Z,T) are merged.
"""
in_image = path.get_spatial()
check_representation_existence(in_image)
channels = ensure_list(channels)
channels = get_channel_indexes(in_image, channels)
hist_info = []
htype = in_image.histogram_type()
hist_filter = operator.itemgetter(*channels)
channels_bounds = hist_filter(in_image.channels_bounds())
if len(channels) == 1:
channels_bounds = [channels_bounds]
for channel, bounds in zip(channels, channels_bounds):
mini, maxi = bounds
hist_info.append(
ChannelHistogramInfo(
channel=channel, type=htype,
color=in_image.channels[channel].hex_color,
minimum=mini, maximum=maxi
)
)
return response_list(hist_info)
def deploy_histogram(self, image: Image) -> Histogram:
"""
Deploy an histogram representation of the image so that it can be used for
efficient histogram requests.
"""
self.histogram_path = self.processed_dir / Path(HISTOGRAM_STEM)
self.notify(
ImportEventType.START_HISTOGRAM_DEPLOY,
self.histogram_path, image
)
try:
self.histogram = build_histogram_file(
image, self.histogram_path, HistogramType.FAST
)
except (FileNotFoundError, FileExistsError) as e:
self.notify(
ImportEventType.ERROR_HISTOGRAM, self.histogram_path, image,
exception=e
)
raise FileErrorProblem(self.histogram_path)
assert self.histogram.has_histogram_role()
self.notify(
ImportEventType.END_HISTOGRAM_DEPLOY, self.histogram_path, image
)
return self.histogram
def end_spatial_deploy(self, spatial_path: Path, *args, **kwargs):
if not spatial_path.is_symlink():
# The spatial path is not a symbolic link
# -> a conversion has been performed
uf = self.get_uf(spatial_path)
uf.status = UploadedFile.IMPORTED
uf.update()
def end_unpacking(self,
path: Path,
unpacked_path: Path,
*args,
format: AbstractFormat = None,
is_collection: bool = False,
**kwargs):
parent = self.get_uf(path)
parent.status = UploadedFile.UNPACKED
parent.update()
if not is_collection:
uf = UploadedFile()
uf.status = UploadedFile.UPLOADED # Better status ?
uf.contentType = format.get_identifier() # TODO
uf.size = unpacked_path.size
uf.filename = str(unpacked_path.relative_to(FILE_ROOT_PATH))
uf.originalFilename = str(format.main_path.name)
uf.ext = ""
uf.storage = parent.storage
uf.user = parent.user
uf.parent = parent.id
uf.imageServer = parent.imageServer
uf.save()
self.path_uf_mapping[str(unpacked_path)] = uf
def _show_associated_image(
request: Request,
response: Response, # required for @cache # noqa
path: Path,
height,
width,
length,
associated_key,
headers,
config: Settings):
in_image = path.get_spatial()
check_representation_existence(in_image)
associated = getattr(in_image, f'associated_{associated_key.value}')
if not associated or not associated.exists:
raise NoAppropriateRepresentationProblem(path, associated_key)
out_format, mimetype = get_output_format(OutputExtension.NONE,
headers.accept,
VISUALISATION_MIMETYPES)
req_size = get_thumb_output_dimensions(associated, height, width, length)
out_size = safeguard_output_dimensions(headers.safe_mode,
config.output_size_limit, *req_size)
out_width, out_height = out_size
return AssociatedResponse(in_image, associated_key, out_width, out_height,
out_format).http_response(
mimetype,
extra_headers=add_image_size_limit_header(
dict(), *req_size, *out_size))
def show_associated(path: Path = Depends(imagepath_parameter)):
"""
Get associated file info
"""
original = path.get_original()
check_representation_existence(original)
return response_list(AssociatedInfo.from_image(original))
def show_channels(path: Path = Depends(imagepath_parameter)):
"""
Get image channel info
"""
original = path.get_original()
check_representation_existence(original)
return response_list(ChannelsInfo.from_image(original))
def show_instrument(path: Path = Depends(imagepath_parameter)):
"""
Get image instrument info
"""
original = path.get_original()
check_representation_existence(original)
return InstrumentInfo.from_image(original)
def show_normalized_pyramid(path: Path = Depends(imagepath_parameter)):
"""
Get image normalized pyramid
"""
original = path.get_original()
check_representation_existence(original)
return PyramidInfo.from_pyramid(original.normalized_pyramid)
def show_image(path: Path = Depends(imagepath_parameter)):
"""
Get standard image info
"""
original = path.get_original()
check_representation_existence(original)
return ImageInfo.from_image(original)
def list_representations(path: Path = Depends(imagepath_parameter)):
"""
Get all image representation info
"""
return response_list([
RepresentationInfo.from_path(rpr)
for rpr in path.get_representations()
])
def show_metadata(path: Path = Depends(imagepath_parameter)):
"""
Get image metadata
"""
original = path.get_original()
check_representation_existence(original)
store = original.raw_metadata
return response_list([Metadata.from_metadata(md) for md in store.values()])
def test_extensions(app, settings):
files = ("upload0/myfile.svs", "upload2/processed/myfile.ome.tiff",
"upload5/processed/visualisation.mrxs.format")
extensions = (".svs", ".ome.tiff", ".mrxs.format")
for f, ext in zip(files, extensions):
path = Path(settings.root, f)
assert path.extension == ext
assert path.true_stem == f.split("/")[-1].replace(ext, "")
def show_metadata_annotations(path: Path = Depends(imagepath_parameter)):
"""
Get image annotation metadata
"""
original = path.get_original()
check_representation_existence(original)
return response_list([
MetadataAnnotation.from_metadata_annotation(a)
for a in original.annotations
])
async def _show_drawing(
request: Request,
response: Response, # required for @cache # noqa
path: Path,
annotations,
context_factor,
try_square,
point_cross,
point_envelope_length,
height,
width,
length,
zoom,
level,
channels,
z_slices,
timepoints,
min_intensities,
max_intensities,
filters,
gammas,
threshold,
log,
extension,
headers,
config,
colormaps=None,
c_reduction=ChannelReduction.ADD,
z_reduction=None,
t_reduction=None,
):
in_image = path.get_spatial()
check_representation_existence(in_image)
annots = parse_annotations(ensure_list(annotations),
ignore_fields=['fill_color'],
default={'stroke_width': 1},
point_envelope_length=point_envelope_length,
origin=headers.annot_origin,
im_height=in_image.height)
region = get_annotation_region(in_image, annots, context_factor,
try_square)
annot_style = dict(mode=AnnotationStyleMode.DRAWING,
point_cross=point_cross,
point_envelope_length=point_envelope_length)
return await _show_window(request, response, path, region, height, width,
length, zoom, level, channels, z_slices,
timepoints, min_intensities, max_intensities,
filters, gammas, threshold, 8, Colorspace.AUTO,
annots, annot_style, extension, headers, config,
colormaps, c_reduction, z_reduction, t_reduction)
def export_file(path: Path = Depends(filepath_parameter)):
"""
Export a file. All files in the server base path can be exported.
"""
if path.is_dir():
# TODO: zip and return the folder archive ?
raise NotAFileProblem(path)
return FileResponse(path,
media_type="application/octet-stream",
filename=path.name)
def show_image_histogram_bounds(
path: Path = Depends(imagepath_parameter)
):
"""
Get histogram info for full image where all planes (C,Z,T) are merged.
"""
in_image = path.get_spatial()
check_representation_existence(in_image)
htype = in_image.histogram_type()
mini, maxi = in_image.image_bounds()
return HistogramInfo(type=htype, minimum=mini, maximum=maxi)
async def _show_crop(
request: Request,
response: Response,
path: Path,
annotations,
context_factor,
background_transparency,
height,
width,
length,
zoom,
level,
channels,
z_slices,
timepoints,
min_intensities,
max_intensities,
filters,
gammas,
threshold,
bits,
colorspace,
extension,
headers,
config,
colormaps=None,
c_reduction=ChannelReduction.ADD,
z_reduction=None,
t_reduction=None,
):
in_image = path.get_spatial()
check_representation_existence(in_image)
annots = parse_annotations(ensure_list(annotations),
ignore_fields=['stroke_width', 'stroke_color'],
default={'fill_color': WHITE},
origin=headers.annot_origin,
im_height=in_image.height)
region = get_annotation_region(in_image, annots, context_factor)
annot_style = dict(mode=AnnotationStyleMode.CROP,
background_transparency=background_transparency)
return await _show_window(request, response, path, region, height, width,
length, zoom, level, channels, z_slices,
timepoints, min_intensities, max_intensities,
filters, gammas, threshold, bits, colorspace,
annots, annot_style, extension, headers, config,
colormaps, c_reduction, z_reduction, t_reduction)
def compute_histogram(
response: Response,
background: BackgroundTasks,
path: Path = Depends(imagepath_parameter),
# companion_file_id: Optional[int] = Body(None, description="Cytomine ID for the histogram")
sync: bool = True,
overwrite: bool = True
):
"""
Ask for histogram computation
"""
in_image = path.get_spatial()
check_representation_existence(in_image)
hist_type = HistogramType.FAST # TODO: allow to build complete histograms
hist_path = in_image.processed_root() / Path(HISTOGRAM_STEM)
if sync:
build_histogram_file(in_image, hist_path, hist_type, overwrite)
response.status_code = status.HTTP_201_CREATED
else:
background.add_task(build_histogram_file, in_image, hist_path, hist_type, overwrite)
response.status_code = status.HTTP_202_ACCEPTED
def run_import(
filepath: str, name: str, extra_listeners: Optional[List[ImportListener]] = None,
prefer_copy: bool = False
):
pending_file = Path(filepath)
if extra_listeners is not None:
if not type(extra_listeners) is list:
extra_listeners = list(extra_listeners)
else:
extra_listeners = []
listeners = [StdoutListener(name)] + extra_listeners
fi = FileImporter(pending_file, name, listeners)
fi.run(prefer_copy)
def register_file(self, path: Path, parent_path: Path, *args, **kwargs):
parent = self.get_uf(parent_path)
uf = UploadedFile()
uf.status = UploadedFile.UPLOADED
uf.contentType = ""
uf.size = path.size
uf.filename = str(path.relative_to(FILE_ROOT_PATH))
uf.originalFilename = str(path.name)
uf.ext = ""
uf.storage = parent.storage
uf.user = parent.user
uf.parent = parent.id
uf.imageServer = parent.imageServer
uf.save()
self.path_uf_mapping[str(path)] = uf
def show_info(path: Path = Depends(imagepath_parameter)):
"""
Get all image info
"""
original = path.get_original()
check_representation_existence(original)
data = dict()
data["image"] = ImageInfo.from_image(original)
data["instrument"] = InstrumentInfo.from_image(original)
data["associated"] = AssociatedInfo.from_image(original)
data["channels"] = ChannelsInfo.from_image(original)
data["representations"] = [
RepresentationInfo.from_path(rpr)
for rpr in original.get_representations()
]
return data
