def check_file_validity(experiment_id, acquisition_id):
"""
.. http:post:: /api/experiments/(string:experiment_id)/acquisitions/(string:acquisition_id)/upload/validity-check
Check if a collection of image or metadata file names have the correct format
for the experiment's microscope type. If this is not the case, the files can't be analyzed
and shouldn't be uploaded.
**Example request**:
.. sourcecode:: http
Content-Type: application/json
{
files: ["right_format_1.png", "right_format_2.png", "wrong_format.png"...]
}
**Example response**:
.. sourcecode:: http
Content-Type: application/json
{
"is_valid": [true, true, false, ...]
}
:reqheader Authorization: JWT token issued by the server
:statuscode 200: no error
:statuscode 500: server error
"""
logger.info('check whether files are valid for upload')
data = json.loads(request.data)
if not 'files' in data:
raise MalformedRequestError()
if not type(data['files']) is list:
raise MalformedRequestError('No image files provided.')
with tm.utils.ExperimentSession(experiment_id) as session:
experiment = session.query(tm.Experiment).one()
microscope_type = experiment.microscope_type
imgfile_regex, metadata_regex = get_microscope_type_regex(microscope_type)
def check_file(fname):
is_imgfile = imgfile_regex.search(fname) is not None
is_metadata_file = metadata_regex.search(fname) is not None
return is_metadata_file or is_imgfile
# TODO: check if metadata files are missing
is_valid = [check_file(f['name']) for f in data['files']]
logger.info('%d of %d files are valid', np.sum(is_valid), len(is_valid))
return jsonify({
'is_valid': is_valid
})
def register_upload(experiment_id, acquisition_id):
"""
.. http:post:: /api/experiments/(string:experiment_id)/acquisitions/(string:acquisition_id)/upload/register
Notify the server that an upload for this acquisition is imminent.
The client has to wait for this response before uploading files.
**Example request**:
.. sourcecode:: http
Content-Type: application/json
{
"files": ["file1.png", "file2.png", ...]
}
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"data": ["file1.png", "file2.png", ...]
}
:reqheader Authorization: JWT token issued by the server
:statuscode 200: no error
:statuscode 500: server error
"""
logger.info('register microscope files for upload')
data = request.get_json()
files = data['files']
if len(files) == 0:
raise MalformedRequestError(
'No files supplied. Cannot register upload.'
)
with tm.utils.ExperimentSession(experiment_id) as session:
experiment = session.query(tm.Experiment).one()
microscope_type = experiment.microscope_type
img_regex, metadata_regex = get_microscope_type_regex(microscope_type)
acquisition = session.query(tm.Acquisition).get(acquisition_id)
img_filenames = [f.name for f in acquisition.microscope_image_files]
img_files = [
tm.MicroscopeImageFile(
name=secure_filename(f), acquisition_id=acquisition.id
)
for f in files
if img_regex.search(f) and
secure_filename(f) not in img_filenames
]
meta_filenames = [f.name for f in acquisition.microscope_metadata_files]
meta_files = [
tm.MicroscopeMetadataFile(
name=secure_filename(f), acquisition_id=acquisition.id
)
for f in files
if metadata_regex.search(f) and
secure_filename(f) not in meta_filenames
]
session.bulk_save_objects(img_files + meta_files)
# Trigger creation of directories
acquisition.location
acquisition.microscope_images_location
acquisition.microscope_metadata_location
with tm.utils.ExperimentSession(experiment_id) as session:
image_filenames = session.query(tm.MicroscopeImageFile.name).\
filter(tm.MicroscopeImageFile.status != FileUploadStatus.COMPLETE).\
all()
metadata_filenames = session.query(tm.MicroscopeMetadataFile.name).\
filter(tm.MicroscopeMetadataFile.status != FileUploadStatus.COMPLETE).\
all()
all_filenames = image_filenames + metadata_filenames
logger.info('registered %d files', len(all_filenames))
return jsonify(data=[f.name for f in all_filenames])
def register(experiment_id, acquisition_id):
"""
.. http:post:: /api/experiments/(string:experiment_id)/acquisitions/(string:acquisition_id)/register
Register a single directory (passed as parameter `path` in the
JSON request) as the directory containing all the files for
the given acquisition.
Calling this method twice with different `path` parameters
will result in an error (so you cannot combine the contents of
two directories by 'registering' them). On the other hand,
the *same* directory can be registered over and over again
without adverse side-effects; the method is idempotent in this
sense.
**Example request**:
.. sourcecode:: http
Content-Type: application/json
{
"path": "/fullpath/to/serversidedata"
}
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"data": "/fullpath/to/serversidedata"
}
:reqheader Authorization: JWT token issued by the server
:statuscode 200: no error
:statuscode 500: server error
"""
data = request.get_json()
path = data['path']
logger.info('Registering microscope files from path `%s` ...', path)
try:
filenames = [
f for f in os.listdir(path)
if (not f.startswith('.')
and not os.path.isdir(os.path.join(path, f)))
]
except OSError as err:
logger.error("Cannot list directory `%s`: %s", path, err)
raise
with tm.utils.ExperimentSession(experiment_id) as session:
experiment = session.query(tm.Experiment).one()
microscope_type = experiment.microscope_type
img_regex, metadata_regex = get_microscope_type_regex(microscope_type)
acquisition = session.query(tm.Acquisition).get(acquisition_id)
plate = session.query(tm.Plate).get(acquisition.plate_id)
# check for image files already registered
img_filenames = [f.name for f in acquisition.microscope_image_files]
img_files = [
tm.MicroscopeImageFile(
name=f, acquisition_id=acquisition.id
)
for f in filenames
if img_regex.search(f) and
f not in img_filenames
]
# check for metadata already registered
meta_filenames = [f.name for f in acquisition.microscope_metadata_files]
meta_files = [
tm.MicroscopeMetadataFile(
name=secure_filename(f), acquisition_id=acquisition.id
)
for f in filenames
if metadata_regex.search(f) and
f not in meta_filenames
]
session.bulk_save_objects(img_files + meta_files)
# trigger creation of directories
acquisition.location
acquisition.microscope_metadata_location
# link root directory
path_to_link = os.path.join(
LibraryConfig().storage_home,
('experiment_{}'
'/plates/plate_{}'
'/acquisitions/acquisition_{}'
'/microscope_images'
.format(experiment.id, plate.id, acquisition.id)))
if not os.path.exists(path_to_link):
try:
os.symlink(path, path_to_link)
except Exception as err:
logger.debug(
"Error linking source directory `%s` to TM directory `%s`: %s",
path, path_to_link, err)
raise
else:
# path exists, check if it is correct
p1 = os.path.realpath(path)
p2 = os.path.realpath(path_to_link)
if p1 != p2:
raise ValueError(
"Acquisition {a} of plate {p} of experiment {e}"
" is already linked to directory `{d}`"
.format(
a=acquisition.id,
p=plate.id,
e=epxeriment.id,
d=p1))
microscope_files = session.query(tm.MicroscopeImageFile).filter_by(acquisition_id=acquisition.id).all()
for microscope_file in microscope_files:
microscope_file.location
microscope_file.status = 'COMPLETE'
return jsonify(message='ok')
def run_job(self, batch, assume_clean_state=False):
'''Configures OMEXML metadata extracted from microscope image files and
complements it with metadata retrieved from additional microscope
metadata files and/or user input.
The actual processing is delegated to a format-specific implementation of
:class:`MetadataHandler <tmlib.workflow.metaconfig.base.MetadataHandler>`.
Parameters
----------
batch: dict
job description
assume_clean_state: bool, optional
assume that output of previous runs has already been cleaned up
See also
--------
:mod:`tmlib.workflow.metaconfig.cellvoyager`
'''
regexp = batch.get('regex', '')
if not regexp:
regexp = get_microscope_type_regex(
batch['microscope_type'], as_string=True
)[0]
with tm.utils.ExperimentSession(self.experiment_id) as session:
experiment = session.query(tm.Experiment).one()
plate_dimensions = experiment.plates[0].dimensions
acquisition = session.query(tm.Acquisition).\
get(batch['acquisition_id'])
metadata_files = session.query(tm.MicroscopeMetadataFile.location).\
filter_by(acquisition_id=batch['acquisition_id']).\
all()
metadata_filenames = [f.location for f in metadata_files]
image_files = session.query(
tm.MicroscopeImageFile.name, tm.MicroscopeImageFile.omexml
).\
filter_by(acquisition_id=batch['acquisition_id']).\
all()
omexml_images = {
f.name: bioformats.OMEXML(f.omexml) for f in image_files
}
MetadataReader = metadata_reader_factory(batch['microscope_type'])
if MetadataReader is not None:
with MetadataReader() as mdreader:
omexml_metadata = mdreader.read(
metadata_filenames, omexml_images.keys()
)
else:
omexml_metadata = None
MetadataHandler = metadata_handler_factory(batch['microscope_type'])
mdhandler = MetadataHandler(omexml_images, omexml_metadata)
mdhandler.configure_from_omexml()
missing = mdhandler.determine_missing_metadata()
if missing:
logger.warning(
'required metadata information is missing: "%s"',
'", "'.join(missing)
)
logger.info(
'try to retrieve missing metadata from filenames '
'using regular expression'
)
if regexp is None:
logger.warn('no regular expression provided')
mdhandler.configure_from_filenames(
plate_dimensions=plate_dimensions, regex=regexp
)
missing = mdhandler.determine_missing_metadata()
if missing:
raise MetadataError(
'The following metadata information is missing:\n"%s"\n'
% '", "'.join(missing)
)
# Once we have collected basic metadata such as information about
# channels and focal planes, we try to determine the relative position
# of images within the acquisition grid
try:
logger.info(
'try to determine grid coordinates from microscope '
'stage positions'
)
mdhandler.determine_grid_coordinates_from_stage_positions()
except MetadataError as error:
logger.warning(
'microscope stage positions are not available: "%s"'
% str(error)
)
logger.info(
'try to determine grid coordinates from provided stitch layout'
)
# In general, the values of these arguments can be ``None``, because
# they are not required and may not be used.
# However, in case the grid coordinates should be determined based
# on user interput, these arguments are required.
if not isinstance(batch['n_vertical'], int):
raise TypeError(
'Value of argument "n_vertical" must be an integer.'
)
if not isinstance(batch['n_horizontal'], int):
raise TypeError(
'Value of argument "n_horizontal" must be an integer.'
)
mdhandler.determine_grid_coordinates_from_layout(
stitch_layout=batch['stitch_layout'],
stitch_dimensions=(batch['n_vertical'], batch['n_horizontal'])
)
if batch['perform_mip']:
mdhandler.group_metadata_per_zstack()
# Create consistent zero-based ids
mdhandler.update_indices()
mdhandler.assign_acquisition_site_indices()
md = mdhandler.remove_redundant_columns()
fmaps = mdhandler.create_image_file_mappings()
logger.info('create database entries')
with tm.utils.ExperimentSession(self.experiment_id) as session:
channels = dict()
bit_depth = md['bit_depth'][0]
for ch_name in np.unique(md['channel_name']):
logger.info('create channel "%s"', ch_name)
ch = session.get_or_create(
tm.Channel, experiment_id=self.experiment_id,
name=ch_name, wavelength=ch_name, bit_depth=bit_depth,
)
channels[ch_name] = ch.id
for w in np.unique(md.well_name):
with tm.utils.ExperimentSession(self.experiment_id) as session:
acquisition = session.query(tm.Acquisition).\
get(batch['acquisition_id'])
logger.info('create well "%s"', w)
w_index = (md.well_name == w)
well = session.get_or_create(
tm.Well, plate_id=acquisition.plate.id, name=w
)
channel_image_files = []
for s in np.unique(md.loc[w_index, 'site']):
logger.debug('create site #%d', s)
s_index = (md.site == s)
y = md.loc[s_index, 'well_position_y'].values[0]
x = md.loc[s_index, 'well_position_x'].values[0]
height = md.loc[s_index, 'height'].values[0]
width = md.loc[s_index, 'width'].values[0]
site = session.get_or_create(
tm.Site, y=y, x=x, height=height, width=width,
well_id=well.id
)
for index, i in md.ix[s_index].iterrows():
channel_image_files.append(
tm.ChannelImageFile(
tpoint=i.tpoint, zplane=i.zplane,
channel_id=channels[i.channel_name],
site_id=site.id, acquisition_id=acquisition.id,
file_map=fmaps[index],
)
)
session.bulk_save_objects(channel_image_files)
def run_job(self, batch, assume_clean_state=False):
'''Configures OMEXML metadata extracted from microscope image files and
complements it with metadata retrieved from additional microscope
metadata files and/or user input.
The actual processing is delegated to a format-specific implementation of
:class:`MetadataHandler <tmlib.workflow.metaconfig.base.MetadataHandler>`.
Parameters
----------
batch: dict
job description
assume_clean_state: bool, optional
assume that output of previous runs has already been cleaned up
See also
--------
:mod:`tmlib.workflow.metaconfig.cellvoyager`
'''
regexp = batch.get('regex', '')
if not regexp:
regexp = get_microscope_type_regex(
batch['microscope_type'], as_string=True
)[0]
with tm.utils.ExperimentSession(self.experiment_id) as session:
experiment = session.query(tm.Experiment).one()
plate_dimensions = experiment.plates[0].dimensions
acquisition = session.query(tm.Acquisition).\
get(batch['acquisition_id'])
metadata_files = session.query(tm.MicroscopeMetadataFile.location).\
filter_by(acquisition_id=batch['acquisition_id']).\
all()
metadata_filenames = [f.location for f in metadata_files]
image_files = session.query(
tm.MicroscopeImageFile.name, tm.MicroscopeImageFile.omexml
).\
filter_by(acquisition_id=batch['acquisition_id']).\
all()
omexml_images = {
f.name: bioformats.OMEXML(f.omexml) for f in image_files
}
MetadataReader = metadata_reader_factory(batch['microscope_type'])
if MetadataReader is not None:
with MetadataReader() as mdreader:
omexml_metadata = mdreader.read(
metadata_filenames, omexml_images.keys()
)
else:
omexml_metadata = None
MetadataHandler = metadata_handler_factory(batch['microscope_type'])
mdhandler = MetadataHandler(omexml_images, omexml_metadata)
mdhandler.configure_from_omexml()
missing = mdhandler.determine_missing_metadata()
if missing:
logger.warning(
'required metadata information is missing: "%s"',
'", "'.join(missing)
)
logger.info(
'try to retrieve missing metadata from filenames '
'using regular expression'
)
if regexp is None:
logger.warn('no regular expression provided')
mdhandler.configure_from_filenames(
plate_dimensions=plate_dimensions, regex=regexp
)
missing = mdhandler.determine_missing_metadata()
if missing:
raise MetadataError(
'The following metadata information is missing:\n"%s"\n'
% '", "'.join(missing)
)
# Once we have collected basic metadata such as information about
# channels and focal planes, we try to determine the relative position
# of images within the acquisition grid
try:
logger.info(
'try to determine grid coordinates from microscope '
'stage positions'
)
mdhandler.determine_grid_coordinates_from_stage_positions()
except MetadataError as error:
logger.warning(
'microscope stage positions are not available: "%s"'
% str(error)
)
logger.info(
'try to determine grid coordinates from provided stitch layout'
)
# In general, the values of these arguments can be ``None``, because
# they are not required and may not be used.
# However, in case the grid coordinates should be determined based
# on user interput, these arguments are required.
if not isinstance(batch['n_vertical'], int):
raise TypeError(
'Value of argument "n_vertical" must be an integer.'
)
if not isinstance(batch['n_horizontal'], int):
raise TypeError(
'Value of argument "n_horizontal" must be an integer.'
)
mdhandler.determine_grid_coordinates_from_layout(
stitch_layout=batch['stitch_layout'],
stitch_dimensions=(batch['n_vertical'], batch['n_horizontal'])
)
if batch['perform_mip']:
mdhandler.group_metadata_per_zstack()
# Create consistent zero-based ids
mdhandler.update_indices()
mdhandler.assign_acquisition_site_indices()
md = mdhandler.remove_redundant_columns()
fmaps = mdhandler.create_image_file_mappings()
logger.info('create database entries')
with tm.utils.ExperimentSession(self.experiment_id) as session:
channels = dict()
bit_depth = md['bit_depth'][0]
for ch_name in np.unique(md['channel_name']):
logger.info('create channel "%s"', ch_name)
ch = session.get_or_create(
tm.Channel, experiment_id=self.experiment_id,
name=ch_name, wavelength=ch_name, bit_depth=bit_depth,
)
channels[ch_name] = ch.id
for w in np.unique(md.well_name):
with tm.utils.ExperimentSession(self.experiment_id) as session:
acquisition = session.query(tm.Acquisition).\
get(batch['acquisition_id'])
logger.info('create well "%s"', w)
w_index = md.well_name == w
well = session.get_or_create(
tm.Well, plate_id=acquisition.plate.id, name=w
)
channel_image_files = list()
for s in np.unique(md.loc[w_index, 'site']):
logger.debug('create site #%d', s)
s_index = md.site == s
y = md.loc[s_index, 'well_position_y'].values[0]
x = md.loc[s_index, 'well_position_x'].values[0]
height = md.loc[s_index, 'height'].values[0]
width = md.loc[s_index, 'width'].values[0]
site = session.get_or_create(
tm.Site, y=y, x=x, height=height, width=width,
well_id=well.id
)
for index, i in md.ix[s_index].iterrows():
channel_image_files.append(
tm.ChannelImageFile(
tpoint=i.tpoint, zplane=i.zplane,
channel_id=channels[i.channel_name],
site_id=site.id, acquisition_id=acquisition.id,
file_map=fmaps[index],
)
)
session.bulk_save_objects(channel_image_files)
def register(experiment_id, acquisition_id):
"""
.. http:post:: /api/experiments/(string:experiment_id)/acquisitions/(string:acquisition_id)/register
Pass the NFS path to the data to the server.
The client has to wait for this response before uploading files.
**Example request**:
.. sourcecode:: http
Content-Type: application/json
{
"path": "/fullpath/to/serversidedata"
}
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"data": "/fullpath/to/serversidedata"
}
:reqheader Authorization: JWT token issued by the server
:statuscode 200: no error
:statuscode 500: server error
"""
logger.info('register microscope files')
data = request.get_json()
path = data['path']
logger.info('path given by the client: %s', path)
# Parse storage_home variable from tissuemaps.cfg file
parser = SafeConfigParser()
parser.read('/home/tissuemaps/.tmaps/tissuemaps.cfg')
storage_home_path = parser.get('tmlib', 'storage_home')
path_to_link = os.path.join(
storage_home_path,
'experiment_{}/plates/plate_{}/acquisitions/acquisition_{}/microscope_images'
)
filenames = [
f for f in os.listdir(path)
if (not f.startswith('.') and not os.path.isdir(os.path.join(path, f)))
]
with tm.utils.ExperimentSession(experiment_id) as session:
experiment = session.query(tm.Experiment).one()
microscope_type = experiment.microscope_type
img_regex, metadata_regex = get_microscope_type_regex(microscope_type)
acquisition = session.query(tm.Acquisition).get(acquisition_id)
# single plate only
plate = session.query(tm.Plate).one()
logger.info(
'path to link: %s',
path_to_link.format(experiment.id, plate.id, acquisition.id))
logger.info('experiment %s', experiment.id)
logger.info('acquisition %s', acquisition)
logger.info('plate %s', plate)
# check for image files already registered
img_filenames = [f.name for f in acquisition.microscope_image_files]
logger.debug('img_filenames %s', img_filenames)
img_files = [
tm.MicroscopeImageFile(name=f, acquisition_id=acquisition.id)
for f in filenames
if img_regex.search(f) and f not in img_filenames
]
# check for metadata already registered
meta_filenames = [
f.name for f in acquisition.microscope_metadata_files
]
meta_files = [
tm.MicroscopeMetadataFile(name=secure_filename(f),
acquisition_id=acquisition.id)
for f in filenames
if metadata_regex.search(f) and f not in meta_filenames
]
session.bulk_save_objects(img_files + meta_files)
# Trigger creation of directories
acquisition.location
#acquisition.microscope_images_location
acquisition.microscope_metadata_location
os.symlink(
path, path_to_link.format(experiment.id, plate.id, acquisition.id))
microscope_files = session.query(tm.MicroscopeImageFile).filter_by(
acquisition_id=acquisition.id).all()
for microscope_file in microscope_files:
microscope_file.location
microscope_file.status = 'COMPLETE'
return jsonify(message='ok')
