def check_file_validity(experiment_id, acquisition_id): """ .. http:post:: /api/experiments/(string:experiment_id)/acquisitions/(string:acquisition_id)/upload/validity-check Check if a collection of image or metadata file names have the correct format for the experiment's microscope type. If this is not the case, the files can't be analyzed and shouldn't be uploaded. **Example request**: .. sourcecode:: http Content-Type: application/json { files: ["right_format_1.png", "right_format_2.png", "wrong_format.png"...] } **Example response**: .. sourcecode:: http Content-Type: application/json { "is_valid": [true, true, false, ...] } :reqheader Authorization: JWT token issued by the server :statuscode 200: no error :statuscode 500: server error """ logger.info('check whether files are valid for upload') data = json.loads(request.data) if not 'files' in data: raise MalformedRequestError() if not type(data['files']) is list: raise MalformedRequestError('No image files provided.') with tm.utils.ExperimentSession(experiment_id) as session: experiment = session.query(tm.Experiment).one() microscope_type = experiment.microscope_type imgfile_regex, metadata_regex = get_microscope_type_regex(microscope_type) def check_file(fname): is_imgfile = imgfile_regex.search(fname) is not None is_metadata_file = metadata_regex.search(fname) is not None return is_metadata_file or is_imgfile # TODO: check if metadata files are missing is_valid = [check_file(f['name']) for f in data['files']] logger.info('%d of %d files are valid', np.sum(is_valid), len(is_valid)) return jsonify({ 'is_valid': is_valid })
def register_upload(experiment_id, acquisition_id): """ .. http:post:: /api/experiments/(string:experiment_id)/acquisitions/(string:acquisition_id)/upload/register Notify the server that an upload for this acquisition is imminent. The client has to wait for this response before uploading files. **Example request**: .. sourcecode:: http Content-Type: application/json { "files": ["file1.png", "file2.png", ...] } **Example response**: .. sourcecode:: http HTTP/1.1 200 OK Content-Type: application/json { "data": ["file1.png", "file2.png", ...] } :reqheader Authorization: JWT token issued by the server :statuscode 200: no error :statuscode 500: server error """ logger.info('register microscope files for upload') data = request.get_json() files = data['files'] if len(files) == 0: raise MalformedRequestError( 'No files supplied. Cannot register upload.' ) with tm.utils.ExperimentSession(experiment_id) as session: experiment = session.query(tm.Experiment).one() microscope_type = experiment.microscope_type img_regex, metadata_regex = get_microscope_type_regex(microscope_type) acquisition = session.query(tm.Acquisition).get(acquisition_id) img_filenames = [f.name for f in acquisition.microscope_image_files] img_files = [ tm.MicroscopeImageFile( name=secure_filename(f), acquisition_id=acquisition.id ) for f in files if img_regex.search(f) and secure_filename(f) not in img_filenames ] meta_filenames = [f.name for f in acquisition.microscope_metadata_files] meta_files = [ tm.MicroscopeMetadataFile( name=secure_filename(f), acquisition_id=acquisition.id ) for f in files if metadata_regex.search(f) and secure_filename(f) not in meta_filenames ] session.bulk_save_objects(img_files + meta_files) # Trigger creation of directories acquisition.location acquisition.microscope_images_location acquisition.microscope_metadata_location with tm.utils.ExperimentSession(experiment_id) as session: image_filenames = session.query(tm.MicroscopeImageFile.name).\ filter(tm.MicroscopeImageFile.status != FileUploadStatus.COMPLETE).\ all() metadata_filenames = session.query(tm.MicroscopeMetadataFile.name).\ filter(tm.MicroscopeMetadataFile.status != FileUploadStatus.COMPLETE).\ all() all_filenames = image_filenames + metadata_filenames logger.info('registered %d files', len(all_filenames)) return jsonify(data=[f.name for f in all_filenames])
def register(experiment_id, acquisition_id): """ .. http:post:: /api/experiments/(string:experiment_id)/acquisitions/(string:acquisition_id)/register Register a single directory (passed as parameter `path` in the JSON request) as the directory containing all the files for the given acquisition. Calling this method twice with different `path` parameters will result in an error (so you cannot combine the contents of two directories by 'registering' them). On the other hand, the *same* directory can be registered over and over again without adverse side-effects; the method is idempotent in this sense. **Example request**: .. sourcecode:: http Content-Type: application/json { "path": "/fullpath/to/serversidedata" } **Example response**: .. sourcecode:: http HTTP/1.1 200 OK Content-Type: application/json { "data": "/fullpath/to/serversidedata" } :reqheader Authorization: JWT token issued by the server :statuscode 200: no error :statuscode 500: server error """ data = request.get_json() path = data['path'] logger.info('Registering microscope files from path `%s` ...', path) try: filenames = [ f for f in os.listdir(path) if (not f.startswith('.') and not os.path.isdir(os.path.join(path, f))) ] except OSError as err: logger.error("Cannot list directory `%s`: %s", path, err) raise with tm.utils.ExperimentSession(experiment_id) as session: experiment = session.query(tm.Experiment).one() microscope_type = experiment.microscope_type img_regex, metadata_regex = get_microscope_type_regex(microscope_type) acquisition = session.query(tm.Acquisition).get(acquisition_id) plate = session.query(tm.Plate).get(acquisition.plate_id) # check for image files already registered img_filenames = [f.name for f in acquisition.microscope_image_files] img_files = [ tm.MicroscopeImageFile( name=f, acquisition_id=acquisition.id ) for f in filenames if img_regex.search(f) and f not in img_filenames ] # check for metadata already registered meta_filenames = [f.name for f in acquisition.microscope_metadata_files] meta_files = [ tm.MicroscopeMetadataFile( name=secure_filename(f), acquisition_id=acquisition.id ) for f in filenames if metadata_regex.search(f) and f not in meta_filenames ] session.bulk_save_objects(img_files + meta_files) # trigger creation of directories acquisition.location acquisition.microscope_metadata_location # link root directory path_to_link = os.path.join( LibraryConfig().storage_home, ('experiment_{}' '/plates/plate_{}' '/acquisitions/acquisition_{}' '/microscope_images' .format(experiment.id, plate.id, acquisition.id))) if not os.path.exists(path_to_link): try: os.symlink(path, path_to_link) except Exception as err: logger.debug( "Error linking source directory `%s` to TM directory `%s`: %s", path, path_to_link, err) raise else: # path exists, check if it is correct p1 = os.path.realpath(path) p2 = os.path.realpath(path_to_link) if p1 != p2: raise ValueError( "Acquisition {a} of plate {p} of experiment {e}" " is already linked to directory `{d}`" .format( a=acquisition.id, p=plate.id, e=epxeriment.id, d=p1)) microscope_files = session.query(tm.MicroscopeImageFile).filter_by(acquisition_id=acquisition.id).all() for microscope_file in microscope_files: microscope_file.location microscope_file.status = 'COMPLETE' return jsonify(message='ok')
def run_job(self, batch, assume_clean_state=False): '''Configures OMEXML metadata extracted from microscope image files and complements it with metadata retrieved from additional microscope metadata files and/or user input. The actual processing is delegated to a format-specific implementation of :class:`MetadataHandler <tmlib.workflow.metaconfig.base.MetadataHandler>`. Parameters ---------- batch: dict job description assume_clean_state: bool, optional assume that output of previous runs has already been cleaned up See also -------- :mod:`tmlib.workflow.metaconfig.cellvoyager` ''' regexp = batch.get('regex', '') if not regexp: regexp = get_microscope_type_regex( batch['microscope_type'], as_string=True )[0] with tm.utils.ExperimentSession(self.experiment_id) as session: experiment = session.query(tm.Experiment).one() plate_dimensions = experiment.plates[0].dimensions acquisition = session.query(tm.Acquisition).\ get(batch['acquisition_id']) metadata_files = session.query(tm.MicroscopeMetadataFile.location).\ filter_by(acquisition_id=batch['acquisition_id']).\ all() metadata_filenames = [f.location for f in metadata_files] image_files = session.query( tm.MicroscopeImageFile.name, tm.MicroscopeImageFile.omexml ).\ filter_by(acquisition_id=batch['acquisition_id']).\ all() omexml_images = { f.name: bioformats.OMEXML(f.omexml) for f in image_files } MetadataReader = metadata_reader_factory(batch['microscope_type']) if MetadataReader is not None: with MetadataReader() as mdreader: omexml_metadata = mdreader.read( metadata_filenames, omexml_images.keys() ) else: omexml_metadata = None MetadataHandler = metadata_handler_factory(batch['microscope_type']) mdhandler = MetadataHandler(omexml_images, omexml_metadata) mdhandler.configure_from_omexml() missing = mdhandler.determine_missing_metadata() if missing: logger.warning( 'required metadata information is missing: "%s"', '", "'.join(missing) ) logger.info( 'try to retrieve missing metadata from filenames ' 'using regular expression' ) if regexp is None: logger.warn('no regular expression provided') mdhandler.configure_from_filenames( plate_dimensions=plate_dimensions, regex=regexp ) missing = mdhandler.determine_missing_metadata() if missing: raise MetadataError( 'The following metadata information is missing:\n"%s"\n' % '", "'.join(missing) ) # Once we have collected basic metadata such as information about # channels and focal planes, we try to determine the relative position # of images within the acquisition grid try: logger.info( 'try to determine grid coordinates from microscope ' 'stage positions' ) mdhandler.determine_grid_coordinates_from_stage_positions() except MetadataError as error: logger.warning( 'microscope stage positions are not available: "%s"' % str(error) ) logger.info( 'try to determine grid coordinates from provided stitch layout' ) # In general, the values of these arguments can be ``None``, because # they are not required and may not be used. # However, in case the grid coordinates should be determined based # on user interput, these arguments are required. if not isinstance(batch['n_vertical'], int): raise TypeError( 'Value of argument "n_vertical" must be an integer.' ) if not isinstance(batch['n_horizontal'], int): raise TypeError( 'Value of argument "n_horizontal" must be an integer.' ) mdhandler.determine_grid_coordinates_from_layout( stitch_layout=batch['stitch_layout'], stitch_dimensions=(batch['n_vertical'], batch['n_horizontal']) ) if batch['perform_mip']: mdhandler.group_metadata_per_zstack() # Create consistent zero-based ids mdhandler.update_indices() mdhandler.assign_acquisition_site_indices() md = mdhandler.remove_redundant_columns() fmaps = mdhandler.create_image_file_mappings() logger.info('create database entries') with tm.utils.ExperimentSession(self.experiment_id) as session: channels = dict() bit_depth = md['bit_depth'][0] for ch_name in np.unique(md['channel_name']): logger.info('create channel "%s"', ch_name) ch = session.get_or_create( tm.Channel, experiment_id=self.experiment_id, name=ch_name, wavelength=ch_name, bit_depth=bit_depth, ) channels[ch_name] = ch.id for w in np.unique(md.well_name): with tm.utils.ExperimentSession(self.experiment_id) as session: acquisition = session.query(tm.Acquisition).\ get(batch['acquisition_id']) logger.info('create well "%s"', w) w_index = (md.well_name == w) well = session.get_or_create( tm.Well, plate_id=acquisition.plate.id, name=w ) channel_image_files = [] for s in np.unique(md.loc[w_index, 'site']): logger.debug('create site #%d', s) s_index = (md.site == s) y = md.loc[s_index, 'well_position_y'].values[0] x = md.loc[s_index, 'well_position_x'].values[0] height = md.loc[s_index, 'height'].values[0] width = md.loc[s_index, 'width'].values[0] site = session.get_or_create( tm.Site, y=y, x=x, height=height, width=width, well_id=well.id ) for index, i in md.ix[s_index].iterrows(): channel_image_files.append( tm.ChannelImageFile( tpoint=i.tpoint, zplane=i.zplane, channel_id=channels[i.channel_name], site_id=site.id, acquisition_id=acquisition.id, file_map=fmaps[index], ) ) session.bulk_save_objects(channel_image_files)
def run_job(self, batch, assume_clean_state=False): '''Configures OMEXML metadata extracted from microscope image files and complements it with metadata retrieved from additional microscope metadata files and/or user input. The actual processing is delegated to a format-specific implementation of :class:`MetadataHandler <tmlib.workflow.metaconfig.base.MetadataHandler>`. Parameters ---------- batch: dict job description assume_clean_state: bool, optional assume that output of previous runs has already been cleaned up See also -------- :mod:`tmlib.workflow.metaconfig.cellvoyager` ''' regexp = batch.get('regex', '') if not regexp: regexp = get_microscope_type_regex( batch['microscope_type'], as_string=True )[0] with tm.utils.ExperimentSession(self.experiment_id) as session: experiment = session.query(tm.Experiment).one() plate_dimensions = experiment.plates[0].dimensions acquisition = session.query(tm.Acquisition).\ get(batch['acquisition_id']) metadata_files = session.query(tm.MicroscopeMetadataFile.location).\ filter_by(acquisition_id=batch['acquisition_id']).\ all() metadata_filenames = [f.location for f in metadata_files] image_files = session.query( tm.MicroscopeImageFile.name, tm.MicroscopeImageFile.omexml ).\ filter_by(acquisition_id=batch['acquisition_id']).\ all() omexml_images = { f.name: bioformats.OMEXML(f.omexml) for f in image_files } MetadataReader = metadata_reader_factory(batch['microscope_type']) if MetadataReader is not None: with MetadataReader() as mdreader: omexml_metadata = mdreader.read( metadata_filenames, omexml_images.keys() ) else: omexml_metadata = None MetadataHandler = metadata_handler_factory(batch['microscope_type']) mdhandler = MetadataHandler(omexml_images, omexml_metadata) mdhandler.configure_from_omexml() missing = mdhandler.determine_missing_metadata() if missing: logger.warning( 'required metadata information is missing: "%s"', '", "'.join(missing) ) logger.info( 'try to retrieve missing metadata from filenames ' 'using regular expression' ) if regexp is None: logger.warn('no regular expression provided') mdhandler.configure_from_filenames( plate_dimensions=plate_dimensions, regex=regexp ) missing = mdhandler.determine_missing_metadata() if missing: raise MetadataError( 'The following metadata information is missing:\n"%s"\n' % '", "'.join(missing) ) # Once we have collected basic metadata such as information about # channels and focal planes, we try to determine the relative position # of images within the acquisition grid try: logger.info( 'try to determine grid coordinates from microscope ' 'stage positions' ) mdhandler.determine_grid_coordinates_from_stage_positions() except MetadataError as error: logger.warning( 'microscope stage positions are not available: "%s"' % str(error) ) logger.info( 'try to determine grid coordinates from provided stitch layout' ) # In general, the values of these arguments can be ``None``, because # they are not required and may not be used. # However, in case the grid coordinates should be determined based # on user interput, these arguments are required. if not isinstance(batch['n_vertical'], int): raise TypeError( 'Value of argument "n_vertical" must be an integer.' ) if not isinstance(batch['n_horizontal'], int): raise TypeError( 'Value of argument "n_horizontal" must be an integer.' ) mdhandler.determine_grid_coordinates_from_layout( stitch_layout=batch['stitch_layout'], stitch_dimensions=(batch['n_vertical'], batch['n_horizontal']) ) if batch['perform_mip']: mdhandler.group_metadata_per_zstack() # Create consistent zero-based ids mdhandler.update_indices() mdhandler.assign_acquisition_site_indices() md = mdhandler.remove_redundant_columns() fmaps = mdhandler.create_image_file_mappings() logger.info('create database entries') with tm.utils.ExperimentSession(self.experiment_id) as session: channels = dict() bit_depth = md['bit_depth'][0] for ch_name in np.unique(md['channel_name']): logger.info('create channel "%s"', ch_name) ch = session.get_or_create( tm.Channel, experiment_id=self.experiment_id, name=ch_name, wavelength=ch_name, bit_depth=bit_depth, ) channels[ch_name] = ch.id for w in np.unique(md.well_name): with tm.utils.ExperimentSession(self.experiment_id) as session: acquisition = session.query(tm.Acquisition).\ get(batch['acquisition_id']) logger.info('create well "%s"', w) w_index = md.well_name == w well = session.get_or_create( tm.Well, plate_id=acquisition.plate.id, name=w ) channel_image_files = list() for s in np.unique(md.loc[w_index, 'site']): logger.debug('create site #%d', s) s_index = md.site == s y = md.loc[s_index, 'well_position_y'].values[0] x = md.loc[s_index, 'well_position_x'].values[0] height = md.loc[s_index, 'height'].values[0] width = md.loc[s_index, 'width'].values[0] site = session.get_or_create( tm.Site, y=y, x=x, height=height, width=width, well_id=well.id ) for index, i in md.ix[s_index].iterrows(): channel_image_files.append( tm.ChannelImageFile( tpoint=i.tpoint, zplane=i.zplane, channel_id=channels[i.channel_name], site_id=site.id, acquisition_id=acquisition.id, file_map=fmaps[index], ) ) session.bulk_save_objects(channel_image_files)
def register(experiment_id, acquisition_id): """ .. http:post:: /api/experiments/(string:experiment_id)/acquisitions/(string:acquisition_id)/register Pass the NFS path to the data to the server. The client has to wait for this response before uploading files. **Example request**: .. sourcecode:: http Content-Type: application/json { "path": "/fullpath/to/serversidedata" } **Example response**: .. sourcecode:: http HTTP/1.1 200 OK Content-Type: application/json { "data": "/fullpath/to/serversidedata" } :reqheader Authorization: JWT token issued by the server :statuscode 200: no error :statuscode 500: server error """ logger.info('register microscope files') data = request.get_json() path = data['path'] logger.info('path given by the client: %s', path) # Parse storage_home variable from tissuemaps.cfg file parser = SafeConfigParser() parser.read('/home/tissuemaps/.tmaps/tissuemaps.cfg') storage_home_path = parser.get('tmlib', 'storage_home') path_to_link = os.path.join( storage_home_path, 'experiment_{}/plates/plate_{}/acquisitions/acquisition_{}/microscope_images' ) filenames = [ f for f in os.listdir(path) if (not f.startswith('.') and not os.path.isdir(os.path.join(path, f))) ] with tm.utils.ExperimentSession(experiment_id) as session: experiment = session.query(tm.Experiment).one() microscope_type = experiment.microscope_type img_regex, metadata_regex = get_microscope_type_regex(microscope_type) acquisition = session.query(tm.Acquisition).get(acquisition_id) # single plate only plate = session.query(tm.Plate).one() logger.info( 'path to link: %s', path_to_link.format(experiment.id, plate.id, acquisition.id)) logger.info('experiment %s', experiment.id) logger.info('acquisition %s', acquisition) logger.info('plate %s', plate) # check for image files already registered img_filenames = [f.name for f in acquisition.microscope_image_files] logger.debug('img_filenames %s', img_filenames) img_files = [ tm.MicroscopeImageFile(name=f, acquisition_id=acquisition.id) for f in filenames if img_regex.search(f) and f not in img_filenames ] # check for metadata already registered meta_filenames = [ f.name for f in acquisition.microscope_metadata_files ] meta_files = [ tm.MicroscopeMetadataFile(name=secure_filename(f), acquisition_id=acquisition.id) for f in filenames if metadata_regex.search(f) and f not in meta_filenames ] session.bulk_save_objects(img_files + meta_files) # Trigger creation of directories acquisition.location #acquisition.microscope_images_location acquisition.microscope_metadata_location os.symlink( path, path_to_link.format(experiment.id, plate.id, acquisition.id)) microscope_files = session.query(tm.MicroscopeImageFile).filter_by( acquisition_id=acquisition.id).all() for microscope_file in microscope_files: microscope_file.location microscope_file.status = 'COMPLETE' return jsonify(message='ok')