def test_populate_metadata_for_screen(self): sid = super(TestImportScripts, self).get_script(populate_metadata) assert sid > 0 client, user = self.new_client_and_user() update_service = client.getSession().getUpdateService() plates = self.import_plates(client, plate_cols=3, plate_rows=1) plate = plates[0] name = plate.name.val screen = omero.model.ScreenI() screen.name = omero.rtypes.rstring("test_for_screen") spl = omero.model.ScreenPlateLinkI() spl.setParent(screen) spl.setChild(plate) spl = update_service.saveAndReturnObject(spl) screen_id = spl.getParent().id.val assert screen_id > 0 assert spl.getChild().id.val == plate.id.val cvs_file = create_path("test_screen", ".csv") # create a file annotation with open(cvs_file.abspath(), 'w+') as f: f.write("Well,Plate, Well Type, Facility-Salt-Batch-ID\n") f.write("A01,%s,Treatment,FOOL10041-101-2\n" % name) f.write("A02,%s,Control,\n" % name) f.write("A03,%s,Treatment,FOOL10041-101-2\n" % name) conn = BlitzGateway(client_obj=client) fa = conn.createFileAnnfromLocalFile(cvs_file, mimetype="text/csv") assert fa is not None assert fa.id > 0 link = omero.model.ScreenAnnotationLinkI() link.setParent(omero.model.ScreenI(screen_id, False)) link.setChild(omero.model.FileAnnotationI(fa.id, False)) link = update_service.saveAndReturnObject(link) assert link.id.val > 0 # run the script screen_ids = [] screen_ids.append(spl.getParent().id) args = { "Data_Type": omero.rtypes.rstring("Screen"), "IDs": omero.rtypes.rlist(screen_ids), "File_Annotation": omero.rtypes.rstring(str(fa.id)) } message = run_script(client, sid, args, "Message") assert message is not None assert message.getValue().startswith('Table data populated') conn.close()
project = conn.getObject("Project", projectId) # NB: only link a client map annotation to a single object project.linkAnnotation(mapAnn) # How to create a file annotation and link to a Dataset # ===================================================== dataset = conn.getObject("Dataset", datasetId) # Specify a local file e.g. could be result of some analysis fileToUpload = "README.txt" # This file should already exist with open(fileToUpload, 'w') as f: f.write('annotation test') # create the original file and file annotation (uploads the file etc.) namespace = "imperial.training.demo" print "\nCreating an OriginalFile and FileAnnotation" fileAnn = conn.createFileAnnfromLocalFile( fileToUpload, mimetype="text/plain", ns=namespace, desc=None) print "Attaching FileAnnotation to Dataset: ", "File ID:", fileAnn.getId(), \ ",", fileAnn.getFile().getName(), "Size:", fileAnn.getFile().getSize() dataset.linkAnnotation(fileAnn) # link it to dataset. os.remove(fileToUpload) # Download a file annotation linked to a Dataset # ============================================== # make a location to download the file. "download" folder. path = os.path.join(os.path.dirname(__file__), "download") if not os.path.exists(path): os.makedirs(path) # Go through all the annotations on the Dataset. Download any file annotations # we find. print "\nAnnotations on Dataset:", dataset.getName() for ann in dataset.listAnnotations():
class Omg(object): """ OMERO gateway that wraps Blitz gateway and CLI, intended for scripting and interactive work. Attributes ---------- conn : Blitz gateway connection """ def __init__(self, conn=None, user=None, passwd=None, server=SERVER, port=PORT, skey=None): """ Requires active Blitz connection OR username plus password or sesskey """ if conn is None and (user is None or (passwd is None and skey is None)): raise ValueError("Bad parameters," + self.__init__.__doc__) if conn is not None: if conn.isConnected(): self.conn = conn else: raise ValueError("Cannot initialize with closed connection!") else: if passwd is not None: self.conn = BlitzGateway(user, passwd, host=server, port=port) self.conn.connect() else: self.conn = BlitzGateway(user, host=server, port=port) self.conn.connect(skey) if self.conn.isConnected(): self._server = self.conn.host self._port = self.conn.port self._user = self.conn.getUser().getName() self._key = self.conn.getSession().getUuid().getValue() print("Connected to {0} (port {1}) as {2}, session key={3}".format( self._server, self._port, self._user, self._key)) else: print("Failed to open connection :-(") def ls(self): """ Print groups, then projects/datasets/images for current group. """ print("Groups for {0}:-".format(self.conn.getUser().getName())) for gid, gname in self._ls_groups(): print(" {0} ({1})".format(gname, str(gid))) curr_grp = self.conn.getGroupFromContext() gid, gname = curr_grp.getId(), curr_grp.getName() print("\nData for current group, {0} ({1}):-".format(gname, gid)) for pid, pname in self._ls_projects(): print(" Project: {0} ({1})".format(pname, str(pid))) for did, dname in self._ls_datasets(pid): print(" Dataset: {0} ({1})".format(dname, str(did))) for iid, iname in self._ls_images(did): print(" Image: {0} ({1})".format(iname, str(iid))) # TODO, list orphaned Datasets and Images def _ls_groups(self): """list groups (id, name) this session is a member of""" groups = self.conn.getGroupsMemberOf() return [(group.getId(), group.getName()) for group in groups] def _ls_projects(self): """list projects (id, name) in the current session group""" projs = self.conn.listProjects(self.conn.getUserId()) return [(proj.getId(), proj.getName()) for proj in projs] def _ls_datasets(self, proj_id): """list datasets (id, name) within the project id given""" dsets = self.conn.getObject("Project", proj_id).listChildren() return [(dset.getId(), dset.getName()) for dset in dsets] def _ls_images(self, dset_id): """list images (id, name) within the dataset id given""" imgs = self.conn.getObject("Dataset", dset_id).listChildren() return [(img.getId(), img.getName()) for img in imgs] def chgrp(self, group_id): """ Change group for this session to the group_id given. """ self.conn.setGroupForSession(group_id) def get(self, im_id, get_att=True): """ Download the specified image as an OME-TIFF to current directory, with attachments also downloaded to folder: img_path + '_attachments' Return : path to downloaded image """ img = self.conn.getObject("Image", oid=im_id) img_name = self._unique_name(img.getName(), im_id) img_path = os.path.join(os.getcwd(), img_name) img_file = open(str(img_path + ".ome.tiff"), "wb") fsize, blockgen = img.exportOmeTiff(bufsize=65536) for block in blockgen: img_file.write(block) img_file.close() fa_type = omero.model.FileAnnotationI attachments = [ann for ann in img.listAnnotations() if ann.OMERO_TYPE == fa_type] if get_att and len(attachments) > 0: att_dir = img_path + "_attachments" os.mkdir(att_dir) def download_attachment(att, att_dir): """download OMERO file annotation to att_dir""" att_file = open(os.path.join(att_dir, att.getFileName()), "wb") for att_chunk in att.getFileInChunks(): att_file.write(att_chunk) att_file.close() for att in attachments: download_attachment(att, att_dir) return img_path def _unique_name(self, img_name, im_id): """Make unique name combining a file basename & OMERO Image id""" path_and_base, ext = os.path.splitext(img_name) base = os.path.basename(path_and_base) # name in OMERO can has path return "{0}_{1}".format(base, str(im_id)) def dget(self, dataset_id): """ Download an entire OMERO Dataset to the current directory. """ downloads = [] wdir = os.getcwd() dset_name = self.conn.getObject("Dataset", dataset_id).getName() dset_path = os.path.join(wdir, dset_name + "_D" + str(dataset_id)) os.mkdir(dset_path) os.chdir(dset_path) for img_id, img_name in self._ls_images(dataset_id): downloads.append(self.get(img_id)) os.chdir(wdir) return downloads def pget(self, project_id): """ Download an entire OMERO Project to the current directory. """ downloads = [] wdir = os.getcwd() proj_name = self.conn.getObject("Project", project_id).getName() proj_path = os.path.join(wdir, proj_name + "_P" + str(project_id)) os.mkdir(proj_path) os.chdir(proj_path) for dset_id, dset_name in self._ls_datasets(project_id): downloads.extend(self.dget(dset_id)) os.chdir(wdir) return downloads def put(self, filename, name=None, dataset=None): """ Import filename using OMERO CLI, optionally with a specified name to a specified dataset (dataset_id). Return : OMERO image Id """ cli = omero.cli.CLI() cli.loadplugins() import_args = ["import"] import_args.extend(["-s", str(self._server)]) import_args.extend(["-k", str(self._key)]) if dataset is not None: import_args.extend(["-d", str(dataset)]) if name is not None: import_args.extend(["-n", str(name)]) clio = "cli.out" clie = "cli.err" import_args.extend(["---errs=" + clie, "---file=" + clio, "--"]) import_args.append(filename) cli.invoke(import_args, strict=True) pix_id = int(open(clio, 'r').read().rstrip()) im_id = self.conn.getQueryService().get("Pixels", pix_id).image.id.val os.remove(clio) os.remove(clie) return im_id def describe(self, im_id, description): """ Append to image description. """ img = self.conn.getObject("Image", oid=im_id) old_description = img.getDescription() or "" img.setDescription(old_description + "\n" + description) img.save() def attach(self, im_id, attachments): """ Attach a list of files to an image. """ img = self.conn.getObject("Image", oid=im_id) for attachment in attachments.split(): fann = self.conn.createFileAnnfromLocalFile(attachment) img.linkAnnotation(fann) img.save() # TODO: ls_tags() and tag() methods? def mkp(self, project_name, description=None): """ Make new OMERO project in current group, returning the new project Id. """ # see: omero/lib/python/omeroweb/webclient/controller/container.py proj = omero.model.ProjectI() proj.name = omero.rtypes.rstring(str(project_name)) if description is not None and description != "": proj.description = omero.rtypes.rstring(str(description)) return self._save_and_return_id(proj) def mkd(self, dataset_name, project_id=None, description=None): """ Make new OMERO dataset, returning the new dataset Id. """ dset = omero.model.DatasetI() dset.name = omero.rtypes.rstring(str(dataset_name)) if description is not None and description != "": dset.description = omero.rtypes.rstring(str(description)) if project_id is not None: l_proj_dset = omero.model.ProjectDatasetLinkI() proj = self.conn.getObject("Project", project_id) l_proj_dset.setParent(proj._obj) l_proj_dset.setChild(dset) dset.addProjectDatasetLink(l_proj_dset) return self._save_and_return_id(dset) def _save_and_return_id(self, obj): """Save new omero object and return id assgined to it""" # see: OmeroWebGateway.saveAndReturnId # in: lib/python/omeroweb/webclient/webclient_gateway.py u_s = self.conn.getUpdateService() res = u_s.saveAndReturnObject(obj, self.conn.SERVICE_OPTS) res.unload() return res.id.val def im(self, im_id): """ Return an Im object for the image id specified. """ img = self.conn.getObject("Image", im_id) # build pixel np.ndarray nx, ny = img.getSizeX(), img.getSizeY() nz, nt, nc = img.getSizeZ(), img.getSizeT(), img.getSizeC() planes = [(z, c, t) for c in range(nc) for t in range(nt) for z in range(nz)] pix_gen = img.getPrimaryPixels().getPlanes(planes) pix = np.array([i for i in pix_gen]).reshape((nc, nt, nz, ny, nx)) # initialize Im using pix and extracted metadata meta = self._extract_meta(img, im_id) return Im(pix=pix, meta=meta) def _extract_meta(self, img, im_id): """Extract metadata attributes from OMERO Blitz gateway Image""" meta = {} meta['name'] = self._unique_name(img.getName(), im_id) meta['description'] = img.getDescription() def _extract_ch_info(ch): """extract core metadata for for channel, return as dict""" ch_info = {'label': ch.getLabel()} ch_info['ex_wave'] = ch.getExcitationWave() ch_info['em_wave'] = ch.getEmissionWave() ch_info['color'] = ch.getColor().getRGB() return ch_info meta['channels'] = [_extract_ch_info(ch) for ch in img.getChannels()] meta['pixel_size'] = {'x': img.getPixelSizeX(), 'y': img.getPixelSizeY(), 'z': img.getPixelSizeZ(), 'units': "um"} tag_type = omero.model.TagAnnotationI tags = [ann for ann in img.listAnnotations() if ann.OMERO_TYPE == tag_type] meta['tags'] = {tag.getValue() + " (" + str(tag.getId()) + ")": tag.getDescription() for tag in tags} fa_type = omero.model.FileAnnotationI attachments = [ann for ann in img.listAnnotations() if ann.OMERO_TYPE == fa_type] meta['attachments'] = [att.getFileName() + " (" + str(att.getId()) + ")" for att in attachments] user_id = self.conn.getUser().getName() + " (" + \ str(self.conn.getUser().getId()) + ") @" + self.conn.host meta_ext = {} meta_ext['user_id'] = user_id meta['meta_ext'] = meta_ext # TODO: ROIs, display settings? # objective: Image.loadOriginalMetadata()[1][find 'Lens ID Number'][1], return meta def imput(self, im, dataset_id=None): """ Create a new OMERO Image using an Im object, returning new image id. """ # see: omero/lib/python/omero/util/script_utils.py # see: omero/lib/python/omeroweb/webclient/webclient_gateway.py # see: https://gist.github.com/will-moore/4141708 if not isinstance(im, Im): raise TypeError("first imput argument must be of type Im") nc, nt, nz, ny, nx = im.shape ch_nums = range(nc) q_s = self.conn.getQueryService() p_s = self.conn.getPixelsService() c_s = self.conn.getContainerService() u_s = self.conn.getUpdateService() pu_s = self.conn.c.sf.createRawPixelsStore() q_ptype = "from PixelsType as p where p.value='{0}'".format( str(im.dtype)) pixelsType = q_s.findByQuery(q_ptype, None) im_id = p_s.createImage(nx, ny, nz, nt, ch_nums, pixelsType, im.name, im.description) img_i = c_s.getImages("Image", [im_id.getValue()], None)[0] img = self.conn.getObject("Image", im_id.getValue()) pix_id = img_i.getPrimaryPixels().getId().getValue() pu_s.setPixelsId(pix_id, True) for c in range(nc): for t in range(nt): for z in range(nz): plane = im.pix[c, t, z, :, :] script_utils.uploadPlaneByRow(pu_s, plane, z, c, t) l_dset_im = omero.model.DatasetImageLinkI() dset = self.conn.getObject("Dataset", dataset_id) l_dset_im.setParent(dset._obj) l_dset_im.setChild(img._obj) self._update_meta(im, im_id) u_s.saveObject(l_dset_im, self.conn.SERVICE_OPTS) return im_id.getValue() def _update_meta(self, im, im_id): """Set OMERO Image metadata using Im metadata"""
project = conn.getObject("Project", projectId) # NB: only link a client map annotation to a single object project.linkAnnotation(mapAnn) # How to create a file annotation and link to a Dataset # ===================================================== dataset = conn.getObject("Dataset", datasetId) # Specify a local file e.g. could be result of some analysis fileToUpload = "README.txt" # This file should already exist with open(fileToUpload, 'w') as f: f.write('annotation test') # create the original file and file annotation (uploads the file etc.) namespace = "imperial.training.demo" print "\nCreating an OriginalFile and FileAnnotation" fileAnn = conn.createFileAnnfromLocalFile(fileToUpload, mimetype="text/plain", ns=namespace, desc=None) print "Attaching FileAnnotation to Dataset: ", "File ID:", fileAnn.getId(), \ ",", fileAnn.getFile().getName(), "Size:", fileAnn.getFile().getSize() dataset.linkAnnotation(fileAnn) # link it to dataset. os.remove(fileToUpload) # Download a file annotation linked to a Dataset # ============================================== # make a location to download the file. "download" folder. path = os.path.join(os.path.dirname(__file__), "download") if not os.path.exists(path): os.makedirs(path) # Go through all the annotations on the Dataset. Download any file annotations # we find. print "\nAnnotations on Dataset:", dataset.getName()
# ** BONUS ** # stack the numpy columns horizontally. hstack is a numpy function kymograph_data = hstack(col_data) print "kymograph_data", kymograph_data.shape if kymograph_data.dtype.name not in ('uint8', 'int8'): # we need to scale... minVal = kymograph_data.min() maxVal = kymograph_data.max() valRange = maxVal - minVal scaled = (kymograph_data - minVal) * (float(255) / valRange) convArray = zeros(kymograph_data.shape, dtype=uint8) convArray += scaled print "using converted int8 plane: dtype: %s min: %s max: %s" % (convArray.dtype.name, convArray.min(), convArray.max()) i = Image.fromarray(convArray) else: i = Image.fromarray(plane) #i.show() i.save("kymograph.png", 'PNG') # attach the png to the image fileAnn = conn.createFileAnnfromLocalFile("kymograph.png", mimetype="image/png") print "Attaching kymograph.png to image" image.linkAnnotation(fileAnn) message = "Tile average value: %s" % average client.setOutput("Message", rstring(message)) client.setOutput("Kymograph", robject(fileAnn._obj)) client.closeSession()
# ** BONUS ** # stack the numpy columns horizontally. hstack is a numpy function kymograph_data = hstack(col_data) print "kymograph_data", kymograph_data.shape if kymograph_data.dtype.name not in ('uint8', 'int8'): # we need to scale... minVal = kymograph_data.min() maxVal = kymograph_data.max() valRange = maxVal - minVal scaled = (kymograph_data - minVal) * (float(255) / valRange) convArray = zeros(kymograph_data.shape, dtype=uint8) convArray += scaled print("using converted int8 plane: dtype: %s min: %s max: %s" % (convArray.dtype.name, convArray.min(), convArray.max())) i = Image.fromarray(convArray) else: i = Image.fromarray(kymograph_data) i.show() i.save("kymograph.png", 'PNG') # attach the png to the image fileAnn = conn.createFileAnnfromLocalFile("kymograph.png", mimetype="image/png") print "Attaching kymograph.png to image" image.linkAnnotation(fileAnn) message = "Tile average value: %s" % average # client.setOutput("Message", rstring(message)) # client.setOutput("Kymograph", robject(fileAnn._obj)) # client.closeSession()
width = 10 # we were asked for '1' but this looks nicer height = size_x tile = (x, y, width, height) col_data = [] # let's collect the column data for each time point for t in range(size_t): print("Getting data for T, tile:", t, tile) col = pixels.getTile(z, c, t, tile) col_data.append(col) # ** BONUS ** # stack the numpy columns horizontally. hstack is a numpy function kymograph_data = hstack(col_data) print("kymograph_data", kymograph_data.shape) name = "kymograph.png" min_max = (kymograph_data.min(), kymograph_data.max()) scriptUtil.numpy_save_as_image(kymograph_data, min_max, int32, name) # attach the png to the image file_ann = conn.createFileAnnfromLocalFile(name, mimetype="image/png") print("Attaching %s to image" % name) image.linkAnnotation(file_ann) message = "Tile average value: %s" % average # Close connection: # ================================================================= # When you are done, close the session to free up server resources. conn.close()
ids = unwrap(client.getInput("IDs")) images = conn.getObjects("Image", ids) with open("selected_images_and_paths_names.csv", "w") as f: for i in images: print i.name image_paths = i.getImportedImageFilePaths() f.write(i.name) f.write(" ") f.write(", ".join(image_paths['client_paths'])) f.write(" ") f.write(", ".join(image_paths['server_paths'])) f.write('\n') file_ann = conn.createFileAnnfromLocalFile("selected_images_and_paths_names.csv", mimetype="text/csv", ns="image.names.foo") image = conn.getObject("Image", ids[0]) image.linkAnnotation(file_ann) # Return some value(s). # Here, we return anything useful the script has produced. # NB: The Insight and web clients will display the "Message" output. msg = "Script ran OK" client.setOutput("Message", rstring(msg)) client.setOutput("File_Annotation", robject(file_ann._obj)) client.closeSession()
tags.append([tag.textValue, desc, str(tag.id), str(owner)]) #sort the tags in descending order to allow to see duplicates tags.sort(key=lambda tag: tag[0].islower()) #create the csv file in which to copy the tag list with open("tags_to_check-" + str(date.today()) + ".csv", "w") as f: f.write("tag name, description, tag ID, owner ID\n") for t in tags: f.write(",".join(t)) f.write('\n') # set group to save file to. NB: hard-coded as stystem group conn.SERVICE_OPTS.setOmeroGroup('0') file_ann = conn.createFileAnnfromLocalFile("tags_to_check-" + str(date.today()) + ".csv", mimetype="text/csv", ns="tags.to.check") # Return some value(s). # Here, we return anything useful the script has produced. # NB: The Insight and web clients will display the "Message" output. msg = "Script ran OK" client.setOutput("Message", rstring(msg)) client.setOutput("File_Annotation", robject(file_ann._obj)) client.closeSession()