def test_populate_metadata_for_screen(self):
sid = super(TestImportScripts, self).get_script(populate_metadata)
assert sid > 0
client, user = self.new_client_and_user()
update_service = client.getSession().getUpdateService()
plates = self.import_plates(client, plate_cols=3, plate_rows=1)
plate = plates[0]
name = plate.name.val
screen = omero.model.ScreenI()
screen.name = omero.rtypes.rstring("test_for_screen")
spl = omero.model.ScreenPlateLinkI()
spl.setParent(screen)
spl.setChild(plate)
spl = update_service.saveAndReturnObject(spl)
screen_id = spl.getParent().id.val
assert screen_id > 0
assert spl.getChild().id.val == plate.id.val
cvs_file = create_path("test_screen", ".csv")
# create a file annotation
with open(cvs_file.abspath(), 'w+') as f:
f.write("Well,Plate, Well Type, Facility-Salt-Batch-ID\n")
f.write("A01,%s,Treatment,FOOL10041-101-2\n" % name)
f.write("A02,%s,Control,\n" % name)
f.write("A03,%s,Treatment,FOOL10041-101-2\n" % name)
conn = BlitzGateway(client_obj=client)
fa = conn.createFileAnnfromLocalFile(cvs_file, mimetype="text/csv")
assert fa is not None
assert fa.id > 0
link = omero.model.ScreenAnnotationLinkI()
link.setParent(omero.model.ScreenI(screen_id, False))
link.setChild(omero.model.FileAnnotationI(fa.id, False))
link = update_service.saveAndReturnObject(link)
assert link.id.val > 0
# run the script
screen_ids = []
screen_ids.append(spl.getParent().id)
args = {
"Data_Type": omero.rtypes.rstring("Screen"),
"IDs": omero.rtypes.rlist(screen_ids),
"File_Annotation": omero.rtypes.rstring(str(fa.id))
}
message = run_script(client, sid, args, "Message")
assert message is not None
assert message.getValue().startswith('Table data populated')
conn.close()
project = conn.getObject("Project", projectId)
# NB: only link a client map annotation to a single object
project.linkAnnotation(mapAnn)
# How to create a file annotation and link to a Dataset
# =====================================================
dataset = conn.getObject("Dataset", datasetId)
# Specify a local file e.g. could be result of some analysis
fileToUpload = "README.txt" # This file should already exist
with open(fileToUpload, 'w') as f:
f.write('annotation test')
# create the original file and file annotation (uploads the file etc.)
namespace = "imperial.training.demo"
print "\nCreating an OriginalFile and FileAnnotation"
fileAnn = conn.createFileAnnfromLocalFile(
fileToUpload, mimetype="text/plain", ns=namespace, desc=None)
print "Attaching FileAnnotation to Dataset: ", "File ID:", fileAnn.getId(), \
",", fileAnn.getFile().getName(), "Size:", fileAnn.getFile().getSize()
dataset.linkAnnotation(fileAnn) # link it to dataset.
os.remove(fileToUpload)
# Download a file annotation linked to a Dataset
# ==============================================
# make a location to download the file. "download" folder.
path = os.path.join(os.path.dirname(__file__), "download")
if not os.path.exists(path):
os.makedirs(path)
# Go through all the annotations on the Dataset. Download any file annotations
# we find.
print "\nAnnotations on Dataset:", dataset.getName()
for ann in dataset.listAnnotations():
class Omg(object):
"""
OMERO gateway that wraps Blitz gateway and CLI, intended for
scripting and interactive work.
Attributes
----------
conn : Blitz gateway connection
"""
def __init__(self, conn=None, user=None, passwd=None,
server=SERVER, port=PORT, skey=None):
"""
Requires active Blitz connection OR username plus password or sesskey
"""
if conn is None and (user is None or (passwd is None and skey is None)):
raise ValueError("Bad parameters," + self.__init__.__doc__)
if conn is not None:
if conn.isConnected():
self.conn = conn
else:
raise ValueError("Cannot initialize with closed connection!")
else:
if passwd is not None:
self.conn = BlitzGateway(user, passwd, host=server, port=port)
self.conn.connect()
else:
self.conn = BlitzGateway(user, host=server, port=port)
self.conn.connect(skey)
if self.conn.isConnected():
self._server = self.conn.host
self._port = self.conn.port
self._user = self.conn.getUser().getName()
self._key = self.conn.getSession().getUuid().getValue()
print("Connected to {0} (port {1}) as {2}, session key={3}".format(
self._server, self._port, self._user, self._key))
else:
print("Failed to open connection :-(")
def ls(self):
"""
Print groups, then projects/datasets/images for current group.
"""
print("Groups for {0}:-".format(self.conn.getUser().getName()))
for gid, gname in self._ls_groups():
print(" {0} ({1})".format(gname, str(gid)))
curr_grp = self.conn.getGroupFromContext()
gid, gname = curr_grp.getId(), curr_grp.getName()
print("\nData for current group, {0} ({1}):-".format(gname, gid))
for pid, pname in self._ls_projects():
print(" Project: {0} ({1})".format(pname, str(pid)))
for did, dname in self._ls_datasets(pid):
print(" Dataset: {0} ({1})".format(dname, str(did)))
for iid, iname in self._ls_images(did):
print(" Image: {0} ({1})".format(iname, str(iid)))
# TODO, list orphaned Datasets and Images
def _ls_groups(self):
"""list groups (id, name) this session is a member of"""
groups = self.conn.getGroupsMemberOf()
return [(group.getId(), group.getName()) for group in groups]
def _ls_projects(self):
"""list projects (id, name) in the current session group"""
projs = self.conn.listProjects(self.conn.getUserId())
return [(proj.getId(), proj.getName()) for proj in projs]
def _ls_datasets(self, proj_id):
"""list datasets (id, name) within the project id given"""
dsets = self.conn.getObject("Project", proj_id).listChildren()
return [(dset.getId(), dset.getName()) for dset in dsets]
def _ls_images(self, dset_id):
"""list images (id, name) within the dataset id given"""
imgs = self.conn.getObject("Dataset", dset_id).listChildren()
return [(img.getId(), img.getName()) for img in imgs]
def chgrp(self, group_id):
"""
Change group for this session to the group_id given.
"""
self.conn.setGroupForSession(group_id)
def get(self, im_id, get_att=True):
"""
Download the specified image as an OME-TIFF to current directory,
with attachments also downloaded to folder: img_path + '_attachments'
Return : path to downloaded image
"""
img = self.conn.getObject("Image", oid=im_id)
img_name = self._unique_name(img.getName(), im_id)
img_path = os.path.join(os.getcwd(), img_name)
img_file = open(str(img_path + ".ome.tiff"), "wb")
fsize, blockgen = img.exportOmeTiff(bufsize=65536)
for block in blockgen:
img_file.write(block)
img_file.close()
fa_type = omero.model.FileAnnotationI
attachments = [ann for ann in img.listAnnotations()
if ann.OMERO_TYPE == fa_type]
if get_att and len(attachments) > 0:
att_dir = img_path + "_attachments"
os.mkdir(att_dir)
def download_attachment(att, att_dir):
"""download OMERO file annotation to att_dir"""
att_file = open(os.path.join(att_dir, att.getFileName()), "wb")
for att_chunk in att.getFileInChunks():
att_file.write(att_chunk)
att_file.close()
for att in attachments:
download_attachment(att, att_dir)
return img_path
def _unique_name(self, img_name, im_id):
"""Make unique name combining a file basename & OMERO Image id"""
path_and_base, ext = os.path.splitext(img_name)
base = os.path.basename(path_and_base) # name in OMERO can has path
return "{0}_{1}".format(base, str(im_id))
def dget(self, dataset_id):
"""
Download an entire OMERO Dataset to the current directory.
"""
downloads = []
wdir = os.getcwd()
dset_name = self.conn.getObject("Dataset", dataset_id).getName()
dset_path = os.path.join(wdir, dset_name + "_D" + str(dataset_id))
os.mkdir(dset_path)
os.chdir(dset_path)
for img_id, img_name in self._ls_images(dataset_id):
downloads.append(self.get(img_id))
os.chdir(wdir)
return downloads
def pget(self, project_id):
"""
Download an entire OMERO Project to the current directory.
"""
downloads = []
wdir = os.getcwd()
proj_name = self.conn.getObject("Project", project_id).getName()
proj_path = os.path.join(wdir, proj_name + "_P" + str(project_id))
os.mkdir(proj_path)
os.chdir(proj_path)
for dset_id, dset_name in self._ls_datasets(project_id):
downloads.extend(self.dget(dset_id))
os.chdir(wdir)
return downloads
def put(self, filename, name=None, dataset=None):
"""
Import filename using OMERO CLI, optionally with a specified name
to a specified dataset (dataset_id).
Return : OMERO image Id
"""
cli = omero.cli.CLI()
cli.loadplugins()
import_args = ["import"]
import_args.extend(["-s", str(self._server)])
import_args.extend(["-k", str(self._key)])
if dataset is not None:
import_args.extend(["-d", str(dataset)])
if name is not None:
import_args.extend(["-n", str(name)])
clio = "cli.out"
clie = "cli.err"
import_args.extend(["---errs=" + clie, "---file=" + clio, "--"])
import_args.append(filename)
cli.invoke(import_args, strict=True)
pix_id = int(open(clio, 'r').read().rstrip())
im_id = self.conn.getQueryService().get("Pixels", pix_id).image.id.val
os.remove(clio)
os.remove(clie)
return im_id
def describe(self, im_id, description):
"""
Append to image description.
"""
img = self.conn.getObject("Image", oid=im_id)
old_description = img.getDescription() or ""
img.setDescription(old_description + "\n" + description)
img.save()
def attach(self, im_id, attachments):
"""
Attach a list of files to an image.
"""
img = self.conn.getObject("Image", oid=im_id)
for attachment in attachments.split():
fann = self.conn.createFileAnnfromLocalFile(attachment)
img.linkAnnotation(fann)
img.save()
# TODO: ls_tags() and tag() methods?
def mkp(self, project_name, description=None):
"""
Make new OMERO project in current group, returning the new project Id.
"""
# see: omero/lib/python/omeroweb/webclient/controller/container.py
proj = omero.model.ProjectI()
proj.name = omero.rtypes.rstring(str(project_name))
if description is not None and description != "":
proj.description = omero.rtypes.rstring(str(description))
return self._save_and_return_id(proj)
def mkd(self, dataset_name, project_id=None, description=None):
"""
Make new OMERO dataset, returning the new dataset Id.
"""
dset = omero.model.DatasetI()
dset.name = omero.rtypes.rstring(str(dataset_name))
if description is not None and description != "":
dset.description = omero.rtypes.rstring(str(description))
if project_id is not None:
l_proj_dset = omero.model.ProjectDatasetLinkI()
proj = self.conn.getObject("Project", project_id)
l_proj_dset.setParent(proj._obj)
l_proj_dset.setChild(dset)
dset.addProjectDatasetLink(l_proj_dset)
return self._save_and_return_id(dset)
def _save_and_return_id(self, obj):
"""Save new omero object and return id assgined to it"""
# see: OmeroWebGateway.saveAndReturnId
# in: lib/python/omeroweb/webclient/webclient_gateway.py
u_s = self.conn.getUpdateService()
res = u_s.saveAndReturnObject(obj, self.conn.SERVICE_OPTS)
res.unload()
return res.id.val
def im(self, im_id):
"""
Return an Im object for the image id specified.
"""
img = self.conn.getObject("Image", im_id)
# build pixel np.ndarray
nx, ny = img.getSizeX(), img.getSizeY()
nz, nt, nc = img.getSizeZ(), img.getSizeT(), img.getSizeC()
planes = [(z, c, t) for c in range(nc)
for t in range(nt)
for z in range(nz)]
pix_gen = img.getPrimaryPixels().getPlanes(planes)
pix = np.array([i for i in pix_gen]).reshape((nc, nt, nz, ny, nx))
# initialize Im using pix and extracted metadata
meta = self._extract_meta(img, im_id)
return Im(pix=pix, meta=meta)
def _extract_meta(self, img, im_id):
"""Extract metadata attributes from OMERO Blitz gateway Image"""
meta = {}
meta['name'] = self._unique_name(img.getName(), im_id)
meta['description'] = img.getDescription()
def _extract_ch_info(ch):
"""extract core metadata for for channel, return as dict"""
ch_info = {'label': ch.getLabel()}
ch_info['ex_wave'] = ch.getExcitationWave()
ch_info['em_wave'] = ch.getEmissionWave()
ch_info['color'] = ch.getColor().getRGB()
return ch_info
meta['channels'] = [_extract_ch_info(ch) for ch in img.getChannels()]
meta['pixel_size'] = {'x': img.getPixelSizeX(),
'y': img.getPixelSizeY(),
'z': img.getPixelSizeZ(),
'units': "um"}
tag_type = omero.model.TagAnnotationI
tags = [ann for ann in img.listAnnotations()
if ann.OMERO_TYPE == tag_type]
meta['tags'] = {tag.getValue() + " (" + str(tag.getId()) + ")":
tag.getDescription() for tag in tags}
fa_type = omero.model.FileAnnotationI
attachments = [ann for ann in img.listAnnotations()
if ann.OMERO_TYPE == fa_type]
meta['attachments'] = [att.getFileName() + " (" + str(att.getId()) +
")" for att in attachments]
user_id = self.conn.getUser().getName() + " (" + \
str(self.conn.getUser().getId()) + ") @" + self.conn.host
meta_ext = {}
meta_ext['user_id'] = user_id
meta['meta_ext'] = meta_ext
# TODO: ROIs, display settings?
# objective: Image.loadOriginalMetadata()[1][find 'Lens ID Number'][1],
return meta
def imput(self, im, dataset_id=None):
"""
Create a new OMERO Image using an Im object, returning new image id.
"""
# see: omero/lib/python/omero/util/script_utils.py
# see: omero/lib/python/omeroweb/webclient/webclient_gateway.py
# see: https://gist.github.com/will-moore/4141708
if not isinstance(im, Im):
raise TypeError("first imput argument must be of type Im")
nc, nt, nz, ny, nx = im.shape
ch_nums = range(nc)
q_s = self.conn.getQueryService()
p_s = self.conn.getPixelsService()
c_s = self.conn.getContainerService()
u_s = self.conn.getUpdateService()
pu_s = self.conn.c.sf.createRawPixelsStore()
q_ptype = "from PixelsType as p where p.value='{0}'".format(
str(im.dtype))
pixelsType = q_s.findByQuery(q_ptype, None)
im_id = p_s.createImage(nx, ny, nz, nt, ch_nums, pixelsType,
im.name, im.description)
img_i = c_s.getImages("Image", [im_id.getValue()], None)[0]
img = self.conn.getObject("Image", im_id.getValue())
pix_id = img_i.getPrimaryPixels().getId().getValue()
pu_s.setPixelsId(pix_id, True)
for c in range(nc):
for t in range(nt):
for z in range(nz):
plane = im.pix[c, t, z, :, :]
script_utils.uploadPlaneByRow(pu_s, plane, z, c, t)
l_dset_im = omero.model.DatasetImageLinkI()
dset = self.conn.getObject("Dataset", dataset_id)
l_dset_im.setParent(dset._obj)
l_dset_im.setChild(img._obj)
self._update_meta(im, im_id)
u_s.saveObject(l_dset_im, self.conn.SERVICE_OPTS)
return im_id.getValue()
def _update_meta(self, im, im_id):
"""Set OMERO Image metadata using Im metadata"""
project = conn.getObject("Project", projectId)
# NB: only link a client map annotation to a single object
project.linkAnnotation(mapAnn)
# How to create a file annotation and link to a Dataset
# =====================================================
dataset = conn.getObject("Dataset", datasetId)
# Specify a local file e.g. could be result of some analysis
fileToUpload = "README.txt" # This file should already exist
with open(fileToUpload, 'w') as f:
f.write('annotation test')
# create the original file and file annotation (uploads the file etc.)
namespace = "imperial.training.demo"
print "\nCreating an OriginalFile and FileAnnotation"
fileAnn = conn.createFileAnnfromLocalFile(fileToUpload,
mimetype="text/plain",
ns=namespace,
desc=None)
print "Attaching FileAnnotation to Dataset: ", "File ID:", fileAnn.getId(), \
",", fileAnn.getFile().getName(), "Size:", fileAnn.getFile().getSize()
dataset.linkAnnotation(fileAnn) # link it to dataset.
os.remove(fileToUpload)
# Download a file annotation linked to a Dataset
# ==============================================
# make a location to download the file. "download" folder.
path = os.path.join(os.path.dirname(__file__), "download")
if not os.path.exists(path):
os.makedirs(path)
# Go through all the annotations on the Dataset. Download any file annotations
# we find.
print "\nAnnotations on Dataset:", dataset.getName()
# ** BONUS **
# stack the numpy columns horizontally. hstack is a numpy function
kymograph_data = hstack(col_data)
print "kymograph_data", kymograph_data.shape
if kymograph_data.dtype.name not in ('uint8', 'int8'): # we need to scale...
minVal = kymograph_data.min()
maxVal = kymograph_data.max()
valRange = maxVal - minVal
scaled = (kymograph_data - minVal) * (float(255) / valRange)
convArray = zeros(kymograph_data.shape, dtype=uint8)
convArray += scaled
print "using converted int8 plane: dtype: %s min: %s max: %s" % (convArray.dtype.name, convArray.min(), convArray.max())
i = Image.fromarray(convArray)
else:
i = Image.fromarray(plane)
#i.show()
i.save("kymograph.png", 'PNG')
# attach the png to the image
fileAnn = conn.createFileAnnfromLocalFile("kymograph.png", mimetype="image/png")
print "Attaching kymograph.png to image"
image.linkAnnotation(fileAnn)
message = "Tile average value: %s" % average
client.setOutput("Message", rstring(message))
client.setOutput("Kymograph", robject(fileAnn._obj))
client.closeSession()
# ** BONUS **
# stack the numpy columns horizontally. hstack is a numpy function
kymograph_data = hstack(col_data)
print "kymograph_data", kymograph_data.shape
if kymograph_data.dtype.name not in ('uint8', 'int8'): # we need to scale...
minVal = kymograph_data.min()
maxVal = kymograph_data.max()
valRange = maxVal - minVal
scaled = (kymograph_data - minVal) * (float(255) / valRange)
convArray = zeros(kymograph_data.shape, dtype=uint8)
convArray += scaled
print("using converted int8 plane: dtype: %s min: %s max: %s" %
(convArray.dtype.name, convArray.min(), convArray.max()))
i = Image.fromarray(convArray)
else:
i = Image.fromarray(kymograph_data)
i.show()
i.save("kymograph.png", 'PNG')
# attach the png to the image
fileAnn = conn.createFileAnnfromLocalFile("kymograph.png",
mimetype="image/png")
print "Attaching kymograph.png to image"
image.linkAnnotation(fileAnn)
message = "Tile average value: %s" % average
# client.setOutput("Message", rstring(message))
# client.setOutput("Kymograph", robject(fileAnn._obj))
# client.closeSession()
width = 10 # we were asked for '1' but this looks nicer
height = size_x
tile = (x, y, width, height)
col_data = [] # let's collect the column data for each time point
for t in range(size_t):
print("Getting data for T, tile:", t, tile)
col = pixels.getTile(z, c, t, tile)
col_data.append(col)
# ** BONUS **
# stack the numpy columns horizontally. hstack is a numpy function
kymograph_data = hstack(col_data)
print("kymograph_data", kymograph_data.shape)
name = "kymograph.png"
min_max = (kymograph_data.min(), kymograph_data.max())
scriptUtil.numpy_save_as_image(kymograph_data, min_max, int32, name)
# attach the png to the image
file_ann = conn.createFileAnnfromLocalFile(name, mimetype="image/png")
print("Attaching %s to image" % name)
image.linkAnnotation(file_ann)
message = "Tile average value: %s" % average
# Close connection:
# =================================================================
# When you are done, close the session to free up server resources.
conn.close()
ids = unwrap(client.getInput("IDs"))
images = conn.getObjects("Image", ids)
with open("selected_images_and_paths_names.csv", "w") as f:
for i in images:
print i.name
image_paths = i.getImportedImageFilePaths()
f.write(i.name)
f.write(" ")
f.write(", ".join(image_paths['client_paths']))
f.write(" ")
f.write(", ".join(image_paths['server_paths']))
f.write('\n')
file_ann = conn.createFileAnnfromLocalFile("selected_images_and_paths_names.csv", mimetype="text/csv", ns="image.names.foo")
image = conn.getObject("Image", ids[0])
image.linkAnnotation(file_ann)
# Return some value(s).
# Here, we return anything useful the script has produced.
# NB: The Insight and web clients will display the "Message" output.
msg = "Script ran OK"
client.setOutput("Message", rstring(msg))
client.setOutput("File_Annotation", robject(file_ann._obj))
client.closeSession()
tags.append([tag.textValue, desc, str(tag.id), str(owner)])
#sort the tags in descending order to allow to see duplicates
tags.sort(key=lambda tag: tag[0].islower())
#create the csv file in which to copy the tag list
with open("tags_to_check-" + str(date.today()) + ".csv", "w") as f:
f.write("tag name, description, tag ID, owner ID\n")
for t in tags:
f.write(",".join(t))
f.write('\n')
# set group to save file to. NB: hard-coded as stystem group
conn.SERVICE_OPTS.setOmeroGroup('0')
file_ann = conn.createFileAnnfromLocalFile("tags_to_check-" +
str(date.today()) + ".csv",
mimetype="text/csv",
ns="tags.to.check")
# Return some value(s).
# Here, we return anything useful the script has produced.
# NB: The Insight and web clients will display the "Message" output.
msg = "Script ran OK"
client.setOutput("Message", rstring(msg))
client.setOutput("File_Annotation", robject(file_ann._obj))
client.closeSession()
