def find_images(self):
session = self.client.getSession()
query_service = session.getQueryService()
dataset_ids = [v.id.val for v in self.project.linkedDatasetList()]
image_counts_per_dataset = self.get_image_counts_per_dataset(
query_service)
for dataset_id in dataset_ids:
if image_counts_per_dataset.get(dataset_id, 0) < 1:
log.info(
'Skipping Dataset:%d Project:%d, contains no Images!' %
(dataset_id, self.project.id.val))
continue
offset = 0
count = limit = 100
params = ParametersI().addId(dataset_id).page(offset, limit)
while count == limit:
t0 = time.time()
images = query_service.findAllByQuery(QUERY_IMAGES, params,
{'omero.group': '-1'})
log.info('Found %d Images in Dataset:%d Project:%d (%dms)' %
(len(images), dataset_id, self.project.id.val,
(time.time() - t0) * 1000))
count = len(images)
offset += count
params.page(offset, limit)
for image in images:
yield image
def test_pagination(self, gateway, dtype):
"""Query should paginate."""
offset = 1
limit = 100
p = ParametersI()
p.page(offset, limit)
# Test using 'params' argument
with_params = gateway.buildQuery(dtype, params=p)
# Test using 'opts' dictionary
opts = {'offset': offset, 'limit': limit}
with_opts = gateway.buildQuery(dtype, opts=opts)
for result in [with_params, with_opts]:
query, params, wrapper = result
assert isinstance(query, str)
assert isinstance(params, Parameters)
assert isinstance(wrapper(), BlitzObjectWrapper)
assert params.theFilter.offset.val == offset
assert params.theFilter.limit.val == limit
def find_wells(self):
session = self.client.getSession()
query_service = session.getQueryService()
offset = 0
count = limit = 100
params = ParametersI().addId(self.plate.id.val).page(offset, limit)
while count == limit:
t0 = time.time()
wells = query_service.findAllByQuery(QUERY_WELLS, params,
{'omero.group': '-1'})
log.info('Found %d Wells in Plate:%d (%dms)' %
(len(wells), self.plate.id.val,
(time.time() - t0) * 1000))
count = len(wells)
offset += count
params.page(offset, limit)
for well in wells:
yield well
def test_groups_experimenters(self, user1):
"""
Test listing groups.
We simply list existing Groups since we have no way to filter
and show only those created in the test.
"""
conn = get_connection(user1)
user_name = conn.getUser().getName()
django_client = self.new_django_client(user_name, user_name)
request_url = reverse(
'api_experimentergroups',
kwargs={'api_version': api_settings.API_VERSIONS[-1]})
data = {'limit': 10}
rsp = get_json(django_client, request_url, data)
groups_json = rsp['data']
query = """select obj from ExperimenterGroup as obj order by
lower(obj.name), obj.id"""
params = ParametersI()
params.page(0, 10)
groups = conn.getQueryService().findAllByQuery(query, params)
assert_objects(conn, groups_json, groups, dtype="ExperimenterGroup")
# Check experimenters_url for all groups above
for group_json in groups_json:
# Check we can follow link to Experimenters for first Group
expimenters_url = group_json["url:experimenters"]
rsp = get_json(django_client, expimenters_url)
exps_json = rsp['data']
exp_ids = [e['@id'] for e in exps_json]
# Check if eids are same for group (won't be ordered)
grp = conn.getObject("ExperimenterGroup", group_json['@id'])
eids = [
link.child.id.val for link in grp.copyGroupExperimenterMap()
]
assert set(eids) == set(exp_ids)
if len(exp_ids) > 0:
assert_objects(conn, exps_json, exp_ids, dtype="Experimenter")
def test_experimenters_groups(self, user1):
"""
Test listing experimenters.
We simply list existing Experimenters since we have no way to filter
and show only those created in the test.
"""
conn = get_connection(user1)
user_name = conn.getUser().getName()
django_client = self.new_django_client(user_name, user_name)
request_url = reverse(
'api_experimenters',
kwargs={'api_version': api_settings.API_VERSIONS[-1]})
data = {'limit': 10}
rsp = get_json(django_client, request_url, data)
exp_json = rsp['data']
query = """select obj from Experimenter as obj order by
lower(obj.lastName), lower(obj.firstName), obj.id"""
params = ParametersI()
params.page(0, 10)
exps = conn.getQueryService().findAllByQuery(query, params)
assert_objects(conn, exp_json, exps, dtype="Experimenter")
# Check we can follow link to Groups for first Experimenter
groups_url = exp_json[0]["url:experimentergroups"]
rsp = get_json(django_client, groups_url)
groups_json = rsp['data']
grp_ids = [g['@id'] for g in groups_json]
# Check if gids are same for experimenter (won't be ordered)
gids = [g.id for g in conn.getOtherGroups(exp_json[0]['@id'])]
assert set(gids) == set(grp_ids)
assert_objects(conn, groups_json, grp_ids, dtype="ExperimenterGroup")
def stat_plates(query, screen, images=False):
params = ParametersI()
params.addString("screen", screen)
obj = query.findByQuery((
"select s from Screen s "
"where s.name = :screen"), params)
if not obj:
raise Exception("unknown screen: %s" % screen)
if images:
q = ("select %s from Image i "
"join i.wellSamples ws join ws.well w "
"join w.plate p join p.screenLinks sl "
"join sl.parent s where s.name = :screen")
limit = 1000
found = 0
count = unwrap(query.projection(
q % "count(distinct i.id)", params
))[0][0]
print >>stderr, count
params.page(0, limit)
q = q % "distinct i.id"
q = "%s order by i.id" % q
while count > 0:
rv = unwrap(query.projection(q, params))
count -= len(rv)
found += len(rv)
params.page(found, limit)
for x in rv:
yield x[0]
return
plates = glob(join(screen, "plates", "*"))
plates = map(basename, plates)
tb = TableBuilder("Plate")
tb.cols(["PID", "Wells", "Images"])
well_count = 0
image_count = 0
for plate in plates:
params.addString("plate", plate)
rv = unwrap(query.projection((
"select p.id, count(distinct w.id), count(distinct i.id)"
" from Screen s "
"left outer join s.plateLinks spl join spl.child as p "
"left outer join p.wells as w "
"left outer join w.wellSamples as ws "
"left outer join ws.image as i "
"where s.name = :screen and p.name = :plate "
"group by p.id"), params))
if not rv:
tb.row(plate, "MISSING", "", "")
else:
for x in rv:
plate_id, wells, images = x
well_count += wells
image_count += images
tb.row(plate, plate_id, wells, images)
tb.row("Total", "", well_count, image_count)
print str(tb.build())
def sets(self, args):
"""List filesets by various criteria
Filesets are bundles of original data imported into OMERO 5 and above
which represent 1 *or more* images.
Examples:
bin/omero fs sets --order=newest # Default
bin/omero fs sets --order=oldest
bin/omero fs sets --order=largest
bin/omero fs sets --without-images # Corrupt filesets
bin/omero fs sets --with-transfer=ln_s # Symlinked filesets
bin/omero fs sets --check # Proof the checksums
"""
from omero.constants.namespaces import NSFILETRANSFER
from omero_sys_ParametersI import ParametersI
from omero.rtypes import unwrap
client = self.ctx.conn(args)
service = client.sf.getQueryService()
admin = client.sf.getAdminService()
if args.check and not admin.getEventContext().isAdmin:
self.error_admin_only(fatal=True)
annselect = (
"(select ann.textValue from Fileset f4 "
"join f4.annotationLinks fal join fal.child ann "
"where f4.id = fs.id and ann.ns =:ns) ")
select = (
"select fs.id, fs.templatePrefix, "
"(select size(f2.images) from Fileset f2 "
"where f2.id = fs.id),"
"(select size(f3.usedFiles) from Fileset f3 "
"where f3.id = fs.id),") \
+ annselect
query1 = (
"from Fileset fs "
"where 1 = 1 ")
query2 = (
"group by fs.id, fs.templatePrefix ")
if args.order:
if args.order == "newest":
query2 += "order by fs.id desc"
elif args.order == "oldest":
query2 += "order by fs.id asc"
elif args.order == "prefix":
query2 += "order by fs.templatePrefix"
if args.without_images:
query = "%s and fs.images is empty %s" % (query1, query2)
else:
query = "%s %s" % (query1, query2)
params = ParametersI()
params.addString("ns", NSFILETRANSFER)
count = service.projection("select count(fs) " + query1,
params, {"omero.group": "-1"})
params.page(args.offset, args.limit)
objs = service.projection(select + query,
params, {"omero.group": "-1"})
objs = unwrap(objs)
count = unwrap(count)[0][0]
cols = ["Id", "Prefix", "Images", "Files", "Transfer"]
if args.check:
cols.append("Check")
tb = self._table(args)
tb.cols(cols)
tb.page(args.offset, args.limit, count)
# Map any requested transfers as well
if args.with_transfer:
restricted = [TRANSFERS.get(x, x) for x in args.with_transfer[0]]
else:
restricted = None
for idx, obj in enumerate(objs):
# Map the transfer name to the CLI symbols
ns = obj[-1]
if ns is None:
ns = ""
elif ns in TRANSFERS:
ns = TRANSFERS[ns]
obj[-1] = ns
# Filter based on the ns symbols
if restricted and ns not in restricted:
continue
# Now perform check if required
if args.check:
from omero.grid import RawAccessRequest
desc, prx = self.get_managed_repo(client)
ctx = client.getContext(group=-1)
check_params = ParametersI()
check_params.addId(obj[0])
rows = service.projection((
"select h.value, f.hash, "
"f.path || '/' || f.name "
"from Fileset fs join fs.usedFiles uf "
"join uf.originalFile f join f.hasher h "
"where fs.id = :id"
), check_params, ctx)
if not rows:
obj.append("Empty")
err = None
for row in rows:
row = unwrap(row)
raw = RawAccessRequest()
raw.repoUuid = desc.hash.val
raw.command = "checksum"
raw.args = map(str, row)
try:
cb = client.submit(raw)
cb.close(True)
except CmdError, ce:
err = ce.err
self.ctx.dbg(err)
if err:
obj.append("ERROR!")
elif rows:
obj.append("OK")
tb.row(idx, *tuple(obj))
def sets(self, args):
"""List filesets by various criteria
Filesets are bundles of original data imported into OMERO 5 and above
which represent 1 *or more* images.
Examples:
bin/omero fs sets --order=newest # Default
bin/omero fs sets --order=oldest
bin/omero fs sets --order=largest
bin/omero fs sets --without-images # Corrupt filesets
bin/omero fs sets --with-transfer=ln_s # Symlinked filesets
bin/omero fs sets --check # Proof the checksums
"""
from omero.constants.namespaces import NSFILETRANSFER
from omero_sys_ParametersI import ParametersI
from omero.rtypes import unwrap
from omero.cmd import OK
client = self.ctx.conn(args)
service = client.sf.getQueryService()
select = (
"select fs.id, fs.templatePrefix, "
"(select size(f2.images) from Fileset f2 where f2.id = fs.id),"
"(select size(f3.usedFiles) from Fileset f3 where f3.id = fs.id),"
"ann.textValue ")
query1 = (
"from Fileset fs "
"left outer join fs.annotationLinks fal "
"left outer join fal.child ann "
"where (ann is null or ann.ns = :ns) ")
query2 = (
"group by fs.id, fs.templatePrefix, ann.textValue ")
if args.order:
if args.order == "newest":
query2 += "order by fs.id desc"
elif args.order == "oldest":
query2 += "order by fs.id asc"
elif args.order == "prefix":
query2 += "order by fs.templatePrefix"
if args.without_images:
query = "%s and fs.images is empty %s" % (query1, query2)
else:
query = "%s %s" % (query1, query2)
params = ParametersI()
params.addString("ns", NSFILETRANSFER)
count = service.projection("select count(fs) " + query1,
params, {"omero.group": "-1"})
params.page(args.offset, args.limit)
objs = service.projection(select + query,
params, {"omero.group": "-1"})
objs = unwrap(objs)
count = unwrap(count)[0][0]
cols = ["Id", "Prefix", "Images", "Files", "Transfer"]
if args.check:
cols.append("Check")
tb = self._table(args)
tb.cols(cols)
tb.page(args.offset, args.limit, count)
for idx, obj in enumerate(objs):
# Map the transfer name to the CLI symbols
ns = obj[-1]
if ns is None:
ns = ""
elif ns in TRANSFERS:
ns = TRANSFERS[ns]
obj[-1] = ns
# Map any requested transfers as well
allowed = args.with_transfer is not None \
and args.with_transfer or []
for idx, x in enumerate(allowed):
x = x[0] # Strip argparse wrapper
x = TRANSFERS.get(x, x) # map
allowed[idx] = x
# Filter based on the ns symbols
if allowed:
if ns not in allowed:
continue
# Now perform check if required
if args.check:
from omero.grid import RawAccessRequest
desc, prx = self.get_managed_repo(client)
ctx = client.getContext(group=-1)
check_params = ParametersI()
check_params.addId(obj[0])
rows = service.projection((
"select h.value, f.hash, "
"f.path || '/' || f.name "
"from Fileset fs join fs.usedFiles uf "
"join uf.originalFile f join f.hasher h "
"where fs.id = :id"
), check_params, ctx)
if not rows:
obj.append("Empty")
err = None
for row in rows:
row = unwrap(row)
raw = RawAccessRequest()
raw.repoUuid = desc.hash.val
raw.command = "checksum"
raw.args = map(str, row)
cb = client.submit(raw)
try:
rsp = cb.getResponse()
if not isinstance(rsp, OK):
err = rsp
break
finally:
cb.close(True)
if err:
obj.append("ERROR!")
elif rows:
obj.append("OK")
tb.row(idx, *tuple(obj))
self.ctx.out(str(tb.build()))
def sets(self, args):
"""List filesets by various criteria
Filesets are bundles of original data imported into OMERO 5 and above
which represent 1 *or more* images.
Examples:
bin/omero fs sets --order=newest # Default
bin/omero fs sets --order=oldest
bin/omero fs sets --order=largest
bin/omero fs sets --without-images # Corrupt filesets
bin/omero fs sets --with-transfer=ln_s # Symlinked filesets
bin/omero fs sets --check # Proof the checksums
"""
from omero.constants.namespaces import NSFILETRANSFER
from omero_sys_ParametersI import ParametersI
from omero.rtypes import unwrap
client = self.ctx.conn(args)
service = client.sf.getQueryService()
admin = client.sf.getAdminService()
if args.check and not admin.getEventContext().isAdmin:
self.error_admin_only(fatal=True)
annselect = (
"(select ann.textValue from Fileset f4 "
"join f4.annotationLinks fal join fal.child ann "
"where f4.id = fs.id and ann.ns =:ns) "
)
select = (
"select fs.id, fs.templatePrefix, "
"(select size(f2.images) from Fileset f2 "
"where f2.id = fs.id),"
"(select size(f3.usedFiles) from Fileset f3 "
"where f3.id = fs.id),"
) + annselect
query1 = "from Fileset fs " "where 1 = 1 "
query2 = "group by fs.id, fs.templatePrefix "
if args.order:
if args.order == "newest":
query2 += "order by fs.id desc"
elif args.order == "oldest":
query2 += "order by fs.id asc"
elif args.order == "prefix":
query2 += "order by fs.templatePrefix"
if args.without_images:
query = "%s and fs.images is empty %s" % (query1, query2)
else:
query = "%s %s" % (query1, query2)
params = ParametersI()
params.addString("ns", NSFILETRANSFER)
count = service.projection("select count(fs) " + query1, params, {"omero.group": "-1"})
params.page(args.offset, args.limit)
objs = service.projection(select + query, params, {"omero.group": "-1"})
objs = unwrap(objs)
count = unwrap(count)[0][0]
cols = ["Id", "Prefix", "Images", "Files", "Transfer"]
if args.check:
cols.append("Check")
tb = self._table(args)
tb.cols(cols)
tb.page(args.offset, args.limit, count)
# Map any requested transfers as well
if args.with_transfer:
restricted = [TRANSFERS.get(x, x) for x in args.with_transfer[0]]
else:
restricted = None
for idx, obj in enumerate(objs):
# Map the transfer name to the CLI symbols
ns = obj[-1]
if ns is None:
ns = ""
elif ns in TRANSFERS:
ns = TRANSFERS[ns]
obj[-1] = ns
# Filter based on the ns symbols
if restricted and ns not in restricted:
continue
# Now perform check if required
if args.check:
from omero.grid import RawAccessRequest
desc, prx = self.get_managed_repo(client)
ctx = client.getContext(group=-1)
check_params = ParametersI()
check_params.addId(obj[0])
rows = service.projection(
(
"select h.value, f.hash, "
"f.path || '/' || f.name "
"from Fileset fs join fs.usedFiles uf "
"join uf.originalFile f join f.hasher h "
"where fs.id = :id"
),
check_params,
ctx,
)
if not rows:
obj.append("Empty")
err = None
for row in rows:
row = unwrap(row)
raw = RawAccessRequest()
raw.repoUuid = desc.hash.val
raw.command = "checksum"
raw.args = map(str, row)
try:
cb = client.submit(raw)
cb.close(True)
except CmdError, ce:
err = ce.err
self.ctx.dbg(err)
if err:
obj.append("ERROR!")
elif rows:
obj.append("OK")
tb.row(idx, *tuple(obj))
def stat_plates(query, screen, images=False):
params = ParametersI()
params.addString("screen", screen)
obj = query.findByQuery(("select s from Screen s "
"where s.name = :screen"), params)
if not obj:
raise Exception("unknown screen: %s" % screen)
if images:
q = ("select %s from Image i "
"join i.wellSamples ws join ws.well w "
"join w.plate p join p.screenLinks sl "
"join sl.parent s where s.name = :screen")
limit = 1000
found = 0
count = unwrap(query.projection(q % "count(distinct i.id)",
params))[0][0]
print >> stderr, count
params.page(0, limit)
q = q % "distinct i.id"
q = "%s order by i.id" % q
while count > 0:
rv = unwrap(query.projection(q, params))
count -= len(rv)
found += len(rv)
params.page(found, limit)
for x in rv:
yield x[0]
return
plates = glob(join(screen, "plates", "*"))
plates = map(basename, plates)
tb = TableBuilder("Plate")
tb.cols(["PID", "Wells", "Images"])
well_count = 0
image_count = 0
for plate in plates:
params.addString("plate", plate)
rv = unwrap(
query.projection(
("select p.id, count(distinct w.id), count(distinct i.id)"
" from Screen s "
"left outer join s.plateLinks spl join spl.child as p "
"left outer join p.wells as w "
"left outer join w.wellSamples as ws "
"left outer join ws.image as i "
"where s.name = :screen and p.name = :plate "
"group by p.id"), params))
if not rv:
tb.row(plate, "MISSING", "", "")
else:
for x in rv:
plate_id, wells, images = x
well_count += wells
image_count += images
tb.row(plate, plate_id, wells, images)
tb.row("Total", "", well_count, image_count)
print str(tb.build())
