Example #1
0
    def find_images(self):
        session = self.client.getSession()
        query_service = session.getQueryService()
        dataset_ids = [v.id.val for v in self.project.linkedDatasetList()]

        image_counts_per_dataset = self.get_image_counts_per_dataset(
            query_service)

        for dataset_id in dataset_ids:
            if image_counts_per_dataset.get(dataset_id, 0) < 1:
                log.info(
                    'Skipping Dataset:%d Project:%d, contains no Images!' %
                    (dataset_id, self.project.id.val))
                continue

            offset = 0
            count = limit = 100
            params = ParametersI().addId(dataset_id).page(offset, limit)
            while count == limit:
                t0 = time.time()
                images = query_service.findAllByQuery(QUERY_IMAGES, params,
                                                      {'omero.group': '-1'})
                log.info('Found %d Images in Dataset:%d Project:%d (%dms)' %
                         (len(images), dataset_id, self.project.id.val,
                          (time.time() - t0) * 1000))
                count = len(images)
                offset += count
                params.page(offset, limit)
                for image in images:
                    yield image
 def test_pagination(self, gateway, dtype):
     """Query should paginate."""
     offset = 1
     limit = 100
     p = ParametersI()
     p.page(offset, limit)
     # Test using 'params' argument
     with_params = gateway.buildQuery(dtype, params=p)
     # Test using 'opts' dictionary
     opts = {'offset': offset, 'limit': limit}
     with_opts = gateway.buildQuery(dtype, opts=opts)
     for result in [with_params, with_opts]:
         query, params, wrapper = result
         assert isinstance(query, str)
         assert isinstance(params, Parameters)
         assert isinstance(wrapper(), BlitzObjectWrapper)
         assert params.theFilter.offset.val == offset
         assert params.theFilter.limit.val == limit
Example #3
0
 def find_wells(self):
     session = self.client.getSession()
     query_service = session.getQueryService()
     offset = 0
     count = limit = 100
     params = ParametersI().addId(self.plate.id.val).page(offset, limit)
     while count == limit:
         t0 = time.time()
         wells = query_service.findAllByQuery(QUERY_WELLS, params,
                                              {'omero.group': '-1'})
         log.info('Found %d Wells in Plate:%d (%dms)' %
                  (len(wells), self.plate.id.val,
                   (time.time() - t0) * 1000))
         count = len(wells)
         offset += count
         params.page(offset, limit)
         for well in wells:
             yield well
Example #4
0
    def test_groups_experimenters(self, user1):
        """
        Test listing groups.

        We simply list existing Groups since we have no way to filter
        and show only those created in the test.
        """
        conn = get_connection(user1)
        user_name = conn.getUser().getName()
        django_client = self.new_django_client(user_name, user_name)
        request_url = reverse(
            'api_experimentergroups',
            kwargs={'api_version': api_settings.API_VERSIONS[-1]})
        data = {'limit': 10}
        rsp = get_json(django_client, request_url, data)
        groups_json = rsp['data']

        query = """select obj from ExperimenterGroup as obj order by
                   lower(obj.name), obj.id"""
        params = ParametersI()
        params.page(0, 10)
        groups = conn.getQueryService().findAllByQuery(query, params)

        assert_objects(conn, groups_json, groups, dtype="ExperimenterGroup")

        # Check experimenters_url for all groups above
        for group_json in groups_json:
            # Check we can follow link to Experimenters for first Group
            expimenters_url = group_json["url:experimenters"]
            rsp = get_json(django_client, expimenters_url)
            exps_json = rsp['data']
            exp_ids = [e['@id'] for e in exps_json]

            # Check if eids are same for group (won't be ordered)
            grp = conn.getObject("ExperimenterGroup", group_json['@id'])
            eids = [
                link.child.id.val for link in grp.copyGroupExperimenterMap()
            ]
            assert set(eids) == set(exp_ids)

            if len(exp_ids) > 0:
                assert_objects(conn, exps_json, exp_ids, dtype="Experimenter")
Example #5
0
    def test_experimenters_groups(self, user1):
        """
        Test listing experimenters.

        We simply list existing Experimenters since we have no way to filter
        and show only those created in the test.
        """
        conn = get_connection(user1)
        user_name = conn.getUser().getName()
        django_client = self.new_django_client(user_name, user_name)
        request_url = reverse(
            'api_experimenters',
            kwargs={'api_version': api_settings.API_VERSIONS[-1]})
        data = {'limit': 10}
        rsp = get_json(django_client, request_url, data)
        exp_json = rsp['data']

        query = """select obj from Experimenter as obj order by
                   lower(obj.lastName), lower(obj.firstName), obj.id"""
        params = ParametersI()
        params.page(0, 10)
        exps = conn.getQueryService().findAllByQuery(query, params)

        assert_objects(conn, exp_json, exps, dtype="Experimenter")

        # Check we can follow link to Groups for first Experimenter
        groups_url = exp_json[0]["url:experimentergroups"]
        rsp = get_json(django_client, groups_url)
        groups_json = rsp['data']
        grp_ids = [g['@id'] for g in groups_json]

        # Check if gids are same for experimenter (won't be ordered)
        gids = [g.id for g in conn.getOtherGroups(exp_json[0]['@id'])]
        assert set(gids) == set(grp_ids)

        assert_objects(conn, groups_json, grp_ids, dtype="ExperimenterGroup")
Example #6
0
def stat_plates(query, screen, images=False):

    params = ParametersI()
    params.addString("screen", screen)

    obj = query.findByQuery((
        "select s from Screen s "
        "where s.name = :screen"), params)

    if not obj:
        raise Exception("unknown screen: %s" % screen)

    if images:
        q = ("select %s from Image i "
             "join i.wellSamples ws join ws.well w "
             "join w.plate p join p.screenLinks sl "
             "join sl.parent s where s.name = :screen")

        limit = 1000
        found = 0
        count = unwrap(query.projection(
            q % "count(distinct i.id)", params
        ))[0][0]
        print >>stderr, count
        params.page(0, limit)

        q = q % "distinct i.id"
        q = "%s order by i.id" % q
        while count > 0:
            rv = unwrap(query.projection(q, params))
            count -= len(rv)
            found += len(rv)
            params.page(found, limit)
            for x in rv:
                yield x[0]
        return

    plates = glob(join(screen, "plates", "*"))
    plates = map(basename, plates)

    tb = TableBuilder("Plate")
    tb.cols(["PID", "Wells", "Images"])

    well_count = 0
    image_count = 0
    for plate in plates:
        params.addString("plate", plate)
        rv = unwrap(query.projection((
            "select p.id, count(distinct w.id), count(distinct i.id)"
            "  from Screen s "
            "left outer join s.plateLinks spl join spl.child as p "
            "left outer join p.wells as w "
            "left outer join w.wellSamples as ws "
            "left outer join ws.image as i "
            "where s.name = :screen and p.name = :plate "
            "group by p.id"), params))
        if not rv:
            tb.row(plate, "MISSING", "", "")
        else:
            for x in rv:
                plate_id, wells, images = x
                well_count += wells
                image_count += images
                tb.row(plate, plate_id, wells, images)
    tb.row("Total", "", well_count, image_count)
    print str(tb.build())
Example #7
0
    def sets(self, args):
        """List filesets by various criteria

Filesets are bundles of original data imported into OMERO 5 and above
which represent 1 *or more* images.

Examples:

    bin/omero fs sets --order=newest        # Default
    bin/omero fs sets --order=oldest
    bin/omero fs sets --order=largest
    bin/omero fs sets --without-images      # Corrupt filesets
    bin/omero fs sets --with-transfer=ln_s  # Symlinked filesets
    bin/omero fs sets --check               # Proof the checksums
        """

        from omero.constants.namespaces import NSFILETRANSFER
        from omero_sys_ParametersI import ParametersI
        from omero.rtypes import unwrap

        client = self.ctx.conn(args)
        service = client.sf.getQueryService()
        admin = client.sf.getAdminService()

        if args.check and not admin.getEventContext().isAdmin:
            self.error_admin_only(fatal=True)

        annselect = (
            "(select ann.textValue from Fileset f4 "
            "join f4.annotationLinks fal join fal.child ann "
            "where f4.id = fs.id and ann.ns =:ns) ")
        select = (
            "select fs.id, fs.templatePrefix, "
            "(select size(f2.images) from Fileset f2 "
            "where f2.id = fs.id),"
            "(select size(f3.usedFiles) from Fileset f3 "
            "where f3.id = fs.id),") \
            + annselect
        query1 = (
            "from Fileset fs "
            "where 1 = 1 ")
        query2 = (
            "group by fs.id, fs.templatePrefix ")

        if args.order:
            if args.order == "newest":
                query2 += "order by fs.id desc"
            elif args.order == "oldest":
                query2 += "order by fs.id asc"
            elif args.order == "prefix":
                query2 += "order by fs.templatePrefix"

        if args.without_images:
            query = "%s and fs.images is empty %s" % (query1, query2)
        else:
            query = "%s %s" % (query1, query2)

        params = ParametersI()
        params.addString("ns", NSFILETRANSFER)
        count = service.projection("select count(fs) " + query1,
                                   params, {"omero.group": "-1"})

        params.page(args.offset, args.limit)
        objs = service.projection(select + query,
                                  params, {"omero.group": "-1"})
        objs = unwrap(objs)
        count = unwrap(count)[0][0]

        cols = ["Id", "Prefix", "Images", "Files", "Transfer"]
        if args.check:
            cols.append("Check")
        tb = self._table(args)
        tb.cols(cols)
        tb.page(args.offset, args.limit, count)

        # Map any requested transfers as well
        if args.with_transfer:
            restricted = [TRANSFERS.get(x, x) for x in args.with_transfer[0]]
        else:
            restricted = None

        for idx, obj in enumerate(objs):

            # Map the transfer name to the CLI symbols
            ns = obj[-1]
            if ns is None:
                ns = ""
            elif ns in TRANSFERS:
                ns = TRANSFERS[ns]
            obj[-1] = ns

            # Filter based on the ns symbols
            if restricted and ns not in restricted:
                continue

            # Now perform check if required
            if args.check:
                from omero.grid import RawAccessRequest
                desc, prx = self.get_managed_repo(client)
                ctx = client.getContext(group=-1)
                check_params = ParametersI()
                check_params.addId(obj[0])
                rows = service.projection((
                    "select h.value, f.hash, "
                    "f.path || '/' || f.name "
                    "from Fileset fs join fs.usedFiles uf "
                    "join uf.originalFile f join f.hasher h "
                    "where fs.id = :id"
                    ), check_params, ctx)

                if not rows:
                    obj.append("Empty")

                err = None
                for row in rows:
                    row = unwrap(row)
                    raw = RawAccessRequest()
                    raw.repoUuid = desc.hash.val
                    raw.command = "checksum"
                    raw.args = map(str, row)
                    try:
                        cb = client.submit(raw)
                        cb.close(True)
                    except CmdError, ce:
                        err = ce.err
                        self.ctx.dbg(err)

                if err:
                    obj.append("ERROR!")
                elif rows:
                    obj.append("OK")

            tb.row(idx, *tuple(obj))
Example #8
0
    def sets(self, args):
        """List filesets by various criteria

Filesets are bundles of original data imported into OMERO 5 and above
which represent 1 *or more* images.

Examples:

    bin/omero fs sets --order=newest        # Default
    bin/omero fs sets --order=oldest
    bin/omero fs sets --order=largest
    bin/omero fs sets --without-images      # Corrupt filesets
    bin/omero fs sets --with-transfer=ln_s  # Symlinked filesets
    bin/omero fs sets --check               # Proof the checksums
        """

        from omero.constants.namespaces import NSFILETRANSFER
        from omero_sys_ParametersI import ParametersI
        from omero.rtypes import unwrap
        from omero.cmd import OK

        client = self.ctx.conn(args)
        service = client.sf.getQueryService()

        select = (
            "select fs.id, fs.templatePrefix, "
            "(select size(f2.images) from Fileset f2 where f2.id = fs.id),"
            "(select size(f3.usedFiles) from Fileset f3 where f3.id = fs.id),"
            "ann.textValue ")
        query1 = (
            "from Fileset fs "
            "left outer join fs.annotationLinks fal "
            "left outer join fal.child ann "
            "where (ann is null or ann.ns = :ns) ")
        query2 = (
            "group by fs.id, fs.templatePrefix, ann.textValue ")

        if args.order:
            if args.order == "newest":
                query2 += "order by fs.id desc"
            elif args.order == "oldest":
                query2 += "order by fs.id asc"
            elif args.order == "prefix":
                query2 += "order by fs.templatePrefix"

        if args.without_images:
            query = "%s and fs.images is empty %s" % (query1, query2)
        else:
            query = "%s %s" % (query1, query2)

        params = ParametersI()
        params.addString("ns", NSFILETRANSFER)
        count = service.projection("select count(fs) " + query1,
                                   params, {"omero.group": "-1"})

        params.page(args.offset, args.limit)
        objs = service.projection(select + query,
                                  params, {"omero.group": "-1"})
        objs = unwrap(objs)
        count = unwrap(count)[0][0]

        cols = ["Id", "Prefix", "Images", "Files", "Transfer"]
        if args.check:
            cols.append("Check")
        tb = self._table(args)
        tb.cols(cols)
        tb.page(args.offset, args.limit, count)
        for idx, obj in enumerate(objs):

            # Map the transfer name to the CLI symbols
            ns = obj[-1]
            if ns is None:
                ns = ""
            elif ns in TRANSFERS:
                ns = TRANSFERS[ns]
            obj[-1] = ns

            # Map any requested transfers as well
            allowed = args.with_transfer is not None \
                and args.with_transfer or []
            for idx, x in enumerate(allowed):
                x = x[0]  # Strip argparse wrapper
                x = TRANSFERS.get(x, x)  # map
                allowed[idx] = x

            # Filter based on the ns symbols
            if allowed:
                if ns not in allowed:
                    continue

            # Now perform check if required
            if args.check:
                from omero.grid import RawAccessRequest
                desc, prx = self.get_managed_repo(client)
                ctx = client.getContext(group=-1)
                check_params = ParametersI()
                check_params.addId(obj[0])
                rows = service.projection((
                    "select h.value, f.hash, "
                    "f.path || '/' || f.name "
                    "from Fileset fs join fs.usedFiles uf "
                    "join uf.originalFile f join f.hasher h "
                    "where fs.id = :id"
                    ), check_params, ctx)

                if not rows:
                    obj.append("Empty")

                err = None
                for row in rows:
                    row = unwrap(row)
                    raw = RawAccessRequest()
                    raw.repoUuid = desc.hash.val
                    raw.command = "checksum"
                    raw.args = map(str, row)
                    cb = client.submit(raw)
                    try:
                        rsp = cb.getResponse()
                        if not isinstance(rsp, OK):
                            err = rsp
                            break
                    finally:
                        cb.close(True)

                if err:
                    obj.append("ERROR!")
                elif rows:
                    obj.append("OK")

            tb.row(idx, *tuple(obj))
        self.ctx.out(str(tb.build()))
Example #9
0
    def sets(self, args):
        """List filesets by various criteria

Filesets are bundles of original data imported into OMERO 5 and above
which represent 1 *or more* images.

Examples:

    bin/omero fs sets --order=newest        # Default
    bin/omero fs sets --order=oldest
    bin/omero fs sets --order=largest
    bin/omero fs sets --without-images      # Corrupt filesets
    bin/omero fs sets --with-transfer=ln_s  # Symlinked filesets
    bin/omero fs sets --check               # Proof the checksums
        """

        from omero.constants.namespaces import NSFILETRANSFER
        from omero_sys_ParametersI import ParametersI
        from omero.rtypes import unwrap

        client = self.ctx.conn(args)
        service = client.sf.getQueryService()
        admin = client.sf.getAdminService()

        if args.check and not admin.getEventContext().isAdmin:
            self.error_admin_only(fatal=True)

        annselect = (
            "(select ann.textValue from Fileset f4 "
            "join f4.annotationLinks fal join fal.child ann "
            "where f4.id = fs.id and ann.ns =:ns) "
        )
        select = (
            "select fs.id, fs.templatePrefix, "
            "(select size(f2.images) from Fileset f2 "
            "where f2.id = fs.id),"
            "(select size(f3.usedFiles) from Fileset f3 "
            "where f3.id = fs.id),"
        ) + annselect
        query1 = "from Fileset fs " "where 1 = 1 "
        query2 = "group by fs.id, fs.templatePrefix "

        if args.order:
            if args.order == "newest":
                query2 += "order by fs.id desc"
            elif args.order == "oldest":
                query2 += "order by fs.id asc"
            elif args.order == "prefix":
                query2 += "order by fs.templatePrefix"

        if args.without_images:
            query = "%s and fs.images is empty %s" % (query1, query2)
        else:
            query = "%s %s" % (query1, query2)

        params = ParametersI()
        params.addString("ns", NSFILETRANSFER)
        count = service.projection("select count(fs) " + query1, params, {"omero.group": "-1"})

        params.page(args.offset, args.limit)
        objs = service.projection(select + query, params, {"omero.group": "-1"})
        objs = unwrap(objs)
        count = unwrap(count)[0][0]

        cols = ["Id", "Prefix", "Images", "Files", "Transfer"]
        if args.check:
            cols.append("Check")
        tb = self._table(args)
        tb.cols(cols)
        tb.page(args.offset, args.limit, count)

        # Map any requested transfers as well
        if args.with_transfer:
            restricted = [TRANSFERS.get(x, x) for x in args.with_transfer[0]]
        else:
            restricted = None

        for idx, obj in enumerate(objs):

            # Map the transfer name to the CLI symbols
            ns = obj[-1]
            if ns is None:
                ns = ""
            elif ns in TRANSFERS:
                ns = TRANSFERS[ns]
            obj[-1] = ns

            # Filter based on the ns symbols
            if restricted and ns not in restricted:
                continue

            # Now perform check if required
            if args.check:
                from omero.grid import RawAccessRequest

                desc, prx = self.get_managed_repo(client)
                ctx = client.getContext(group=-1)
                check_params = ParametersI()
                check_params.addId(obj[0])
                rows = service.projection(
                    (
                        "select h.value, f.hash, "
                        "f.path || '/' || f.name "
                        "from Fileset fs join fs.usedFiles uf "
                        "join uf.originalFile f join f.hasher h "
                        "where fs.id = :id"
                    ),
                    check_params,
                    ctx,
                )

                if not rows:
                    obj.append("Empty")

                err = None
                for row in rows:
                    row = unwrap(row)
                    raw = RawAccessRequest()
                    raw.repoUuid = desc.hash.val
                    raw.command = "checksum"
                    raw.args = map(str, row)
                    try:
                        cb = client.submit(raw)
                        cb.close(True)
                    except CmdError, ce:
                        err = ce.err
                        self.ctx.dbg(err)

                if err:
                    obj.append("ERROR!")
                elif rows:
                    obj.append("OK")

            tb.row(idx, *tuple(obj))
Example #10
0
def stat_plates(query, screen, images=False):

    params = ParametersI()
    params.addString("screen", screen)

    obj = query.findByQuery(("select s from Screen s "
                             "where s.name = :screen"), params)

    if not obj:
        raise Exception("unknown screen: %s" % screen)

    if images:
        q = ("select %s from Image i "
             "join i.wellSamples ws join ws.well w "
             "join w.plate p join p.screenLinks sl "
             "join sl.parent s where s.name = :screen")

        limit = 1000
        found = 0
        count = unwrap(query.projection(q % "count(distinct i.id)",
                                        params))[0][0]
        print >> stderr, count
        params.page(0, limit)

        q = q % "distinct i.id"
        q = "%s order by i.id" % q
        while count > 0:
            rv = unwrap(query.projection(q, params))
            count -= len(rv)
            found += len(rv)
            params.page(found, limit)
            for x in rv:
                yield x[0]
        return

    plates = glob(join(screen, "plates", "*"))
    plates = map(basename, plates)

    tb = TableBuilder("Plate")
    tb.cols(["PID", "Wells", "Images"])

    well_count = 0
    image_count = 0
    for plate in plates:
        params.addString("plate", plate)
        rv = unwrap(
            query.projection(
                ("select p.id, count(distinct w.id), count(distinct i.id)"
                 "  from Screen s "
                 "left outer join s.plateLinks spl join spl.child as p "
                 "left outer join p.wells as w "
                 "left outer join w.wellSamples as ws "
                 "left outer join ws.image as i "
                 "where s.name = :screen and p.name = :plate "
                 "group by p.id"), params))
        if not rv:
            tb.row(plate, "MISSING", "", "")
        else:
            for x in rv:
                plate_id, wells, images = x
                well_count += wells
                image_count += images
                tb.row(plate, plate_id, wells, images)
    tb.row("Total", "", well_count, image_count)
    print str(tb.build())