Beispiel #1
0
    def _lookup_key(self, key_id, group_ids):
        eid = elliptics.Id(key_id.encode('utf-8'))

        lookups = []
        for group_id in group_ids:
            s = self.session.clone()
            s.set_exceptions_policy(elliptics.exceptions_policy.no_exceptions)
            s.add_groups([group_id])
            lookups.append((s.lookup(eid), eid, group_id))

        lookup_by_group = {}
        not_found_count = 0

        def set_group_lookup(lookup, group_id,
                             elapsed_time=None, end_time=None):
            global not_found_count
            if lookup.error.code:
                if lookup.error.code == -2:
                    logger.warn(
                        'Key {}: lookup returned -2, group {}/{}'.format(
                            key_id, lookup.group_id, group_id))
                    not_found_count += 1
                    return
                else:
                    raise lookup.error
            lookup_by_group[group_id] = lookup

        logger.debug('Key {}: performing lookups on groups {}'.format(
            key_id, group_ids))

        for result, eid, group_id in lookups:
            try:
                h.process_elliptics_async_result(
                    result, set_group_lookup, group_id, raise_on_error=False)
            except Exception as e:
                logger.exception(
                    'Failed to lookup key {0} on group {1}'.format(
                        eid, group_id))
                continue

        if not lookup_by_group:
            if len(lookups) == not_found_count:
                raise ValueError('key has already been removed from couple')
            else:
                raise RuntimeError('all lookups for key failed')

        return lookup_by_group
Beispiel #2
0
    def monitor_top_stats(self):
        try:
            start_ts = time.time()
            logger.info('Monitor top stats update started')

            logger.info('Before calculating routes')
            host_addrs = set(r.address
                             for r in self.session.routes.get_unique_routes())
            logger.info('Unique routes calculated')

            requests = []
            for address in host_addrs:
                session = self.session.clone()
                session.set_direct_id(address)
                logger.debug('Request for top of node {0}'.format(address))
                requests.append((session.monitor_stat(address,
                                                      self.STAT_CATEGORIES),
                                 address))

            new_top_keys = {}
            for result, address in requests:
                try:
                    h.process_elliptics_async_result(
                        result, self.update_top, new_top_keys)
                except Exception as e:
                    logger.error(
                        'Failed to request monitor_stat for node {0}: '
                        '{1}\n{2}'.format(address, e, traceback.format_exc()))
                    continue

            self.top_keys = new_top_keys

            self._try_distribute_keys()

        except Exception as e:
            logger.error('Failed to update monitor top stats: {0}\n{1}'.format(
                e, traceback.format_exc()))
            pass
        finally:
            logger.info(
                'Monitor top stats update finished, time: {0:.3f}'.format(
                    time.time() - start_ts))
            self.__tq.add_task_at(
                CacheManager.MONITOR_TOP_STATS,
                self.top_update_timer.next(),
                self.monitor_top_stats)
Beispiel #3
0
    def _key_by_dc(self, key):
        eid = elliptics.Id(key['id'].encode('utf-8'))

        lookups = []

        # add other cache groups as a source when key size is fixed
        for group_id in key['data_groups']:
            s = self.session.clone()
            s.set_exceptions_policy(elliptics.exceptions_policy.no_exceptions)
            s.add_groups([group_id])
            lookups.append((s.lookup(eid), eid, group_id))

        key_by_dc = {}
        not_found_count = 0

        def set_key_size_by_dc(lookup, group_id,
                               elapsed_time=None, end_time=None):
            global not_found_count
            if lookup.error.code:
                if lookup.error.code == -2:
                    not_found_count += 1
                    return
                else:
                    raise lookup.error

            dc = storage.groups[group_id].node_backends[0].node.host.dc
            key_by_dc[dc] = {
                'group': group_id,
                'size': lookup.size,
            }

        for result, eid, group_id in lookups:
            try:
                h.process_elliptics_async_result(
                    result, set_key_size_by_dc, group_id, raise_on_error=False)
            except Exception as e:
                logger.error(
                    'Failed to lookup key {0} on group {1}: {2}'.format(
                        eid, group_id, e))
                continue

        if len(lookups) == not_found_count:
            logger.info('Key {0}: has already been removed from couple'.format(
                key['id']))

        return key_by_dc
    def monitor_stats(self, groups=None):

        if groups:
            hosts = set((nb.node.host.addr, nb.node.port, nb.node.family)
                        for g in groups for nb in g.node_backends)
            host_addrs = [elliptics.Address(*host) for host in hosts]
        else:
            logger.info('Before calculating routes')
            host_addrs = set(r.address for r in self.__session.routes.get_unique_routes())
            logger.info('Unique routes calculated')

        requests = []
        for address in host_addrs:
            session = self.__session.clone()
            session.set_direct_id(address)
            logger.debug('Request for monitor_stat of node {0}'.format(
                address))
            requests.append((session.monitor_stat(address,
                self.MONITOR_STAT_CATEGORIES), address))

        for result, address in requests:
            try:
                h.process_elliptics_async_result(result, self.update_statistics)
            except Exception as e:
                logger.error('Failed to request monitor_stat for node {0}: '
                    '{1}\n{2}'.format(address, e, traceback.format_exc()))
                continue

        nbs = (groups and [nb for g in groups for nb in g.node_backends]
                      or storage.node_backends.keys())
        for nb in nbs:
            nb.update_statistics_status()
            nb.update_status()

        fss = (groups and set(nb.fs for nb in nbs) or storage.fs.keys())
        for fs in fss:
            fs.update_status()

        for group in groups or storage.groups.keys():
            logger.info('Updating status for group {0}'.format(group.group_id))
            group.update_status()
Beispiel #5
0
    def update_symm_groups_async(self, groups=None):

        _queue = set()

        def _get_data_groups(group):
            return group.meta['couple']

        def _get_lrc_groups(group):
            return group.meta['lrc']['groups']

        def _create_groupset_if_needed(groups, group_type, ns_id):

            for gid in groups:
                if gid not in storage.groups:
                    logger.info(
                        'Group {group} is not found, adding fake group '
                        'for groupset {groups}'.format(
                            group=gid,
                            groups=groups,
                        ))
                    storage.groups.add(gid)

            groupset_str = ':'.join((str(gid) for gid in sorted(groups)))
            if groupset_str not in storage.groupsets:
                # TODO: somehow check that couple type matches group.type
                # for all groups in couple (not very easy when metakey read
                # fails)
                logger.info(
                    'Creating groupset {groups}, group type "{group_type}"'.
                    format(
                        groups=groupset_str,
                        group_type=group_type,
                    ))
                c = storage.groupsets.add(
                    groups=(storage.groups[gid] for gid in groups),
                    group_type=group_type,
                )

                for gid in groups:
                    infrastructure.update_group_history(storage.groups[gid])

                if ns_id not in storage.namespaces:
                    logger.info('Creating storage namespace {}'.format(ns_id))
                    ns = storage.namespaces.add(ns_id)
                else:
                    ns = storage.namespaces[ns_id]

                ns.add_couple(c)
            return storage.groupsets[groupset_str]

        def _process_group_metadata(response,
                                    group,
                                    elapsed_time=None,
                                    end_time=None):
            logger.debug(
                'Cluster updating: group {0} meta key read time: {1}.{2}'.
                format(group.group_id, elapsed_time.tsec, elapsed_time.tnsec))

            if response.error.code:
                if response.error.code == errors.ELLIPTICS_NOT_FOUND:
                    # This group is some kind of uncoupled group, not an error
                    group.parse_meta(None)
                    logger.info(
                        'Group {group} has no metakey'.format(group=group))
                elif response.error.code in (
                        # Route list did not contain the group, expected error
                        errors.ELLIPTICS_GROUP_NOT_IN_ROUTE_LIST,
                        # Timeout in reading metakey from the group, expected error
                        errors.ELLIPTICS_TIMEOUT,
                ):
                    group.reset_meta()
                    logger.error(
                        'Error on updating metakey from group {group}: {error}'
                        .format(
                            group=group,
                            error=response.error.message,
                        ))
                else:
                    raise RuntimeError(response.error.mssage)

                return

            meta = response.data

            group.parse_meta(meta)

            if group.type == storage.Group.TYPE_UNCOUPLED_LRC_8_2_2_V1:
                return

            ns_id = group.meta.get('namespace')
            if ns_id is None:
                logger.error(
                    'Inconsistent meta read from group {group}, missing namespace: {meta}'
                    .format(
                        group=group,
                        meta=group.meta,
                    ))
                return

            if group.type == storage.Group.TYPE_DATA:
                groups = _get_data_groups(group)
            elif group.type == storage.Group.TYPE_LRC_8_2_2_V1:
                groups = _get_lrc_groups(group)
            elif group.type == storage.Group.TYPE_CACHE:
                groups = _get_data_groups(group)
            else:
                raise RuntimeError(
                    'Group {group_id}, unexpected type to process: {type}'.
                    format(
                        group_id=group.group_id,
                        type=group.type,
                    ))

            logger.info('Read symmetric groups from group {}: {}'.format(
                group.group_id, groups))

            for gid in groups:
                if gid != group.group_id:
                    logger.info('Scheduling update for group {}'.format(gid))
                    _queue.add(gid)

            groupset = _create_groupset_if_needed(groups, group.type, ns_id)

            if group.type == storage.Group.TYPE_LRC_8_2_2_V1:
                # TODO: this will become unnecessary when new "Couple" instance
                # is introduced
                data_groups = _get_data_groups(group)
                data_groupset = _create_groupset_if_needed(
                    data_groups, storage.Group.TYPE_DATA, ns_id)
                data_groupset.lrc822v1_groupset = groupset
                # TODO: this should point to a new "Couple" object
                groupset.couple = data_groupset
            return

        try:
            check_groups = groups or storage.groups.keys()

            results = {}
            for group in check_groups:
                session = self.__session.clone()
                session.set_exceptions_policy(
                    elliptics.exceptions_policy.no_exceptions)
                session.set_filter(elliptics.filters.all_with_ack)
                session.add_groups([group.group_id])

                logger.debug('Request to read {0} for group {1}'.format(
                    keys.SYMMETRIC_GROUPS_KEY.replace('\0', '\\0'),
                    group.group_id))
                results[group.group_id] = session.read_data(
                    keys.SYMMETRIC_GROUPS_KEY)

            jobs = {}
            if self.job_finder:
                try:
                    params = {'statuses': Job.ACTIVE_STATUSES}
                    if groups:
                        params['groups'] = [g.group_id for g in groups]
                    for job in self.job_finder.jobs(**params):
                        # TODO: this should definitely be done some other way
                        if hasattr(job, 'group'):
                            jobs[job.group] = job
                except Exception as e:
                    logger.exception(
                        'Failed to fetch pending jobs: {0}'.format(e))
                    pass

            while results:
                # TODO: Think on queue, it does not work well with lrc couples
                if _queue:
                    group_id = _queue.pop()
                    if group_id not in results:
                        continue
                    result = results.pop(group_id)
                else:
                    group_id, result = results.popitem()

                group = storage.groups[group_id]

                try:
                    h.process_elliptics_async_result(
                        result,
                        _process_group_metadata,
                        group,
                        raise_on_error=False,
                    )
                except Exception as e:
                    logger.exception(
                        'Critical error on updating metakey from group {}'.
                        format(group_id))
                    group.parse_meta(None)
                finally:
                    try:
                        group.set_active_job(jobs.get(group.group_id))
                    except Exception as e:
                        logger.exception(
                            'Failed to set group active job: {}'.format(e))
                        pass
                    try:
                        group.update_status_recursive()
                    except Exception as e:
                        logger.exception(
                            'Failed to update group {0} status: {1}'.format(
                                group, e))
                        pass

            if groups is None:
                self.update_couple_settings()
                load_manager.update(storage)
                weight_manager.update(storage)

                infrastructure.schedule_history_update()

        except Exception as e:
            logger.exception('Critical error during symmetric group update')
    def update_symm_groups_async(self, groups=None):

        _queue = set()

        def _process_group_metadata(response,
                                    group,
                                    elapsed_time=None,
                                    end_time=None):
            logger.debug(
                'Cluster updating: group {0} meta key read time: {1}.{2}'.
                format(group.group_id, elapsed_time.tsec, elapsed_time.tnsec))
            meta = response.data

            group.parse_meta(meta)
            couple = group.meta.get('couple')
            if couple is None:
                logger.error(
                    'Read symmetric groups from group {} (no couple data): {}'.
                    format(group.group_id, meta))
                return

            logger.info('Read symmetric groups from group {}: {}'.format(
                group.group_id, couple))
            for gid in couple:
                if gid != group.group_id:
                    logger.info('Scheduling update for group {}'.format(gid))
                    _queue.add(gid)

            couple_str = ':'.join((str(gid) for gid in sorted(couple)))

            logger.debug('{0} in storage.couples: {1}'.format(
                couple_str, couple_str in storage.couples))

            if couple_str not in storage.couples and couple_str not in storage.cache_couples:

                ns_id = group.meta.get('namespace')
                if ns_id is None:
                    logger.error('Inconsistent meta read from group {}, '
                                 'missing namespace: {}'.format(
                                     group, group.meta))
                    return

                if group.type == storage.Group.TYPE_DATA:
                    logger.info('Creating couple {0}'.format(couple_str))
                    for gid in couple:
                        if gid not in storage.groups:
                            logger.info(
                                'Group {} is not found adding fake group for '
                                'couple {}'.format(gid, couple))
                            storage.groups.add(gid)
                    c = storage.couples.add(
                        [storage.groups[gid] for gid in couple])
                    logger.info('Created couple {0} {1}'.format(c, repr(c)))
                elif group.type == storage.Group.TYPE_CACHE:
                    logger.info('Creating cache couple {0}'.format(couple_str))
                    c = storage.cache_couples.add(
                        [storage.groups[gid] for gid in couple])
                    logger.info('Created cache couple {0} {1}'.format(
                        c, repr(c)))
                else:
                    raise ValueError(
                        'Unknown group type for group {}: {}'.format(
                            group, group.type))

                for gid in couple:
                    infrastructure.update_group_history(storage.groups[gid])

                if ns_id not in storage.namespaces:
                    logger.info('Creating storage namespace {}'.format(ns_id))
                    ns = storage.namespaces.add(ns_id)
                else:
                    ns = storage.namespaces[ns_id]

                ns.add_couple(c)
            return

        try:
            check_groups = groups or storage.groups.keys()

            results = {}
            for group in check_groups:
                session = self.__session.clone()
                session.add_groups([group.group_id])

                logger.debug('Request to read {0} for group {1}'.format(
                    keys.SYMMETRIC_GROUPS_KEY.replace('\0', '\\0'),
                    group.group_id))
                results[group.group_id] = session.read_data(
                    keys.SYMMETRIC_GROUPS_KEY)

            jobs = {}
            if self.job_finder:
                try:
                    params = {'statuses': Job.ACTIVE_STATUSES}
                    if groups:
                        params['groups'] = [g.group_id for g in groups]
                    for job in self.job_finder.jobs(**params):
                        jobs[job.group] = job
                except Exception as e:
                    logger.exception(
                        'Failed to fetch pending jobs: {0}'.format(e))
                    pass

            while results:
                if _queue:
                    group_id = _queue.pop()
                    if group_id not in results:
                        continue
                    result = results.pop(group_id)
                else:
                    group_id, result = results.popitem()

                group = storage.groups[group_id]

                try:
                    h.process_elliptics_async_result(result,
                                                     _process_group_metadata,
                                                     group)
                except elliptics.NotFoundError as e:
                    logger.warn(
                        'Failed to read symmetric_groups from group {0}: {1}'.
                        format(group_id, e))
                    group.parse_meta(None)
                except Exception as e:
                    logger.exception(
                        'Failed to read symmetric_groups from group {0}: {1}'.
                        format(group_id, e))
                    group.parse_meta(None)
                finally:
                    try:
                        group.set_active_job(jobs.get(group.group_id))
                    except Exception as e:
                        logger.exception(
                            'Failed to set group active job: {}'.format(e))
                        pass
                    try:
                        group.update_status_recursive()
                    except Exception as e:
                        logger.exception(
                            'Failed to update group {0} status: {1}'.format(
                                group, e))
                        pass

            load_manager.update(storage)
            weight_manager.update(storage)

            infrastructure.schedule_history_update()

        except Exception as e:
            logger.exception('Critical error during symmetric group update')
    def update_symm_groups_async(self, groups=None):

        _queue = set()

        def _process_group_metadata(response, group, elapsed_time=None, end_time=None):
            logger.debug('Cluster updating: group {0} meta key read time: {1}.{2}'.format(
                group.group_id, elapsed_time.tsec, elapsed_time.tnsec))
            meta = response.data

            group.parse_meta(meta)
            couple = group.meta.get('couple')
            if couple is None:
                logger.error('Read symmetric groups from group {} (no couple data): {}'.format(
                    group.group_id, group.meta))
                return

            logger.info('Read symmetric groups from group {}: {}'.format(group.group_id, couple))
            for gid in couple:
                if gid != group.group_id:
                    logger.info('Scheduling update for group {}'.format(gid))
                    _queue.add(gid)

            couple_str = ':'.join((str(gid) for gid in sorted(couple)))

            logger.debug('{0} in storage.couples: {1}'.format(
                couple_str, couple_str in storage.couples))

            if couple_str not in storage.couples and couple_str not in storage.cache_couples:

                ns_id = group.meta.get('namespace')
                if ns_id is None:
                    logger.error('Inconsistent meta read from group {}, '
                                 'missing namespace: {}'.format(group, group.meta))
                    return

                if group.type == storage.Group.TYPE_DATA:
                    logger.info('Creating couple {0}'.format(couple_str))
                    for gid in couple:
                        if gid not in storage.groups:
                            logger.info('Group {} is not found adding fake group for '
                                        'couple {}'.format(gid, couple))
                            storage.groups.add(gid)
                    c = storage.couples.add(storage.groups[gid] for gid in couple)
                    logger.info('Created couple {0} {1}'.format(c, repr(c)))
                elif group.type == storage.Group.TYPE_CACHE:
                    logger.info('Creating cache couple {0}'.format(couple_str))
                    c = storage.cache_couples.add(storage.groups[gid] for gid in couple)
                    logger.info('Created cache couple {0} {1}'.format(c, repr(c)))
                else:
                    raise ValueError('Unknown group type for group {}: {}'.format(
                        group, group.type))

                for gid in couple:
                    infrastructure.update_group_history(storage.groups[gid])

                if ns_id not in storage.namespaces:
                    logger.info('Creating storage namespace {}'.format(ns_id))
                    ns = storage.namespaces.add(ns_id)
                else:
                    ns = storage.namespaces[ns_id]

                ns.add_couple(c)
            return

        try:
            check_groups = groups or storage.groups.keys()

            results = {}
            for group in check_groups:
                session = self.__session.clone()
                session.add_groups([group.group_id])

                logger.debug('Request to read {0} for group {1}'.format(
                    keys.SYMMETRIC_GROUPS_KEY.replace('\0', '\\0'), group.group_id))
                results[group.group_id] = session.read_data(keys.SYMMETRIC_GROUPS_KEY)

            jobs = {}
            if self.job_finder:
                try:
                    params = {'statuses': Job.ACTIVE_STATUSES}
                    if groups:
                        params['groups'] = [g.group_id for g in groups]
                    for job in self.job_finder.jobs(**params):
                        jobs[job.group] = job
                except Exception as e:
                    logger.exception('Failed to fetch pending jobs: {0}'.format(e))
                    pass

            while results:
                if _queue:
                    group_id = _queue.pop()
                    if group_id not in results:
                        continue
                    result = results.pop(group_id)
                else:
                    group_id, result = results.popitem()

                group = storage.groups[group_id]

                try:
                    h.process_elliptics_async_result(result, _process_group_metadata, group)
                except elliptics.NotFoundError as e:
                    logger.warn('Failed to read symmetric_groups from group {0}: {1}'.format(
                        group_id, e))
                    group.parse_meta(None)
                except Exception as e:
                    logger.exception('Failed to read symmetric_groups from group {0}: {1}'.format(
                        group_id, e))
                    group.parse_meta(None)
                finally:
                    try:
                        group.set_active_job(jobs.get(group.group_id))
                    except Exception as e:
                        logger.exception('Failed to set group active job: {}'.format(e))
                        pass
                    try:
                        group.update_status_recursive()
                    except Exception as e:
                        logger.exception('Failed to update group {0} status: {1}'.format(group, e))
                        pass

            if groups is None:
                load_manager.update(storage)
                weight_manager.update(storage)

                infrastructure.schedule_history_update()

        except Exception as e:
            logger.exception('Critical error during symmetric group update')
    def update_symm_groups_async(self, groups=None):

        _queue = set()
        def _process_group_metadata(response, group, elapsed_time=None, end_time=None):
            logger.debug('Cluster updating: group {0} meta key read time: {1}.{2}'.format(
                    group.group_id, elapsed_time.tsec, elapsed_time.tnsec))
            meta = response.data

            group.parse_meta(meta)
            couple = group.meta['couple']
            logger.info('Read symmetric groups from group '
                '{0}: {1}'.format(group.group_id, couple))
            for gid in couple:
                if gid != group.group_id:
                    logger.info('Scheduling update '
                        'for group {0}'.format(gid))
                    _queue.add(gid)

            couple_str = ':'.join((str(gid) for gid in sorted(couple)))

            logger.debug('{0} in storage.couples: {1}'.format(
                couple_str, couple_str in storage.couples))

            if not couple_str in storage.couples:
                logger.info('Creating couple {0}'.format(couple_str))
                for gid in couple:
                    if not gid in storage.groups:
                        logger.info("Group {0} doesn't exist in "
                            "all_groups, add fake data with couple={1}".format(gid, couple))
                        storage.groups.add(gid)
                c = storage.couples.add([storage.groups[gid] for gid in couple])
                logger.info('Created couple {0} {1}'.format(c, repr(c)))
            return

        try:
            groups = groups or storage.groups.keys()

            results = {}
            for group in groups:
                session = self.__session.clone()
                session.add_groups([group.group_id])

                logger.debug('Request to read {0} for group {1}'.format(
                    keys.SYMMETRIC_GROUPS_KEY.replace('\0', '\\0'), group.group_id))
                results[group.group_id] = session.read_data(keys.SYMMETRIC_GROUPS_KEY)

            while results:
                if _queue:
                    group_id = _queue.pop()
                    if group_id not in results:
                        continue
                    result = results.pop(group_id)
                else:
                    group_id, result = results.popitem()

                group = storage.groups[group_id]

                try:
                    h.process_elliptics_async_result(result, _process_group_metadata, group)
                except elliptics.NotFoundError as e:
                    logger.warn('Failed to read symmetric_groups '
                        'from group {0}: {1}'.format(group_id, e))
                    group.parse_meta(None)
                except Exception as e:
                    logger.error('Failed to read symmetric_groups '
                        'from group {0}: {1}\n{2}'.format(
                            group_id, e, traceback.format_exc()))
                    group.parse_meta(None)
                finally:
                    try:
                        group.update_status_recursive()
                    except Exception as e:
                        logger.error('Failed to update group {0} status: '
                            '{1}\n{2}'.format(group, e, traceback.format_exc()))
                        pass

        except Exception as e:
            logger.error('Critical error during symmetric group '
                                 'update, {0}: {1}'.format(str(e),
                                     traceback.format_exc()))
Beispiel #9
0
    def update_symm_groups_async(self, groups=None):

        _queue = set()

        def _get_data_groups(group):
            return group.meta['couple']

        def _get_lrc_groups(group):
            return group.meta['lrc']['groups']

        def _create_groupset_if_needed(groups, group_type, ns_id):

            for gid in groups:
                if gid not in storage.groups:
                    logger.info(
                        'Group {group} is not found, adding fake group '
                        'for groupset {groups}'.format(
                            group=gid,
                            groups=groups,
                        )
                    )
                    storage.groups.add(gid)

            groupset_str = ':'.join((str(gid) for gid in sorted(groups)))
            if groupset_str not in storage.groupsets:
                # TODO: somehow check that couple type matches group.type
                # for all groups in couple (not very easy when metakey read
                # fails)
                logger.info('Creating groupset {groups}, group type "{group_type}"'.format(
                    groups=groupset_str,
                    group_type=group_type,
                ))
                c = storage.groupsets.add(
                    groups=(storage.groups[gid] for gid in groups),
                    group_type=group_type,
                )

                for gid in groups:
                    infrastructure.update_group_history(storage.groups[gid])

                if ns_id not in storage.namespaces:
                    logger.info('Creating storage namespace {}'.format(ns_id))
                    ns = storage.namespaces.add(ns_id)
                else:
                    ns = storage.namespaces[ns_id]

                ns.add_couple(c)
            return storage.groupsets[groupset_str]

        def _process_group_metadata(response, group, elapsed_time=None, end_time=None):
            logger.debug('Cluster updating: group {0} meta key read time: {1}.{2}'.format(
                group.group_id, elapsed_time.tsec, elapsed_time.tnsec))

            if response.error.code:
                if response.error.code == errors.ELLIPTICS_NOT_FOUND:
                    # This group is some kind of uncoupled group, not an error
                    group.parse_meta(None)
                    logger.info(
                        'Group {group} has no metakey'.format(group=group)
                    )
                elif response.error.code in (
                    # Route list did not contain the group, expected error
                    errors.ELLIPTICS_GROUP_NOT_IN_ROUTE_LIST,
                    # Timeout in reading metakey from the group, expected error
                    errors.ELLIPTICS_TIMEOUT,
                ):
                    group.reset_meta()
                    logger.error(
                        'Error on updating metakey from group {group}: {error}'.format(
                            group=group,
                            error=response.error.message,
                        )
                    )
                else:
                    raise RuntimeError(response.error.mssage)

                return

            meta = response.data

            group.parse_meta(meta)

            if group.type == storage.Group.TYPE_UNCOUPLED_LRC_8_2_2_V1:
                return

            ns_id = group.meta.get('namespace')
            if ns_id is None:
                logger.error(
                    'Inconsistent meta read from group {group}, missing namespace: {meta}'.format(
                        group=group,
                        meta=group.meta,
                    )
                )
                return

            if group.type == storage.Group.TYPE_DATA:
                groups = _get_data_groups(group)
            elif group.type == storage.Group.TYPE_LRC_8_2_2_V1:
                groups = _get_lrc_groups(group)
            elif group.type == storage.Group.TYPE_CACHE:
                groups = _get_data_groups(group)
            else:
                raise RuntimeError(
                    'Group {group_id}, unexpected type to process: {type}'.format(
                        group_id=group.group_id,
                        type=group.type,
                    )
                )

            logger.info('Read symmetric groups from group {}: {}'.format(group.group_id, groups))

            for gid in groups:
                if gid != group.group_id:
                    logger.info('Scheduling update for group {}'.format(gid))
                    _queue.add(gid)

            groupset = _create_groupset_if_needed(groups, group.type, ns_id)

            if group.type == storage.Group.TYPE_LRC_8_2_2_V1:
                # TODO: this will become unnecessary when new "Couple" instance
                # is introduced
                data_groups = _get_data_groups(group)
                data_groupset = _create_groupset_if_needed(
                    data_groups,
                    storage.Group.TYPE_DATA,
                    ns_id
                )
                data_groupset.lrc822v1_groupset = groupset
                # TODO: this should point to a new "Couple" object
                groupset.couple = data_groupset
            return

        try:
            check_groups = groups or storage.groups.keys()

            results = {}
            for group in check_groups:
                session = self.__session.clone()
                session.set_exceptions_policy(elliptics.exceptions_policy.no_exceptions)
                session.set_filter(elliptics.filters.all_with_ack)
                session.add_groups([group.group_id])

                logger.debug('Request to read {0} for group {1}'.format(
                    keys.SYMMETRIC_GROUPS_KEY.replace('\0', '\\0'), group.group_id))
                results[group.group_id] = session.read_data(keys.SYMMETRIC_GROUPS_KEY)

            jobs = {}
            if self.job_finder:
                try:
                    params = {'statuses': Job.ACTIVE_STATUSES}
                    if groups:
                        params['groups'] = [g.group_id for g in groups]
                    for job in self.job_finder.jobs(**params):
                        # TODO: this should definitely be done some other way
                        if hasattr(job, 'group'):
                            jobs[job.group] = job
                except Exception as e:
                    logger.exception('Failed to fetch pending jobs: {0}'.format(e))
                    pass

            while results:
                # TODO: Think on queue, it does not work well with lrc couples
                if _queue:
                    group_id = _queue.pop()
                    if group_id not in results:
                        continue
                    result = results.pop(group_id)
                else:
                    group_id, result = results.popitem()

                group = storage.groups[group_id]

                try:
                    h.process_elliptics_async_result(
                        result,
                        _process_group_metadata,
                        group,
                        raise_on_error=False,
                    )
                except Exception as e:
                    logger.exception(
                        'Critical error on updating metakey from group {}'.format(group_id)
                    )
                    group.parse_meta(None)
                finally:
                    try:
                        group.set_active_job(jobs.get(group.group_id))
                    except Exception as e:
                        logger.exception('Failed to set group active job: {}'.format(e))
                        pass
                    try:
                        group.update_status_recursive()
                    except Exception as e:
                        logger.exception('Failed to update group {0} status: {1}'.format(group, e))
                        pass

            if groups is None:
                self.update_couple_settings()
                load_manager.update(storage)
                weight_manager.update(storage)

                infrastructure.schedule_history_update()

        except Exception as e:
            logger.exception('Critical error during symmetric group update')