Beispiel #1
0
    def _init_from_body(self, _body):
        cls = self.__class__

        # H2N
        assert isinstance(_body, dict), _body
        _query = _body['query']

        # Maybe, deserialize chunks
        if 'where' in _query:
            # Copy the dictionary, for not to modify the original
            query = dict(_query)
            _old_where = query['where']

            _CHUNK_FIELDS = cls.CHUNK_FIELDS

            if _CHUNK_FIELDS in _old_where:
                assert _old_where.keys() == [_CHUNK_FIELDS]
                # New condition, with deserialized chunks
                query['where'] = \
                    {_CHUNK_FIELDS: [ChunkInfo.from_json(c)()
                                         for c in _old_where[_CHUNK_FIELDS]]}
            elif 'path' in _old_where:
                query['where'].update({
                    'path': decode_posix_path(_old_where['path']),
                    'rec': bool(_old_where.get('rec', 0))
                })

        else:
            query = _query

        self.query = query
Beispiel #2
0
    def __execute_incoming_query(self, host_uuid, query):
        """
        Given a somehow-resembling-SQL query in the dict form,
        return the result.

        @param host_uuid: the UUID of the host which requests the query
            to be executed.
        @type host_uuid: PeerUUID

        @type query: dict
        @precondition: 'select' in query and 'from' in query # query

        @returns: the result of the query, in form of final C{State} object.
        @rtype: QueryResultData

        @todo: For cloud_stats, cache the values.
        """
        logger.verbose('%r querying node: %r', self, query)

        res_type = query['from']

        # Execute the query, and put the result into the "result" variable.

        if res_type == 'datasets':
            assert 'where' not in query

            with db.RDB() as rdbw:
                result = list(Queries.Datasets.get_just_datasets(host_uuid,
                                                                 rdbw))

        elif res_type == 'files':
            assert 'where' in query, repr(query)
            where_cond = query['where']

            assert len(where_cond.keys()) == 1, repr(where_cond)
            assert 'dataset' in where_cond, repr(where_cond)
            where_dataset_uuid = UUID(where_cond['dataset'])

            with db.RDB() as rdbw:
                files_iter = \
                    Queries.Files.get_files_for_dataset(host_uuid,
                                                        where_dataset_uuid,
                                                        rdbw)

                result = {}
                for f in files_iter:
                    result.setdefault(f.base_dir, []).append(f)

        elif res_type == 'chunks':

            select_what = query['select']
            where_cond = query['where']

            expected_key = '["hash", "size", "uuid"]'

            assert set(select_what) == {'chunks.uuid', 'uuid'}, \
                   repr(select_what)
            assert where_cond.keys() == [expected_key], \
                   (where_cond, expected_key)

            attempt_count = 0
            successfully_saved = False
            while attempt_count < MAX_ATTEMPTS_TO_SAVE_CHUNKS and \
                  not successfully_saved:
                # Try to save chunks, but if saving fails
                # (due to parallel request completed before),
                # retry several times.
                attempt_count += 1
                try:
                    with db.RDB() as rdbw:
                        result = TrustedQueries.TrustedChunks \
                                               .save_chunks_and_get_duplicates(
                                                    where_cond[expected_key],
                                                    rdbw)
                except Exception as e:
                    if attempt_count < MAX_ATTEMPTS_TO_SAVE_CHUNKS:
                        logger.debug('Saving chunks failed on attempt %d, '
                                         'retrying',
                                     attempt_count)
                    else:
                        logger.exception('Could not save chunks during '
                                             '%d attempts',
                                         MAX_ATTEMPTS_TO_SAVE_CHUNKS)
                        raise
                else:
                    successfully_saved = True
                    logger.debug('Saved chunks in %d attempt(s)',
                                 attempt_count)

        elif res_type == 'cloud_stats':
            assert 'where' not in query, repr(query)

            select_what = query['select']

            assert set(select_what) == {'total_hosts_count',
                                        'alive_hosts_count',
                                        'total_mb',
                                        'used_mb'}, repr(select_what)

            _known_hosts = self.manager.app.known_hosts
            total_hosts_count = _known_hosts.peers_count()
            alive_hosts_count = _known_hosts.alive_peers_count()

            total_mb, used_mb = \
                TrustedQueries.TrustedChunks.get_cloud_sizes(
                    ignore_uuid=host_uuid)

            result = {'total_hosts_count': total_hosts_count,
                      'alive_hosts_count': alive_hosts_count,
                      'total_mb': total_mb,
                      'used_mb': used_mb}

        elif res_type == 'data_stats':
            assert 'where' in query, repr(query)

            select_what = query['select']
            where_cond = query['where']

            assert set(select_what) == {'file_count',
                                        'file_size',
                                        'uniq_file_count',
                                        'uniq_file_size',
                                        'full_replicas_count',
                                        'chunk_count',
                                        'chunk_replicas_count',
                                        'hosts_count'}, \
                   repr(select_what)

            ds_uuid, path = (UUID(where_cond['dataset']),
                             decode_posix_path(where_cond['path']))

            result = TrustedQueries.TrustedChunks.get_data_stats(
                         ds_uuid=ds_uuid if ds_uuid != '*'
                                         else None,
                         path=path if path != '*'
                                   else None,
                         path_rec=where_cond['rec'] if path != '*'
                                                    else None)
        else:
            raise NotImplementedError('Unsupported result_type {}'
                                          .format(res_type))

        logger.verbose('For %r, result is %r (%r)',
                       query, result, res_type)

        return QueryResultData(result_type=res_type,
                               result=result)