def _init_from_body(self, _body): cls = self.__class__ # H2N assert isinstance(_body, dict), _body _query = _body['query'] # Maybe, deserialize chunks if 'where' in _query: # Copy the dictionary, for not to modify the original query = dict(_query) _old_where = query['where'] _CHUNK_FIELDS = cls.CHUNK_FIELDS if _CHUNK_FIELDS in _old_where: assert _old_where.keys() == [_CHUNK_FIELDS] # New condition, with deserialized chunks query['where'] = \ {_CHUNK_FIELDS: [ChunkInfo.from_json(c)() for c in _old_where[_CHUNK_FIELDS]]} elif 'path' in _old_where: query['where'].update({ 'path': decode_posix_path(_old_where['path']), 'rec': bool(_old_where.get('rec', 0)) }) else: query = _query self.query = query
def __execute_incoming_query(self, host_uuid, query): """ Given a somehow-resembling-SQL query in the dict form, return the result. @param host_uuid: the UUID of the host which requests the query to be executed. @type host_uuid: PeerUUID @type query: dict @precondition: 'select' in query and 'from' in query # query @returns: the result of the query, in form of final C{State} object. @rtype: QueryResultData @todo: For cloud_stats, cache the values. """ logger.verbose('%r querying node: %r', self, query) res_type = query['from'] # Execute the query, and put the result into the "result" variable. if res_type == 'datasets': assert 'where' not in query with db.RDB() as rdbw: result = list( Queries.Datasets.get_just_datasets(host_uuid, rdbw)) elif res_type == 'files': assert 'where' in query, repr(query) where_cond = query['where'] assert len(where_cond.keys()) == 1, repr(where_cond) assert 'dataset' in where_cond, repr(where_cond) where_dataset_uuid = UUID(where_cond['dataset']) with db.RDB() as rdbw: files_iter = \ Queries.Files.get_files_for_dataset(host_uuid, where_dataset_uuid, rdbw) result = {} for f in files_iter: result.setdefault(f.base_dir, []).append(f) elif res_type == 'chunks': select_what = query['select'] where_cond = query['where'] expected_key = '["hash", "size", "uuid"]' assert set(select_what) == {'chunks.uuid', 'uuid'}, \ repr(select_what) assert where_cond.keys() == [expected_key], \ (where_cond, expected_key) attempt_count = 0 successfully_saved = False while attempt_count < MAX_ATTEMPTS_TO_SAVE_CHUNKS and \ not successfully_saved: # Try to save chunks, but if saving fails # (due to parallel request completed before), # retry several times. attempt_count += 1 try: with db.RDB() as rdbw: result = TrustedQueries.TrustedChunks \ .save_chunks_and_get_duplicates( where_cond[expected_key], rdbw) except Exception as e: if attempt_count < MAX_ATTEMPTS_TO_SAVE_CHUNKS: logger.debug( 'Saving chunks failed on attempt %d, ' 'retrying', attempt_count) else: logger.exception( 'Could not save chunks during ' '%d attempts', MAX_ATTEMPTS_TO_SAVE_CHUNKS) raise else: successfully_saved = True logger.debug('Saved chunks in %d attempt(s)', attempt_count) elif res_type == 'cloud_stats': assert 'where' not in query, repr(query) select_what = query['select'] assert set(select_what) == { 'total_hosts_count', 'alive_hosts_count', 'total_mb', 'used_mb' }, repr(select_what) _known_hosts = self.manager.app.known_hosts total_hosts_count = _known_hosts.peers_count() alive_hosts_count = _known_hosts.alive_peers_count() total_mb, used_mb = \ TrustedQueries.TrustedChunks.get_cloud_sizes( ignore_uuid=host_uuid) result = { 'total_hosts_count': total_hosts_count, 'alive_hosts_count': alive_hosts_count, 'total_mb': total_mb, 'used_mb': used_mb } elif res_type == 'data_stats': assert 'where' in query, repr(query) select_what = query['select'] where_cond = query['where'] assert set(select_what) == {'file_count', 'file_size', 'uniq_file_count', 'uniq_file_size', 'full_replicas_count', 'chunk_count', 'chunk_replicas_count', 'hosts_count'}, \ repr(select_what) ds_uuid, path = (UUID(where_cond['dataset']), decode_posix_path(where_cond['path'])) result = TrustedQueries.TrustedChunks.get_data_stats( ds_uuid=ds_uuid if ds_uuid != '*' else None, path=path if path != '*' else None, path_rec=where_cond['rec'] if path != '*' else None) else: raise NotImplementedError( 'Unsupported result_type {}'.format(res_type)) logger.verbose('For %r, result is %r (%r)', query, result, res_type) return QueryResultData(result_type=res_type, result=result)
def __execute_incoming_query(self, host_uuid, query): """ Given a somehow-resembling-SQL query in the dict form, return the result. @param host_uuid: the UUID of the host which requests the query to be executed. @type host_uuid: PeerUUID @type query: dict @precondition: 'select' in query and 'from' in query # query @returns: the result of the query, in form of final C{State} object. @rtype: QueryResultData @todo: For cloud_stats, cache the values. """ logger.verbose('%r querying node: %r', self, query) res_type = query['from'] # Execute the query, and put the result into the "result" variable. if res_type == 'datasets': assert 'where' not in query with db.RDB() as rdbw: result = list(Queries.Datasets.get_just_datasets(host_uuid, rdbw)) elif res_type == 'files': assert 'where' in query, repr(query) where_cond = query['where'] assert len(where_cond.keys()) == 1, repr(where_cond) assert 'dataset' in where_cond, repr(where_cond) where_dataset_uuid = UUID(where_cond['dataset']) with db.RDB() as rdbw: files_iter = \ Queries.Files.get_files_for_dataset(host_uuid, where_dataset_uuid, rdbw) result = {} for f in files_iter: result.setdefault(f.base_dir, []).append(f) elif res_type == 'chunks': select_what = query['select'] where_cond = query['where'] expected_key = '["hash", "size", "uuid"]' assert set(select_what) == {'chunks.uuid', 'uuid'}, \ repr(select_what) assert where_cond.keys() == [expected_key], \ (where_cond, expected_key) attempt_count = 0 successfully_saved = False while attempt_count < MAX_ATTEMPTS_TO_SAVE_CHUNKS and \ not successfully_saved: # Try to save chunks, but if saving fails # (due to parallel request completed before), # retry several times. attempt_count += 1 try: with db.RDB() as rdbw: result = TrustedQueries.TrustedChunks \ .save_chunks_and_get_duplicates( where_cond[expected_key], rdbw) except Exception as e: if attempt_count < MAX_ATTEMPTS_TO_SAVE_CHUNKS: logger.debug('Saving chunks failed on attempt %d, ' 'retrying', attempt_count) else: logger.exception('Could not save chunks during ' '%d attempts', MAX_ATTEMPTS_TO_SAVE_CHUNKS) raise else: successfully_saved = True logger.debug('Saved chunks in %d attempt(s)', attempt_count) elif res_type == 'cloud_stats': assert 'where' not in query, repr(query) select_what = query['select'] assert set(select_what) == {'total_hosts_count', 'alive_hosts_count', 'total_mb', 'used_mb'}, repr(select_what) _known_hosts = self.manager.app.known_hosts total_hosts_count = _known_hosts.peers_count() alive_hosts_count = _known_hosts.alive_peers_count() total_mb, used_mb = \ TrustedQueries.TrustedChunks.get_cloud_sizes( ignore_uuid=host_uuid) result = {'total_hosts_count': total_hosts_count, 'alive_hosts_count': alive_hosts_count, 'total_mb': total_mb, 'used_mb': used_mb} elif res_type == 'data_stats': assert 'where' in query, repr(query) select_what = query['select'] where_cond = query['where'] assert set(select_what) == {'file_count', 'file_size', 'uniq_file_count', 'uniq_file_size', 'full_replicas_count', 'chunk_count', 'chunk_replicas_count', 'hosts_count'}, \ repr(select_what) ds_uuid, path = (UUID(where_cond['dataset']), decode_posix_path(where_cond['path'])) result = TrustedQueries.TrustedChunks.get_data_stats( ds_uuid=ds_uuid if ds_uuid != '*' else None, path=path if path != '*' else None, path_rec=where_cond['rec'] if path != '*' else None) else: raise NotImplementedError('Unsupported result_type {}' .format(res_type)) logger.verbose('For %r, result is %r (%r)', query, result, res_type) return QueryResultData(result_type=res_type, result=result)