def __get_chunk_uuids_by_host_uuid(cls,
chunk_uuids, acceptor_host_uuid, rdbw):
"""Get the actual distribution of chunks to the hosts which have them.
@todo: the complexity of the function depends on the complexity
of C{TrustedQueries.TrustedChunks.get_host_uuids_for_chunks()}.
@param chunk_uuids: the UUIDs of the chunks whose location (on the
peers) we want to know.
@type chunk_uuids: col.Iterable
@param acceptor_host_uuid: the UUID of the host for which the restore
is executed.
@type acceptor_host_uuid: PeerUUID
@type rdbw: DatabaseWrapperSQLAlchemy
@return: the mapping from the host UUID to the chunks (ones from
C{chunk_uuids}) which are stored on that host.
If the "p2p_storage" feature is supported,
all hosts which have these chunks are returned;
otherwise, only the Trusted Hosts and the acceptor host
are returned.
@rtype: col.Mapping
"""
# Get the hosts which carry these chunks, but differently
# depending upon the availability of the P2P storage.
_chunk_uuids_by_host_uuid = \
TrustedQueries.TrustedChunks.get_host_uuids_for_chunks(
chunk_uuids, rdbw)
# Now filter the result depending on the system capabilities.
feature_set = settings.get_feature_set()
if feature_set.p2p_storage:
# With "p2p_storage" feature, each host may serve some chunks
# separately.
result = _chunk_uuids_by_host_uuid
else:
# Without "p2p_storage" feature, let's consider that only
# Trusted Hosts serve the chunks.
_trusted_storage_host_uuids = \
{h.uuid
for h in TrustedQueries.HostAtNode.get_all_trusted_hosts(
for_storage=True, rdbw=rdbw)}
_wanted_host_uuids = \
_trusted_storage_host_uuids | {acceptor_host_uuid}
result = {h_uuid: c_uuids
for h_uuid, c_uuids
in _chunk_uuids_by_host_uuid.iteritems()
if h_uuid in _wanted_host_uuids}
return result
def __find_hosts(self):
"""
@rtype: PBHResultData
"""
cls = self.__class__
result_code = ProvideBackupHostsMessage.ResultCodes.OK
result = {}
if __debug__:
rng = random.Random(42)
else:
rng = None
host = self.message.src
feature_set = settings.get_feature_set()
if feature_set.p2p_storage:
_hosts = self.manager.app.known_hosts.alive_peers(
filter_uuid=host.uuid)
else:
_hosts = self.manager.app.known_hosts.alive_trusted_hosts(
for_storage=True)
# Make sure _hosts is reiterable
hosts = list(_hosts)
logger.debug('For backup, will be using the following hosts: %r',
hosts)
uuids_to_urls = {h.uuid: h.urls for h in hosts}
# To be safe, let's distribute only the hosts which have urls defined
# and non-empty
host_uuids_to_distribute = frozenset(h.uuid for h in hosts if h.urls)
hosts_free_space_map = \
{host_uuid: stat.free_size
for host_uuid, stat
in TrustedQueries.SpaceStat.get_hosts_space_stat()
.iteritems()
if host_uuid in host_uuids_to_distribute}
data = self.message.chunks_by_size_code
if not host_uuids_to_distribute:
# Probably, we cannot complete a backup.
# But if there is not chunk data to backup,... yes we can!
if sum(data.itervalues()) == 0:
result_code = ProvideBackupHostsMessage.ResultCodes.OK
logger.debug(
'%r attempts a backup with no chunks, '
'when there are no hosts', host)
else:
result_code = \
ProvideBackupHostsMessage.ResultCodes.NO_SPACE_ON_CLOUD
logger.warning('%r attempts a backup, but there is no hosts!',
host)
else:
# We should try to complete a backup.
logger.debug('Distributing to: %r', host_uuids_to_distribute)
try:
_result = cls._calculate_distribution(
chunks_map=data,
hosts_free_space_map=hosts_free_space_map,
rng=rng)
except NoSpaceError:
result_code = ProvideBackupHostsMessage.ResultCodes \
.NO_SPACE_ON_CLOUD
result = {}
logger.debug(
'Could not allocate all the required chunks '
'due to error %i! %r, %r', result_code,
hosts_free_space_map, data)
else:
result_code = ProvideBackupHostsMessage.ResultCodes.OK
result = {
host_uuid: PerHostChunksData(urls=uuids_to_urls[host_uuid],
chunks_by_size=per_host)
for host_uuid, per_host in _result.iteritems()
}
logger.debug('Allocated the chunks! %r, %r', data, _result)
logger.debug('Final result %r', result)
# By now, we have calculated "result_code" and "result" variables.
return PBHResultData(result_code=result_code, result=result)
def __find_hosts(self):
"""
@rtype: PBHResultData
"""
cls = self.__class__
result_code = ProvideBackupHostsMessage.ResultCodes.OK
result = {}
if __debug__:
rng = random.Random(42)
else:
rng = None
host = self.message.src
feature_set = settings.get_feature_set()
if feature_set.p2p_storage:
_hosts = self.manager.app.known_hosts.alive_peers(
filter_uuid=host.uuid)
else:
_hosts = self.manager.app.known_hosts.alive_trusted_hosts(
for_storage=True)
# Make sure _hosts is reiterable
hosts = list(_hosts)
logger.debug('For backup, will be using the following hosts: %r',
hosts)
uuids_to_urls = {h.uuid: h.urls for h in hosts}
# To be safe, let's distribute only the hosts which have urls defined
# and non-empty
host_uuids_to_distribute = frozenset(h.uuid for h in hosts
if h.urls)
hosts_free_space_map = \
{host_uuid: stat.free_size
for host_uuid, stat
in TrustedQueries.SpaceStat.get_hosts_space_stat()
.iteritems()
if host_uuid in host_uuids_to_distribute}
data = self.message.chunks_by_size_code
if not host_uuids_to_distribute:
# Probably, we cannot complete a backup.
# But if there is not chunk data to backup,... yes we can!
if sum(data.itervalues()) == 0:
result_code = ProvideBackupHostsMessage.ResultCodes.OK
logger.debug('%r attempts a backup with no chunks, '
'when there are no hosts',
host)
else:
result_code = \
ProvideBackupHostsMessage.ResultCodes.NO_SPACE_ON_CLOUD
logger.warning('%r attempts a backup, but there is no hosts!',
host)
else:
# We should try to complete a backup.
logger.debug('Distributing to: %r', host_uuids_to_distribute)
try:
_result = cls._calculate_distribution(
chunks_map=data,
hosts_free_space_map=hosts_free_space_map,
rng=rng)
except NoSpaceError:
result_code = ProvideBackupHostsMessage.ResultCodes \
.NO_SPACE_ON_CLOUD
result = {}
logger.debug('Could not allocate all the required chunks '
'due to error %i! %r, %r',
result_code, hosts_free_space_map, data)
else:
result_code = ProvideBackupHostsMessage.ResultCodes.OK
result = {host_uuid: PerHostChunksData(
urls=uuids_to_urls[host_uuid],
chunks_by_size=per_host)
for host_uuid, per_host in _result.iteritems()}
logger.debug('Allocated the chunks! %r, %r',
data, _result)
logger.debug('Final result %r', result)
# By now, we have calculated "result_code" and "result" variables.
return PBHResultData(result_code=result_code,
result=result)