def add(self, dst_peer, chunks, end_ts, duration):
"""
Add a notification that some chunks were successfully uploaded
to the dst_peer on end_ts, taking duration.
@type chunks: (list, set)
@precondition: consists_of(chunks, Chunk)
@param dst_peer: The Host,
to which the chunks were uploaded before.
@type dst_peer: Host
@param end_ts: When the upload of these chunks was completed?
@type end_ts: datetime
@param duration: How much time did the upload of the chunks took?
@type duration: timedelta
@rtype: Deferred
"""
assert not in_main_thread()
result_deferred = Deferred()
_notification = \
ProgressNotificationPerHostWithDeferred(chunks=chunks,
end_ts=end_ts,
duration=duration,
deferred=result_deferred)
logger.debug('Adding progress... ')
should_launch_calllater = False
with self.__notification_storage_lock:
# First, create and add the information about the group of chunks.
self.__notification_storage.setdefault(dst_peer, []) \
.append(_notification)
# Then, if the transaction is not yet pending,
# let's plan to run the transaction in several seconds.
if not self.__waiting_for_transaction:
should_launch_calllater = self.__waiting_for_transaction = True
# ... and leave the lock ASAP!
# Since this moment, self.__notification_storage_lock
# is redundant.
# Outside of the lock!!!
if should_launch_calllater:
callLaterInThread(WAIT_TO_COLLECT_PROGRESS_BEFORE_SEND,
self.__try_start_transaction)
logger.debug('Will wait for %i chunks to %r with %r', len(chunks),
dst_peer, result_deferred)
return result_deferred
def add(self, dst_peer, chunks, end_ts, duration):
"""
Add a notification that some chunks were successfully uploaded
to the dst_peer on end_ts, taking duration.
@type chunks: (list, set)
@precondition: consists_of(chunks, Chunk)
@param dst_peer: The Host,
to which the chunks were uploaded before.
@type dst_peer: Host
@param end_ts: When the upload of these chunks was completed?
@type end_ts: datetime
@param duration: How much time did the upload of the chunks took?
@type duration: timedelta
@rtype: Deferred
"""
assert not in_main_thread()
result_deferred = Deferred()
_notification = \
ProgressNotificationPerHostWithDeferred(chunks=chunks,
end_ts=end_ts,
duration=duration,
deferred=result_deferred)
logger.debug('Adding progress... ')
should_launch_calllater = False
with self.__notification_storage_lock:
# First, create and add the information about the group of chunks.
self.__notification_storage.setdefault(dst_peer, []) \
.append(_notification)
# Then, if the transaction is not yet pending,
# let's plan to run the transaction in several seconds.
if not self.__waiting_for_transaction:
should_launch_calllater = self.__waiting_for_transaction = True
# ... and leave the lock ASAP!
# Since this moment, self.__notification_storage_lock
# is redundant.
# Outside of the lock!!!
if should_launch_calllater:
callLaterInThread(WAIT_TO_COLLECT_PROGRESS_BEFORE_SEND,
self.__try_start_transaction)
logger.debug('Will wait for %i chunks to %r with %r',
len(chunks), dst_peer, result_deferred)
return result_deferred
def __upload_more_chunks(self):
"""
Try upload some more chunks. If no more chunks can be uploaded,
unpause the transaction.
"""
assert not in_main_thread()
logger.debug('Should we upload more chunks? '
'Current ack_result_code is %r',
self.ack_result_code)
if BackupMessage.ResultCodes.is_good(self.ack_result_code):
logger.debug('So far so good...')
if not self.paused:
self.__notify_about_backup_running()
if not self.__try_upload_next_chunks():
self._no_more_chunks_but_wait_for_progresses()
else:
logger.debug('%r paused, retry in a second', self)
callLaterInThread(1.0, self.__upload_more_chunks)
else:
logger.debug('Must wait for progresses')
self._no_more_chunks_but_wait_for_progresses()
def on_begin(self):
"""
This method is called when the HEARTBEAT request together with its body
is received completely.
@todo: Check that the received settings are among the ones
that are allowed to update.
"""
cls = self.__class__
_message = self.message
me = _message.dst
from_host = _message.src
_host_uuid = from_host.uuid
_app = self.manager.app # NodeApp
_known_hosts = _app.known_hosts
# We've probably received some settings,
# let's update them in the database if needed.
incoming_settings = _message.settings_getter()
if not _message.revive and incoming_settings:
logger.warning('Non-revival heartbeat sent some settings: %r',
incoming_settings)
if _message.revive and incoming_settings:
# Filter out the special-case settings
assert 'urls' in incoming_settings, repr(incoming_settings)
del incoming_settings['urls']
# Don't update some settings which only the node is allowed
# to update, and which are not real settings.
real_settings = \
{k: v
for k, v in incoming_settings.iteritems()
if k != 'urls' and
k not in Queries.Settings.SETTINGS_NODE_UPDATE_ONLY}
if real_settings:
# Need to write some new settings into the DB
_now = datetime.utcnow()
_last_update_time = \
max(time for value, time in real_settings.itervalues())
with self.open_state(for_update=True) as state:
state.last_update_time = _last_update_time
logger.debug('Received settings %r', real_settings)
TrustedQueries.TrustedSettings.update_host_settings_directly(
_host_uuid,
{k: v
for k, (v, _time_sink)
in real_settings.iteritems()},
_now)
# As we've just received the settings, let's also send our ones
# to the Host.
self.__do_nested_update_configuration(real_settings)
if _message.revive:
self.__update_inbox_status_from_host()
self.__init_or_update_last_auth_token_sync_ts()
_app.maybe_refresh_chunks_info(node=me,
host=from_host,
parent_tr=self)
# End transaction.
# We do nothing at all and stay paused,
# so that when the timer fires and the transaction is unpaused,
# the on_end will be called.
with db.RDB() as rdbw:
hb_delay = \
TrustedQueries.TrustedSettings.get_setting_or_default(
rdbw, Queries.Settings.HEARTBEAT_INTERVAL, _host_uuid)
logger.debug('Inside %r, delaying for %0.02f seconds', self, hb_delay)
# This is where Node throttles the Heartbeat processing.
callLaterInThread(hb_delay, self.__try_to_unpause_myself)
def __request_restoring_more_chunks(self, state):
"""
Try the next host among the available ones, and attempt
to request it to send all the chunks it can.
@todo: each call of C{__request_restoring_more_chunks()}
should be pretty fast, less than a second, because it holds
the state context open for writing during the whole call.
If it ever happens to be that long, this needs to be refactored,
so that it opens a state context for writing only for the duration
of actual writing.
@param state: transaction state already opened for update.
@type state: AbstractTransaction.State
@return: whether everything goes so far so good. In the opposite case,
the caller must call C{self.__restore_ops_failed()}
(and do that outside of the state context).
@rtype: bool
"""
_message = self.message
_manager = self.manager
me = _message.src
host = _message.dst
so_far_so_good = True
logger.debug('%s::Pending hosts (%i), %r',
self.uuid,
len(state.pending_host_uuids),
state.pending_host_uuids)
logger.debug('%s::Pending chunks for restore (%i), %r',
self.uuid,
len(state.pending_chunk_uuids),
state.pending_chunk_uuids)
if not state.pending_chunk_uuids:
logger.debug('%s::Seems done with restoring...', self.uuid)
so_far_so_good = \
self.__no_more_chunks_to_restore(state, success=True)
else:
# What host shall we use? Use only alive ones.
host_uuids = \
deque(ifilter(self.manager.app.known_hosts.is_peer_alive,
state.pending_host_uuids))
if not host_uuids:
# If no hosts now, they still may occur in some time.
logger.warning('%s::Cannot restore: no more hosts '
'on attempt %d!',
self.uuid, state.no_donors_retries)
state.no_donors_retries += 1
if state.no_donors_retries <= MAX_RETRIES_IF_NO_DONORS:
logger.debug('Pausing for %s...',
RESTORE_RETRY_PERIOD_IF_NO_HOSTS)
callLaterInThread(
RESTORE_RETRY_PERIOD_IF_NO_HOSTS.total_seconds(),
self.__on_restore_retry_delay_elapsed)
so_far_so_good = True
else:
logger.error('In %d attempts, '
"couldn't find any donors, cancelling",
MAX_RETRIES_IF_NO_DONORS)
so_far_so_good = \
self.__no_more_chunks_to_restore(state, success=False)
else:
# No matter how many retries we could've made, but now we found
# some donors; reset the retries counter.
state.no_donors_retries = 0
# Let's use the first host in the list.
restoring_from_host_uuid = host_uuids.popleft()
# ... but in case of possible failure, move it
# to the end of loop.
host_uuids.append(restoring_from_host_uuid)
restoring_from_host = \
self.manager.app.known_hosts[restoring_from_host_uuid]
logger.debug("%s::Let's restore some chunks from %r to %r",
self.uuid, restoring_from_host, host)
with db.RDB() as rdbw:
all_chunk_uuids_on_from_host = \
set(TrustedQueries.TrustedChunks
.get_chunk_uuids_for_host(
restoring_from_host_uuid, rdbw))
# Among all the chunks on the SRC host,
# we need only several ones, which are needed for restore
available_chunk_uuids = \
all_chunk_uuids_on_from_host & state.pending_chunk_uuids
del all_chunk_uuids_on_from_host # help GC
logger.verbose('%s::Chunks available at host %s are: %r',
self.uuid,
restoring_from_host_uuid,
available_chunk_uuids)
with db.RDB() as rdbw:
restoring_chunks = \
list(Queries.Chunks.get_chunks_by_uuids(
available_chunk_uuids, rdbw))
if restoring_chunks:
logger.verbose('%s::Restoring chunks from %r to %r: %r',
self.uuid,
restoring_from_host,
host,
available_chunk_uuids)
# Start the nested RECEIVE_CHUNKS transaction.
rc_tr = _manager.create_new_transaction(
name='RECEIVE_CHUNKS',
src=me,
dst=host,
parent=self,
# RECEIVE_CHUNKS-specific
chunks_to_restore={
restoring_from_host.uuid: restoring_chunks
})
rc_tr.completed.addCallbacks(
partial(self._on_receive_chunks_success,
what_chunks=restoring_chunks,
from_what_host=restoring_from_host),
partial(self._on_receive_chunks_error,
from_what_host=restoring_from_host))
so_far_so_good = True
else:
logger.debug("%s::Host %s doesn't have any chunks for %s,"
'removing it from the set',
self.uuid, restoring_from_host_uuid, host)
state.pending_host_uuids.remove(restoring_from_host_uuid)
so_far_so_good = \
self.__request_restoring_more_chunks(state)
return so_far_so_good