Example #1
0
    def add(self, dst_peer, chunks, end_ts, duration):
        """
        Add a notification that some chunks were successfully uploaded
        to the dst_peer on end_ts, taking duration.

        @type chunks: (list, set)
        @precondition: consists_of(chunks, Chunk)

        @param dst_peer: The Host,
                         to which the chunks were uploaded before.
        @type dst_peer: Host

        @param end_ts: When the upload of these chunks was completed?
        @type end_ts: datetime

        @param duration: How much time did the upload of the chunks took?
        @type duration: timedelta

        @rtype: Deferred
        """
        assert not in_main_thread()
        result_deferred = Deferred()
        _notification = \
            ProgressNotificationPerHostWithDeferred(chunks=chunks,
                                                    end_ts=end_ts,
                                                    duration=duration,
                                                    deferred=result_deferred)

        logger.debug('Adding progress... ')

        should_launch_calllater = False

        with self.__notification_storage_lock:
            # First, create and add the information about the group of chunks.
            self.__notification_storage.setdefault(dst_peer, []) \
                                     .append(_notification)

            # Then, if the transaction is not yet pending,
            # let's plan to run the transaction in several seconds.
            if not self.__waiting_for_transaction:
                should_launch_calllater = self.__waiting_for_transaction = True
                # ... and leave the lock ASAP!

                # Since this moment, self.__notification_storage_lock
                # is redundant.

        # Outside of the lock!!!
        if should_launch_calllater:
            callLaterInThread(WAIT_TO_COLLECT_PROGRESS_BEFORE_SEND,
                              self.__try_start_transaction)

        logger.debug('Will wait for %i chunks to %r with %r', len(chunks),
                     dst_peer, result_deferred)

        return result_deferred
Example #2
0
    def add(self, dst_peer, chunks, end_ts, duration):
        """
        Add a notification that some chunks were successfully uploaded
        to the dst_peer on end_ts, taking duration.

        @type chunks: (list, set)
        @precondition: consists_of(chunks, Chunk)

        @param dst_peer: The Host,
                         to which the chunks were uploaded before.
        @type dst_peer: Host

        @param end_ts: When the upload of these chunks was completed?
        @type end_ts: datetime

        @param duration: How much time did the upload of the chunks took?
        @type duration: timedelta

        @rtype: Deferred
        """
        assert not in_main_thread()
        result_deferred = Deferred()
        _notification = \
            ProgressNotificationPerHostWithDeferred(chunks=chunks,
                                                    end_ts=end_ts,
                                                    duration=duration,
                                                    deferred=result_deferred)

        logger.debug('Adding progress... ')

        should_launch_calllater = False

        with self.__notification_storage_lock:
            # First, create and add the information about the group of chunks.
            self.__notification_storage.setdefault(dst_peer, []) \
                                     .append(_notification)

            # Then, if the transaction is not yet pending,
            # let's plan to run the transaction in several seconds.
            if not self.__waiting_for_transaction:
                should_launch_calllater = self.__waiting_for_transaction = True
                # ... and leave the lock ASAP!

                # Since this moment, self.__notification_storage_lock
                # is redundant.

        # Outside of the lock!!!
        if should_launch_calllater:
            callLaterInThread(WAIT_TO_COLLECT_PROGRESS_BEFORE_SEND,
                              self.__try_start_transaction)

        logger.debug('Will wait for %i chunks to %r with %r',
                     len(chunks), dst_peer, result_deferred)

        return result_deferred
Example #3
0
    def __upload_more_chunks(self):
        """
        Try upload some more chunks. If no more chunks can be uploaded,
        unpause the transaction.
        """
        assert not in_main_thread()

        logger.debug('Should we upload more chunks? '
                         'Current ack_result_code is %r',
                     self.ack_result_code)
        if BackupMessage.ResultCodes.is_good(self.ack_result_code):
            logger.debug('So far so good...')
            if not self.paused:
                self.__notify_about_backup_running()

                if not self.__try_upload_next_chunks():
                    self._no_more_chunks_but_wait_for_progresses()
            else:
                logger.debug('%r paused, retry in a second', self)
                callLaterInThread(1.0, self.__upload_more_chunks)
        else:
            logger.debug('Must wait for progresses')
            self._no_more_chunks_but_wait_for_progresses()
Example #4
0
    def on_begin(self):
        """
        This method is called when the HEARTBEAT request together with its body
        is received completely.

        @todo: Check that the received settings are among the ones
               that are allowed to update.
        """
        cls = self.__class__

        _message = self.message
        me = _message.dst
        from_host = _message.src
        _host_uuid = from_host.uuid

        _app = self.manager.app  # NodeApp
        _known_hosts = _app.known_hosts

        # We've probably received some settings,
        # let's update them in the database if needed.
        incoming_settings = _message.settings_getter()

        if not _message.revive and incoming_settings:
            logger.warning('Non-revival heartbeat sent some settings: %r',
                           incoming_settings)

        if _message.revive and incoming_settings:
            # Filter out the special-case settings
            assert 'urls' in incoming_settings, repr(incoming_settings)
            del incoming_settings['urls']

            # Don't update some settings which only the node is allowed
            # to update, and which are not real settings.
            real_settings = \
                {k: v
                     for k, v in incoming_settings.iteritems()
                     if k != 'urls' and
                        k not in Queries.Settings.SETTINGS_NODE_UPDATE_ONLY}

            if real_settings:
                # Need to write some new settings into the DB
                _now = datetime.utcnow()

                _last_update_time = \
                    max(time for value, time in real_settings.itervalues())

                with self.open_state(for_update=True) as state:
                    state.last_update_time = _last_update_time

                logger.debug('Received settings %r', real_settings)
                TrustedQueries.TrustedSettings.update_host_settings_directly(
                    _host_uuid,
                    {k: v
                         for k, (v, _time_sink)
                             in real_settings.iteritems()},
                    _now)

                # As we've just received the settings, let's also send our ones
                # to the Host.
                self.__do_nested_update_configuration(real_settings)

        if _message.revive:
            self.__update_inbox_status_from_host()
            self.__init_or_update_last_auth_token_sync_ts()

        _app.maybe_refresh_chunks_info(node=me,
                                       host=from_host,
                                       parent_tr=self)

        # End transaction.
        # We do nothing at all and stay paused,
        # so that when the timer fires and the transaction is unpaused,
        # the on_end will be called.
        with db.RDB() as rdbw:
            hb_delay = \
                TrustedQueries.TrustedSettings.get_setting_or_default(
                    rdbw, Queries.Settings.HEARTBEAT_INTERVAL, _host_uuid)
        logger.debug('Inside %r, delaying for %0.02f seconds', self, hb_delay)

        # This is where Node throttles the Heartbeat processing.
        callLaterInThread(hb_delay, self.__try_to_unpause_myself)
Example #5
0
    def __request_restoring_more_chunks(self, state):
        """
        Try the next host among the available ones, and attempt
        to request it to send all the chunks it can.

        @todo: each call of C{__request_restoring_more_chunks()}
            should be pretty fast, less than a second, because it holds
            the state context open for writing during the whole call.
            If it ever happens to be that long, this needs to be refactored,
            so that it opens a state context for writing only for the duration
            of actual writing.

        @param state: transaction state already opened for update.
        @type state: AbstractTransaction.State

        @return: whether everything goes so far so good. In the opposite case,
            the caller must call C{self.__restore_ops_failed()}
            (and do that outside of the state context).
        @rtype: bool
        """
        _message = self.message
        _manager = self.manager
        me = _message.src
        host = _message.dst

        so_far_so_good = True

        logger.debug('%s::Pending hosts (%i), %r',
                     self.uuid,
                     len(state.pending_host_uuids),
                     state.pending_host_uuids)
        logger.debug('%s::Pending chunks for restore (%i), %r',
                     self.uuid,
                     len(state.pending_chunk_uuids),
                     state.pending_chunk_uuids)

        if not state.pending_chunk_uuids:
            logger.debug('%s::Seems done with restoring...', self.uuid)
            so_far_so_good = \
                self.__no_more_chunks_to_restore(state, success=True)
        else:
            # What host shall we use? Use only alive ones.
            host_uuids = \
                deque(ifilter(self.manager.app.known_hosts.is_peer_alive,
                              state.pending_host_uuids))

            if not host_uuids:
                # If no hosts now, they still may occur in some time.
                logger.warning('%s::Cannot restore: no more hosts '
                                   'on attempt %d!',
                               self.uuid, state.no_donors_retries)

                state.no_donors_retries += 1

                if state.no_donors_retries <= MAX_RETRIES_IF_NO_DONORS:
                    logger.debug('Pausing for %s...',
                                 RESTORE_RETRY_PERIOD_IF_NO_HOSTS)
                    callLaterInThread(
                        RESTORE_RETRY_PERIOD_IF_NO_HOSTS.total_seconds(),
                        self.__on_restore_retry_delay_elapsed)
                    so_far_so_good = True
                else:
                    logger.error('In %d attempts, '
                                     "couldn't find any donors, cancelling",
                                 MAX_RETRIES_IF_NO_DONORS)
                    so_far_so_good = \
                        self.__no_more_chunks_to_restore(state, success=False)

            else:
                # No matter how many retries we could've made, but now we found
                # some donors; reset the retries counter.
                state.no_donors_retries = 0

                # Let's use the first host in the list.
                restoring_from_host_uuid = host_uuids.popleft()
                # ... but in case of possible failure, move it
                # to the end of loop.
                host_uuids.append(restoring_from_host_uuid)

                restoring_from_host = \
                    self.manager.app.known_hosts[restoring_from_host_uuid]

                logger.debug("%s::Let's restore some chunks from %r to %r",
                             self.uuid, restoring_from_host, host)

                with db.RDB() as rdbw:
                    all_chunk_uuids_on_from_host = \
                        set(TrustedQueries.TrustedChunks
                                          .get_chunk_uuids_for_host(
                                               restoring_from_host_uuid, rdbw))

                # Among all the chunks on the SRC host,
                # we need only several ones, which are needed for restore
                available_chunk_uuids = \
                    all_chunk_uuids_on_from_host & state.pending_chunk_uuids
                del all_chunk_uuids_on_from_host  # help GC

                logger.verbose('%s::Chunks available at host %s are: %r',
                               self.uuid,
                               restoring_from_host_uuid,
                               available_chunk_uuids)

                with db.RDB() as rdbw:
                    restoring_chunks = \
                        list(Queries.Chunks.get_chunks_by_uuids(
                                 available_chunk_uuids, rdbw))

                if restoring_chunks:
                    logger.verbose('%s::Restoring chunks from %r to %r: %r',
                                   self.uuid,
                                   restoring_from_host,
                                   host,
                                   available_chunk_uuids)

                    # Start the nested RECEIVE_CHUNKS transaction.
                    rc_tr = _manager.create_new_transaction(
                                name='RECEIVE_CHUNKS',
                                src=me,
                                dst=host,
                                parent=self,
                                # RECEIVE_CHUNKS-specific
                                chunks_to_restore={
                                    restoring_from_host.uuid: restoring_chunks
                                })
                    rc_tr.completed.addCallbacks(
                        partial(self._on_receive_chunks_success,
                                what_chunks=restoring_chunks,
                                from_what_host=restoring_from_host),
                        partial(self._on_receive_chunks_error,
                                from_what_host=restoring_from_host))
                    so_far_so_good = True
                else:
                    logger.debug("%s::Host %s doesn't have any chunks for %s,"
                                     'removing it from the set',
                                 self.uuid, restoring_from_host_uuid, host)
                    state.pending_host_uuids.remove(restoring_from_host_uuid)

                    so_far_so_good = \
                        self.__request_restoring_more_chunks(state)

        return so_far_so_good