Exemplo n.º 1
0
    def _send_state_to_control_service(self, state_changes):
        context = LOG_SEND_TO_CONTROL_SERVICE(
            self.fsm.logger,
            connection=self.client,
            local_changes=list(state_changes),
        )
        with context.context():
            d = DeferredContext(
                self.client.callRemote(NodeStateCommand,
                                       state_changes=state_changes,
                                       eliot_context=context))

            def record_acknowledged_state(ignored):
                self._last_acknowledged_state = state_changes

            def clear_acknowledged_state(failure):
                # We don't know if the control service has processed the update
                # or not. So we clear the last acknowledged state so that we
                # always send the state on the next iteration.
                self._last_acknowledged_state = None
                return failure

            d.addCallbacks(record_acknowledged_state, clear_acknowledged_state)
            d.addErrback(writeFailure, self.fsm.logger,
                         u"Failed to send local state to control node.")
            return d.addActionFinish()
Exemplo n.º 2
0
    def _send_state_to_control_service(self, state_changes):
        context = LOG_SEND_TO_CONTROL_SERVICE(
            self.fsm.logger, connection=self.client,
            local_changes=list(state_changes),
        )
        with context.context():
            d = DeferredContext(self.client.callRemote(
                NodeStateCommand,
                state_changes=state_changes,
                eliot_context=context)
            )

            def record_acknowledged_state(ignored):
                self._last_acknowledged_state = state_changes

            def clear_acknowledged_state(failure):
                # We don't know if the control service has processed the update
                # or not. So we clear the last acknowledged state so that we
                # always send the state on the next iteration.
                self._last_acknowledged_state = None
                return failure

            d.addCallbacks(record_acknowledged_state, clear_acknowledged_state)
            d.addErrback(
                writeFailure, self.fsm.logger,
                u"Failed to send local state to control node.")
            return d.addActionFinish()
Exemplo n.º 3
0
def proxy(upstream, endpoint, header):
    """
    Establish a new connection to ``endpoint`` and begin proxying between that
    connection and ``upstream``.

    :param IProtocol upstream: A connected protocol.  All data received by
        this protocol from this point on will be sent along to another newly
        established connection.

    :param IStreamClientEndpoint endpoint: An endpoint to use to establish a
        new connection.  All data received over this connection will be sent
        along to the upstream connection.

    :param bytes header: Some extra data to write to the new downstream
        connection before proxying begins.
    """
    def failed(reason):
        upstream.transport.resumeProducing()
        upstream.transport.abortConnection()
        return reason

    upstream.transport.pauseProducing()

    peer = upstream.transport.getPeer()
    action = start_action(
        action_type=u"grid-router:proxy",
        **{u"from": (peer.host, peer.port)}
    )
    with action.context():
        d = DeferredContext(endpoint.connect(Factory.forProtocol(_Proxy)))
        d.addCallbacks(
            lambda downstream: DeferredContext(downstream.take_over(upstream, header)),
            failed,
        )
        return d.addActionFinish()
Exemplo n.º 4
0
def proxy(upstream, endpoint, header):
    """
    Establish a new connection to ``endpoint`` and begin proxying between that
    connection and ``upstream``.

    :param IProtocol upstream: A connected protocol.  All data received by
        this protocol from this point on will be sent along to another newly
        established connection.

    :param IStreamClientEndpoint endpoint: An endpoint to use to establish a
        new connection.  All data received over this connection will be sent
        along to the upstream connection.

    :param bytes header: Some extra data to write to the new downstream
        connection before proxying begins.
    """
    def failed(reason):
        upstream.transport.resumeProducing()
        upstream.transport.abortConnection()
        return reason

    upstream.transport.pauseProducing()

    peer = upstream.transport.getPeer()
    action = start_action(action_type=u"grid-router:proxy",
                          **{u"from": (peer.host, peer.port)})
    with action.context():
        d = DeferredContext(endpoint.connect(Factory.forProtocol(_Proxy)))
        d.addCallbacks(
            lambda downstream: DeferredContext(
                downstream.take_over(upstream, header)),
            failed,
        )
        return d.addActionFinish()
Exemplo n.º 5
0
    def create_stateful_container(self, node, count):
        """
        Configure a stateful container to mount a new dataset, and wait for
        it to be running.
        """
        with start_action(
            action_type=u'flocker:benchmark:create_stateful_container',
            node=unicode(node.uuid),
            count=count
        ):
            d = DeferredContext(
                self.client.create_dataset(
                    primary=node.uuid,
                    maximum_size=self.max_size,
                )
            )

            def start_container(dataset):
                volume = MountedDataset(
                    dataset_id=dataset.dataset_id,
                    mountpoint=self.mountpoint
                )
                d = create_container(
                    self.reactor,
                    control_service=self.client,
                    node_uuid=node.uuid,
                    name=unicode(uuid4()),
                    image=self.image,
                    volumes=[volume],
                    timeout=self.timeout)

                # If container creation fails, delete dataset as well
                def delete_dataset(failure):
                    d = self.client.delete_dataset(dataset.dataset_id)
                    d.addErrback(write_failure)
                    d.addBoth(lambda _ignore: failure)
                    return d
                d.addErrback(delete_dataset)

                return d
            d.addCallback(start_container)

            def update_container_count(container):
                self.container_count += 1

            def update_error_count(failure):
                self.error_count += 1
                failure.printTraceback(sys.stderr)
                write_failure(failure)

            d.addCallbacks(update_container_count, update_error_count)

            return d.addActionFinish()
Exemplo n.º 6
0
    def create_stateful_container(self, node, count):
        """
        Configure a stateful container to mount a new dataset, and wait for
        it to be running.
        """
        with start_action(
            action_type=u'flocker:benchmark:create_stateful_container',
            node=unicode(node.uuid),
            count=count
        ):
            d = DeferredContext(
                self.client.create_dataset(
                    primary=node.uuid,
                    maximum_size=self.max_size,
                )
            )

            def start_container(dataset):
                volume = MountedDataset(
                    dataset_id=dataset.dataset_id,
                    mountpoint=self.mountpoint
                )
                d = create_container(
                    self.reactor,
                    control_service=self.client,
                    node_uuid=node.uuid,
                    name=unicode(uuid4()),
                    image=self.image,
                    volumes=[volume],
                    timeout=self.timeout)

                # If container creation fails, delete dataset as well
                def delete_dataset(failure):
                    d = self.client.delete_dataset(dataset.dataset_id)
                    d.addErrback(write_failure)
                    d.addBoth(lambda _ignore: failure)
                    return d
                d.addErrback(delete_dataset)

                return d
            d.addCallback(start_container)

            def update_container_count(container):
                self.container_count += 1

            def update_error_count(failure):
                self.error_count += 1
                failure.printTraceback(sys.stderr)
                write_failure(failure)

            d.addCallbacks(update_container_count, update_error_count)

            return d.addActionFinish()
Exemplo n.º 7
0
    def output_START(self, context):
        """
        Create a node.
        """
        action = start_action(
            action_type="flocker_bb:ec2:start",
            name=self.identifier())
        with action.context():

            def thread_start(task_id):
                # Since loading the image metadata is done separately from
                # booting the node, it's possible the metadata here won't
                # actually match the metadata of the image the node ends up
                # running (someone could replace the image with a different one
                # between this call and the node being started).  Since
                # generating images is currently a manual step, this probably
                # won't happen very often and if it does there's a person there
                # who can deal with it.  Also it will be resolved after the
                # next node restart.  It would be better to extract the image
                # metadata from the booted node, though.
                # FLOC-1905
                with Action.continue_task(task_id=task_id):
                    self.image_metadata = self.driver.get_image_metadata()
                    return self.driver.create()

            d = DeferredContext(
                deferToThread(thread_start, action.serialize_task_id())
            )

            def started(node):
                self.node = node
                instance_metadata = {
                    'instance_id': node.id,
                    'instance_name': node.name,
                }
                self._fsm.receive(InstanceStarted(
                    instance_id=node.id,
                    image_metadata=self.image_metadata,
                    instance_metadata=instance_metadata,
                ))
                self.instance_metadata = instance_metadata

            def failed(f):
                # We log the exception twice.
                # For Zulip
                log.err(f, "while starting %s" % (self.identifier(),))
                self._fsm.receive(StartFailed())
                # For eliot
                return f

            d.addCallbacks(started, failed)
            d.addActionFinish()
Exemplo n.º 8
0
def _retry_exception_async(reactor, f, steps=(0.1,) * 10):
    """
    Retry a function if it raises an exception.

    :return: Deferred that fires with whatever the function returns or the
        last raised exception if the function never succeeds.
    """
    # Any failure is recorded and converted to False so that loop_until keeps
    # trying.  Any success is recorded and converted to True so that
    # loop_until completes even if the result evaluates to False.
    # If loop_until() succeeds then the recorded result is returned, otherwise
    # the last recorded failure is returned.
    saved_failure = [None]
    saved_result = [None]

    def handle_success(result):
        saved_result[0] = result
        return True

    def handle_failure(failure):
        Message.log(
            message_type=(
                u"flocker:provision:libcloud:retry_exception:got_exception"
            ),
        )
        write_failure(failure)
        saved_failure[0] = failure
        return False

    def make_call():
        d = maybeDeferred(f)
        d = DeferredContext(d)
        d.addCallbacks(handle_success, errback=handle_failure)
        return d.result

    action = start_action(
        action_type=u"flocker:provision:libcloud:retry_exception",
        function=function_serializer(f),
    )
    with action.context():
        d = loop_until(reactor, make_call, steps)
        d = DeferredContext(d)
        d.addCallbacks(
            lambda _: saved_result[0],
            errback=lambda _: saved_failure[0],
        )
        return d.addActionFinish()
Exemplo n.º 9
0
def _retry_exception_async(reactor, f, steps=(0.1,) * 10):
    """
    Retry a function if it raises an exception.

    :return: Deferred that fires with whatever the function returns or the
        last raised exception if the function never succeeds.
    """
    # Any failure is recorded and converted to False so that loop_until keeps
    # trying.  Any success is recorded and converted to True so that
    # loop_until completes even if the result evaluates to False.
    # If loop_until() succeeds then the recorded result is returned, otherwise
    # the last recorded failure is returned.
    saved_failure = [None]
    saved_result = [None]

    def handle_success(result):
        saved_result[0] = result
        return True

    def handle_failure(failure):
        Message.log(
            message_type=(
                u"flocker:provision:libcloud:retry_exception:got_exception"
            ),
        )
        write_failure(failure)
        saved_failure[0] = failure
        return False

    def make_call():
        d = maybeDeferred(f)
        d = DeferredContext(d)
        d.addCallbacks(handle_success, errback=handle_failure)
        return d.result

    action = start_action(
        action_type=u"flocker:provision:libcloud:retry_exception",
        function=function_serializer(f),
    )
    with action.context():
        d = loop_until(reactor, make_call, steps)
        d = DeferredContext(d)
        d.addCallbacks(
            lambda _: saved_result[0],
            errback=lambda _: saved_failure[0],
        )
        return d.addActionFinish()
Exemplo n.º 10
0
    def g(self, name, req):
        # Bind the method to the instance so it has a better
        # fullyQualifiedName later on.  This is not necessary on Python 3.
        bound_getChild = getChild.__get__(self, type(self))

        action = start_action(
            action_type=u"allmydata:web:common-getChild",
            uri=req.uri,
            method=req.method,
            name=name,
            handler=fullyQualifiedName(bound_getChild),
        )
        with action.context():
            result = DeferredContext(maybeDeferred(bound_getChild, name, req))
            result.addCallbacks(
                _getChild_done,
                _getChild_failed,
                callbackArgs=(self, ),
            )
            result = result.addActionFinish()
        return DeferredResource(result)
Exemplo n.º 11
0
 def make_call():
     d = maybeDeferred(f)
     d = DeferredContext(d)
     d.addCallbacks(handle_success, errback=handle_failure)
     return d.result
Exemplo n.º 12
0
    def create_nodes(self, reactor, names, distribution, metadata={}):
        """
        Create nodes with the given names.

        :param reactor: The reactor.
        :param name: The names of the nodes.
        :type name: list of str
        :param str distribution: The name of the distribution to
            install on the nodes.
        :param dict metadata: Metadata to associate with the nodes.

        :return: A list of ``Deferred``s each firing with an INode
            when the corresponding node is created.   The list has
            the same order as :param:`names`.

        """
        size = self._default_size
        image_name = self._image_names[distribution]
        create_node_arguments = self._create_node_arguments()

        def handle_create_error(failure, name):
            # XXX This could be dangerous... What about a pre-existing
            # node with the same name (or even multiple nodes if the name
            # does not have to be unique)?
            Message.log(
                message_type="flocker:provision:libcloud:create_node:failed",
                node_name=name,
            )
            write_failure(failure)
            d = self._async_cleanup_node_named(reactor, name)
            d.addCallback(lambda _: failure)
            return d

        def make_node(node):
            public_address = _filter_ipv4(node.public_ips)[0]
            if isinstance(public_address, unicode):
                public_address = public_address.encode("ascii")

            if self._use_private_addresses:
                private_address = _filter_ipv4(node.private_ips)[0]
            else:
                private_address = None

            if isinstance(private_address, unicode):
                private_address = private_address.encode("ascii")

            Message.log(
                message_type="flocker:provision:libcloud:node_created",
                name=node.name,
                id=node.id,
                public_address=public_address,
                private_address=private_address,
            )
            return LibcloudNode(
                provisioner=self,
                node=node, address=public_address,
                private_address=private_address,
                distribution=distribution)

        action = start_action(
            action_type=u"flocker:provision:libcloud:create_nodes",
            instance_count=len(names),
            distribution=distribution,
            size=size,
            metadata=metadata,
        )
        with action.context():
            results = []
            for name in names:
                Message.log(
                    message_type=u"flocker:provision:libcloud:creating_node",
                    node_name=name,
                )
                d = maybeDeferred(
                    self._driver.create_node,
                    name=name,
                    image=get_image(self._driver, image_name),
                    size=get_size(self._driver, size),
                    ex_keyname=self._keyname,
                    ex_metadata=metadata,
                    **create_node_arguments
                )
                d = DeferredContext(d)
                d.addCallbacks(
                    lambda node: self._wait_until_running(reactor, node),
                    errback=handle_create_error,
                    errbackArgs=(name,),
                )
                d.addCallback(make_node)
                results.append(d.result)

            action_completion = DeferredContext(DeferredList(results))
            action_completion.addActionFinish()
            # Individual results and errors should be consumed by the caller,
            # so we can leave action_completion alone now.
            return results
Exemplo n.º 13
0
 def make_call():
     d = maybeDeferred(f)
     d = DeferredContext(d)
     d.addCallbacks(handle_success, errback=handle_failure)
     return d.result
Exemplo n.º 14
0
    def create_nodes(self, reactor, names, distribution, metadata={}):
        """
        Create nodes with the given names.

        :param reactor: The reactor.
        :param name: The names of the nodes.
        :type name: list of str
        :param str distribution: The name of the distribution to
            install on the nodes.
        :param dict metadata: Metadata to associate with the nodes.

        :return: A list of ``Deferred``s each firing with an INode
            when the corresponding node is created.   The list has
            the same order as :param:`names`.

        """
        size = self._default_size
        image_name = self._image_names[distribution]
        create_node_arguments = self._create_node_arguments()

        def handle_create_error(failure, name):
            # XXX This could be dangerous... What about a pre-existing
            # node with the same name (or even multiple nodes if the name
            # does not have to be unique)?
            Message.log(
                message_type="flocker:provision:libcloud:create_node:failed",
                node_name=name,
            )
            write_failure(failure)
            d = self._async_cleanup_node_named(reactor, name)
            d.addCallback(lambda _: failure)
            return d

        def make_node(node):
            public_address = _filter_ipv4(node.public_ips)[0]
            if isinstance(public_address, unicode):
                public_address = public_address.encode("ascii")

            if self._use_private_addresses:
                private_address = _filter_ipv4(node.private_ips)[0]
            else:
                private_address = None

            if isinstance(private_address, unicode):
                private_address = private_address.encode("ascii")

            Message.log(
                message_type="flocker:provision:libcloud:node_created",
                name=node.name,
                id=node.id,
                public_address=public_address,
                private_address=private_address,
            )
            return LibcloudNode(provisioner=self,
                                node=node,
                                address=public_address,
                                private_address=private_address,
                                distribution=distribution)

        action = start_action(
            action_type=u"flocker:provision:libcloud:create_nodes",
            instance_count=len(names),
            distribution=distribution,
            size=size,
            metadata=metadata,
        )
        with action.context():
            results = []
            for name in names:
                Message.log(
                    message_type=u"flocker:provision:libcloud:creating_node",
                    node_name=name,
                )
                d = maybeDeferred(self._driver.create_node,
                                  name=name,
                                  image=get_image(self._driver, image_name),
                                  size=get_size(self._driver, size),
                                  ex_keyname=self._keyname,
                                  ex_metadata=metadata,
                                  **create_node_arguments)
                d = DeferredContext(d)
                d.addCallbacks(
                    lambda node: self._wait_until_running(reactor, node),
                    errback=handle_create_error,
                    errbackArgs=(name, ),
                )
                d.addCallback(make_node)
                results.append(d.result)

            action_completion = DeferredContext(DeferredList(results))
            action_completion.addActionFinish()
            # Individual results and errors should be consumed by the caller,
            # so we can leave action_completion alone now.
            return results