def stop(self): """ Stop the scenario from being maintained by stopping all the loops that may be executing. :return Deferred[Optional[Dict[unicode, Any]]]: Scenario metrics. """ self.is_started = False if self.monitor_loop.running: self.monitor_loop.stop() if self.loop.running: self.loop.stop() outstanding_requests = self.rate_measurer.outstanding() if outstanding_requests > 0: msg = ( "There are {num_requests} outstanding requests. " "Waiting {num_seconds} seconds for them to complete." ).format(num_requests=outstanding_requests, num_seconds=self.timeout) Message.log(key="outstanding_requests", value=msg) with start_action(action_type=u"flocker:benchmark:scenario:stop", scenario="request_load"): def no_outstanding_requests(): return self.rate_measurer.outstanding() == 0 scenario_stopped = loop_until(self.reactor, no_outstanding_requests, repeat(1)) timeout(self.reactor, scenario_stopped, self.timeout) scenario = DeferredContext(scenario_stopped) def handle_timeout(failure): failure.trap(CancelledError) msg = ("Force stopping the scenario. " "There are {num_requests} outstanding requests").format( num_requests=outstanding_requests ) Message.log(key="force_stop_request", value=msg) scenario.addErrback(handle_timeout) def scenario_cleanup(ignored): """ Calls the scenario cleanup, and wraps it inside an eliot start action, so we can see the logs if something goes wrong within the cleanup :return Deferred: that will fire once the cleanup has been completed """ with start_action(action_type=u"flocker:benchmark:scenario:cleanup", scenario="request_load"): return self.request.run_cleanup() scenario.addBoth(scenario_cleanup) def return_metrics(_ignore): return self.rate_measurer.get_metrics() scenario.addCallback(return_metrics) return scenario.addActionFinish()
def _stopInstance(self): """ Shutdown the slave and then stop the instance. We need to do both, to avoid the following sequence: - Shutdown instance - Pending build triggers new instance - When new slave connects, duplicate slave detection kicks in, causing the original slave to disconnect. That disconnect triggers the new slave instance to shutdown. - Loop. https://clusterhq.atlassian.net/browse/FLOC-1938 """ with start_action( action_type="ondemand_slave:stop_instance", slave=self.slavename, ).context(): with start_action( action_type="ondemand_slave:shutdown" ).context(): d = DeferredContext(self.shutdown()) timeoutDeferred(reactor, d, 60) d = d.addActionFinish() d = DeferredContext(d) d.addBoth(lambda _: self.instance_booter.stop()) d.addActionFinish()
def g(self, request): # Bind the method to the instance so it has a better # fullyQualifiedName later on. This is not necessary on Python 3. bound_render = render.__get__(self, type(self)) action = start_action( action_type=u"allmydata:web:common-render", uri=request.uri, method=request.method, handler=fullyQualifiedName(bound_render), ) if getattr(request, "dont_apply_extra_processing", False): with action: return bound_render(request) with action.context(): result = DeferredContext(maybeDeferred(bound_render, request)) # Apply `_finish` all of our result handling logic to whatever it # returned. result.addBoth(_finish, bound_render, request) d = result.addActionFinish() # If the connection is lost then there's no point running our _finish # logic because it has nowhere to send anything. There may also be no # point in finishing whatever operation was being performed because # the client cannot be informed of its result. Also, Twisted Web # raises exceptions from some Request methods if they're used after # the connection is lost. request.notifyFinish().addErrback(lambda ignored: d.cancel(), ) return NOT_DONE_YET
def _stopInstance(self): """ Shutdown the slave and then stop the instance. We need to do both, to avoid the following sequence: - Shutdown instance - Pending build triggers new instance - When new slave connects, duplicate slave detection kicks in, causing the original slave to disconnect. That disconnect triggers the new slave instance to shutdown. - Loop. https://clusterhq.atlassian.net/browse/FLOC-1938 """ with start_action( action_type="ondemand_slave:stop_instance", slave=self.slavename, ).context(): with start_action(action_type="ondemand_slave:shutdown").context(): d = DeferredContext(self.shutdown()) timeoutDeferred(reactor, d, 60) d = d.addActionFinish() d = DeferredContext(d) d.addBoth(lambda _: self.instance_booter.stop()) d.addActionFinish()
def ensure_agent_has_ssh_key(reactor, key): """ Check that the running ssh-agent has the private key corresponding to the provided key. :param reactor: The reactor to use to connect to the agent. :param Key key: The ssh key to check for in the agent. :return Deferred: That fires with a successful result if the key is found. Otherwise, fails with ``AgentNotFound`` or ``KeyNotFound``. """ try: agent_socket = os.environ["SSH_AUTH_SOCK"] except KeyError: return fail(AgentNotFound()) if not key.isPublic(): key = key.public() action = start_action( action_type="flocker.provision.ssh:check_agent_has_ssh_keys", key_fingerprint=key.fingerprint(), agent_socket=agent_socket, ) with action.context(): agent_endpoint = UNIXClientEndpoint(reactor, agent_socket) agent_protocol = SSHAgentClient() connected = DeferredContext(connectProtocol(agent_endpoint, agent_protocol)) connected.addCallback(lambda _: agent_protocol.requestIdentities()) def check_keys(results): for key_data, comment in results: agent_key = Key.fromString(key_data, type="blob") Message.new( message_type="flocker.provision.ssh:agent_key", key_fingerprint=agent_key.fingerprint(), commnet=comment, ).write() if agent_key == key: return True raise KeyNotFound(expected_key=key) connected.addCallback(check_keys) def disconnect(result): agent_protocol.transport.loseConnection() return result connected.addBoth(disconnect) return connected.addActionFinish()
def ensure_agent_has_ssh_key(reactor, key): """ Check that the running ssh-agent has the private key corresponding to the provided key. :param reactor: The reactor to use to connect to the agent. :param Key key: The ssh key to check for in the agent. :return Deferred: That fires with a successful result if the key is found. Otherwise, fails with ``AgentNotFound`` or ``KeyNotFound``. """ try: agent_socket = os.environ["SSH_AUTH_SOCK"] except KeyError: return fail(AgentNotFound()) if not key.isPublic(): key = key.public() action = start_action( action_type="flocker.provision.ssh:check_agent_has_ssh_keys", key_fingerprint=key.fingerprint(), agent_socket=agent_socket) with action.context(): agent_endpoint = UNIXClientEndpoint(reactor, agent_socket) agent_protocol = SSHAgentClient() connected = DeferredContext( connectProtocol(agent_endpoint, agent_protocol)) connected.addCallback(lambda _: agent_protocol.requestIdentities()) def check_keys(results): for key_data, comment in results: agent_key = Key.fromString(key_data, type='blob') Message.new(message_type="flocker.provision.ssh:agent_key", key_fingerprint=agent_key.fingerprint(), commnet=comment).write() if agent_key == key: return True raise KeyNotFound(expected_key=key) connected.addCallback(check_keys) def disconnect(result): agent_protocol.transport.loseConnection() return result connected.addBoth(disconnect) return connected.addActionFinish()
def stop(self): """ Stop the scenario from being maintained by stopping all the loops that may be executing. :return Deferred[Optional[Dict[unicode, Any]]]: Scenario metrics. """ self.is_started = False if self.monitor_loop.running: self.monitor_loop.stop() if self.loop.running: self.loop.stop() outstanding_requests = self.rate_measurer.outstanding() if outstanding_requests > 0: msg = ( "There are {num_requests} outstanding requests. " "Waiting {num_seconds} seconds for them to complete." ).format( num_requests=outstanding_requests, num_seconds=self.timeout ) Message.log(key='outstanding_requests', value=msg) with start_action( action_type=u'flocker:benchmark:scenario:stop', scenario='request_load' ): def no_outstanding_requests(): return self.rate_measurer.outstanding() == 0 scenario_stopped = loop_until(self.reactor, no_outstanding_requests, repeat(1)) timeout(self.reactor, scenario_stopped, self.timeout) scenario = DeferredContext(scenario_stopped) def handle_timeout(failure): failure.trap(CancelledError) msg = ( "Force stopping the scenario. " "There are {num_requests} outstanding requests" ).format( num_requests=outstanding_requests ) Message.log(key='force_stop_request', value=msg) scenario.addErrback(handle_timeout) def scenario_cleanup(ignored): """ Calls the scenario cleanup, and wraps it inside an eliot start action, so we can see the logs if something goes wrong within the cleanup :return Deferred: that will fire once the cleanup has been completed """ with start_action( action_type=u'flocker:benchmark:scenario:cleanup', scenario='request_load' ): return self.request.run_cleanup() scenario.addBoth(scenario_cleanup) def return_metrics(_ignore): return self.rate_measurer.get_metrics() scenario.addCallback(return_metrics) return scenario.addActionFinish()