def stop(self):
"""
Stop the scenario from being maintained by stopping all the
loops that may be executing.
:return Deferred[Optional[Dict[unicode, Any]]]: Scenario metrics.
"""
self.is_started = False
if self.monitor_loop.running:
self.monitor_loop.stop()
if self.loop.running:
self.loop.stop()
outstanding_requests = self.rate_measurer.outstanding()
if outstanding_requests > 0:
msg = (
"There are {num_requests} outstanding requests. " "Waiting {num_seconds} seconds for them to complete."
).format(num_requests=outstanding_requests, num_seconds=self.timeout)
Message.log(key="outstanding_requests", value=msg)
with start_action(action_type=u"flocker:benchmark:scenario:stop", scenario="request_load"):
def no_outstanding_requests():
return self.rate_measurer.outstanding() == 0
scenario_stopped = loop_until(self.reactor, no_outstanding_requests, repeat(1))
timeout(self.reactor, scenario_stopped, self.timeout)
scenario = DeferredContext(scenario_stopped)
def handle_timeout(failure):
failure.trap(CancelledError)
msg = ("Force stopping the scenario. " "There are {num_requests} outstanding requests").format(
num_requests=outstanding_requests
)
Message.log(key="force_stop_request", value=msg)
scenario.addErrback(handle_timeout)
def scenario_cleanup(ignored):
"""
Calls the scenario cleanup, and wraps it inside an eliot
start action, so we can see the logs if something goes
wrong within the cleanup
:return Deferred: that will fire once the cleanup has been
completed
"""
with start_action(action_type=u"flocker:benchmark:scenario:cleanup", scenario="request_load"):
return self.request.run_cleanup()
scenario.addBoth(scenario_cleanup)
def return_metrics(_ignore):
return self.rate_measurer.get_metrics()
scenario.addCallback(return_metrics)
return scenario.addActionFinish()
def _stopInstance(self):
"""
Shutdown the slave and then stop the instance.
We need to do both, to avoid the following sequence:
- Shutdown instance
- Pending build triggers new instance
- When new slave connects, duplicate slave detection kicks in, causing
the original slave to disconnect. That disconnect triggers the new
slave instance to shutdown.
- Loop.
https://clusterhq.atlassian.net/browse/FLOC-1938
"""
with start_action(
action_type="ondemand_slave:stop_instance",
slave=self.slavename,
).context():
with start_action(
action_type="ondemand_slave:shutdown"
).context():
d = DeferredContext(self.shutdown())
timeoutDeferred(reactor, d, 60)
d = d.addActionFinish()
d = DeferredContext(d)
d.addBoth(lambda _: self.instance_booter.stop())
d.addActionFinish()
def g(self, request):
# Bind the method to the instance so it has a better
# fullyQualifiedName later on. This is not necessary on Python 3.
bound_render = render.__get__(self, type(self))
action = start_action(
action_type=u"allmydata:web:common-render",
uri=request.uri,
method=request.method,
handler=fullyQualifiedName(bound_render),
)
if getattr(request, "dont_apply_extra_processing", False):
with action:
return bound_render(request)
with action.context():
result = DeferredContext(maybeDeferred(bound_render, request))
# Apply `_finish` all of our result handling logic to whatever it
# returned.
result.addBoth(_finish, bound_render, request)
d = result.addActionFinish()
# If the connection is lost then there's no point running our _finish
# logic because it has nowhere to send anything. There may also be no
# point in finishing whatever operation was being performed because
# the client cannot be informed of its result. Also, Twisted Web
# raises exceptions from some Request methods if they're used after
# the connection is lost.
request.notifyFinish().addErrback(lambda ignored: d.cancel(), )
return NOT_DONE_YET
def _stopInstance(self):
"""
Shutdown the slave and then stop the instance.
We need to do both, to avoid the following sequence:
- Shutdown instance
- Pending build triggers new instance
- When new slave connects, duplicate slave detection kicks in, causing
the original slave to disconnect. That disconnect triggers the new
slave instance to shutdown.
- Loop.
https://clusterhq.atlassian.net/browse/FLOC-1938
"""
with start_action(
action_type="ondemand_slave:stop_instance",
slave=self.slavename,
).context():
with start_action(action_type="ondemand_slave:shutdown").context():
d = DeferredContext(self.shutdown())
timeoutDeferred(reactor, d, 60)
d = d.addActionFinish()
d = DeferredContext(d)
d.addBoth(lambda _: self.instance_booter.stop())
d.addActionFinish()
def ensure_agent_has_ssh_key(reactor, key):
"""
Check that the running ssh-agent has the private key corresponding to the
provided key.
:param reactor: The reactor to use to connect to the agent.
:param Key key: The ssh key to check for in the agent.
:return Deferred: That fires with a successful result if the key is found.
Otherwise, fails with ``AgentNotFound`` or ``KeyNotFound``.
"""
try:
agent_socket = os.environ["SSH_AUTH_SOCK"]
except KeyError:
return fail(AgentNotFound())
if not key.isPublic():
key = key.public()
action = start_action(
action_type="flocker.provision.ssh:check_agent_has_ssh_keys",
key_fingerprint=key.fingerprint(),
agent_socket=agent_socket,
)
with action.context():
agent_endpoint = UNIXClientEndpoint(reactor, agent_socket)
agent_protocol = SSHAgentClient()
connected = DeferredContext(connectProtocol(agent_endpoint, agent_protocol))
connected.addCallback(lambda _: agent_protocol.requestIdentities())
def check_keys(results):
for key_data, comment in results:
agent_key = Key.fromString(key_data, type="blob")
Message.new(
message_type="flocker.provision.ssh:agent_key",
key_fingerprint=agent_key.fingerprint(),
commnet=comment,
).write()
if agent_key == key:
return True
raise KeyNotFound(expected_key=key)
connected.addCallback(check_keys)
def disconnect(result):
agent_protocol.transport.loseConnection()
return result
connected.addBoth(disconnect)
return connected.addActionFinish()
def ensure_agent_has_ssh_key(reactor, key):
"""
Check that the running ssh-agent has the private key corresponding to the
provided key.
:param reactor: The reactor to use to connect to the agent.
:param Key key: The ssh key to check for in the agent.
:return Deferred: That fires with a successful result if the key is found.
Otherwise, fails with ``AgentNotFound`` or ``KeyNotFound``.
"""
try:
agent_socket = os.environ["SSH_AUTH_SOCK"]
except KeyError:
return fail(AgentNotFound())
if not key.isPublic():
key = key.public()
action = start_action(
action_type="flocker.provision.ssh:check_agent_has_ssh_keys",
key_fingerprint=key.fingerprint(),
agent_socket=agent_socket)
with action.context():
agent_endpoint = UNIXClientEndpoint(reactor, agent_socket)
agent_protocol = SSHAgentClient()
connected = DeferredContext(
connectProtocol(agent_endpoint, agent_protocol))
connected.addCallback(lambda _: agent_protocol.requestIdentities())
def check_keys(results):
for key_data, comment in results:
agent_key = Key.fromString(key_data, type='blob')
Message.new(message_type="flocker.provision.ssh:agent_key",
key_fingerprint=agent_key.fingerprint(),
commnet=comment).write()
if agent_key == key:
return True
raise KeyNotFound(expected_key=key)
connected.addCallback(check_keys)
def disconnect(result):
agent_protocol.transport.loseConnection()
return result
connected.addBoth(disconnect)
return connected.addActionFinish()
def stop(self):
"""
Stop the scenario from being maintained by stopping all the
loops that may be executing.
:return Deferred[Optional[Dict[unicode, Any]]]: Scenario metrics.
"""
self.is_started = False
if self.monitor_loop.running:
self.monitor_loop.stop()
if self.loop.running:
self.loop.stop()
outstanding_requests = self.rate_measurer.outstanding()
if outstanding_requests > 0:
msg = (
"There are {num_requests} outstanding requests. "
"Waiting {num_seconds} seconds for them to complete."
).format(
num_requests=outstanding_requests,
num_seconds=self.timeout
)
Message.log(key='outstanding_requests', value=msg)
with start_action(
action_type=u'flocker:benchmark:scenario:stop',
scenario='request_load'
):
def no_outstanding_requests():
return self.rate_measurer.outstanding() == 0
scenario_stopped = loop_until(self.reactor,
no_outstanding_requests,
repeat(1))
timeout(self.reactor, scenario_stopped, self.timeout)
scenario = DeferredContext(scenario_stopped)
def handle_timeout(failure):
failure.trap(CancelledError)
msg = (
"Force stopping the scenario. "
"There are {num_requests} outstanding requests"
).format(
num_requests=outstanding_requests
)
Message.log(key='force_stop_request', value=msg)
scenario.addErrback(handle_timeout)
def scenario_cleanup(ignored):
"""
Calls the scenario cleanup, and wraps it inside an eliot
start action, so we can see the logs if something goes
wrong within the cleanup
:return Deferred: that will fire once the cleanup has been
completed
"""
with start_action(
action_type=u'flocker:benchmark:scenario:cleanup',
scenario='request_load'
):
return self.request.run_cleanup()
scenario.addBoth(scenario_cleanup)
def return_metrics(_ignore):
return self.rate_measurer.get_metrics()
scenario.addCallback(return_metrics)
return scenario.addActionFinish()
