def ping(self):
if self.socket is not None :
pingid = self.ping_counter.next()
defer = Deferred()
self.ping_callbacks[pingid] = defer
log.msg('Ping {0} devicehive server.'.format(pingid))
self.socket.send_frame(True, WS_OPCODE_PING, pingid)
# I cannot move it into decorator logic because I need a reference to
# a function which is clousered to a local scope.
def on_timeout():
if pingid in self.ping_callbacks :
defer = self.ping_callbacks.pop(pingid)
defer.errback(WebSocketError('Ping {0} timeout.'.format(pingid)))
timeout_defer = reactor.callLater(self.timeout, on_timeout)
def cancel_timeout(result, *args, **kwargs):
if timeout_defer.active() :
log.msg('Cancelling timeout function call for ping {0}.'.format(pingid))
timeout_defer.cancel()
return result
defer.addBoth(cancel_timeout)
return defer
else :
return fail(WebSocketError('Failed to send ping to the server. Websocket is not established.'))
def test_ws_unregister_fail(self):
patcher = patch('autopush.websocket.log', spec=True)
mock_log = patcher.start()
self._connect()
self.proto.uaid = str(uuid.uuid4())
chid = str(uuid.uuid4())
d = Deferred()
d.addBoth(lambda x: patcher.stop())
# Replace storage delete with call to fail
table = self.proto.ap_settings.storage.table
delete = table.delete_item
def raise_exception(*args, **kwargs):
# Stick the original back
table.delete_item = delete
raise Exception("Connection problem?")
table.delete_item = Mock(side_effect=raise_exception)
self._send_message(dict(messageType="unregister",
channelID=chid))
def wait_for_times(): # pragma: nocover
if len(mock_log.mock_calls) > 0:
eq_(len(mock_log.mock_calls), 1)
d.callback(True)
return
reactor.callLater(0.1, wait_for_times)
reactor.callLater(0.1, wait_for_times)
return d
def news_response(response):
if response.code != 200:
news_received_fail(response)
else:
def process_body(response_content):
news_stats = json.loads(response_content)
log.msg('News stats from server: %s' % news_stats)
if not news_stats:
log.err('Could not find news stats with ids %s' % news_ids)
else:
if action:
assert (len(news_stats) == 1)
for stat in news_stats:
news_id = stat['news_id']
news = self._news[news_id]
news.news_id = news_id
news.app_ids.update(stat['app_ids'])
news.read_count += stat['read_count']
news.users_that_rogered = set(stat['users_that_rogered'])
self._news_read_updates.add(news_id)
if action:
if action == 'roger':
news.users_that_rogered.add(account)
elif action == 'read':
news.read_count += 1
if callback:
stats = {news_info['news_id']: NewsInfo.serialize(self._news[news_info['news_id']], friends,
account) for news_info in news_stats}
callback(stats)
finished = Deferred()
response.deliverBody(ReceiverProtocol(finished))
finished.addBoth(process_body)
def make_deferred_with_timeout(canceller=None, timeout=None, timeout_msg=None,
**timeout_msg_kwargs):
d = Deferred(canceller=canceller)
if (timeout is not None) and (timeout >= 0):
from twisted.internet import reactor
timeout_msg_kwargs.update(timeout=timeout)
if not timeout_msg:
timeout_msg = 'User timeout of %(timeout)s caused failure.'
timeout_msg %= timeout_msg_kwargs
def on_timeout():
if not d.called:
d.errback(PoolTimeout(timeout_msg))
timeout_call = reactor.callLater(timeout, on_timeout)
def cancel_timeout(result):
if timeout_call.active():
timeout_call.cancel()
return result
d.addBoth(cancel_timeout)
return d
def test_disconnectedOnError(self):
"""
If the event handler raises an exception, the event is removed from the
reactor and the handler's C{connectionLost} method is called in the I/O
thread and the exception is logged.
"""
reactorThreadID = getThreadID()
reactor = self.buildReactor()
event = win32event.CreateEvent(None, False, False, None)
result = []
finished = Deferred()
finished.addBoth(result.append)
finished.addBoth(lambda ignored: reactor.stop())
listener = Listener(finished)
reactor.addEvent(event, listener, 'brokenOccurred')
reactor.callWhenRunning(win32event.SetEvent, event)
self.runReactor(reactor)
self.assertIsInstance(result[0], Failure)
result[0].trap(RuntimeError)
self.assertEqual(reactorThreadID, listener.connLostThreadID)
self.assertEqual(1, len(self.flushLoggedErrors(RuntimeError)))
class _BufferedResponse(proxyForInterface(IResponse)):
def __init__(self, original):
self.original = original
self._buffer = []
self._waiters = []
self._waiting = None
self._finished = False
self._reason = None
def _deliverWaiting(self, reason):
self._reason = reason
self._finished = True
for waiter in self._waiters:
for segment in self._buffer:
waiter.dataReceived(segment)
waiter.connectionLost(reason)
def deliverBody(self, protocol):
if self._waiting is None and not self._finished:
self._waiting = Deferred()
self._waiting.addBoth(self._deliverWaiting)
self.original.deliverBody(
_BodyBufferingProtocol(
protocol,
self._buffer,
self._waiting
)
)
elif self._finished:
for segment in self._buffer:
protocol.dataReceived(segment)
protocol.connectionLost(self._reason)
else:
self._waiters.append(protocol)
def fetched(self, response):
d = Deferred()
response.deliverBody(BodyReturner(d, verbose=self.verbose))
d.addCallback(json.loads)
if self.verbose or self.debug:
d.addBoth(self.logLoaded)
return d.addCallback(IDrawingEngine(self).update)
def callRemote(rpc_client, method_name, *params, **kwargs):
assert isinstance(rpc_client, JSONClient), 'rpc_client must be an instance of JSONClient'
deferred = rpc_client.callRemote(method_name, *params)
print'%s' % consoleRootName(rpc_client.protocolName), method_name, params
onSuccess = kwargs.get('onSuccess', None)
if onSuccess and callable(onSuccess):
deferred.addCallback(onSuccess)
else:
deferred.addCallback(onResponseSuccess)
onError = kwargs.get('onError', None)
if onError and callable(onError):
deferred.addErrback(onError)
else:
deferred.addErrback(onResponseError)
# chain try stop reactor, so we can free the response functions to call it
stopDeferred = Deferred()
stopDeferred.addBoth(tryStopReactor, rpc_client)
# when results came back success or error, fire stopDeferred callBacks
deferred.chainDeferred(stopDeferred)
return deferred
def run(reactor, command, **kwargs):
"""
Run a process and kill it if the reactor stops.
:param reactor: Reactor to use.
:param list command: The command to run.
:return Deferred: Deferred that fires when the process is ended.
"""
if 'env' not in kwargs:
kwargs['env'] = os.environ
action = RUN_ACTION(command=command)
protocol_done = Deferred()
protocol = CommandProtocol(deferred=protocol_done, action=action)
with action.context():
protocol_done = DeferredContext(protocol_done)
reactor.spawnProcess(protocol, command[0], command, **kwargs)
def unregister_killer(result, trigger_id):
try:
reactor.removeSystemEventTrigger(trigger_id)
except:
# If we can't remove the trigger, presumably it has already
# been removed (or run). In any case, there is nothing sensible
# to do if this fails.
pass
return result
trigger_id = reactor.addSystemEventTrigger(
'before', 'shutdown', protocol.transport.signalProcess, 'TERM')
protocol_done.addBoth(unregister_killer, trigger_id)
return protocol_done.addActionFinish()
class ProgramExecutionService(InternalServiceMixin, Service):
"""
"""
def __init__(self, program_exec, time_between_exec=0,
protocol=DefaultProcessProtocol, **kwargs):
super(ProgramExecutionService, self).__init__(**kwargs)
self.program_exec = program_exec
self.delay = time_between_exec
self.protocol = protocol
self.started = False
self.stopDeferred = None
self.process = None
def startService(self):
super(ProgramExecutionService, self).startService()
self.started = True
self._run_process()
def stopService(self):
super(ProgramExecutionService, self).stopService()
self.started = False
if self.process and self.process.pid:
self.stopDeferred = Deferred()
def cb(arg):
self.stopDeferred = None
self.stopDeferred.addBoth(cb)
try:
self.process.signalProcess("TERM")
except ProcessExitedAlready:
pass # No op(prevent race condition causing problem)
return self.stopDeferred
def _run_process(self):
if self.stopDeferred is not None:
self.stopDeferred.callback(True)
return # Shutting down. So we fire the callback when the process has terminated.
# We won't need to do any more processing so return.
if not self.started:
return # Service not running, so don't start the process
if self.process and self.process.pid:
return # Existing process still running.
protocol = _InternalProcessProtocol(_run_process, self.protocol())
if self.delay > 0:
self._reactor.callLater(_spawn_process, self.delay, protocol)
else:
_spawn_process(protocol)
def _spawn_process(self, protocol):
self.process = self._reactor.spawnProcess(protocol, self.command[0], self.command);
def got_response(response):
status_code = response.code
finished = Deferred()
response.deliverBody(ReceiverProtocol(finished))
if status_code == 200:
finished.addBoth(lambda body: _send_reply('result', body))
else:
finished.addBoth(lambda body: _send_reply('error', body))
return finished
def onResponse(response):
if response.code != 200:
onRequestComplete(None)
return long_url
bodyReceived = Deferred()
response.deliverBody(_ResponseCollector(bodyReceived))
bodyReceived.addBoth(onRequestComplete)
return bodyReceived
def got_response(response):
status_code = response.code
finished = Deferred()
response.deliverBody(ReceiverProtocol(finished))
if status_code == 200:
finished.addBoth(lambda body: log.msg("Posted response to appengine:\nstatus: %s\nbody:\n%s" % (status_code, body)))
else:
finished.addBoth(lambda body: log.err("Error posting response to appengine:\nstatus: %s\nbody:\n%s" % (status_code, body)))
return finished
def got_response(response):
status_code = response.code
finished = Deferred()
response.deliverBody(ReceiverProtocol(finished))
if status_code == 200:
finished.addCallback(lambda _: self.flag_message_flow_as_running(message_flow_run_key))
else:
finished.addBoth(lambda body: log.err("Error starting message flow:\nstatus: %s\nbody:\n%s" % (status_code, body)))
return finished
def got_response(response):
status_code = response.code
finished = Deferred()
response.deliverBody(ReceiverProtocol(finished))
if status_code == 200:
finished.addBoth(lambda body: log.msg("Successfully flagged message flow run %s as started." % message_flow_run_key))
else:
finished.addBoth(lambda body: log.err("Error flagging message flow run %s as started." % (status_code, body)))
return finished
def getFile(self, url, filepath):
d = Deferred()
self.requests.append(url)
self.called += 1
def finish(ignored):
return True
d.addBoth(finish)
return d
def testControlReadvarExchangeEmptyStatusSet(self):
def final(result):
self.assertEqual(self.protocol.status, STATE_UNKNOWN)
d = Deferred()
d.addBoth(final)
self.protocol.d = d
self.protocol.controlReadvarExchange()
d.cancel()
return d
def wait_for_active(log,
server_endpoint,
auth_token,
server_id,
interval=5,
clock=None):
"""
Wait until the server specified by server_id's status is 'ACTIVE'
@TODO: Timeouts
:param log: A bound logger.
:param str server_endpoint: Server endpoint URI.
:param str auth_token: Keystone Auth token.
:param str server_id: Opaque nova server id.
:param str expected_status: Nova status string.
:param int interval: Polling interval. Default: 5.
:return: Deferred that fires when the expected status has been seen.
"""
log.msg("Checking instance status every {interval} seconds",
interval=interval)
d = Deferred()
def poll():
def check_status(server):
status = server['server']['status']
log.msg("Waiting for 'ACTIVE' got {status}.", status=status)
if server['server']['status'] == 'ACTIVE':
d.callback(server)
elif server['server']['status'] != 'BUILDING':
d.errback(UnexpectedServerStatus(
server_id,
status,
'ACTIVE'))
sd = server_details(server_endpoint, auth_token, server_id)
sd.addCallback(check_status)
return sd
lc = LoopingCall(poll)
if clock is not None: # pragma: no cover
lc.clock = clock
def stop(r):
lc.stop()
return r
d.addBoth(stop)
return lc.start(interval).addCallback(lambda _: d)
def run(reactor, command, handle_stdout=None, handle_stderr=None, **kwargs):
"""
Run a process and kill it if the reactor stops.
:param reactor: Reactor to use.
:param list command: The command to run.
:param handle_stdout: Callable that will be called with lines parsed
from the command stdout. By default logs an Eliot message.
:param handle_stderr: Callable that will be called with lines parsed
from the command stderr. By default logs an Eliot message.
:return Deferred: Deferred that fires when the process is ended.
"""
if 'env' not in kwargs:
kwargs['env'] = os.environ
action = RUN_ACTION(command=command)
if handle_stdout is None:
def handle_stdout(line):
RUN_OUTPUT_MESSAGE(
line=line,
).write(action=action)
if handle_stderr is None:
def handle_stderr(line):
RUN_ERROR_MESSAGE(
line=line,
).write(action=action)
protocol_done = Deferred()
protocol = CommandProtocol(
deferred=protocol_done,
handle_stdout=handle_stdout,
handle_stderr=handle_stderr,
)
with action.context():
protocol_done = DeferredContext(protocol_done)
reactor.spawnProcess(protocol, command[0], command, **kwargs)
def unregister_killer(result, trigger_id):
try:
reactor.removeSystemEventTrigger(trigger_id)
except:
# If we can't remove the trigger, presumably it has already
# been removed (or run). In any case, there is nothing sensible
# to do if this fails.
pass
return result
trigger_id = reactor.addSystemEventTrigger(
'before', 'shutdown', protocol.transport.signalProcess, 'TERM')
protocol_done.addBoth(unregister_killer, trigger_id)
return protocol_done.addActionFinish()
def on_response(self, response):
from twisted.internet import reactor
if response.code != 200:
log.error("non 200 response received: %d" % response.code)
reactor.callLater(self.on_error_retry, self.connect)
else:
finished = Deferred()
protocol = EventSourceProtocol(self.on_event, finished)
finished.addBoth(self.reconnect)
response.deliverBody(protocol)
return finished
def api_received(self, response):
if response.code == 200 :
result_proto = Deferred()
result_proto.addCallbacks(self.api_succeed, self.api_failed)
response.deliverBody(JsonDataConsumer(result_proto))
else :
def get_response_text(reason):
self.api_failed(reason)
response_defer = Deferred()
response_defer.addBoth(get_response_text)
response.deliverBody(TextDataConsumer(response_defer))
def submit(self, callable, *args, **kw):
"""
submit a callable to be executed. A deferred will be returned with the
the result of the callable.
"""
deferred = Deferred()
deferred.addBoth(self._taskFinished)
task = ExecutorTask(deferred, callable, *args, **kw)
self._taskQueue.append(task)
reactor.callLater(0, self._runTask)
return deferred
def _do(self,job,*a,**k):
"""Wrapper for calling the background thread."""
self.count += 1
debug = k.get("_debug",_DEBUG)
d = Deferred()
debug_(debug,"QUEUE",self.tid,job,a,k)
self.q.put((d,job,a,k))
def _log(r):
debug_(debug,"DEQUEUE",self.tid,r)
return r
d.addBoth(_log)
return d
def commit(self,res=None):
d = Deferred()
if self.count:
debug("CALL COMMIT",self.tid,d,res)
self.q.put((d,"COMMIT",[],{'res':res}))
else:
debug("NO CALL COMMIT",self.tid,d,res)
d.callback(res)
d.addCallback(self._run_committed)
d.addErrback(self._run_rolledback)
d.addBoth(self._done)
return d
def callInThreadWithCallback(self, onResult, function, *args, **kwargs):
def _(res):
if isinstance(res, Failure):
onResult(False, res)
else:
onResult(True, res)
d = Deferred()
d.addCallback(lambda x: function(*args, **kwargs))
d.addBoth(_)
self._reactor.callLater(0, d.callback, True)
return d
def testStartProtocolWithoutHostMsg(self):
def final(err):
errMsg = "Host is not specified"
self.assertEqual(err.getErrorMessage(), errMsg)
d = Deferred()
d.addBoth(final)
self.protocol.d = d
self.protocol.host = None
self.protocol.startProtocol()
return d
def callInThreadWithCallback(self, onResult, function, *args, **kwargs):
def _(res):
if isinstance(res, Failure):
onResult(False, res)
else:
onResult(True, res)
d = Deferred()
d.addCallback(lambda x: function(*args, **kwargs))
d.addBoth(_)
clock._reactor.callLater(0, d.callback, True)
return d
def testControlReadvarExchangePop(self):
def final(result):
self.assertFalse(self.protocol.peersToCheck)
d = Deferred()
d.addBoth(final)
self.protocol.d = d
self.protocol.peersToCheck = self.peersToCheck
self.protocol.minPeerSource = 100 # wrong value (too high)
self.protocol.controlReadvarExchange()
d.cancel()
def run(reactor, command, handle_stdout=None, handle_stderr=None, **kwargs):
"""
Run a process and kill it if the reactor stops.
:param reactor: Reactor to use.
:param list command: The command to run.
:param handle_stdout: Callable that will be called with lines parsed
from the command stdout. By default logs an Eliot message.
:param handle_stderr: Callable that will be called with lines parsed
from the command stderr. By default logs an Eliot message.
:return Deferred: Deferred that fires when the process is ended.
"""
if 'env' not in kwargs:
kwargs['env'] = os.environ
action = RUN_ACTION(command=command)
if handle_stdout is None:
def handle_stdout(line):
RUN_OUTPUT_MESSAGE(line=line, ).write(action=action)
if handle_stderr is None:
def handle_stderr(line):
RUN_ERROR_MESSAGE(line=line, ).write(action=action)
protocol_done = Deferred()
protocol = CommandProtocol(
deferred=protocol_done,
handle_stdout=handle_stdout,
handle_stderr=handle_stderr,
)
with action.context():
protocol_done = DeferredContext(protocol_done)
reactor.spawnProcess(protocol, command[0], command, **kwargs)
def unregister_killer(result, trigger_id):
try:
reactor.removeSystemEventTrigger(trigger_id)
except:
# If we can't remove the trigger, presumably it has already
# been removed (or run). In any case, there is nothing sensible
# to do if this fails.
pass
return result
trigger_id = reactor.addSystemEventTrigger(
'before', 'shutdown', protocol.transport.signalProcess, 'TERM')
protocol_done.addBoth(unregister_killer, trigger_id)
return protocol_done.addActionFinish()
def commit(self, res=None):
d = Deferred()
if self.count:
debug("CALL COMMIT", self.tid, d, res)
self.q.put((d, "COMMIT", [], {'res': res}))
else:
debug("NO CALL COMMIT", self.tid, d, res)
d.callback(res)
d.addCallback(self._run_committed)
d.addErrback(self._run_rolledback)
d.addBoth(self._done)
return d
def testStartProtocolWithoutHost(self):
def final(err):
ex = err.value
self.assertIsInstance(ex, NtpException)
d = Deferred()
d.addBoth(final)
self.protocol.d = d
self.protocol.host = None
self.protocol.startProtocol()
return d
def stop_tasks(self):
logger.info("Stopping periodic tasks.")
if self.task.running:
self.task.stop()
if self._flush_states_task.running:
self._flush_states_task.stop()
if self._logging_task.running:
self._logging_task.stop()
d = Deferred()
d.addBoth(self._perform_shutdown)
d.addBoth(self._stop_reactor)
return d
def api_received(self, response):
if response.code == 200:
result_proto = Deferred()
result_proto.addCallbacks(self.api_succeed, self.api_failed)
response.deliverBody(JsonDataConsumer(result_proto))
else:
def get_response_text(reason):
self.api_failed(reason)
response_defer = Deferred()
response_defer.addBoth(get_response_text)
response.deliverBody(TextDataConsumer(response_defer))
def stop_tasks(self):
logger.info("Stopping periodic tasks.")
if self.task.running:
self.task.stop()
if self._flush_states_task.running:
self._flush_states_task.stop()
if self._logging_task.running:
self._logging_task.stop()
d = Deferred()
d.addBoth(self._perform_shutdown)
d.addBoth(self._stop_reactor)
return d
def testStartProtocolConnected(self):
def final(err):
# err is a Failure instance because of d.cancel()
if self.protocol.timeoutCall.active():
self.protocol.timeoutCall.cancel()
self.assertTrue(self.protocol.transport.connected)
d = Deferred()
d.addBoth(final)
self.protocol.d = d
self.protocol.startProtocol()
d.cancel()
return d
def rollback(self, res=None):
d = Deferred()
if self.q is None:
d.callback(res)
return d
if self.count:
debug("CALL ROLLBACK", self.tid, d, res)
self.q.put((d, "ROLLBACK", [], {'res': res}))
else:
debug("NO CALL ROLLBACK", self.tid, d, res)
d.callback(res)
d.addBoth(self._run_rolledback)
d.addBoth(self._done)
return d
def _do(self, job, *a, **k):
"""Wrapper for calling the background thread."""
self.count += 1
debug = k.get("_debug", _DEBUG)
d = Deferred()
debug_(debug, "QUEUE", self.tid, job, a, k)
self.q.put((d, job, a, k))
def _log(r):
debug_(debug, "DEQUEUE", self.tid, r)
return r
d.addBoth(_log)
return d
async def _wait_for(self, event):
assert event not in self._events
d = Deferred()
# We might get callbacked, we might get cancelled; either way we want
# to clean up then pass through the result:
def cleanup(obj):
assert self._events[event] is d
del self._events[event]
return obj
d.addBoth(cleanup)
self._events[event] = d
await d
def testSendReadStatRequestSent(self):
def final(err):
# err is a Failure instance because of d.cancel()
if self.protocol.timeoutCall.active():
self.protocol.timeoutCall.cancel()
self.assertTrue(self.protocol.transport.written)
d = Deferred()
d.addBoth(final)
self.protocol.d = d
self.protocol.sendReadstatRequest()
d.cancel()
return d
def ftp_STOR(self, path):
if self.dtpInstance is None:
raise BadCmdSequenceError('PORT or PASV required before STOR')
try:
newsegs = toSegments(self.workingDirectory, path)
except InvalidPath:
return defer.fail(FileNotFoundError(path))
# parent_path = '/' + ('/'.join(newsegs[:-1]))
# parent_item = backup_fs.GetByPath(parent_path)
# if not parent_item:
# return defer.fail(FileNotFoundError(parent_path))
# XXX For now, just disable the timeout. Later we'll want to
# leave it active and have the DTP connection reset it
# periodically.
self.setTimeout(None)
# Put it back later
# , prefix=(newsegs[-1] + '_')
upload_dir = tmpfile.make_dir('backup', extension='.ftp')
if not upload_dir:
return defer.fail(FileNotFoundError(path))
upload_filename = os.path.join(upload_dir, newsegs[-1])
fp = filepath.FilePath(upload_filename)
try:
fObj = fp.open('w')
except (IOError, OSError) as e:
return errnoToFailure(e.errno, path)
except:
return defer.fail(FileNotFoundError(path))
# TODO: remove file after all
# d.addCallback(lambda ignored: file.close(); and remove file)
# d = Deferred()
d = Deferred()
d.addCallback(self._cbWriteOpened, upload_filename, newsegs)
d.addErrback(self._ebWriteOpened, newsegs)
d.addBoth(self._enableTimeoutLater)
d.callback(_FileWriter(fObj))
# d.addCallbacks(self._cbFileRecevied, self._ebFileReceived)
# fw = _FileWriter(fObj)
# receive_defer = fw.receive()
# receive_defer.addBoth(self._enableTimeoutLater)
# receive_defer.addCallback(self._prepareConsumer)
# receive_defer.addCallback(self._startFileBackup, upload_filename, newsegs, d)
# receive_defer.addErrback(lambda err: d.errback(FileNotFoundError(path)))
return d
def success(self, response):
LOG_MSG('Got command poll response from the server for device {0}.'.format(self.owner.info))
if response.code in [200, 201]:
def err(reason):
LOG_ERR('Failed to parse command request response. Reason: <{0}>.'.format(reason))
self.failure(reason)
result = Deferred()
result.addCallbacks(self.command_received, err)
response.deliverBody(JsonDataConsumer(result))
else:
def on_get_response_text(error_text):
LOG_ERR('Invalid response has been received during command polling. Reason: {0}.'.format(error_text))
self.failure(DhError(error_text))
d = Deferred()
d.addBoth(on_get_response_text)
response.deliverBody(TextDataConsumer(d))
def testControlReadvarExchangePeerSet(self):
def final(err):
# err is a Failure instance because of d.cancel()
if self.protocol.timeoutCall.active():
self.protocol.timeoutCall.cancel()
self.assertEqual(self.protocol.currentPeer, self.peer)
d = Deferred()
d.addBoth(final)
self.protocol.d = d
self.protocol.peersToCheck = self.peersToCheck
self.protocol.controlReadvarExchange()
d.cancel()
return d
def deferWithTimeout(timeout, func=None, *args, **kwargs):
"""Call `func`, returning a `Deferred`.
The `Deferred` will be cancelled after `timeout` seconds if not otherwise
called.
If `func` is not specified, or None, this will return a new
:py:class:`Deferred` instance that will be cancelled after `timeout`
seconds. Do not specify `args` or `kwargs` if `func` is `None`.
:param timeout: The number of seconds before cancelling `d`.
:param func: A callable, or `None`.
:param args: Positional arguments to pass to `func`.
:param kwargs: Keyword arguments to pass to `func`.
"""
if func is None and len(args) == len(kwargs) == 0:
d = Deferred()
else:
d = maybeDeferred(func, *args, **kwargs)
from twisted.internet import reactor
timeoutCall = reactor.callLater(timeout, d.cancel)
def done(result):
if timeoutCall.active():
timeoutCall.cancel()
return result
return d.addBoth(done)
def deferred_timeout(deferred, timeout):
"""
If the deferred takes too long, attempt to cancel it and raise an error.
:param deferred: The deferred to monitor.
:type deferred: :class:`twisted.internet.defer.Deferred`
:param timeout: Seconds to wait before cancelling the deferred
:type timeout: float
:raises TimeoutError: if the deferred times out.
"""
d = Deferred()
deferred.chainDeferred(d)
def trigger_timeout():
log.msg(
'Deferred did not return before timeout, raising TimeoutError.'
)
d.errback(Failure(TimeoutError(), TimeoutError))
def callback(result):
delay.cancel()
return result
delay = reactor.callLater(timeout, trigger_timeout)
return d.addBoth(callback)
def testSendReadStatRequestCorrect(self):
def final(err):
# err is a Failure instance because of d.cancel()
if self.protocol.timeoutCall.active():
self.protocol.timeoutCall.cancel()
written_data = self.protocol.transport.written[-1][0]
expected_data = '\x16\x01\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00'
self.assertEqual(written_data, expected_data)
d = Deferred()
d.addBoth(final)
self.protocol.d = d
self.protocol.sendReadstatRequest()
d.cancel()
return d
def run(args, reactor=None, sysinfo=None):
"""
@param reactor: The reactor to (optionally) run the sysinfo plugins in.
"""
try:
setup_logging()
except IOError as e:
sys.exit("Unable to setup logging. %s" % e)
if sysinfo is None:
sysinfo = SysInfoPluginRegistry()
config = SysInfoConfiguration()
# landscape-sysinfo needs to work where there's no
# /etc/landscape/client.conf See lp:1293990
config.load(args, accept_nonexistent_default_config=True)
for plugin in config.get_plugins():
sysinfo.add(plugin)
def show_output(result):
print(
format_sysinfo(sysinfo.get_headers(),
sysinfo.get_notes(),
sysinfo.get_footnotes(),
indent=" "))
def run_sysinfo():
return sysinfo.run().addCallback(show_output)
if reactor is not None:
# In case any plugins run processes or do other things that require the
# reactor to already be started, we delay them until the reactor is
# running.
done = Deferred()
reactor.callWhenRunning(
lambda: maybeDeferred(run_sysinfo).chainDeferred(done))
def stop_reactor(result):
# We won't need to use callLater here once we use Twisted >8.
# tm:3011
reactor.callLater(0, reactor.stop)
return result
done.addBoth(stop_reactor)
reactor.run()
else:
done = run_sysinfo()
return done
def stop_tasks(self):
logger.info("Stopping periodic tasks.")
self._logging_task.stop()
d = Deferred()
d.addBoth(self._stop_slot)
d.addBoth(self._close_slot)
d.addBoth(self._perform_shutdown)
d.addBoth(self._stop_reactor)
return d
def test__logs_failures_from_cancelled_hooks(self):
logger = self.useFixture(TwistedLoggerFixture())
error = factory.make_exception()
dhooks = DeferredHooks()
d = Deferred()
d.addBoth(lambda _: Failure(error))
dhooks.add(d)
dhooks.reset()
self.assertThat(dhooks.hooks, HasLength(0))
self.assertThat(d, IsFiredDeferred())
self.assertDocTestMatches(
dedent("""\
Failure when cancelling hook.
Traceback (most recent call last):
...
maastesting.factory.TestException#...
"""), logger.output)
def testTimeout(self):
def final(err):
errMsg = 'Timeout. No response from NTP server'
ex = err.value
self.assertEqual(err.getErrorMessage(), errMsg)
self.assertIsInstance(ex, NtpException)
testTimeout = 0.5
self.protocol = NtpProtocol(host="127.0.0.1", timeout=testTimeout)
self.protocol.transport = TestableDatagramTransportWithTimeout(testTimeout)
d = Deferred()
d.addBoth(final)
self.protocol.d = d
# normally invoked by reactor.listenUDP method
self.protocol.startProtocol()
return d
def testSendReadvarRequestCorrect(self):
def final(err):
# err is a Failure instance because of d.cancel()
if self.protocol.timeoutCall.active():
self.protocol.timeoutCall.cancel()
written_data = self.protocol.transport.written[-1][0]
expected_data = '\x16\x02\x00\x02\x00\x00g\xf3\x00\x00\x00\x06offset\x00\x00\x00\x00\x00\x00'
self.assertEqual(written_data, expected_data)
d = Deferred()
d.addBoth(final)
self.protocol.d = d
self.protocol.currentPeer = self.peer
self.protocol.sequenceCounter = 2
self.protocol.sendReadvarRequest()
d.cancel()
return d
def smmpValuablePduSendAndWaitResp(self, pdu):
log.msg("smmpValuablePduSendAndWaitResp")
if SmppGeneric._isRespPdu(pdu): raise Exception("PDU can't be resp")
if not (self._canBeIssuedByHost(pdu)):
raise Exception("PDU can't be sent by the host")
d = Deferred()
try:
self.smmpValuablePduSend(pdu)
except Exception as e:
return fail(e)
self.waiting_valuable[self.sequence_number_tx] = (pdu, d)
def onRemoveDef(x):
if self.sequence_number_tx in self.waiting_valuable:
del self.waiting_valuable[self.sequence_number_tx]
return x
d.addBoth(onRemoveDef)
return d
def service(self):
creator = ClientCreator(reactor,
SmppServerNode,
host=self.host,
port=self.port,
login=self.login,
password=self.password,
vhost=self.vhost,
spec=self.spec,
exchange=self.exchange,
routing_key=self.routing_key)
while True:
try:
self.protocol = yield creator.connectTCP(self.host, self.port)
yield self.protocol.start({
"LOGIN": self.login,
"PASSWORD": self.password
})
log.msg("Connected")
try:
yield self.protocol.register("host")
yield self.protocol.work()
except Exception as e:
log.msg("Exception %s" % (repr(e), ))
log.msg("Disconnecting")
yield self.protocol.cancel()
log.msg("Disconnected")
#break
except ConnectionRefusedError as e:
log.msg("Connection refused. Will try again after time-out...")
defer = Deferred()
timer = reactor.callLater(10, defer.callback, None)
defer.addBoth(lambda _: timer.cancel()
if timer.active() else None)
yield defer
except Exception as e:
log.msg("Exception %s" % (repr(e), ))
#break
returnValue(None)
def test_upgrade(self):
"""
The L{ReleaseUpgrader.upgrade} method spawns the appropropriate
upgrade-tool script and reports the result.
"""
self.upgrader.logs_directory = self.makeDir()
upgrade_tool_directory = self.config.upgrade_tool_directory
upgrade_tool_filename = os.path.join(upgrade_tool_directory, "karmic")
fd = open(upgrade_tool_filename, "w")
fd.write("#!/bin/sh\n"
"echo $@\n"
"echo FOO=$FOO\n"
"echo PWD=$PWD\n"
"echo out\n")
fd.close()
os.chmod(upgrade_tool_filename, 0o755)
env_backup = os.environ.copy()
os.environ.clear()
os.environ.update({"FOO": "bar"})
deferred = Deferred()
def do_test():
result = self.upgrader.upgrade("karmic", 100)
def check_result(ignored):
self.assertIn(
"INFO: Queuing message with release upgrade "
"results to exchange urgently.", self.logfile.getvalue())
result_text = (u"=== Standard output ===\n\n"
"--frontend DistUpgradeViewNonInteractive\n"
"FOO=bar\n"
"PWD=%s\nout\n\n\n" % upgrade_tool_directory)
self.assertMessages(self.get_pending_messages(),
[{
"type": "operation-result",
"operation-id": 100,
"status": SUCCEEDED,
"result-text": result_text,
"result-code": 0
}])
result.addCallback(check_result)
result.chainDeferred(deferred)
reactor.callWhenRunning(do_test)
def cleanup(ignored):
os.environ = env_backup
return ignored
return deferred.addBoth(cleanup)
def _main(loops):
reactor = make_reactor()
file_out = StringIO()
with redirect_stderr(file_out):
d = Deferred()
d.addCallback(lambda _: ensureDeferred(main(reactor, loops)))
def on_done(_):
if isinstance(_, Failure):
_.printTraceback()
print(file_out.getvalue())
reactor.stop()
return _
d.addBoth(on_done)
reactor.callWhenRunning(lambda: d.callback(True))
reactor.run()
return d.result
class Client(object):
def __init__(self, reactor):
self._reactor = reactor
self._requestCount = 0
def run(self, concurrency, duration):
self._reactor.callLater(duration, self._stop, None)
self._finished = Deferred()
self._finished.addBoth(self._cleanup)
for i in range(concurrency):
self._request()
return self._finished
def cleanup(self):
pass
def _continue(self, ignored):
self._requestCount += 1
if self._finished is not None:
self._request()
def _stop(self, reason):
if self._finished is not None:
finished = self._finished
self._finished = None
if reason is not None:
finished.errback(reason)
else:
finished.callback(self._requestCount)
def _cleanup(self, passthrough):
self.cleanup()
return passthrough
def test_exception_restart(self):
import aplt.runner as runner
import aplt.scenarios as scenarios
# give the client time to finish disconnecting.
import time
time.sleep(1)
scenarios._RESTARTS = 0
h = runner.run_scenario([
"--log_output=none",
"--websocket_url={}".format(AUTOPUSH_SERVER),
"aplt.scenarios:_explode",
], run=False)
f = Deferred()
d = Deferred()
eq_(scenarios._RESTARTS, 0)
def check_restarts(result):
self.flushLoggedErrors()
eq_(scenarios._RESTARTS, 3)
f.callback(True)
d.addBoth(check_restarts)
reactor.callLater(0, self._check_testplan_done, h, d)
return f
def main():
def getServData(data):
log.msg("main>> callback: getServData " + data)
def notgetServData(er):
log.msg("main>> errback: notgetServData" + er.value.args[0])
def actionDone(data):
if data is not None:
log.msg("main>> actionDone :" + data)
else:
log.msg("main>> actionDone: err None")
reactor.stop()
log.startLogging(open("my.log", "w"))
d = Deferred()
d.addCallback(getServData)
d.addErrback(notgetServData)
d.addBoth(actionDone)
factory = Myfactory(d)
reactor.connectTCP(host, port, factory)
reactor.run()
def getProcessOutputAndValue(executable,
args=(),
env={},
path=None,
reactor=None):
"""Utility to create a process in the reactor and get all the output and
return code of that process.
This is very similar to the
`twisted.internet.utils.getProcessOutputAndValue` except this ensures that
the process is reaped at the end. Without this defunct processes will hang
around until this process dies.
"""
if reactor is None:
from twisted.internet import reactor
d = Deferred()
proc = reactor.spawnProcess(
_EverythingGetter(d),
executable,
(executable, ) + tuple(args),
env,
None,
)
def _cleanup(result):
# Always reap the process
try:
proc.reapProcess()
except Exception:
# Allow the error to occur, in the case the process has
# already been reaped.
pass
return result
d.addBoth(_cleanup)
return d
