def _startReapingProcesses(self): """ Start a LoopingCall that calls reapAllProcesses. """ lc = LoopingCall(self._reapAllProcesses) lc.clock = self._reactor lc.start(0.1, False)
def wait_for_volume(self, name): """ Wait for a volume by the given name, owned by thus service, to exist. Polls the storage pool for the specified volume to appear. :param unicode name: The name of the volume. :return: A ``Deferred`` that fires with a :class:`Volume`. """ volume = Volume(uuid=self.uuid, name=name, _pool=self._pool) def check_for_volume(volumes): if volume in volumes: call.stop() def loop(): d = self.enumerate() d.addCallback(check_for_volume) return d call = LoopingCall(loop) call.clock = self._reactor d = call.start(WAIT_FOR_VOLUME_INTERVAL) d.addCallback(lambda _: volume) return d
def make_lc(self, reactor, func): if DEBUG: self.stdout_length = 0 self.stderr_length = 0 def _(lc, reactor): if DEBUG: stdout = self.stdout.getvalue() stderr = self.stderr.getvalue() if self.stdout.getvalue()[self.stdout_length:]: print(self.stdout.getvalue()[self.stdout_length:], file=sys.__stdout__) if self.stderr.getvalue()[self.stderr_length:]: print(self.stderr.getvalue()[self.stderr_length:], file=sys.__stderr__) self.stdout_length = len(stdout) self.stderr_length = len(stderr) return func(lc, reactor) lc = LoopingCall(_) lc.a = (lc, reactor) lc.clock = reactor lc.start(0.1) return lc
def wait_for_volume(self, name): """ Wait for a volume by the given name, owned by thus service, to exist. Polls the storage pool for the specified volume to appear. :param VolumeName name: The name of the volume. :return: A ``Deferred`` that fires with a :class:`Volume`. """ # Change this to not create the Volume instance right away. Instead, # try to find it by uuid/name in `check_for_volume`. If a Volume # instance there matches, use that object as the final result of the # Deferred returned by this method (it will have its other attributes # set correctly because they will be set correctly by enumerate). # FLOC-976 volume = Volume(uuid=self.uuid, name=name, service=self) def check_for_volume(volumes): if volume in volumes: call.stop() def loop(): d = self.enumerate() d.addCallback(check_for_volume) return d call = LoopingCall(loop) call.clock = self._reactor d = call.start(WAIT_FOR_VOLUME_INTERVAL) d.addCallback(lambda _: volume) return d
def _start_worker(self): consumer = TwistedSubscribeMessageWorker(self._pubnub, self._listener_manager, self._message_queue, None) looping_call = LoopingCall(consumer.run) if self.clock is not None: looping_call.clock = self.clock self.worker_loop = looping_call.start(0.1, False)
def add_update_loop(self): """ Setup the LoopingCall to poll Consul every ``self.poll_interval``; helper for testing. """ l = LoopingCall(self.update_active_node) l.clock = self.reactor logger.warning('Setting Consul poll interval to %s seconds', self.poll_interval) l.start(self.poll_interval)
def schedule(self, function, interval): """ Schedule C{function} to be called every C{interval} """ task = LoopingCall(function) if self.clock is not None: task.clock = self.clock self.tasks.append((task, interval)) if self.running: task.start(interval, now=True)
def wait_for_active(log, server_endpoint, auth_token, server_id, interval=5, clock=None): """ Wait until the server specified by server_id's status is 'ACTIVE' @TODO: Timeouts :param log: A bound logger. :param str server_endpoint: Server endpoint URI. :param str auth_token: Keystone Auth token. :param str server_id: Opaque nova server id. :param str expected_status: Nova status string. :param int interval: Polling interval. Default: 5. :return: Deferred that fires when the expected status has been seen. """ log.msg("Checking instance status every {interval} seconds", interval=interval) d = Deferred() def poll(): def check_status(server): status = server['server']['status'] log.msg("Waiting for 'ACTIVE' got {status}.", status=status) if server['server']['status'] == 'ACTIVE': d.callback(server) elif server['server']['status'] != 'BUILDING': d.errback(UnexpectedServerStatus( server_id, status, 'ACTIVE')) sd = server_details(server_endpoint, auth_token, server_id) sd.addCallback(check_status) return sd lc = LoopingCall(poll) if clock is not None: # pragma: no cover lc.clock = clock def stop(r): lc.stop() return r d.addBoth(stop) return lc.start(interval).addCallback(lambda _: d)
def createExpirationChecker(self): if self._expirationLoop is not None: self._expirationLoop.stop() check_expired_interval = self.check_expired_interval if check_expired_interval == 0: self._expirationLoop = None else: expirationLoop = LoopingCall(self._clean_expired) expirationLoop.clock = self.reactor expirationLoop.start(self.check_expired_interval, now=False) self._expirationLoop = expirationLoop
def _add_update_loop(self): """ Setup the LoopingCall to poll MumbleLink every ``self.poll_interval``; helper for testing. The LoopingCall will simply call :py:meth:`~._read`. """ logger.debug("Creating LoopingCall") l = LoopingCall(self._read) l.clock = self.server.reactor logger.info('Setting poll interval to %s seconds', self._poll_interval) self._looping_deferred = l.start(self._poll_interval) self._looping_deferred.addErrback(logger.error)
def connectTheEndpoints(endpoints): doneTrying = [] outstanding = [] errors = [] succeeded = [] actuallyDidIt = Deferred() def removeMe(result, attempt): outstanding.remove(attempt) return result def connectingDone(result): if lc.running: lc.stop() succeeded.append(True) for o in outstanding[::]: o.cancel() actuallyDidIt.callback(result) return None def lastChance(): if doneTrying and not outstanding and not succeeded: # We've issued our last attempts. There are no remaining # outstanding attempts; they've all failed. We haven't # succeeded. Time... to die. actuallyDidIt.errback(MultiFailure(errors)) def connectingFailed(why): errors.append(why) lastChance() return None def nextOne(): try: endpoint = endpoints.next() except StopIteration: # Out of endpoints to try! Now it's time to wait for all of # the outstanding attempts to complete, and, if none of them # have been successful, then to give up with a relevant # error. They'll all be dealt with by connectingDone or # connectingFailed. doneTrying.append(True) lc.stop() lastChance() else: attempt = endpoint.connect(factory) attempt.addBoth(removeMe, attempt) attempt.addCallbacks(connectingDone, connectingFailed) outstanding.append(attempt) lc = LoopingCall(nextOne) lc.clock = self.reactor lc.start(0.0) return actuallyDidIt
def wait_for_status(log, server_endpoint, auth_token, server_id, expected_status, interval=5, clock=None): """ Wait until the server specified by server_id's status is expected_status. @TODO: Timeouts @TODO: Errback on error statuses. :param log: A bound logger. :param str server_endpoint: Server endpoint URI. :param str auth_token: Keystone Auth token. :param str server_id: Opaque nova server id. :param str expected_status: Nova status string. :param int interval: Polling interval. Default: 5. :return: Deferred that fires when the expected status has been seen. """ log.msg("Checking instance status every {interval} seconds", interval=interval) d = Deferred() def poll(): def _check_status(server): log.msg("Waiting for status {expected_status} got {status}.", expected_status=expected_status, status=server['server']['status']) if server['server']['status'] == expected_status: d.callback(server) sd = server_details(server_endpoint, auth_token, server_id) sd.addCallback(_check_status) return sd lc = LoopingCall(poll) if clock is not None: # pragma: no cover lc.clock = clock def _stop(r): lc.stop() return r d.addCallback(_stop) return lc.start(interval).addCallback(lambda _: d)
def _add_update_loop(self): """ Setup the LoopingCall to poll MumbleLink every ``self.poll_interval``; helper for testing. """ logger.debug("Creating LoopingCall") l = LoopingCall(self._wine_protocol.ask_for_output) l.clock = self.server.reactor logger.info('Setting poll interval to %s seconds', self._poll_interval) self._looping_deferred = l.start(self._poll_interval) self._looping_deferred.addErrback(logger.error)
def every(self, interval, f, *args, **kwargs): """Call f every interval ticks""" if not self.__lcs: self.__lcs = [] tick = kwargs.pop('tick', True) call = LoopingCall(f, *args, **kwargs) if tick: call.clock = self.bot.tick call.start(interval, False) self.__lcs.append(call) return call
def schedule(self, function, interval, report_function): """ Schedule C{function} to be called every C{interval} seconds and then report gathered metrics to C{Graphite} using C{report_function}. If C{report_function} is C{None}, it just calls the function without reporting the metrics. """ if report_function is not None: call = self.wrapped(function, report_function) else: call = function task = LoopingCall(call) if self.clock is not None: task.clock = self.clock self.tasks.append((task, interval)) if self.running: task.start(interval, now=True)
def test_delayed_looping_call_register_wait_and_cancel(self): self.assertFalse(self.tm.is_pending_task_active("test")) lc = LoopingCall(self.count) lc.clock = self.tm._reactor self.tm.register_task("test", lc, delay=1, interval=1) self.assertTrue(self.tm.is_pending_task_active("test")) # After one second, the counter has increased by one and the task is still active. self.tm._reactor.advance(1) self.assertEquals(1, self.counter) self.assertTrue(self.tm.is_pending_task_active("test")) # After one more second, the counter should be 2 self.tm._reactor.advance(1) self.assertEquals(2, self.counter) # After canceling the task the counter should stop increasing self.tm.cancel_pending_task("test") self.assertFalse(self.tm.is_pending_task_active("test")) self.tm._reactor.advance(10) self.assertEquals(2, self.counter)
def looping_call(self, *args, **kwargs): looping_call = LoopingCall(*args, **kwargs) looping_call.clock = self.clock return looping_call
def schedule(self, func, *args, **kw): task = LoopingCall(CountingProxy(func), *args, **kw) task.clock = self.clock task.f.scheduled_event = scheduled = ScheduledEvent(task, self.clock, self.reactor) return scheduled
def __init__(self, core_reactor, options, config): log_timer = Log(logging.DEBUG).start() Log(logging.INFO).log("Service", "state", lambda: "[anode] initialising") self.core_reactor = core_reactor self.options = options self.config = config self.plugins = {} self.certificate = pem.twisted.certificateOptionsFromFiles(self.options.certificate) \ if os.path.isfile(self.options.certificate) else None self.web_ws = WebWsFactory(u"ws" + ("" if self.certificate is None else "s") + "://" + self.config["host"] + ":" + str(self.config["port"]), self, self.certificate) self.web_ws.protocol = WebWs self.web_rest = WebRest(self, "http" + ("" if self.certificate is None else "s") + "://" + self.config["host"] + ":" + str(self.config["port"])) self.web_pool = HTTPConnectionPool(reactor, persistent=True) self.publish_service = None self.publish = "publish_host" in self.config and len(self.config["publish_host"]) > 0 and \ "publish_port" in self.config and self.config["publish_port"] > 0 if self.publish: access_key = config["profile"]["VERNEMQ_ACCESS_KEY"] if "VERNEMQ_ACCESS_KEY" in config["profile"] else None secret_key = config["profile"]["VERNEMQ_SECRET_KEY"] if "VERNEMQ_SECRET_KEY" in config["profile"] else None mqtt_client_string = clientFromString(reactor, "tcp:" + self.config["publish_host"] + ":" + str(self.config["publish_port"])) self.publish_service = MqttPublishService(mqtt_client_string, MQTTFactory(profile=MQTTFactory.PUBLISHER), KEEPALIVE_DEFAULT_SECONDS, access_key, secret_key) def looping_call(loop_function, loop_seconds): loop_call = LoopingCall(loop_function) loop_call.clock = self.core_reactor loop_call.start(loop_seconds) if "model_pull_seconds" in self.config and self.config["model_pull_seconds"] > 0: model_pull = ModelPull(self, "pullmodel", { "pool": self.web_pool, "db_dir": self.options.db_dir, "profile": self.config["profile"], "model_pull_region": self.config["model_pull_region"] if "model_pull_region" in self.config else S3_REGION, "model_pull_bucket": (self.config["model_pull_bucket"] if "model_pull_bucket" in self.config else S3_BUCKET) + ( self.config["model_pull_bucket_snapshot"] if ("model_pull_bucket_snapshot" in self.config and APP_VERSION.endswith("-SNAPSHOT")) else "")}, self.core_reactor) looping_call(model_pull.poll, self.config["model_pull_seconds"]) if "plugin" in self.config and self.config["plugin"] is not None: for plugin_name in self.config["plugin"]: self.config["plugin"][plugin_name]["pool"] = self.web_pool self.config["plugin"][plugin_name]["db_dir"] = self.options.db_dir self.config["plugin"][plugin_name]["profile"] = self.config["profile"] if self.publish_service is not None: self.config["plugin"][plugin_name]["publish_service"] = self.publish_service if "publish_batch_seconds" in self.config: self.config["plugin"][plugin_name]["publish_batch_seconds"] = self.config["publish_batch_seconds"] if "publish_status_topic" in self.config: self.config["plugin"][plugin_name]["publish_status_topic"] = self.config["publish_status_topic"] if "publish_push_data_topic" in self.config: self.config["plugin"][plugin_name]["publish_push_data_topic"] = self.config["publish_push_data_topic"] if "publish_push_metadata_topic" in self.config: self.config["plugin"][plugin_name]["publish_push_metadata_topic"] = self.config["publish_push_metadata_topic"] if "publish_batch_datum_topic" in self.config: self.config["plugin"][plugin_name]["publish_batch_datum_topic"] = self.config["publish_batch_datum_topic"] self.plugins[plugin_name] = Plugin.get(self, plugin_name, self.config["plugin"][plugin_name], self.core_reactor) if "poll_seconds" in self.config["plugin"][plugin_name] and self.config["plugin"][plugin_name]["poll_seconds"] > 0: looping_call(self.plugins[plugin_name].poll, self.config["plugin"][plugin_name]["poll_seconds"]) if "repeat_seconds" in self.config["plugin"][plugin_name] and self.config["plugin"][plugin_name]["repeat_seconds"] > 0: looping_call(self.plugins[plugin_name].repeat, self.config["plugin"][plugin_name]["repeat_seconds"]) for plugin in self.plugins.itervalues(): if "history_partition_seconds" in self.config["plugin"][plugin.name] and \ self.config["plugin"][plugin.name]["history_partition_seconds"] > 0 and \ "repeat_seconds" in self.config["plugin"][plugin_name] and \ self.config["plugin"][plugin_name]["repeat_seconds"] >= 0: time_current = plugin.get_time() time_partition = self.config["plugin"][plugin.name]["history_partition_seconds"] time_partition_next = time_partition - (time_current - plugin.get_time_period(time_current, time_partition)) plugin_partition_call = LoopingCall(self.plugins[plugin.name].repeat, force=True) plugin_partition_call.clock = self.core_reactor self.core_reactor.callLater(time_partition_next, lambda _plugin_partition_call, _time_partition: _plugin_partition_call.start(_time_partition), plugin_partition_call, time_partition) if self.publish and "publish_batch_seconds" in self.config and self.config["publish_batch_seconds"] > 0: looping_call(self.publish_datums, self.config["publish_batch_seconds"]) if "save_seconds" in self.config and self.config["save_seconds"] > 0: looping_call(self.store_state, self.config["save_seconds"]) log_timer.log("Service", "timer", lambda: "[anode] initialised", context=self.__init__)
def _test_start_run(self): """ A basic start, that enters the reactor. """ code_location = os.path.abspath(self.mktemp()) os.mkdir(code_location) with open(self.config, "w") as f: f.write("""{ "controller": { }, "workers": [ { "type": "router", "options": { "pythonpath": ["."] }, "realms": [ { "name": "realm1", "roles": [ { "name": "anonymous", "permissions": [ { "uri": "*", "publish": true, "subscribe": true, "call": true, "register": true } ] } ] } ], "transports": [ { "type": "web", "endpoint": { "type": "tcp", "port": 8080 }, "paths": { "/": { "directory": ".", "type": "static" }, "ws": { "type": "websocket" } } } ] }, { "type": "container", "options": { "pythonpath": ["%s"] }, "components": [ { "type": "class", "classname": "test.AppSession", "realm": "realm1", "transport": { "type": "websocket", "endpoint": { "type": "tcp", "host": "127.0.0.1", "port": 8080 }, "url": "ws://127.0.0.1:8080/ws" } } ] } ] } """ % ("/".join(code_location.split(os.sep), ))) with open(code_location + "/test.py", "w") as f: f.write("""#!/usr/bin/env python from twisted.internet.defer import inlineCallbacks from twisted.logger import Logger from autobahn.twisted.wamp import ApplicationSession from autobahn.wamp.exception import ApplicationError class AppSession(ApplicationSession): log = Logger() @inlineCallbacks def onJoin(self, details): self.log.info("Loaded the component!") yield self.publish("com.bar", "test") """) reactor = SelectReactor() def _check(lc): if "Loaded the component!" in self.stdout.getvalue(): if reactor.running: reactor.stop() lc.stop() lc = LoopingCall(_check) lc.a = (lc, ) lc.clock = reactor # In case it hard-locks reactor.callLater(self._subprocess_timeout, reactor.stop) lc.start(0.1) cli.run( "crossbar", ["start", "--cbdir={}".format(self.cbdir), "--logformat=syslogd"], reactor=reactor) self.assertIn("Entering reactor event loop", self.stdout.getvalue()) self.assertIn("Loaded the component!", self.stdout.getvalue())
def _test_start_run(self): """ A basic start, that enters the reactor. """ code_location = os.path.abspath(self.mktemp()) os.mkdir(code_location) with open(self.config, "w") as f: f.write("""{ "controller": { }, "workers": [ { "type": "router", "options": { "pythonpath": ["."] }, "realms": [ { "name": "realm1", "roles": [ { "name": "anonymous", "permissions": [ { "uri": "*", "publish": true, "subscribe": true, "call": true, "register": true } ] } ] } ], "transports": [ { "type": "web", "endpoint": { "type": "tcp", "port": 8080 }, "paths": { "/": { "directory": ".", "type": "static" }, "ws": { "type": "websocket" } } } ] }, { "type": "container", "options": { "pythonpath": ["%s"] }, "components": [ { "type": "class", "classname": "test.AppSession", "realm": "realm1", "transport": { "type": "websocket", "endpoint": { "type": "tcp", "host": "127.0.0.1", "port": 8080 }, "url": "ws://127.0.0.1:8080/ws" } } ] } ] } """ % ("/".join(code_location.split(os.sep),))) with open(code_location + "/test.py", "w") as f: f.write("""#!/usr/bin/env python from twisted.internet.defer import inlineCallbacks from twisted.logger import Logger from autobahn.twisted.wamp import ApplicationSession from autobahn.wamp.exception import ApplicationError class AppSession(ApplicationSession): log = Logger() @inlineCallbacks def onJoin(self, details): self.log.info("Loaded the component!") yield self.publish("com.bar", "test") """) reactor = SelectReactor() def _check(lc): if "Loaded the component!" in self.stdout.getvalue(): if reactor.running: reactor.stop() lc.stop() lc = LoopingCall(_check) lc.a = (lc,) lc.clock = reactor # In case it hard-locks reactor.callLater(self._subprocess_timeout, reactor.stop) lc.start(0.1) cli.run("crossbar", ["start", "--cbdir={}".format(self.cbdir), "--logformat=syslogd"], reactor=reactor) self.assertIn("Entering reactor event loop", self.stdout.getvalue()) self.assertIn("Loaded the component!", self.stdout.getvalue())
def looping_call(loop_function, loop_seconds): loop_call = LoopingCall(loop_function) loop_call.clock = self.core_reactor loop_call.start(loop_seconds)
def test_tls_auth_denied(self): """ A MQTT client offering the wrong certificate won't be authenticated. """ reactor, router, server_factory, session_factory = build_mqtt_server() real_reactor = selectreactor.SelectReactor() logger = make_logger() session, pump = connect_application_session( server_factory, ObservingSession, component_config=ComponentConfig(realm=u"mqtt")) endpoint = create_listening_endpoint_from_config({ "type": "tcp", "port": 1099, "interface": "0.0.0.0", "tls": { "certificate": "server.crt", "key": "server.key", "dhparam": "dhparam", "ca_certificates": [ "ca.cert.pem", "intermediate.cert.pem" ]}, }, FilePath(__file__).sibling('certs').path, real_reactor, logger) client_endpoint = create_connecting_endpoint_from_config({ "type": "tcp", "host": "127.0.0.1", "port": 1099, "tls": { # BAD key: trusted by the CA, but wrong ID "certificate": "client_1.crt", "hostname": u"localhost", "key": "client_1.key", "ca_certificates": [ "ca.cert.pem", "intermediate.cert.pem" ]}, }, FilePath(__file__).sibling('certs').path, real_reactor, logger) p = [] l = endpoint.listen(server_factory) class TestProtocol(Protocol): data = b"" expected = ( ConnACK(session_present=False, return_code=1).serialise()) def dataReceived(self_, data): self_.data = self_.data + data if len(self_.data) == len(self_.expected): self.assertEqual(self_.data, self_.expected) real_reactor.stop() @l.addCallback def _listening(factory): d = client_endpoint.connect(Factory.forProtocol(TestProtocol)) @d.addCallback def _(proto): p.append(proto) proto.transport.write( Connect(client_id=u"test123", flags=ConnectFlags(clean_session=False)).serialise()) proto.transport.write( Publish(duplicate=False, qos_level=1, retain=False, topic_name=u"test", payload=b"{}", packet_identifier=1).serialise()) lc = LoopingCall(pump.flush) lc.clock = real_reactor lc.start(0.01) def timeout(): print("Timing out :(") real_reactor.stop() print(self.logs.log_text.getvalue()) # Timeout, just in case real_reactor.callLater(10, timeout) real_reactor.run() client_protocol = p[0] # We get a CONNECT self.assertEqual(client_protocol.data, ConnACK(session_present=False, return_code=1).serialise()) client_protocol.data = b"" pump.flush() # No events! self.assertEqual(len(session.events), 0)
def _test_tls_auth(self): """ A MQTT client can connect using mutually authenticated TLS authentication. """ reactor, router, server_factory, session_factory = build_mqtt_server() real_reactor = selectreactor.SelectReactor() logger = make_logger() session, pump = connect_application_session( server_factory, ObservingSession, component_config=ComponentConfig(realm="mqtt", controller=MockContainer())) endpoint = create_listening_endpoint_from_config( { "type": "tcp", "port": 1099, "interface": "0.0.0.0", "tls": { "certificate": "server.crt", "key": "server.key", "dhparam": "dhparam", "ca_certificates": ["ca.cert.pem", "intermediate.cert.pem"] }, }, FilePath(__file__).sibling('certs').path, real_reactor, logger) client_endpoint = create_connecting_endpoint_from_config( { "type": "tcp", "host": "127.0.0.1", "port": 1099, "tls": { "certificate": "client.crt", "hostname": "localhost", "key": "client.key", "ca_certificates": ["ca.cert.pem", "intermediate.cert.pem"] }, }, FilePath(__file__).sibling('certs').path, real_reactor, logger) p = [] l = endpoint.listen(server_factory) class TestProtocol(Protocol): data = b"" expected = ( ConnACK(session_present=False, return_code=0).serialise() + PubACK(packet_identifier=1).serialise()) def dataReceived(self_, data): self_.data = self_.data + data if len(self_.data) == len(self_.expected): self.assertEqual(self_.data, self_.expected) real_reactor.stop() @l.addCallback def _listening(factory): d = client_endpoint.connect(Factory.forProtocol(TestProtocol)) @d.addCallback def _(proto): p.append(proto) proto.transport.write( Connect( client_id="test123", flags=ConnectFlags(clean_session=False)).serialise()) proto.transport.write( Publish(duplicate=False, qos_level=1, retain=False, topic_name="test", payload=b"{}", packet_identifier=1).serialise()) lc = LoopingCall(pump.flush) lc.clock = real_reactor lc.start(0.01) def timeout(): print("Timing out :(") real_reactor.stop() print(self.logs.log_text.getvalue()) # Timeout, just in case real_reactor.callLater(10, timeout) real_reactor.run() client_protocol = p[0] # We get a CONNECT self.assertEqual( client_protocol.data, ConnACK(session_present=False, return_code=0).serialise() + PubACK(packet_identifier=1).serialise()) client_protocol.data = b"" pump.flush() # This needs to be replaced with the real deal, see https://github.com/crossbario/crossbar/issues/885 self.assertEqual(len(session.events), 1) self.assertEqual(session.events, [{"args": tuple(), "kwargs": {}}])