def _setup_workers(self, bucketters, aggregators, bucket_size): broker = FakeAMQPBroker() self.broker = BrokerWrapper(broker) bucket_config = { 'buckets': aggregators, 'bucket_size': bucket_size, } for _i in range(bucketters): worker = get_stubbed_worker(metrics.MetricTimeBucket, config=bucket_config, broker=broker) yield worker.startWorker() self.bucket_workers.append(worker) aggregator_config = { 'bucket_size': bucket_size, } for i in range(aggregators): config = aggregator_config.copy() config['bucket'] = i worker = get_stubbed_worker(metrics.MetricAggregator, config=config, broker=broker) worker._time = self.fake_time yield worker.startWorker() self.aggregator_workers.append(worker)
def _setup_workers(self, bucketters, aggregators, bucket_size): broker = FakeAMQPBroker() self.broker = BrokerWrapper(broker) bucket_config = { 'buckets': aggregators, 'bucket_size': bucket_size, } for _i in range(bucketters): worker = get_stubbed_worker(metrics_workers.MetricTimeBucket, config=bucket_config, broker=broker) yield worker.startWorker() self.bucket_workers.append(worker) aggregator_config = { 'bucket_size': bucket_size, } for i in range(aggregators): config = aggregator_config.copy() config['bucket'] = i worker = get_stubbed_worker(metrics_workers.MetricAggregator, config=config, broker=broker) worker._time = self.fake_time yield worker.startWorker() self.aggregator_workers.append(worker)
def setUp(self): self.config = { 'web_port': 9999, 'web_receive_path': '/t/receive', 'web_receipt_path': '/t/receipt', 'url': 'http://localhost:9998/t/send', } self.worker = get_stubbed_worker(TestFakeVas2NetsWorker, self.config) self.test_worker = get_stubbed_worker(TestWorker, self.config) self.today = datetime.utcnow().date()
def test_bucketing(self): config = {'buckets': 4, 'bucket_size': 5} worker = get_stubbed_worker(metrics_workers.MetricTimeBucket, config=config) broker = BrokerWrapper(worker._amqp_client.broker) yield worker.startWorker() datapoints = [ ("vumi.test.foo", ("agg", ), [(1230, 1.5), (1235, 2.0)]), ("vumi.test.bar", ("sum", ), [(1240, 1.0)]), ] broker.send_datapoints("vumi.metrics", "vumi.metrics", datapoints) yield broker.kick_delivery() buckets = [ broker.recv_datapoints("vumi.metrics.buckets", "bucket.%d" % i) for i in range(4) ] expected_buckets = [ [], [[[u'vumi.test.bar', ['sum'], [[1240, 1.0]]]]], [[[u'vumi.test.foo', ['agg'], [[1230, 1.5]]]], [[u'vumi.test.foo', ['agg'], [[1235, 2.0]]]]], [], ] self.assertEqual(buckets, expected_buckets) yield worker.stopWorker()
def get_worker(self): worker = get_stubbed_worker(RockPaperScissorsWorkerStub, { 'transport_name': 'foo', 'ussd_code': '99999', }) worker.startWorker() return worker
def test_send_to_with_no_send_to_tags(self): config = {'transport_name': 'notags_app'} notags_worker = get_stubbed_worker(ApplicationWorker, config=config) yield notags_worker.startWorker() self.assertRaises(ValueError, notags_worker.send_to, '+12345', "Hi!")
def test_bucketing(self): config = {'buckets': 4, 'bucket_size': 5} worker = get_stubbed_worker(metrics.MetricTimeBucket, config=config) broker = BrokerWrapper(worker._amqp_client.broker) yield worker.startWorker() datapoints = [ ("vumi.test.foo", ("agg",), [(1230, 1.5), (1235, 2.0)]), ("vumi.test.bar", ("sum",), [(1240, 1.0)]), ] broker.send_datapoints("vumi.metrics", "vumi.metrics", datapoints) yield broker.kick_delivery() buckets = [broker.recv_datapoints("vumi.metrics.buckets", "bucket.%d" % i) for i in range(4)] expected_buckets = [ [], [[[u'vumi.test.bar', ['sum'], [[1240, 1.0]]]]], [[[u'vumi.test.foo', ['agg'], [[1230, 1.5]]]], [[u'vumi.test.foo', ['agg'], [[1235, 2.0]]]]], [], ] self.assertEqual(buckets, expected_buckets) yield worker.stopWorker()
def get_application(self, config, cls=None, start=True): """ Get an instance of a worker class. :param config: Config dict. :param cls: The Application class to instantiate. Defaults to :attr:`application_class` :param start: True to start the application (default), False otherwise. Some default config values are helpfully provided in the interests of reducing boilerplate: * ``transport_name`` defaults to :attr:`self.transport_name` * ``send_to`` defaults to a dictionary with config for each tag defined in worker's SEND_TO_TAGS attribute. Each tag's config contains a transport_name set to ``<tag>_outbound``. """ if cls is None: cls = self.application_class config.setdefault('transport_name', self.transport_name) if 'send_to' not in config and cls.SEND_TO_TAGS: config['send_to'] = {} for tag in cls.SEND_TO_TAGS: config['send_to'][tag] = { 'transport_name': '%s_outbound' % tag} worker = get_stubbed_worker(cls, config, self._amqp) self._workers.append(worker) if start: yield worker.startWorker() returnValue(worker)
def mk_failure_worker(self, config, broker, redis): w = get_stubbed_worker(Vas2NetsFailureWorker, config, broker) self.workers.append(w) yield w.startWorker() w.retry_publisher = yield self.worker.publish_to("foo") w.r_server = redis returnValue(w)
def get_worker(self, **config_extras): if getattr(self, 'worker', None) is not None: yield self.worker.stopWorker() config = { "transport_names": [ "transport1", "transport2", "transport3", ], "exposed_names": [ "app1", "app2", "app3", ], "router_class": "vumi.dispatchers.base.SimpleDispatchRouter", "route_mappings": { "transport1": ["app1"], "transport2": ["app2"], "transport3": ["app1", "app3"] }, } config.update(config_extras) self.worker = get_stubbed_worker(BaseDispatchWorker, config) self._amqp = self.worker._amqp_client.broker yield self.worker.startWorker()
def setUp(self): self.transport_name = 'test' self.config = {'transport_name': self.transport_name} self.worker = get_stubbed_worker(DummyApplicationWorker, config=self.config) self.broker = self.worker._amqp_client.broker yield self.worker.startWorker()
def mk_resource_worker(self, body, headers=None, code=http.OK): w = get_stubbed_worker(TestResourceWorker, {}, self.broker) self.workers.append(w) w.set_resources([(self.path, BadVas2NetsResource, (body, headers, code))]) yield w.startWorker() returnValue(w)
def get_worker(self): worker = get_stubbed_worker(MemoWorker, { # 'transport_name': 'foo', # 'ussd_code': '99999', }) yield worker.startWorker() returnValue(worker)
def mk_failure_worker(self, config, broker, redis): w = get_stubbed_worker(FailureWorker, config, broker) self.workers.append(w) w.retry_publisher = yield self.worker.publish_to("foo") w.r_server = redis yield w.startWorker() returnValue(w)
def get_application(self, config, cls=None, start=True): """ Get an instance of a worker class. :param config: Config dict. :param cls: The Application class to instantiate. Defaults to :attr:`application_class` :param start: True to start the application (default), False otherwise. Some default config values are helpfully provided in the interests of reducing boilerplate: * ``transport_name`` defaults to :attr:`self.transport_name` * ``send_to`` defaults to a dictionary with config for each tag defined in worker's SEND_TO_TAGS attribute. Each tag's config contains a transport_name set to ``<tag>_outbound``. """ if cls is None: cls = self.application_class config.setdefault('transport_name', self.transport_name) if 'send_to' not in config and cls.SEND_TO_TAGS: config['send_to'] = {} for tag in cls.SEND_TO_TAGS: config['send_to'][tag] = { 'transport_name': '%s_outbound' % tag } worker = get_stubbed_worker(cls, config, self._amqp) self._workers.append(worker) if start: yield worker.startWorker() returnValue(worker)
def get_worker(self): worker = get_stubbed_worker(RockPaperScissorsWorker, { 'transport_name': 'foo', 'ussd_code': '99999', }, self._amqp) self._workers.append(worker) worker.startWorker() return worker
def make_resource_worker(self, code=http.OK, content='', headers={}, callback=None): w = get_stubbed_worker(TestResourceWorker, {}) w.set_resources([ (self.path, TestResource, (code, content, headers, callback))]) self._workers.append(w) yield w.startWorker() returnValue(w)
def get_worker(self, config=None): if not config: config = { 'worker_name': 'wikitest', 'sms_transport': 'sphex', } config.setdefault('transport_name', self.transport_name) worker = get_stubbed_worker(WikipediaWorker, config, self.broker) self._workers.append(worker) yield worker.startWorker() returnValue(worker)
def make_resource_worker(self, code=http.OK, content='', headers={}, callback=None): w = get_stubbed_worker(TestResourceWorker, {}) w.set_resources([(self.path, TestResource, (code, content, headers, callback))]) self._workers.append(w) yield w.startWorker() returnValue(w)
def test_start_publisher(self): """The publisher should publish""" worker = get_stubbed_worker(Worker) publisher = yield worker.publish_to('test.routing.key') self.assertEquals(publisher.routing_key, 'test.routing.key') publisher.publish_message(Message(key="value")) [published_msg] = publisher.channel.broker.get_dispatched( 'vumi', 'test.routing.key') self.assertEquals(published_msg.body, '{"key": "value"}') self.assertEquals(published_msg.properties, {'delivery mode': 2})
def setUp(self): config = { "route_mappings": { "test1.inbound": ["test2.outbound", "test3.outbound"], "test2.inbound": ["test1.outbound"], "test3.inbound": ["test1.outbound"] } } self.worker = get_stubbed_worker(SimpleDispatcher, config) self.broker = self.worker._amqp_client.broker yield self.worker.startWorker()
def setUp(self): config = { 'transport_name': 'test_infobip', 'transport_type': 'ussd', 'web_path': "/session/", 'web_port': 0, } self.worker = get_stubbed_worker(InfobipTransport, config) self.broker = self.worker._amqp_client.broker yield self.worker.startWorker() self.worker_url = self.worker.get_transport_url() self.worker.r_server = FakeRedis()
def make_worker(self, retry_delivery_period=0): self.config = { 'transport_name': 'sphex', 'retry_routing_key': 'sms.outbound.%(transport_name)s', 'failures_routing_key': 'sms.failures.%(transport_name)s', 'retry_delivery_period': retry_delivery_period, } self.worker = get_stubbed_worker(FailureWorker, self.config) self.worker.r_server = FakeRedis() self.worker.r_server.flushdb() self.redis = self.worker.r_server yield self.worker.startWorker() self.broker = self.worker._amqp_client.broker self.retry_timestamps_key = self.worker._retry_timestamps_key
def test_send_to_with_bad_config(self): config = {'transport_name': 'badconfig_app', 'send_to': { 'default': {}, # missing transport_name 'outbound1': {}, # also missing transport_name }, } badcfg_worker = get_stubbed_worker(DummyApplicationWorker, config=config) errors = [] d = badcfg_worker.startWorker() d.addErrback(lambda result: errors.append(result)) yield d self.assertEqual(errors[0].type, ConfigError)
def make_worker(self, retry_delivery_period=0): self.config = { "transport_name": "sphex", "retry_routing_key": "sms.outbound.%(transport_name)s", "failures_routing_key": "sms.failures.%(transport_name)s", "retry_delivery_period": retry_delivery_period, } self.worker = get_stubbed_worker(FailureWorker, self.config) self.worker.r_server = FakeRedis() self.worker.r_server.flushdb() self.redis = self.worker.r_server yield self.worker.startWorker() self.broker = self.worker._amqp_client.broker self.retry_timestamps_key = self.worker._retry_timestamps_key
def setUp(self): config = { "transport_names": [ "transport1", "transport2", ], "exposed_names": [], "router_class": "vumi.dispatchers.base.TransportToTransportRouter", "route_mappings": { "transport1": ["transport2"], }, } self.worker = get_stubbed_worker(BaseDispatchWorker, config) self._amqp = self.worker._amqp_client.broker yield self.worker.startWorker()
def test_subclassing_api(self): worker = get_stubbed_worker(ApplicationWorker, {'transport_name': 'test'}) ack = TransportEvent(event_type='ack', sent_message_id='remote-id', user_message_id='ack-uuid') dr = TransportEvent(event_type='delivery_report', delivery_status='pending', user_message_id='dr-uuid') worker.consume_ack(ack) worker.consume_delivery_report(dr) worker.consume_unknown_event(FakeUserMessage()) worker.consume_user_message(FakeUserMessage()) worker.new_session(FakeUserMessage()) worker.close_session(FakeUserMessage())
def setUp(self): config = { "route_mappings": { "test1.inbound": [ "test2.outbound", "test3.outbound"], "test2.inbound": [ "test1.outbound"], "test3.inbound": [ "test1.outbound"] } } self.worker = get_stubbed_worker(SimpleDispatcher, config) self.broker = self.worker._amqp_client.broker yield self.worker.startWorker()
def test_single_message(self): worker = get_stubbed_worker(metrics_workers.GraphiteMetricsCollector) broker = BrokerWrapper(worker._amqp_client.broker) yield worker.startWorker() datapoints = [("vumi.test.foo", "", [(1234, 1.5)])] broker.send_datapoints("vumi.metrics.aggregates", "vumi.metrics.aggregates", datapoints) yield broker.kick_delivery() content, = broker.get_dispatched("graphite", "vumi.test.foo") parts = content.body.split() value, ts = float(parts[0]), int(parts[1]) self.assertEqual(value, 1.5) self.assertEqual(ts, 1234)
def test_send_to_with_bad_config(self): config = { 'transport_name': 'badconfig_app', 'send_to': { 'default': {}, # missing transport_name 'outbound1': {}, # also missing transport_name }, } badcfg_worker = get_stubbed_worker(DummyApplicationWorker, config=config) errors = [] d = badcfg_worker.startWorker() d.addErrback(lambda result: errors.append(result)) yield d self.assertEqual(errors[0].type, ConfigError)
def test_single_message(self): worker = get_stubbed_worker(metrics.GraphiteMetricsCollector) broker = BrokerWrapper(worker._amqp_client.broker) yield worker.startWorker() datapoints = [("vumi.test.foo", "", [(1234, 1.5)])] broker.send_datapoints("vumi.metrics.aggregates", "vumi.metrics.aggregates", datapoints) yield broker.kick_delivery() content, = broker.get_dispatched("graphite", "vumi.test.foo") parts = content.body.split() value, ts = float(parts[0]), int(parts[1]) self.assertEqual(value, 1.5) self.assertEqual(ts, 1234)
def test_in_worker(self): worker = get_stubbed_worker(Worker) broker = worker._amqp_client.broker mm = yield worker.start_publisher(metrics.MetricManager, "vumi.test.", 0.1, self.on_publish) acc = mm.register(metrics.Metric("my.acc")) try: self.assertTrue(mm._task is not None) self._check_msg(broker, acc, None) acc.set(1.5) acc.set(1.0) yield self.wait_publish() self._check_msg(broker, acc, [1.5, 1.0]) finally: mm.stop()
def setUp(self): self.integrat_calls = DeferredQueue() self.mock_integrat = MockHttpServer(self.handle_request) yield self.mock_integrat.start() config = { 'transport_name': 'testgrat', 'web_path': "foo", 'web_port': "0", 'url': self.mock_integrat.url, 'username': '******', 'password': '******', } self.worker = get_stubbed_worker(IntegratTransport, config) self.broker = self.worker._amqp_client.broker yield self.worker.startWorker() addr = self.worker.web_resource.getHost() self.worker_url = "http://%s:%s/" % (addr.host, addr.port)
def setUp(self): self.transport_name = 'test' self.config = { 'transport_name': self.transport_name, 'send_to': { 'default': { 'transport_name': 'default_transport', }, 'outbound1': { 'transport_name': 'outbound1_transport', }, }, } self.worker = get_stubbed_worker(DummyApplicationWorker, config=self.config) self.broker = self.worker._amqp_client.broker yield self.worker.startWorker()
def setUp(self): root = Resource() # data is elephant with a UTF-8 encoded BOM # it is a sad elephant (as seen in the wild) root.putChild("word", Data('\xef\xbb\xbfelephant\r\n', 'text/html')) site_factory = Site(root) self.webserver = yield reactor.listenTCP(0, site_factory) addr = self.webserver.getHost() random_word_url = "http://%s:%s/word" % (addr.host, addr.port) self.worker = get_stubbed_worker(HangmanWorkerStub, { 'transport_name': 'foo', 'ussd_code': '99999', 'random_word_url': random_word_url, }) yield self.worker.startWorker() self.worker.r_server = FakeRedis()
def test_consume(self): """The consume helper should direct all incoming messages matching the specified routing_key, queue_name & exchange to the given callback""" message = fake_amq_message({"key": "value"}) worker = get_stubbed_worker(Worker) # buffer to check messages consumed log = [Message.from_json(message.content.body)] # consume all messages on the given routing key and append # them to the log worker.consume('test.routing.key', lambda msg: log.append(msg)) # if all works well then the consume method should funnel the test # message straight to the callback, the callback will apend it to the # log and we can test it. worker._amqp_client.broker.basic_publish('vumi', 'test.routing.key', message.content) self.assertEquals(log, [Message(key="value")])
def setUp(self): DelayedCall.debug = True self.ok_transport_calls = DeferredQueue() self.mock_service = MockHttpServer(self.handle_request) yield self.mock_service.start() config = { 'transport_name': 'test_ok_transport', 'transport_type': 'ok', 'ussd_string_prefix': '', 'web_path': "foo", 'web_port': 0, 'url': self.mock_service.url, 'username': '******', 'password': '******', } self.worker = get_stubbed_worker(OkTransport, config) self.broker = self.worker._amqp_client.broker yield self.worker.startWorker() self.worker_url = self.worker.get_transport_url()
def test_one_run(self): worker = get_stubbed_worker(metrics.RandomMetricsGenerator, config={ "manager_period": "0.1", "generator_period": "0.1", }) worker.on_run = self.on_run broker = BrokerWrapper(worker._amqp_client.broker) yield worker.startWorker() yield self.wake_after_run() yield self.wake_after_run() datapoints, = broker.recv_datapoints('vumi.metrics', 'vumi.metrics') self.assertEqual(sorted(d[0] for d in datapoints), ["vumi.random.count", "vumi.random.timer", "vumi.random.value"]) yield worker.stopWorker()
def setUp(self): self.transport_name = 'test' self.control_name = 'mycontrol' self.database_name = 'test_program_db' self.vusion_database_name = 'test_vusion_db' self.config = {'transport_name': self.transport_name, 'database_name': self.database_name, 'vusion_database_name': self.vusion_database_name, 'control_name': self.control_name, 'dispatcher_name': 'dispatcher'} self.worker = get_stubbed_worker(TtcGenericWorker, config=self.config) self.broker = self.worker._amqp_client.broker self.broker.exchange_declare('vumi', 'direct') self.broker.queue_declare("%s.outbound" % self.transport_name) self.broker.queue_bind("%s.outbound" % self.transport_name, "vumi", "%s.outbound" % self.transport_name) #Database# connection = pymongo.Connection("localhost", 27017) self.db = connection[self.config['database_name']] self.collections = {} self.setup_collections(['dialogues', 'participants', 'history', 'schedules', 'shortcodes', 'program_settings', 'unattached_messages', 'requests']) self.db = connection[self.config['vusion_database_name']] self.setup_collections(['templates']) self.drop_collections() #Let's rock" self.worker.startService() yield self.worker.startWorker()
def setUp(self): self.config = { 'database_name': 'test', 'application_name': 'garbage', 'transport_name': 'garbage' } connection = pymongo.Connection('localhost', 27017) db = connection[self.config['database_name']] self.unmatchable_replies_collection = db['unmatchable_reply'] self.unmatchable_replies_collection.drop() self.templates_collection = db['templates'] self.templates_collection.drop() self.shortcodes_collection = db['shortcodes'] self.shortcodes_collection.drop() self.worker = get_stubbed_worker(GarbageWorker, config=self.config) self.broker = self.worker._amqp_client.broker self.worker.startService() yield self.worker.startWorker()
def test_one_run(self): worker = get_stubbed_worker(metrics_workers.RandomMetricsGenerator, config={ "manager_period": "0.1", "generator_period": "0.1", }) self._workers.append(worker) worker.on_run = self.on_run broker = BrokerWrapper(worker._amqp_client.broker) yield worker.startWorker() yield self.wake_after_run() yield self.wake_after_run() datasets = broker.recv_datapoints('vumi.metrics', 'vumi.metrics') # there should be a least one but there may be more # than one if the tests are running slowly datapoints = datasets[0] self.assertEqual( sorted(d[0] for d in datapoints), ["vumi.random.count", "vumi.random.timer", "vumi.random.value"])
def get_transport(self, config, cls=None, start=True): """ Get an instance of a transport class. :param config: Config dict. :param cls: The transport class to instantiate. Defaults to :attr:`transport_class` :param start: True to start the transport (default), False otherwise. Some default config values are helpfully provided in the interests of reducing boilerplate: * ``transport_name`` defaults to :attr:`self.transport_name` """ if cls is None: cls = self.transport_class config.setdefault('transport_name', self.transport_name) worker = get_stubbed_worker(cls, config, self._amqp) self._workers.append(worker) if start: yield worker.startWorker() returnValue(worker)
def get_worker(self): config = { 'dispatcher_name': 'vusion', 'router_class': 'vumi.dispatchers.ContentKeywordRouter', 'exposed_names': ['app1', 'fallback_app'], 'keyword_mappings': { 'app1': 'keyword1' }, 'rules': [{ 'app': 'app1', 'keyword': 'keyword2', 'to_addr': '8181' }], 'transport_names': ['transport1'], 'transport_mappings': { '8181': 'transport1' }, 'fallback_application': 'fallback_app', 'expire_routing_memory': '1' } self.worker = get_stubbed_worker(DynamicDispatchWorker, config) self._amqp = self.worker._amqp_client.broker yield self.worker.startWorker()
def setUp(self): self.transport_name = 'test' self.control_name = 'mycontrol' self.database_name = 'test_program_db' self.vusion_database_name = 'test_vusion_db' self.config = { 'transport_name': self.transport_name, 'database_name': self.database_name, 'vusion_database_name': self.vusion_database_name, 'control_name': self.control_name, 'dispatcher_name': 'dispatcher' } self.worker = get_stubbed_worker(TtcGenericWorker, config=self.config) self.broker = self.worker._amqp_client.broker self.broker.exchange_declare('vumi', 'direct') self.broker.queue_declare("%s.outbound" % self.transport_name) self.broker.queue_bind("%s.outbound" % self.transport_name, "vumi", "%s.outbound" % self.transport_name) #Database# connection = pymongo.Connection("localhost", 27017) self.db = connection[self.config['database_name']] self.collections = {} self.setup_collections([ 'dialogues', 'participants', 'history', 'schedules', 'shortcodes', 'program_settings', 'unattached_messages', 'requests' ]) self.db = connection[self.config['vusion_database_name']] self.setup_collections(['templates']) self.drop_collections() #Let's rock" self.worker.startService() yield self.worker.startWorker()
def test_get_stubbed_worker(self): worker = get_stubbed_worker(ToyWorker) self.assertEqual("poke", worker.poke()) self.assertTrue(isinstance(worker._amqp_client, FakeAMQClient))
def test_get_stubbed_worker_with_config(self): options = {'key': 'value'} worker = get_stubbed_worker(ToyWorker, options) self.assertEqual({}, worker._amqp_client.vumi_options) self.assertEqual(options, worker.config)
def make_resource_worker(self, msg, code=http.OK, send_id=None): w = get_stubbed_worker(TestResourceWorker, {}) w.set_resources([ (self.send_path, TestResource, (msg, code, send_id))]) self._workers.append(w) return w.startWorker()
def get_multiworker(self, config): self.worker = get_stubbed_worker(StubbedMultiWorker, config) self.worker.startService() self.broker = self.worker._amqp_client.broker yield self.worker.wait_for_workers() returnValue(self.worker)