def test_emitter_exception(self): emitter = Emitter(urls=['file:///tmp/test_emitter'], emitter_args={'extra': '{"a2}', 'extra_all_features': True, 'uuid': 'aaaaaa'}, format='csv') emitter.__enter__() emitter.__exit__(None, ValueError('a'), None)
def test_emitter_json_http_simple(self, mock_sleep, mock_post): metadata = {} metadata['namespace'] = 'namespace777' retries = 5 with Emitter(urls=['http://1.1.1.1/good'], emitter_args=metadata, format='json', max_emit_retries=retries) as emitter: emitter.emit("dummy_feature", {'test': 'bla', 'test2': 12345, 'test3': 12345.0, 'test4': 12345.00000}, 'dummy_feature') emitter.emit("dummy_feature", {'test': 'bla', 'test2': 12345, 'test3': 12345.0, 'test4': 12345.00000}, 'dummy_feature') ''' we expect call_count to be equal to number of emit data + 1 (metadata) ''' self.assertEqual(mock_post.call_count, 3)
def test_emitter_csv_compressed_stdout(self): with Capturing() as _output: with Emitter(urls=['stdout://'], emitter_args={'namespace': '123', 'compress': True}) as emitter: emitter.emit("dummy", {'test': 'bla'}, 'dummy') output = "%s" % _output
def test_emitter_csv_kafka_invalid_url( self, MockKafkaClient1, MockKafkaClient2): metadata = {} metadata['namespace'] = 'namespace777' with self.assertRaises(crawler.crawler_exceptions.EmitterBadURL): with Emitter(urls=['kafka://abc'], max_emit_retries=1) as emitter: emitter.emit("dummy_feature", {'test': 'bla'}, 'dummy_feature')
def test_emitter_graphite_simple_file(self): metadata = {} metadata['namespace'] = 'namespace777' with Emitter(urls=['file:///tmp/test_emitter'], emitter_args=metadata, format='graphite') as emitter: emitter.emit("dummy_feature", {'test': 'bla', 'test2': 12345, 'test3': 12345.0, 'test4': 12345.00000}, 'dummy_feature') with open('/tmp/test_emitter') as f: _output = f.readlines() output = "%s" % _output # should look like this: # ['namespace777.dummy-feature.test3 3.000000 1449870719', # 'namespace777.dummy-feature.test2 2.000000 1449870719', # 'namespace777.dummy-feature.test4 4.000000 1449870719'] assert len(_output) == 3 assert "dummy_feature" not in output # can't have '_' assert "dummy-feature" in output # can't have '_' assert "metadata" not in output assert 'namespace777.dummy-feature.test2' in output assert 'namespace777.dummy-feature.test3' in output assert 'namespace777.dummy-feature.test4' in output # three fields in graphite format assert len(_output[0].split(' ')) == 3 # three fields in graphite format assert len(_output[1].split(' ')) == 3 # three fields in graphite format assert len(_output[2].split(' ')) == 3 assert float(_output[0].split(' ')[1]) == 12345.0 assert float(_output[1].split(' ')[1]) == 12345.0 assert float(_output[2].split(' ')[1]) == 12345.0
def _test_emitter_csv_simple_stdout(self): with Emitter(urls=['stdout://']) as emitter: emitter.emit("dummy_feature", {'test': 'bla', 'test2': 12345, 'test3': 12345.0, 'test4': 12345.00000}, 'dummy_feature')
def test_emitter_mtgraphite(self, mock_sleep, MockMTGraphiteClient): metadata = {} metadata['namespace'] = 'namespace777' retries = 2 with Emitter(urls=['mtgraphite://1.1.1.1:123/topic1'], emitter_args=metadata, max_emit_retries=retries) as emitter: emitter.emit("dummy_feature", {'test': 'bla', 'test2': 12345, 'test3': 12345.0, 'test4': 12345.00000}, 'dummy_feature') emitter.emit("dummy_feature", {'test': 'bla', 'test2': 12345, 'test3': 12345.0, 'test4': 12345.00000}, 'dummy_feature') metadata = {} metadata['namespace'] = 'namespace777' retries = 2 with Emitter(urls=['mtgraphite://1.1.1.1:123/topic1'], emitter_args=metadata, format='graphite', max_emit_retries=retries) as emitter: emitter.emit("dummy_feature", {'test': 'bla', 'test2': 12345, 'test3': 12345.0, 'test4': 12345.00000}, 'dummy_feature') emitter.emit("dummy_feature", {'test': 'bla', 'test2': 12345, 'test3': 12345.0, 'test4': 12345.00000}, 'dummy_feature') """ The thing with the mtgraphite client is that it's a static long standin connection, so if you instantiate lots of Emitter's, the connection will be created once; i.e. the client will be instantiated once. """ self.assertEqual(MockMTGraphiteClient.call_count, 1)
def test_emitter_graphite_invalid_feature(self): metadata = {} metadata['namespace'] = 'namespace777' with Emitter(urls=['file:///tmp/test_emitter'], emitter_args=metadata, format='graphite') as emitter: with self.assertRaises(AttributeError): emitter.emit("dummy", {'blabla'}, 'dummy') with self.assertRaises(AttributeError): emitter.emit("dummy", 12, 'dummy')
def test_emitter_csv_kafka_failed_new_process(self, mock_process): metadata = {} metadata['namespace'] = 'namespace777' retries = 2 with self.assertRaises(OSError): with Emitter(urls=['kafka://1.1.1.1:123/timeouttopic'], emitter_args=metadata, max_emit_retries=retries, kafka_timeout_secs=0.1) as emitter: emitter.emit("dummy", {'test': 'bla'}, 'dummy')
def test_emitter_csv_kafka_emit_timeout(self, mock_sleep, MockC1, MockC2): metadata = {} metadata['namespace'] = 'namespace777' retries = 2 with self.assertRaises(crawler.crawler_exceptions.EmitterEmitTimeout): with Emitter(urls=['kafka://1.1.1.1:123/timeouttopic'], emitter_args=metadata, max_emit_retries=retries, kafka_timeout_secs=0.1) as emitter: emitter.emit("dummy", {'test': 'bla'}, 'dummy')
def _test_emitter_graphite_simple_stdout(self): metadata = {} metadata['namespace'] = 'namespace777' with Emitter(urls=['stdout://'], emitter_args=metadata, format='graphite') as emitter: emitter.emit("dummy_feature", {'test': 'bla', 'test2': 12345, 'test3': 12345.0, 'test4': 12345.00000}, 'dummy_feature')
def test_emitter_incorrect_json(self): try: with Emitter(urls=['file:///tmp/test_emitter'], emitter_args={'extra': '{"a2}', 'extra_all_features': True, 'uuid': 'aaaaaa'}, format='csv') as emitter: emitter.emit("memory", {'test3': 12345}, 'memory') except ValueError: pass except Exception: raise
def test_emitter_failed_emit(self): metadata = {} metadata['namespace'] = 'namespace777' try: with Emitter(urls=['file:///tmp/test_emitter'], format='csv') as emitter: with mock.patch('crawler.emitter.json.dumps') as mock_dumps: mock_dumps.side_effect = ValueError() emitter.emit("memory", {'test3': 12345}, 'memory') except ValueError: pass except Exception: raise
def test_emitter_csv_kafka_failed_emit_no_retries(self, MockC1, MockC2): metadata = {} metadata['namespace'] = 'namespace777' retries = 1 with self.assertRaises(RandomKafkaException): with Emitter(urls=['kafka://1.1.1.1:123/badtopic'], emitter_args=metadata, max_emit_retries=retries) as emitter: emitter.emit("dummy_feature", {'test': 'bla', 'test2': 12345, 'test3': 12345.0, 'test4': 12345.00000}, 'dummy_feature')
def test_emitter_csv_kafka( self, mock_sleep, MockKafkaClient1, MockKafkaClient2): metadata = {} metadata['namespace'] = 'namespace777' retries = 2 with Emitter(urls=['kafka://1.1.1.1:123/topic1'], emitter_args=metadata, max_emit_retries=retries) as emitter: emitter.emit("dummy_feature", {'test': 'bla', 'test2': 12345, 'test3': 12345.0, 'test4': 12345.00000}, 'dummy_feature')
def test_emitter_csv_simple_file(self): with Emitter(urls=['file:///tmp/test_emitter'], emitter_args={'namespace': 'bla'}) as emitter: emitter.emit("dummy_feature", {'test': 'bla', 'test2': 12345, 'test3': 12345.0, 'test4': 12345.00000}, 'dummy_feature') with open('/tmp/test_emitter') as f: _output = f.readlines() output = "%s" % _output assert len(_output) == 2 assert "dummy_feature" in output assert "metadata" in output
def test_emitter_csv_kafka_unsupported_format( self, mock_sleep, MockC1, MockC2): metadata = {} metadata['namespace'] = 'namespace777' retries = 2 with self.assertRaises(crawler.crawler_exceptions.EmitterUnsupportedFormat): with Emitter(urls=['kafka://1.1.1.1:123/badtopic'], emitter_args=metadata, format='blablafformat', max_emit_retries=retries) as emitter: emitter.emit("dummy_feature", {'test': 'bla', 'test2': 12345, 'test3': 12345.0, 'test4': 12345.00000}, 'dummy_feature')
def test_emitter_unsupported_format(self): metadata = {} metadata['namespace'] = 'namespace777' try: with Emitter(urls=['file:///tmp/test_emitter'], emitter_args=metadata, format='unsupported') as emitter: emitter.emit("dummy_feature", {'test': 'bla', 'test2': 12345, 'test3': 12345.0, 'test4': 12345.00000}, 'dummy_feature') except crawler.crawler_exceptions.EmitterUnsupportedFormat: pass except Exception: raise
def test_emitter_exception(self): emitter = Emitter(urls=['file:///tmp/test_emitter'], emitter_args={'extra': '{"a2}', 'extra_all_features': True, 'uuid': 'aaaaaa'}, format='csv') emitter.__enter__() with self.assertRaises(ValueError): emitter.__exit__(None, ValueError('a'), None) with self.assertRaises(ValueError): with Emitter(urls=['file:///tmp/test_emitter'], emitter_args={'extra': '{"a2}', 'extra_all_features': True, 'uuid': 'aaaaaa'}, format='csv') as emitter: raise ValueError('bla')
def test_emitter_all_features_graphite(self): with Emitter(urls=['file:///tmp/test_emitter'], emitter_args={'extra': '{"a":"1", "b":2}', 'extra_all_features': True, 'uuid': 'aaaaaa'}, format='graphite') as emitter: emitter.emit("memory", {'test3': 12345}, 'memory') emitter.emit("memory_0", {'test3': 12345}, 'memory') emitter.emit("load", {'load': 12345}, 'load') emitter.emit("cpu", {'test3': 12345}, 'cpu') emitter.emit("cpu_0", {'test3': 12345}, 'cpu') emitter.emit("eth0", {'if_tx': 12345}, 'interface') emitter.emit("eth0", {'if_rx': 12345}, 'interface') emitter.emit("bla/bla", {'ble/ble': 12345}, 'disk') with open('/tmp/test_emitter') as f: _output = f.readlines() output = "%s" % _output print output assert len(_output) == 24
def test_emitter_graphite_broker_server_error(self, mock_sleep, mock_post): metadata = {} metadata['namespace'] = 'namespace777' retries = 2 with Emitter(urls=['http://1.1.1.1/bad'], emitter_args=metadata, format='graphite', max_emit_retries=retries) as emitter: emitter.emit("dummy_feature", {'test': 'bla', 'test2': 12345, 'test3': 12345.0, 'test4': 12345.00000}, 'dummy_feature') emitter.emit("dummy_feature", {'test': 'bla', 'test2': 12345, 'test3': 12345.0, 'test4': 12345.00000}, 'dummy_feature') self.assertEqual(mock_post.call_count, retries)