def test_emitter_exception(self):
emitter = Emitter(urls=['file:///tmp/test_emitter'],
emitter_args={'extra': '{"a2}',
'extra_all_features': True,
'uuid': 'aaaaaa'},
format='csv')
emitter.__enter__()
emitter.__exit__(None, ValueError('a'), None)
def test_emitter_json_http_simple(self, mock_sleep, mock_post):
metadata = {}
metadata['namespace'] = 'namespace777'
retries = 5
with Emitter(urls=['http://1.1.1.1/good'],
emitter_args=metadata,
format='json',
max_emit_retries=retries) as emitter:
emitter.emit("dummy_feature",
{'test': 'bla',
'test2': 12345,
'test3': 12345.0,
'test4': 12345.00000},
'dummy_feature')
emitter.emit("dummy_feature",
{'test': 'bla',
'test2': 12345,
'test3': 12345.0,
'test4': 12345.00000},
'dummy_feature')
'''
we expect call_count to be equal to number of
emit data + 1 (metadata)
'''
self.assertEqual(mock_post.call_count, 3)
def test_emitter_csv_compressed_stdout(self):
with Capturing() as _output:
with Emitter(urls=['stdout://'],
emitter_args={'namespace': '123',
'compress': True}) as emitter:
emitter.emit("dummy", {'test': 'bla'}, 'dummy')
output = "%s" % _output
def test_emitter_csv_kafka_invalid_url(
self, MockKafkaClient1, MockKafkaClient2):
metadata = {}
metadata['namespace'] = 'namespace777'
with self.assertRaises(crawler.crawler_exceptions.EmitterBadURL):
with Emitter(urls=['kafka://abc'], max_emit_retries=1) as emitter:
emitter.emit("dummy_feature", {'test': 'bla'}, 'dummy_feature')
def test_emitter_graphite_simple_file(self):
metadata = {}
metadata['namespace'] = 'namespace777'
with Emitter(urls=['file:///tmp/test_emitter'],
emitter_args=metadata,
format='graphite') as emitter:
emitter.emit("dummy_feature",
{'test': 'bla',
'test2': 12345,
'test3': 12345.0,
'test4': 12345.00000},
'dummy_feature')
with open('/tmp/test_emitter') as f:
_output = f.readlines()
output = "%s" % _output
# should look like this:
# ['namespace777.dummy-feature.test3 3.000000 1449870719',
# 'namespace777.dummy-feature.test2 2.000000 1449870719',
# 'namespace777.dummy-feature.test4 4.000000 1449870719']
assert len(_output) == 3
assert "dummy_feature" not in output # can't have '_'
assert "dummy-feature" in output # can't have '_'
assert "metadata" not in output
assert 'namespace777.dummy-feature.test2' in output
assert 'namespace777.dummy-feature.test3' in output
assert 'namespace777.dummy-feature.test4' in output
# three fields in graphite format
assert len(_output[0].split(' ')) == 3
# three fields in graphite format
assert len(_output[1].split(' ')) == 3
# three fields in graphite format
assert len(_output[2].split(' ')) == 3
assert float(_output[0].split(' ')[1]) == 12345.0
assert float(_output[1].split(' ')[1]) == 12345.0
assert float(_output[2].split(' ')[1]) == 12345.0
def _test_emitter_csv_simple_stdout(self):
with Emitter(urls=['stdout://']) as emitter:
emitter.emit("dummy_feature",
{'test': 'bla',
'test2': 12345,
'test3': 12345.0,
'test4': 12345.00000},
'dummy_feature')
def test_emitter_mtgraphite(self, mock_sleep, MockMTGraphiteClient):
metadata = {}
metadata['namespace'] = 'namespace777'
retries = 2
with Emitter(urls=['mtgraphite://1.1.1.1:123/topic1'],
emitter_args=metadata,
max_emit_retries=retries) as emitter:
emitter.emit("dummy_feature",
{'test': 'bla',
'test2': 12345,
'test3': 12345.0,
'test4': 12345.00000},
'dummy_feature')
emitter.emit("dummy_feature",
{'test': 'bla',
'test2': 12345,
'test3': 12345.0,
'test4': 12345.00000},
'dummy_feature')
metadata = {}
metadata['namespace'] = 'namespace777'
retries = 2
with Emitter(urls=['mtgraphite://1.1.1.1:123/topic1'],
emitter_args=metadata,
format='graphite',
max_emit_retries=retries) as emitter:
emitter.emit("dummy_feature",
{'test': 'bla',
'test2': 12345,
'test3': 12345.0,
'test4': 12345.00000},
'dummy_feature')
emitter.emit("dummy_feature",
{'test': 'bla',
'test2': 12345,
'test3': 12345.0,
'test4': 12345.00000},
'dummy_feature')
"""
The thing with the mtgraphite client is that it's a static long standin
connection, so if you instantiate lots of Emitter's, the connection
will be created once; i.e. the client will be instantiated once.
"""
self.assertEqual(MockMTGraphiteClient.call_count, 1)
def test_emitter_graphite_invalid_feature(self):
metadata = {}
metadata['namespace'] = 'namespace777'
with Emitter(urls=['file:///tmp/test_emitter'],
emitter_args=metadata,
format='graphite') as emitter:
with self.assertRaises(AttributeError):
emitter.emit("dummy", {'blabla'}, 'dummy')
with self.assertRaises(AttributeError):
emitter.emit("dummy", 12, 'dummy')
def test_emitter_csv_kafka_failed_new_process(self, mock_process):
metadata = {}
metadata['namespace'] = 'namespace777'
retries = 2
with self.assertRaises(OSError):
with Emitter(urls=['kafka://1.1.1.1:123/timeouttopic'],
emitter_args=metadata,
max_emit_retries=retries,
kafka_timeout_secs=0.1) as emitter:
emitter.emit("dummy", {'test': 'bla'}, 'dummy')
def test_emitter_csv_kafka_emit_timeout(self, mock_sleep, MockC1, MockC2):
metadata = {}
metadata['namespace'] = 'namespace777'
retries = 2
with self.assertRaises(crawler.crawler_exceptions.EmitterEmitTimeout):
with Emitter(urls=['kafka://1.1.1.1:123/timeouttopic'],
emitter_args=metadata,
max_emit_retries=retries,
kafka_timeout_secs=0.1) as emitter:
emitter.emit("dummy", {'test': 'bla'}, 'dummy')
def _test_emitter_graphite_simple_stdout(self):
metadata = {}
metadata['namespace'] = 'namespace777'
with Emitter(urls=['stdout://'],
emitter_args=metadata,
format='graphite') as emitter:
emitter.emit("dummy_feature",
{'test': 'bla',
'test2': 12345,
'test3': 12345.0,
'test4': 12345.00000},
'dummy_feature')
def test_emitter_incorrect_json(self):
try:
with Emitter(urls=['file:///tmp/test_emitter'],
emitter_args={'extra': '{"a2}',
'extra_all_features': True,
'uuid': 'aaaaaa'},
format='csv') as emitter:
emitter.emit("memory", {'test3': 12345}, 'memory')
except ValueError:
pass
except Exception:
raise
def test_emitter_failed_emit(self):
metadata = {}
metadata['namespace'] = 'namespace777'
try:
with Emitter(urls=['file:///tmp/test_emitter'],
format='csv') as emitter:
with mock.patch('crawler.emitter.json.dumps') as mock_dumps:
mock_dumps.side_effect = ValueError()
emitter.emit("memory", {'test3': 12345}, 'memory')
except ValueError:
pass
except Exception:
raise
def test_emitter_exception(self):
emitter = Emitter(urls=['file:///tmp/test_emitter'],
emitter_args={'extra': '{"a2}',
'extra_all_features': True,
'uuid': 'aaaaaa'},
format='csv')
emitter.__enter__()
emitter.__exit__(None, ValueError('a'), None)
def test_emitter_csv_kafka_failed_emit_no_retries(self, MockC1, MockC2):
metadata = {}
metadata['namespace'] = 'namespace777'
retries = 1
with self.assertRaises(RandomKafkaException):
with Emitter(urls=['kafka://1.1.1.1:123/badtopic'],
emitter_args=metadata,
max_emit_retries=retries) as emitter:
emitter.emit("dummy_feature",
{'test': 'bla',
'test2': 12345,
'test3': 12345.0,
'test4': 12345.00000},
'dummy_feature')
def test_emitter_csv_kafka(
self, mock_sleep, MockKafkaClient1, MockKafkaClient2):
metadata = {}
metadata['namespace'] = 'namespace777'
retries = 2
with Emitter(urls=['kafka://1.1.1.1:123/topic1'],
emitter_args=metadata,
max_emit_retries=retries) as emitter:
emitter.emit("dummy_feature",
{'test': 'bla',
'test2': 12345,
'test3': 12345.0,
'test4': 12345.00000},
'dummy_feature')
def test_emitter_csv_simple_file(self):
with Emitter(urls=['file:///tmp/test_emitter'],
emitter_args={'namespace': 'bla'}) as emitter:
emitter.emit("dummy_feature",
{'test': 'bla',
'test2': 12345,
'test3': 12345.0,
'test4': 12345.00000},
'dummy_feature')
with open('/tmp/test_emitter') as f:
_output = f.readlines()
output = "%s" % _output
assert len(_output) == 2
assert "dummy_feature" in output
assert "metadata" in output
def test_emitter_csv_kafka_unsupported_format(
self, mock_sleep, MockC1, MockC2):
metadata = {}
metadata['namespace'] = 'namespace777'
retries = 2
with self.assertRaises(crawler.crawler_exceptions.EmitterUnsupportedFormat):
with Emitter(urls=['kafka://1.1.1.1:123/badtopic'],
emitter_args=metadata,
format='blablafformat',
max_emit_retries=retries) as emitter:
emitter.emit("dummy_feature",
{'test': 'bla',
'test2': 12345,
'test3': 12345.0,
'test4': 12345.00000},
'dummy_feature')
def test_emitter_unsupported_format(self):
metadata = {}
metadata['namespace'] = 'namespace777'
try:
with Emitter(urls=['file:///tmp/test_emitter'],
emitter_args=metadata,
format='unsupported') as emitter:
emitter.emit("dummy_feature",
{'test': 'bla',
'test2': 12345,
'test3': 12345.0,
'test4': 12345.00000},
'dummy_feature')
except crawler.crawler_exceptions.EmitterUnsupportedFormat:
pass
except Exception:
raise
def test_emitter_exception(self):
emitter = Emitter(urls=['file:///tmp/test_emitter'],
emitter_args={'extra': '{"a2}',
'extra_all_features': True,
'uuid': 'aaaaaa'},
format='csv')
emitter.__enter__()
with self.assertRaises(ValueError):
emitter.__exit__(None, ValueError('a'), None)
with self.assertRaises(ValueError):
with Emitter(urls=['file:///tmp/test_emitter'],
emitter_args={'extra': '{"a2}',
'extra_all_features': True,
'uuid': 'aaaaaa'},
format='csv') as emitter:
raise ValueError('bla')
def test_emitter_all_features_graphite(self):
with Emitter(urls=['file:///tmp/test_emitter'],
emitter_args={'extra': '{"a":"1", "b":2}',
'extra_all_features': True,
'uuid': 'aaaaaa'},
format='graphite') as emitter:
emitter.emit("memory", {'test3': 12345}, 'memory')
emitter.emit("memory_0", {'test3': 12345}, 'memory')
emitter.emit("load", {'load': 12345}, 'load')
emitter.emit("cpu", {'test3': 12345}, 'cpu')
emitter.emit("cpu_0", {'test3': 12345}, 'cpu')
emitter.emit("eth0", {'if_tx': 12345}, 'interface')
emitter.emit("eth0", {'if_rx': 12345}, 'interface')
emitter.emit("bla/bla", {'ble/ble': 12345}, 'disk')
with open('/tmp/test_emitter') as f:
_output = f.readlines()
output = "%s" % _output
print output
assert len(_output) == 24
def test_emitter_graphite_broker_server_error(self, mock_sleep, mock_post):
metadata = {}
metadata['namespace'] = 'namespace777'
retries = 2
with Emitter(urls=['http://1.1.1.1/bad'],
emitter_args=metadata,
format='graphite',
max_emit_retries=retries) as emitter:
emitter.emit("dummy_feature",
{'test': 'bla',
'test2': 12345,
'test3': 12345.0,
'test4': 12345.00000},
'dummy_feature')
emitter.emit("dummy_feature",
{'test': 'bla',
'test2': 12345,
'test3': 12345.0,
'test4': 12345.00000},
'dummy_feature')
self.assertEqual(mock_post.call_count, retries)
