def test_reporter_start_recovers_from_data_source_exception(self):
DS = FailingTestDataSource([1, 2, 3], 3)
R = Reporter(DS, db_plugin='DictConnection')
R.start(interval=0, num_samples=1)
C = R.connection
self.assertEqual(2, len(C.get_data(R.topic_id)))
def test_reports_data_until_None(self):
DS = TestDataSource([1, 2, 3], 3)
R = Reporter(DS, db_plugin='DictConnection')
R.start(interval=0, num_samples=1)
C = R.connection
self.assertEqual(3, len(C.get_data(R.topic_id)))
def test_reporter_start_stop_on_errors(self):
DS = FailingTestDataSource([1, 2, 3], 3)
R = Reporter(DS, db_plugin='DictConnection')
with self.assertRaises(ZeroDivisionError) as a:
R.start(interval=0, num_samples=1, stop_on_errors=True)
self.assertEqual(str(a.exception), 'Test error (1)')
def execute(db_connection,
name,
verbose=False,
interval=2,
console_as_json=False):
print(f'Waiting for execution topic with name {name}.')
topic_d = wait_for_topic(db_connection, name, interval)
topic_id = topic_d.get('id')
print(f'Found topic {topic_id}.')
reporter = Reporter(db_connection=db_connection,
topic_id=topic_id,
verbose=verbose)
command, since, until = _get_execution(topic_d)
print(f'Waiting for execution to start at {timestamp(since)}.')
_wait_for_start(since, interval)
print(f'Repeating command until {timestamp(until)}.')
while _should_repeat(until):
if console_as_json:
started, process, elapsed, output = _run_command_json(command)
else:
started, process, elapsed, output = _run_command_raw(command)
data = dict(
started=timestamp(started),
elapsed=elapsed,
exit_status=process.returncode,
output=output,
hostname=gethostname(),
)
reporter.push(data)
print(f'Execution finished at {timestamp()}.')
def test_averaging_wont_push_if_no_valid_samples(self):
DS = TestDataSource([None, None, None], 3)
R = Reporter(DS, db_plugin='DictConnection')
R.start(interval=0, num_samples=6)
C = R.connection
data = C.get_data(R.topic_id)
self.assertEqual(0, len(data))
def test_start_supports_non_numeric_values(self):
DS = TestDataSource(['qwe', 'asd', 'zxc'], 3)
R = Reporter(DS, db_plugin='DictConnection')
R.start(interval=0, num_samples=1)
C = R.connection
self.assertEqual(3, len(C.get_data(R.topic_id)))
self.assertEqual('asd', C.get_data(R.topic_id)[1]['number'])
def test_provides_push_method(self):
DS = TestDataSource([], 0)
R = Reporter(DS, db_plugin='DictConnection')
for i in range(3):
R.push({'number': i})
C = R.connection
self.assertEqual(3, len(C.get_data(R.topic_id)))
self.assertEqual(0, C.get_data(R.topic_id)[0]['number'])
def test_creates_template_on_init(self):
DS = TestDataSourceWithTemplate()
R = Reporter(DS, db_plugin='DictConnection')
C = R.connection
self.assertEqual(len(C.get_topics()), 2)
self.assertEqual(C.get_topics()[0]["type"], "template")
R = Reporter(DS, db_connection=C)
self.assertEqual(len(C.get_topics()), 3)
def test_provides_averaging_over_push_interval(self):
DS = TestDataSource([0, 2, 4, 6, 8, 10], 6)
R = Reporter(DS, db_plugin='DictConnection')
R.start(interval=0, num_samples=6)
C = R.connection
data = C.get_data(R.topic_id)
self.assertEqual(1, len(data))
self.assertAlmostEqual(5, data[0]["number"])
def test_averaging_ignores_samples_with_none(self):
# TODO check for warning
DS = TestDataSource([0, 2, None, 6, None, 10], 6)
R = Reporter(DS, db_plugin='DictConnection')
R.start(interval=0, num_samples=6)
C = R.connection
data = C.get_data(R.topic_id)
self.assertEqual(1, len(data))
self.assertAlmostEqual(4.5, data[0]["number"])
def test_can_use_existing_connection(self):
DS = TestDataSource([1, 2, 3], 3)
C = DictConnection()
R = Reporter(DS, db_connection=C)
self.assertEqual(C.get_topic(R.topic_id)["name"], "topic")
self.assertEqual(C.get_topic(R.topic_id)["fields"], ["number"])
def main():
parser = get_reporter_argparser()
parser.add_argument("--target", "-t", action="append", default=[], type=str, nargs=2, metavar=("name","url"), help="Add target to monitor.")
parser.add_argument("--timeout", default=3, type=float, help="Timeout for network requests.")
parser.add_argument("--target-file", "-f", type=str, help="Add targets from json file to monitor.")
args = parser.parse_args()
targets_cmd = [{'name': name, 'url': url} for name, url in args.target]
if args.target_file is not None:
with open(args.target_file) as f:
targets_file = json.load(f)
else:
targets_file = []
targets = targets_cmd + targets_file
data_sources = (NetStatus(target, args.timeout) for target in targets)
db_connection = create_db_connection(args.db_connection, args.db_parameters)
reporters = (Reporter(data_source, db_connection, verbose=args.verbose) for data_source in data_sources)
threads = []
for reporter in reporters:
threads.append(Thread(target=reporter.start, kwargs=dict(interval=args.interval, num_samples=args.num_samples, stop_on_errors=args.stop_on_errors)))
threads[-1].start()
for thread in threads:
thread.join()
def test_push_automatically_on_report_for_interval(self, add_mock):
DS = TestDataSource([], 0)
R = Reporter(DS,
db_plugin='DictConnection',
verbose=False,
interval=120)
for i in range(1, 10):
with freeze_time(datetime(2020, 1, 1, 1, i)):
R.report(dict(a=i))
if not i % 3:
add_mock.assert_called()
self.assertIsNone(R._data)
add_mock.reset_mock()
else:
add_mock.assert_not_called()
self.assertIsNotNone(R._data)
def test_allows_custom_print_fn(self):
print_mock = Mock()
DS = TestDataSource([], 0)
R = Reporter(DS,
db_plugin='DictConnection',
verbose=False,
print_fn=print_mock)
R._print('asd')
print_mock.assert_not_called()
R = Reporter(DS,
db_plugin='DictConnection',
verbose=True,
print_fn=print_mock)
R._print('asd')
print_mock.assert_called()
def test_averaging_wont_pass_through_exception_on_failed_push(self, mock):
DS = TestDataSource([1, 2, 3], 3)
R = Reporter(DS, db_plugin='DictConnection')
R.start(interval=0, num_samples=1)
C = R.connection
data = C.get_data(R.topic_id)
self.assertEqual(3, len(mock.mock_calls))
self.assertEqual(0, len(data))
R.push({'number': 3})
def test_push_automatically_on_report_for_num_samples(self, add_mock):
DS = TestDataSource([], 0)
R = Reporter(DS,
db_plugin='DictConnection',
verbose=False,
num_samples=2)
for i in range(5):
R.report(dict(a=1))
self.assertEqual(i, len(add_mock.mock_calls))
R.report(dict(a=1))
self.assertEqual(i + 1, len(add_mock.mock_calls))
def test_raises_error_with_invalid_db_connection(self):
DS = TestDataSource([0], 1)
with self.assertRaises(RuntimeError):
R = Reporter(DS, db_plugin='NoConnection')
def test_data_can_be_pushed_to_existing_topic(self):
R = Reporter(None, db_plugin='DictConnection', topic_id='TBD')
topic_id = R.connection.add_topic(**TestDataSource([], 0).topic)
R._topic_id = topic_id
R.push(dict(a=1))
def test_data_collection_fails_without_data_source(self):
R = Reporter(None, db_plugin='DictConnection', topic_id='123')
with self.assertRaises(ValueError):
R.start()
def test_topic_creation_fails_without_data_source(self):
with self.assertRaises(ValueError):
R = Reporter(None, db_plugin='DictConnection')
def test_creates_topic_on_init(self):
DS = TestDataSource([1, 2, 3], 3)
R = Reporter(DS, db_plugin='DictConnection')
C = R.connection
self.assertEqual(C.get_topic(R.topic_id)["name"], "topic")
self.assertEqual(C.get_topic(R.topic_id)["fields"], ["number"])
def test_push_is_skipped_on_empty_data(self, mock):
DS = TestDataSource([1, 2, 3], 3)
R = Reporter(DS, db_plugin='DictConnection')
R.push()
mock.assert_not_called()
def __init__(self, sensors, **kwargs):
self._reporters = {}
for sensor in sensors:
self._reporters[sensor.get("mac")] = Reporter(
topic_id=sensor.get("topic_id"), **kwargs)
return {
"name": self.__topic_name,
"template": "sysstatus",
"type_str": "topic",
}
@property
def data(self):
data = {
'CPU_usage': psutil.cpu_percent(),
'memory_usage': psutil.virtual_memory().percent,
'disk_usage': psutil.disk_usage('/').percent
}
return data
if __name__ == '__main__':
from argparse import ArgumentParser
import json
from fdbk import Reporter
from fdbk.utils import get_reporter_argparser
parser = get_reporter_argparser()
args = parser.parse_args()
SYS_STATUS = SysStatus()
REPORTER = Reporter(SYS_STATUS, db_plugin=args.db_connection, db_parameters=args.db_parameters, verbose=args.verbose)
REPORTER.start(interval=args.interval, num_samples=args.num_samples, stop_on_errors=args.stop_on_errors)
def test_start_method_catches_ctrl_c(self):
DS = TestDataSource([], 5)
R = Reporter(DS, db_plugin='DictConnection')
R.start(interval=0, num_samples=1)