def test_top_level(self):
for file, data in get_sucessful_files(data_dir):
for report, combo in input_combinations(data):
collector = report.get('collector', None)
root_timestamp = report.get('timestamp', None)
data_tags = report.get("data_tags", None)
version = data.get("version", None)
made_data = make_data(report['data'],
data_tags=data_tags,
collector=collector,
root_timestamp=root_timestamp,
version=version)
report_level_data = make_meta_report_level(
data=made_data,
global_tags=data.get('meta_tags'),
uptime=data.get('uptime'),
messages=data.get('messages'),
collector=collector,
destination=data.get('destination'),
language=data.get('language'),
platform=data.get('platform'),
timestamp=root_timestamp,
version=data.get('version'))[0]
fields = report_level_data['fields']
self.assertEqual(
fields['total_metrics'], fields['total_bool_metrics'] +
fields['total_numeric_metrics'] +
fields['total_text_metrics'])
self.assertIsNotNone(fields.values())
def test_data_generation(self):
for file, data in get_sucessful_files(data_dir):
proc_data = self.InfluxClient.process(data)
data = self.InfluxClient._generate_data(proc_data.data)
self.assertGreaterEqual(len(data), 1, msg=f"File {file} failed")
for item in data:
self.assertIsInstance(item, Point)
def test_validation(self):
for file, data in get_sucessful_files(data_dir):
for report, combo in input_combinations(data):
try:
self.assertTrue(Processor().report_is_valid(report))
except Exception:
print(f"Failed on {file.name} with variants {combo}")
raise
def test_basic_write(self):
for file, data in get_sucessful_files(data_dir):
processed_data = self.client.process(data)
try:
self.client.send_data(processed_data.all_meta_data, table_name=self.client.data_table)
self.client.send_data(processed_data.data, table_name=self.client.data_table)
except:
raise
def test_in(self):
"""Test known good inputs"""
for filename, data in get_sucessful_files(data_dir):
try:
self.InputSchema.load(data)
except Exception as e:
print(f"failed on {filename.name}")
print(data)
raise
def test_fields_data(self):
for file, data in get_sucessful_files(data_dir):
for report, combo in input_combinations(data):
try:
P = Processor().process(report)
for item in P.data:
self.assertIn("collector", item['tags'].keys())
except Exception as e:
print(f"Failed with {file} and missing data {combo}")
raise
def test_combination_tester(self):
for file, data in get_sucessful_files(data_dir):
for report, combo in input_combinations(data):
missing_toplevel = combo['missing_toplevel']
missing_datalevel = combo['missing_datalevel']
for key in missing_toplevel:
self.assertNotIn(key, report)
for item in report['data']:
for data_key in missing_datalevel:
self.assertNotIn(data_key, item)
def test_nested_data(self):
NestedFormat = NestedInputData()
for filename, data in get_sucessful_files(data_dir):
data = data['data']
for entry in data:
try:
NestedFormat.load(entry)
except Exception as e:
print(f"failed on {filename.path}")
raise
def test_failure(self):
for filename, data in get_sucessful_files(data_dir):
required_fields = [
x.name for x in self.InputSchema.fields.values()
if x.required == True
]
for field in required_fields:
with self.assertRaises(expected_exception=ValidationError):
d = data.copy().pop(field)
self.InputSchema.load(d)
def test_meta_has_no_string(self):
for file, data in get_sucessful_files(data_dir):
for report, combo in input_combinations(data):
P = Processor().process(report)
for item in P.all_meta_data:
for key, value in item['fields'].items():
self.assertNotIsInstance(
value,
str,
msg=
f"String value found for key {key} in metadata. Metadata should only return numerics"
)
def test_tags_get_added(self):
for file, data in get_sucessful_files(data_dir):
for report, combo in input_combinations(data):
for remove_all_existing_tags in [False, True]:
if remove_all_existing_tags:
for entry in data['data']:
if entry.get("tags", False):
entry.pop("tags")
made_data = make_data(data['data'],
data_tags=dict(should_see_this=True),
collector="True",
root_timestamp=time(),
version="1.2.3.4")
for item in made_data:
self.assertIn("should_see_this", item['tags'])
def test_processor(self):
for file, data in get_sucessful_files(data_dir):
for report, combo in input_combinations(data):
try:
P = Processor().process(report)
self.assertIsNotNone(P)
self.assertIsNotNone(P.data)
self.assertIsNotNone(P.all_meta_data)
self.assertIsNotNone(P.report_level_meta_data)
self.assertIsNotNone(P.data_level_meta_data)
for item in P.all_meta_data + P.data:
# Todo output validator
self.assertIn("timestamp", item)
self.assertIsNotNone(item['timestamp'])
self.assertIn("collector", item['tags'])
except Exception:
print(f"failed on {file.name} with variants {combo}")
raise
def test_write(self) -> None:
for file, data in get_sucessful_files(data_dir):
prod_data = self.InfluxClient.process(data)
self.InfluxClient.send(prod_data)
query_api = self.InfluxClient.client.query_api()
bucket = INFLUX_CI_CONFIG['b']
collector = data['collector']
query_string = f'from(bucket:"{bucket}")|> range(start: -100y)|> filter(fn:(r) => r._measurement == "{collector}")'
start_time = time()
query_result = 0
while time() - start_time < 10:
sleep(0.01)
query_result = query_api.query(query=query_string,
org=INFLUX_CI_CONFIG['o'])
if len(query_result) != 0:
break
self.assertNotEqual(len(query_result), 0)
records = [x.records[0].values for x in query_result]
fields = list(map(itemgetter('_field'), records))
self.assertIn("meta_total_tags", fields) # from data_level
self.assertIn("total_reports", fields) # from report_level
def test_data_level(self):
for file, data in get_sucessful_files(data_dir):
for report, combo in input_combinations(data):
collector = report.get('collector', None)
root_timestamp = report.get('timestamp', None)
data_tags = report.get("data_tags", None)
version = data.get("version", None)
made_data = make_data(report['data'],
data_tags=data_tags,
collector=collector,
root_timestamp=root_timestamp,
version=version)
made_report = make_meta_data_level(made_data,
top_level_tags=report.get(
"meta_tags", None))
for item in made_report:
fields = item['fields']
self.assertEqual(
fields['meta_text_metrics'] +
fields['meta_bool_metrics'] +
fields['meta_numeric_metrics'],
fields['meta_total_metrics'])
def test_optional(self):
for file, data in get_sucessful_files(data_dir):
for report, combo in input_combinations(data):
collector = report.get('collector', None)
root_timestamp = report.get('timestamp', None)
data_tags = report.get("data_tags", None)
version = data.get("version", None)
try:
made_data = make_data(report['data'],
data_tags=data_tags,
collector=collector,
root_timestamp=root_timestamp,
version=version)
self.assertIsInstance(made_data, list)
self.assertNotEqual(len(made_data), 0)
for item in made_data:
self.assertIsInstance(item, dict)
# Todo validate output properly
except Exception as e:
print(
f"Failed for file {file.name} with missing info {combo}"
)
raise
def test_behaviour_only_string(self):
# This is a case where it's failing
failing_file = [data for file, data in get_sucessful_files(data_dir) if 'all_strings' in file.name][0]
processed_data = self.client.process(failing_file)
self.client.send_data(processed_data.data, table_name=self.client.data_table)
def test_full(self):
test_data = [data for file, data in get_sucessful_files(data_dir)]
self.client.send(self.client.process(test_data[0]))
pass
def test_get_successful_files(self):
self.assertGreater(len(list(get_sucessful_files(data_dir))), 0)
for path, file in get_sucessful_files(data_dir):
self.assertIsInstance(path, Path)
self.assertIsInstance(file, dict)
