def test_top_level(self):
     for file, data in get_sucessful_files(data_dir):
         for report, combo in input_combinations(data):
             collector = report.get('collector', None)
             root_timestamp = report.get('timestamp', None)
             data_tags = report.get("data_tags", None)
             version = data.get("version", None)
             made_data = make_data(report['data'],
                                   data_tags=data_tags,
                                   collector=collector,
                                   root_timestamp=root_timestamp,
                                   version=version)
             report_level_data = make_meta_report_level(
                 data=made_data,
                 global_tags=data.get('meta_tags'),
                 uptime=data.get('uptime'),
                 messages=data.get('messages'),
                 collector=collector,
                 destination=data.get('destination'),
                 language=data.get('language'),
                 platform=data.get('platform'),
                 timestamp=root_timestamp,
                 version=data.get('version'))[0]
             fields = report_level_data['fields']
             self.assertEqual(
                 fields['total_metrics'], fields['total_bool_metrics'] +
                 fields['total_numeric_metrics'] +
                 fields['total_text_metrics'])
             self.assertIsNotNone(fields.values())
 def test_data_generation(self):
     for file, data in get_sucessful_files(data_dir):
         proc_data = self.InfluxClient.process(data)
         data = self.InfluxClient._generate_data(proc_data.data)
         self.assertGreaterEqual(len(data), 1, msg=f"File {file} failed")
         for item in data:
             self.assertIsInstance(item, Point)
 def test_validation(self):
     for file, data in get_sucessful_files(data_dir):
         for report, combo in input_combinations(data):
             try:
                 self.assertTrue(Processor().report_is_valid(report))
             except Exception:
                 print(f"Failed on {file.name} with variants {combo}")
                 raise
 def test_basic_write(self):
     for file, data in get_sucessful_files(data_dir):
         processed_data = self.client.process(data)
         try:
             self.client.send_data(processed_data.all_meta_data, table_name=self.client.data_table)
             self.client.send_data(processed_data.data, table_name=self.client.data_table)
         except:
             raise
 def test_in(self):
     """Test known good inputs"""
     for filename, data in get_sucessful_files(data_dir):
         try:
             self.InputSchema.load(data)
         except Exception as e:
             print(f"failed on {filename.name}")
             print(data)
             raise
 def test_fields_data(self):
     for file, data in get_sucessful_files(data_dir):
         for report, combo in input_combinations(data):
             try:
                 P = Processor().process(report)
                 for item in P.data:
                     self.assertIn("collector", item['tags'].keys())
             except Exception as e:
                 print(f"Failed with {file} and missing data {combo}")
                 raise
 def test_combination_tester(self):
     for file, data in get_sucessful_files(data_dir):
         for report, combo in input_combinations(data):
             missing_toplevel = combo['missing_toplevel']
             missing_datalevel = combo['missing_datalevel']
             for key in missing_toplevel:
                 self.assertNotIn(key, report)
             for item in report['data']:
                 for data_key in missing_datalevel:
                     self.assertNotIn(data_key, item)
 def test_nested_data(self):
     NestedFormat = NestedInputData()
     for filename, data in get_sucessful_files(data_dir):
         data = data['data']
         for entry in data:
             try:
                 NestedFormat.load(entry)
             except Exception as e:
                 print(f"failed on {filename.path}")
                 raise
    def test_failure(self):

        for filename, data in get_sucessful_files(data_dir):
            required_fields = [
                x.name for x in self.InputSchema.fields.values()
                if x.required == True
            ]
            for field in required_fields:
                with self.assertRaises(expected_exception=ValidationError):
                    d = data.copy().pop(field)
                    self.InputSchema.load(d)
 def test_meta_has_no_string(self):
     for file, data in get_sucessful_files(data_dir):
         for report, combo in input_combinations(data):
             P = Processor().process(report)
             for item in P.all_meta_data:
                 for key, value in item['fields'].items():
                     self.assertNotIsInstance(
                         value,
                         str,
                         msg=
                         f"String value found for key {key} in metadata. Metadata should only return numerics"
                     )
    def test_tags_get_added(self):
        for file, data in get_sucessful_files(data_dir):
            for report, combo in input_combinations(data):
                for remove_all_existing_tags in [False, True]:
                    if remove_all_existing_tags:
                        for entry in data['data']:
                            if entry.get("tags", False):
                                entry.pop("tags")

                    made_data = make_data(data['data'],
                                          data_tags=dict(should_see_this=True),
                                          collector="True",
                                          root_timestamp=time(),
                                          version="1.2.3.4")
                    for item in made_data:
                        self.assertIn("should_see_this", item['tags'])
    def test_processor(self):
        for file, data in get_sucessful_files(data_dir):
            for report, combo in input_combinations(data):
                try:
                    P = Processor().process(report)
                    self.assertIsNotNone(P)
                    self.assertIsNotNone(P.data)
                    self.assertIsNotNone(P.all_meta_data)
                    self.assertIsNotNone(P.report_level_meta_data)
                    self.assertIsNotNone(P.data_level_meta_data)
                    for item in P.all_meta_data + P.data:
                        # Todo output validator
                        self.assertIn("timestamp", item)
                        self.assertIsNotNone(item['timestamp'])
                        self.assertIn("collector", item['tags'])

                except Exception:
                    print(f"failed on {file.name} with variants {combo}")
                    raise
 def test_write(self) -> None:
     for file, data in get_sucessful_files(data_dir):
         prod_data = self.InfluxClient.process(data)
         self.InfluxClient.send(prod_data)
         query_api = self.InfluxClient.client.query_api()
         bucket = INFLUX_CI_CONFIG['b']
         collector = data['collector']
         query_string = f'from(bucket:"{bucket}")|> range(start: -100y)|> filter(fn:(r) => r._measurement == "{collector}")'
         start_time = time()
         query_result = 0
         while time() - start_time < 10:
             sleep(0.01)
             query_result = query_api.query(query=query_string,
                                            org=INFLUX_CI_CONFIG['o'])
             if len(query_result) != 0:
                 break
         self.assertNotEqual(len(query_result), 0)
         records = [x.records[0].values for x in query_result]
         fields = list(map(itemgetter('_field'), records))
         self.assertIn("meta_total_tags", fields)  # from data_level
         self.assertIn("total_reports", fields)  # from report_level
 def test_data_level(self):
     for file, data in get_sucessful_files(data_dir):
         for report, combo in input_combinations(data):
             collector = report.get('collector', None)
             root_timestamp = report.get('timestamp', None)
             data_tags = report.get("data_tags", None)
             version = data.get("version", None)
             made_data = make_data(report['data'],
                                   data_tags=data_tags,
                                   collector=collector,
                                   root_timestamp=root_timestamp,
                                   version=version)
             made_report = make_meta_data_level(made_data,
                                                top_level_tags=report.get(
                                                    "meta_tags", None))
             for item in made_report:
                 fields = item['fields']
                 self.assertEqual(
                     fields['meta_text_metrics'] +
                     fields['meta_bool_metrics'] +
                     fields['meta_numeric_metrics'],
                     fields['meta_total_metrics'])
    def test_optional(self):
        for file, data in get_sucessful_files(data_dir):
            for report, combo in input_combinations(data):
                collector = report.get('collector', None)
                root_timestamp = report.get('timestamp', None)
                data_tags = report.get("data_tags", None)
                version = data.get("version", None)
                try:
                    made_data = make_data(report['data'],
                                          data_tags=data_tags,
                                          collector=collector,
                                          root_timestamp=root_timestamp,
                                          version=version)
                    self.assertIsInstance(made_data, list)
                    self.assertNotEqual(len(made_data), 0)
                    for item in made_data:
                        self.assertIsInstance(item, dict)

                    # Todo validate output properly
                except Exception as e:
                    print(
                        f"Failed for file {file.name} with missing info {combo}"
                    )
                    raise
 def test_behaviour_only_string(self):
     # This is a case where it's failing
     failing_file = [data for file, data in get_sucessful_files(data_dir) if 'all_strings' in file.name][0]
     processed_data = self.client.process(failing_file)
     self.client.send_data(processed_data.data, table_name=self.client.data_table)
 def test_full(self):
     test_data = [data for file, data in get_sucessful_files(data_dir)]
     self.client.send(self.client.process(test_data[0]))
     pass
    def test_get_successful_files(self):

        self.assertGreater(len(list(get_sucessful_files(data_dir))), 0)
        for path, file in get_sucessful_files(data_dir):
            self.assertIsInstance(path, Path)
            self.assertIsInstance(file, dict)