def test_malformed_data(self): # job_id already exists metadata = {"job_id": "12345", "a-metadata-field": "value"} tests = { "test-malformed": "pass", "testb": { "result": "pass", "log": "the log" }, } metrics = {"metrica": 42} with self.assertLogs(logger='squad_client.core.models', level=logging.ERROR) as cm: success = submit_results( group_project_slug="my_group/my_project", build_version="my_build", env_slug="my_env", tests=tests, metrics=metrics, metadata=metadata, ) self.assertIn( 'ERROR:squad_client.core.models:Failed to submit results: There is already a test run with job_id 12345', cm.output) results = self.squad.tests(name="test-malformed") self.assertTrue(len(results) == 0) self.assertFalse(success)
def send_to_qa_reports(self): if None in (self.qa_reports_server, self.qa_reports_token, self.qa_reports_group, self.qa_reports_project, self.qa_reports_build_version, self.qa_reports_env): self.logger.warning("All parameters for qa reports are not set, results will not be pushed to qa reports") return SquadApi.configure( url=self.qa_reports_server, token=self.qa_reports_token ) tests = {} metrics = {} for metric in self.metrics: if metric['measurement'] != "": metrics["{}/{}".format(self.test['test_name'], metric['test_case_id'])] = metric['measurement'] else: tests["{}/{}".format(self.test['test_name'], metric['test_case_id'])] = metric['result'] with open("{}/stdout.log".format(self.test['test_path']), "r") as logfile: log = logfile.read() metadata = {} if not self.qa_reports_disable_metadata: if self.qa_reports_metadata: metadata.update(self.qa_reports_metadata) if self.qa_reports_metadata_file: try: with open(self.qa_reports_metadata_file, "r") as metadata_file: loaded_metadata = yaml.load(metadata_file, Loader=yaml.SafeLoader) # check if loaded metadata is key=value and both are strings for key, value in loaded_metadata.items(): if type(key) == str and type(value) == str: # only update metadata with simple keys # ignore all other items in the dictionary metadata.update({key: value}) else: self.logger.warning("Ignoring key: %s" % key) except FileNotFoundError: self.logger.warning("Metadata file not found") except PermissionError: self.logger.warning("Insufficient permissions to open metadata file") if submit_results( group_project_slug="{}/{}".format(self.qa_reports_group, self.qa_reports_project), build_version=self.qa_reports_build_version, env_slug=self.qa_reports_env, tests=tests, metrics=metrics, log=log, metadata=metadata, attachments=None): self.logger.info("Results pushed to QA Reports") else: self.logger.warning("Results upload to QA Reports failed!")
def run(self, args): builds = self._load_builds(args.tuxbuild) # log if builds is None: return False try: jsonschema.validate(instance=builds, schema=tuxbuild_schema) except jsonschema.exceptions.ValidationError as ve: logger.error("Failed to validate tuxbuild data: %s", ve) return False data = {} for build in builds: arch = build["target_arch"] description = build["git_describe"] kconfig = build["kconfig"] status = build["build_status"] toolchain = build["toolchain"] test = self._get_test_name(kconfig, toolchain) multi_key = (description, arch) if multi_key not in data: data[multi_key] = {} data[multi_key].update({test: status}) for key, result in data.items(): description, arch = key submit_results( group_project_slug="%s/%s" % (args.group, args.project), build_version=description, env_slug=arch, tests=result, ) return True
def run(self, args): try: builds = load_builds(args.tuxbuild) except InvalidBuildJson as ibj: logger.error("Failed to load build json: %s", ibj) return False except OSError as ose: logger.error("Failed to load build json: %s", ose) return False try: jsonschema.validate(instance=builds, schema=TUXBUILD_SCHEMA) except jsonschema.exceptions.ValidationError as ve: logger.error("Failed to validate tuxbuild data: %s", ve) return False for build in builds: arch = build["target_arch"] description = build["git_describe"] warnings_count = build["warnings_count"] test_name = create_name(build) test_status = build["build_status"] duration = build["duration"] tests = {test_name: test_status} metrics = {test_name + '-warnings': warnings_count} metrics.update({test_name + '-duration': duration}) submit_results( group_project_slug="%s/%s" % (args.group, args.project), build_version=description, env_slug=arch, tests=tests, metrics=metrics, metadata=create_metadata(build), ) return True
def run(self, args): builds = self._load_builds(args.tuxbuild) # log if builds is None: return False try: jsonschema.validate(instance=builds, schema=tuxbuild_schema) except jsonschema.exceptions.ValidationError as ve: logger.error("Failed to validate tuxbuild data: %s", ve) return False for build in builds: arch = build["target_arch"] description = build["git_describe"] kconfig = build["kconfig"] toolchain = build["toolchain"] warnings_count = build["warnings_count"] test_name = self._get_test_name(kconfig, toolchain) test_status = build["build_status"] duration = build["duration"] tests = {test_name: test_status} metrics = {test_name + '-warnings': warnings_count} metrics.update({test_name + '-duration': duration}) submit_results(group_project_slug="%s/%s" % (args.group, args.project), build_version=description, env_slug=arch, tests=tests, metrics=metrics, metadata=self._build_metadata(build)) return True
def test_basic(self): metadata = {"job_id": "12345", "a-metadata-field": "value"} tests = {"testa": "pass", "testb": {"result": "pass", "log": "the log"}} metrics = {"metrica": 42} success = submit_results( group_project_slug="my_group/my_project", build_version="my_build", env_slug="my_env", tests=tests, metrics=metrics, metadata=metadata, ) results = self.squad.tests(name="testa") self.assertTrue(len(results) > 0) self.assertTrue(success)
def test_malformed_data(self): # job_id already exists metadata = {"job_id": "12345", "a-metadata-field": "value"} tests = { "test-malformed": "pass", "testb": { "result": "pass", "log": "the log" }, } metrics = {"metrica": 42} success = submit_results( group_project_slug="my_group/my_project", build_version="my_build", env_slug="my_env", tests=tests, metrics=metrics, metadata=metadata, ) results = self.squad.tests(name="test-malformed") self.assertTrue(len(results) == 0) self.assertFalse(success)
def run(self, args): results_dict = {} metrics_dict = {} metadata_dict = None logs_file = None if args.result_name: if not args.result_value: logger.error("Test result value is required") return False results_dict = {args.result_name: args.result_value} if args.results: if args.results_layout == 'tuxbuild_json': results_dict = self._load_tuxbuild_json(args.results) else: results_dict = self.__read_input_file(args.results) if results_dict is None: return False if args.metrics: metrics_dict = self.__read_input_file(args.metrics) if metrics_dict is None: return False if args.result_name is None and args.results is None and args.metrics is None: logger.error( "At least one of --result-name, --results, --metrics is required" ) return False if args.metadata: metadata_dict = self.__read_input_file(args.metadata) if metadata_dict is None: return False if args.logs: if not self.__check_file(args.logs): return False with open(args.logs, "r") as logs_file_source: logs_file = logs_file_source.read() for filename in args.attachments: if not self.__check_file(filename): return False if results_dict: # check dictionary correctness for key, value in iter(results_dict.items()): if type(key) is not str: logger.error("Non-string key detected") return False if type(value) not in [str, dict]: logger.error("Incompatible results detected") return False if metrics_dict: # check dictionary correctness for key, value in iter(metrics_dict.items()): if type(key) is not str: logger.error("Non-string key detected") return False if type(value) not in [float, int, list]: logger.error("Incompatible metrics detected") return False if metadata_dict: # check dictionary correctness for key, value in iter(metadata_dict.items()): if type(key) is not str: logger.error("Non-string key detected") return False if type(value) not in [str, dict]: logger.error("Incompatible results detected") return False submit_results( group_project_slug="%s/%s" % (args.group, args.project), build_version=args.build, env_slug=args.environment, tests=results_dict, metrics=metrics_dict, log=logs_file, metadata=metadata_dict, ) return True