def get_config(config_file=None, config_values=None, load_project_conf=True): """Load config file and return its content.""" config_values = config_values or {} default_conf = _get_default_conf() user_conf = _get_user_conf(config_file) if config_file else {} # load project configuration only when user configuration was not specified project_conf = {} if user_conf or not load_project_conf else configuration.get_config( ) if not (user_conf or project_conf or config_values): if load_project_conf: raise Dump2PolarionException( "Failed to find configuration file for the project " "and no configuration file or values passed.") raise Dump2PolarionException("No configuration file or values passed.") # merge configuration config_settings = {} config_settings.update(default_conf) utils.merge_dicts(config_settings, user_conf) utils.merge_dicts(config_settings, project_conf) utils.merge_dicts(config_settings, config_values) _populate_urls(config_settings) _set_legacy_project_id(config_settings) _set_legacy_custom_fields(config_settings) _check_config(config_settings) return config_settings
def get_testrun_id(args, config, records_testrun_id): """Returns testrun id.""" config_testrun_id = utils.get_testrun_id_config(config) found_testrun_id = args.testrun_id or records_testrun_id or config_testrun_id if not found_testrun_id: raise Dump2PolarionException( "The testrun id was not specified on command line and not found in the input data " "or config file." ) match = True for tr_id in (args.testrun_id, records_testrun_id, config_testrun_id): if tr_id and tr_id != found_testrun_id: match = False break if not match and args.force: logger.warning("Using '%s' as testrun id.", found_testrun_id) elif not match: raise Dump2PolarionException( "The test run ids found in exported data, config file and/or specified on command line " "differ. If you really want to proceed, add '-f' and test run id '{}' " "will be used.".format(found_testrun_id) ) return found_testrun_id
def get_config(config_file=None): """Loads config file and returns its content.""" default_conf = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dump2polarion.yaml') user_conf = os.path.expanduser(config_file or DEFAULT_USER_CONF) try: with open(user_conf): pass except EnvironmentError: user_conf = None if config_file: raise Dump2PolarionException( "Cannot open config file '{}'".format(config_file)) with io.open(default_conf, encoding='utf-8') as input_file: config_settings = yaml.load(input_file) logger.debug("Default config loaded from '{}'".format(default_conf)) if user_conf: with io.open(user_conf, encoding='utf-8') as input_file: config_settings_user = yaml.load(input_file) logger.info("Config loaded from '{}'".format(user_conf)) # merge default and user configuration try: config_settings.update(config_settings_user) except ValueError as err: raise Dump2PolarionException( "Failed to load the '{}' config file: {}".format( user_conf, err)) _check_config(config_settings) return config_settings
def get_imported_data(csv_file, **kwargs): """Reads the content of the Polarion exported csv file and returns imported data.""" open_args = [] open_kwargs = {} try: # pylint: disable=pointless-statement unicode open_args.append('rb') except NameError: open_kwargs['encoding'] = 'utf-8' with open(os.path.expanduser(csv_file), *open_args, **open_kwargs) as input_file: reader = _get_csv_reader(input_file) fieldnames = _get_csv_fieldnames(reader) if not fieldnames: raise Dump2PolarionException( "Cannot find field names in CSV file '{}'".format(csv_file)) results = _get_results(reader, fieldnames) if not results: raise Dump2PolarionException( "No results read from CSV file '{}'".format(csv_file)) testrun = _get_testrun_from_csv(input_file, reader) return exporter.ImportedData(results=results, testrun=testrun)
def _get_testrun_properties(xml_root): if xml_root.tag in ("testcases", "requirements"): return None if xml_root.tag != "testsuites": raise Dump2PolarionException("{} {}".format(_NOT_EXPECTED_FORMAT_MSG, "- missing <testsuites>")) properties = xml_root.find("properties") if properties is None: raise Dump2PolarionException( "Failed to find <properties> in the XML file") return properties
def _properties_element(self, parent_element): """Returns properties XML element.""" testsuites_properties = etree.SubElement(parent_element, "properties") etree.SubElement( testsuites_properties, "property", {"name": "polarion-testrun-id", "value": str(self.testrun_id)}, ) etree.SubElement( testsuites_properties, "property", {"name": "polarion-project-id", "value": str(self.config["polarion-project-id"])}, ) for name, value in sorted(self.config["xunit_import_properties"].items()): if name == "polarion-lookup-method": lookup_prop = str(value).lower() if lookup_prop not in ("id", "name", "custom"): raise Dump2PolarionException( "Invalid value '{}' for the 'polarion-lookup-method' property".format( str(value) ) ) self._lookup_prop = lookup_prop elif name in ("polarion-testrun-id", "polarion-project-id"): # this was already set continue else: etree.SubElement( testsuites_properties, "property", {"name": name, "value": str(value)} ) return testsuites_properties
def _check_required_columns(csv_file, results): required_columns = {'verdict': 'Verdict'} missing_columns = [required_columns[k] for k in required_columns if k not in results[0]] if missing_columns: raise Dump2PolarionException( "The input file '{}' is missing following columns: {}".format( csv_file, ', '.join(missing_columns)))
def parse(self): """Parse log file produced by the XUnit Iporter.""" existing_items = [] duplicate_items = [] new_items = [] for line in self.fp: line = line.strip() if "Work item: " in line: existing_it = self.get_result(line) if existing_it: existing_items.append(existing_it) elif "Unable to find *unique* work item" in line: duplicate_it = self.get_result_warn(line) if duplicate_it: duplicate_items.append(duplicate_it) elif "Unable to find work item for" in line: new_it = self.get_result_warn(line) if new_it: new_items.append(new_it) outcome = ParsedLog("xunit", new_items, existing_items, duplicate_items) if not outcome: raise Dump2PolarionException( "No valid data found in the log file '{}'".format(self.log_file) ) return outcome
def _properties_element(self, parent_element): """Returns properties XML element.""" testsuites_properties = ElementTree.SubElement(parent_element, 'properties') ElementTree.SubElement(testsuites_properties, 'property', { 'name': 'polarion-testrun-id', 'value': str(self.testrun_id) }) for name, value in sorted( self.config['xunit_import_properties'].items()): if name == 'polarion-lookup-method': lookup_prop = str(value).lower() if lookup_prop not in ('id', 'name', 'custom'): raise Dump2PolarionException( "Invalid value '{}' for the 'polarion-lookup-method' property" .format(str(value))) self._lookup_prop = lookup_prop else: ElementTree.SubElement(testsuites_properties, 'property', { 'name': name, 'value': str(value) }) return testsuites_properties
def _get_xml_root(junit_file): try: tree = etree.parse(os.path.expanduser(junit_file)) except Exception as err: raise Dump2PolarionException("Failed to parse XML file '{}': {}".format(junit_file, err)) return tree.getroot()
def get_testrun_id(args, testrun_id): """Returns testrun id.""" if (args.testrun_id and testrun_id and not args.force and testrun_id != args.testrun_id): raise Dump2PolarionException( "The test run id '{}' found in exported data doesn't match '{}'. " "If you really want to proceed, add '-f'.".format( testrun_id, args.testrun_id)) found_testrun_id = args.testrun_id or testrun_id if not found_testrun_id: raise Dump2PolarionException( "The testrun id was not specified on command line and not found in the input data." ) return found_testrun_id
def get_session(credentials, config): """Gets requests session.""" session = requests.Session() session.verify = False auth_url = config.get("auth_url") if auth_url: cookie = session.post( auth_url, data={ "j_username": credentials[0], "j_password": credentials[1], "submit": "Log In", "rememberme": "true", }, headers={"Content-Type": "application/x-www-form-urlencoded"}, ) if not cookie: raise Dump2PolarionException( "Cookie was not retrieved from {}.".format(auth_url)) else: # TODO: can be removed once basic auth is discontinued on prod session.auth = credentials return session
def _get_importer(input_file): """Select importer based on input file type.""" __, ext = os.path.splitext(input_file) ext = ext.lower() if "ostriz" in input_file: from dump2polarion.results import ostriztools importer = ostriztools.import_ostriz elif ext == ".xml": # expect junit-report from pytest from dump2polarion.results import junittools importer = junittools.import_junit elif ext == ".csv": from dump2polarion.results import csvtools importer = csvtools.import_csv elif ext in dbtools.SQLITE_EXT: importer = dbtools.import_sqlite elif ext == ".json": from dump2polarion.results import jsontools importer = jsontools.import_json else: raise Dump2PolarionException( "Cannot recognize type of input data, add file extension.") return importer
def parse(self): """Parse log file produced by the Requirements Importer.""" existing_items = [] duplicate_items = [] new_items = [] for line in self.fp: line = line.strip() if "Updated requirement" in line: existing_it = self.get_requirement(line) if existing_it: existing_items.append(existing_it) elif "Found multiple work items with the title" in line: duplicate_it = self.get_requirement_warn(line) if duplicate_it: duplicate_items.append(duplicate_it) elif "Created requirement" in line: new_it = self.get_requirement(line) if new_it: new_items.append(new_it) outcome = ParsedLog("requirement", new_items, existing_items, duplicate_items) if not outcome: raise Dump2PolarionException( "No valid data found in the log file '{}'".format(self.log_file) ) return outcome
def _get_json(location): """Reads JSON data from file or URL.""" location = os.path.expanduser(location) try: if os.path.isfile(location): with io.open(location, encoding='utf-8') as json_data: return json.load(json_data, object_pairs_hook=OrderedDict).get('tests') elif 'http' in location: json_data = requests.get(location) if not json_data: raise Dump2PolarionException("Failed to download") return json.loads(json_data.text, object_pairs_hook=OrderedDict).get('tests') else: raise Dump2PolarionException("Invalid location") except Exception as err: raise Dump2PolarionException( "Failed to parse JSON from {}: {}".format(location, err))
def get_xml_root_from_str(xml_str): """Return XML root from string.""" try: xml_root = etree.fromstring(xml_str.encode("utf-8"), NO_BLANKS_PARSER) # pylint: disable=broad-except except Exception as err: raise Dump2PolarionException("Failed to parse XML string: {}".format(err)) return xml_root
def get_xml_root(xml_file): """Return XML root.""" try: xml_root = etree.parse(os.path.expanduser(xml_file), NO_BLANKS_PARSER).getroot() # pylint: disable=broad-except except Exception as err: raise Dump2PolarionException("Failed to parse XML file '{}': {}".format(xml_file, err)) return xml_root
def set_lookup_method(xml_root, value): """Change lookup method.""" if xml_root.tag == "testsuites": _set_property(xml_root, "polarion-lookup-method", value) elif xml_root.tag in ("testcases", "requirements"): _set_property(xml_root, "lookup-method", value) else: raise Dump2PolarionException(_NOT_EXPECTED_FORMAT_MSG)
def _get_xml_root(xml_root, xml_str, xml_file): if xml_root is not None: return xml_root if xml_str: return utils.get_xml_root_from_str(xml_str) if xml_file: return utils.get_xml_root(xml_file) raise Dump2PolarionException("Failed to submit to Polarion - no data supplied")
def import_junit(junit_file, **kwargs): """Reads the content of the junit-results file produced by pytest and returns imported data.""" try: tree = ElementTree.parse(os.path.expanduser(junit_file)) except Exception as err: raise Dump2PolarionException( "Failed to parse XML file '{}': {}".format(junit_file, err)) xml_root = tree.getroot() results = [] for test_data in xml_root: if test_data.tag != 'testcase': continue verdict = None verdict_found = False comment = '' properties = {} for element in test_data: if not verdict_found: if element.tag == 'error': verdict = 'failed' comment = element.get('message') # continue to see if there's more telling verdict for this record elif element.tag == 'failure': verdict = 'failed' comment = element.get('message') verdict_found = True elif element.tag == 'skipped': verdict = 'skipped' comment = element.get('message') verdict_found = True if element.tag == 'properties': for prop in element: properties[prop.get('name')] = prop.get('value') if not verdict: verdict = 'passed' title = test_data.get('name') classname = test_data.get('classname') time = test_data.get('time', 0) filepath = test_data.get('file') data = [ ('title', title), ('classname', classname), ('verdict', verdict), ('comment', comment), ('time', time), ('file', filepath), ] for key in properties: data.append((key, properties[key])) record = OrderedDict(data) results.append(record) return exporter.ImportedData(results=results, testrun=None)
def get_xml_root_from_str(xml): """Returns XML root from string.""" try: xml_root = ElementTree.fromstring(xml.encode('utf-8')) # pylint: disable=broad-except except Exception as err: raise Dump2PolarionException( "Failed to parse XML file: {}".format(err)) return xml_root
def _get_submit_target(xml_root, config): if xml_root.tag == 'testcases': target = config.get('testcase_taget') elif xml_root.tag == 'testsuites': target = config.get('xunit_target') else: raise Dump2PolarionException( "Failed to submit results to Polarion - submit target not found") return target
def get_imported_data(csv_file, **kwargs): """Read the content of the Polarion exported csv file and return imported data.""" with open(os.path.expanduser(csv_file), encoding="utf-8") as input_file: reader = _get_csv_reader(input_file) fieldnames = _get_csv_fieldnames(reader) if not fieldnames: raise Dump2PolarionException( "Cannot find field names in CSV file '{}'".format(csv_file) ) results = _get_results(reader, fieldnames) if not results: raise Dump2PolarionException("No results read from CSV file '{}'".format(csv_file)) testrun = _get_testrun_from_csv(input_file, reader) return xunit_exporter.ImportedData(results=results, testrun=testrun)
def _check_config(config): missing = [] for key in ('testcase_taget', 'xunit_target', 'message_bus'): if not config.get(key): missing.append(key) if missing: raise Dump2PolarionException( "Failed to find following keys in config file: {}\n" "Please see https://mojo.redhat.com/docs/DOC-1098563#config".format(', '.join(missing)))
def import_pytest_collect(json_filename): """Reads the content of the JSON file produced by pytest-polarion-collect file.""" try: results = _load_json(json_filename)["results"] except Exception as err: raise Dump2PolarionException("Cannot load results from {}: {}".format( json_filename, err)) return xunit_exporter.ImportedData(results=results, testrun=None)
def _fill_lookup_prop(self, testcases_properties): """Fills the polarion-lookup-method property.""" if not self._lookup_prop: raise Dump2PolarionException( "Failed to set the 'polarion-lookup-method' property") etree.SubElement(testcases_properties, "property", { "name": "lookup-method", "value": self._lookup_prop })
def _open_sqlite(db_file): """Opens database connection.""" db_file = os.path.expanduser(db_file) try: with open(db_file): # test that the file can be accessed pass return sqlite3.connect(db_file, detect_types=sqlite3.PARSE_DECLTYPES) except (IOError, sqlite3.Error) as err: raise Dump2PolarionException('{}'.format(err))
def set_dry_run(xml_root, value=True): """Set dry-run so records are not updated, only log file is produced.""" value_str = str(value).lower() assert value_str in ("true", "false") if xml_root.tag == "testsuites": _set_property(xml_root, "polarion-dry-run", value_str) elif xml_root.tag in ("testcases", "requirements"): _set_property(xml_root, "dry-run", value_str) else: raise Dump2PolarionException(_NOT_EXPECTED_FORMAT_MSG)
def get_xml_root(xml_file): """Returns XML root.""" try: xml_tree = ElementTree.parse(os.path.expanduser(xml_file)) xml_root = xml_tree.getroot() # pylint: disable=broad-except except Exception as err: raise Dump2PolarionException( "Failed to parse XML file '{}': {}".format(xml_file, err)) return xml_root
def _fill_lookup_prop(self, testsuites_properties): """Fills the polarion-lookup-method property.""" if not self._lookup_prop: raise Dump2PolarionException( "Failed to set the 'polarion-lookup-method' property") ElementTree.SubElement(testsuites_properties, 'property', { 'name': 'polarion-lookup-method', 'value': self._lookup_prop })