def get_config(config_file=None, config_values=None, load_project_conf=True):
"""Load config file and return its content."""
config_values = config_values or {}
default_conf = _get_default_conf()
user_conf = _get_user_conf(config_file) if config_file else {}
# load project configuration only when user configuration was not specified
project_conf = {} if user_conf or not load_project_conf else configuration.get_config(
)
if not (user_conf or project_conf or config_values):
if load_project_conf:
raise Dump2PolarionException(
"Failed to find configuration file for the project "
"and no configuration file or values passed.")
raise Dump2PolarionException("No configuration file or values passed.")
# merge configuration
config_settings = {}
config_settings.update(default_conf)
utils.merge_dicts(config_settings, user_conf)
utils.merge_dicts(config_settings, project_conf)
utils.merge_dicts(config_settings, config_values)
_populate_urls(config_settings)
_set_legacy_project_id(config_settings)
_set_legacy_custom_fields(config_settings)
_check_config(config_settings)
return config_settings
def get_testrun_id(args, config, records_testrun_id):
"""Returns testrun id."""
config_testrun_id = utils.get_testrun_id_config(config)
found_testrun_id = args.testrun_id or records_testrun_id or config_testrun_id
if not found_testrun_id:
raise Dump2PolarionException(
"The testrun id was not specified on command line and not found in the input data "
"or config file."
)
match = True
for tr_id in (args.testrun_id, records_testrun_id, config_testrun_id):
if tr_id and tr_id != found_testrun_id:
match = False
break
if not match and args.force:
logger.warning("Using '%s' as testrun id.", found_testrun_id)
elif not match:
raise Dump2PolarionException(
"The test run ids found in exported data, config file and/or specified on command line "
"differ. If you really want to proceed, add '-f' and test run id '{}' "
"will be used.".format(found_testrun_id)
)
return found_testrun_id
def get_config(config_file=None):
"""Loads config file and returns its content."""
default_conf = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'dump2polarion.yaml')
user_conf = os.path.expanduser(config_file or DEFAULT_USER_CONF)
try:
with open(user_conf):
pass
except EnvironmentError:
user_conf = None
if config_file:
raise Dump2PolarionException(
"Cannot open config file '{}'".format(config_file))
with io.open(default_conf, encoding='utf-8') as input_file:
config_settings = yaml.load(input_file)
logger.debug("Default config loaded from '{}'".format(default_conf))
if user_conf:
with io.open(user_conf, encoding='utf-8') as input_file:
config_settings_user = yaml.load(input_file)
logger.info("Config loaded from '{}'".format(user_conf))
# merge default and user configuration
try:
config_settings.update(config_settings_user)
except ValueError as err:
raise Dump2PolarionException(
"Failed to load the '{}' config file: {}".format(
user_conf, err))
_check_config(config_settings)
return config_settings
def get_imported_data(csv_file, **kwargs):
"""Reads the content of the Polarion exported csv file and returns imported data."""
open_args = []
open_kwargs = {}
try:
# pylint: disable=pointless-statement
unicode
open_args.append('rb')
except NameError:
open_kwargs['encoding'] = 'utf-8'
with open(os.path.expanduser(csv_file), *open_args, **open_kwargs) as input_file:
reader = _get_csv_reader(input_file)
fieldnames = _get_csv_fieldnames(reader)
if not fieldnames:
raise Dump2PolarionException(
"Cannot find field names in CSV file '{}'".format(csv_file))
results = _get_results(reader, fieldnames)
if not results:
raise Dump2PolarionException(
"No results read from CSV file '{}'".format(csv_file))
testrun = _get_testrun_from_csv(input_file, reader)
return exporter.ImportedData(results=results, testrun=testrun)
def _get_testrun_properties(xml_root):
if xml_root.tag in ("testcases", "requirements"):
return None
if xml_root.tag != "testsuites":
raise Dump2PolarionException("{} {}".format(_NOT_EXPECTED_FORMAT_MSG,
"- missing <testsuites>"))
properties = xml_root.find("properties")
if properties is None:
raise Dump2PolarionException(
"Failed to find <properties> in the XML file")
return properties
def _properties_element(self, parent_element):
"""Returns properties XML element."""
testsuites_properties = etree.SubElement(parent_element, "properties")
etree.SubElement(
testsuites_properties,
"property",
{"name": "polarion-testrun-id", "value": str(self.testrun_id)},
)
etree.SubElement(
testsuites_properties,
"property",
{"name": "polarion-project-id", "value": str(self.config["polarion-project-id"])},
)
for name, value in sorted(self.config["xunit_import_properties"].items()):
if name == "polarion-lookup-method":
lookup_prop = str(value).lower()
if lookup_prop not in ("id", "name", "custom"):
raise Dump2PolarionException(
"Invalid value '{}' for the 'polarion-lookup-method' property".format(
str(value)
)
)
self._lookup_prop = lookup_prop
elif name in ("polarion-testrun-id", "polarion-project-id"):
# this was already set
continue
else:
etree.SubElement(
testsuites_properties, "property", {"name": name, "value": str(value)}
)
return testsuites_properties
def _check_required_columns(csv_file, results):
required_columns = {'verdict': 'Verdict'}
missing_columns = [required_columns[k] for k in required_columns if k not in results[0]]
if missing_columns:
raise Dump2PolarionException(
"The input file '{}' is missing following columns: {}".format(
csv_file, ', '.join(missing_columns)))
def parse(self):
"""Parse log file produced by the XUnit Iporter."""
existing_items = []
duplicate_items = []
new_items = []
for line in self.fp:
line = line.strip()
if "Work item: " in line:
existing_it = self.get_result(line)
if existing_it:
existing_items.append(existing_it)
elif "Unable to find *unique* work item" in line:
duplicate_it = self.get_result_warn(line)
if duplicate_it:
duplicate_items.append(duplicate_it)
elif "Unable to find work item for" in line:
new_it = self.get_result_warn(line)
if new_it:
new_items.append(new_it)
outcome = ParsedLog("xunit", new_items, existing_items, duplicate_items)
if not outcome:
raise Dump2PolarionException(
"No valid data found in the log file '{}'".format(self.log_file)
)
return outcome
def _properties_element(self, parent_element):
"""Returns properties XML element."""
testsuites_properties = ElementTree.SubElement(parent_element,
'properties')
ElementTree.SubElement(testsuites_properties, 'property', {
'name': 'polarion-testrun-id',
'value': str(self.testrun_id)
})
for name, value in sorted(
self.config['xunit_import_properties'].items()):
if name == 'polarion-lookup-method':
lookup_prop = str(value).lower()
if lookup_prop not in ('id', 'name', 'custom'):
raise Dump2PolarionException(
"Invalid value '{}' for the 'polarion-lookup-method' property"
.format(str(value)))
self._lookup_prop = lookup_prop
else:
ElementTree.SubElement(testsuites_properties, 'property', {
'name': name,
'value': str(value)
})
return testsuites_properties
def _get_xml_root(junit_file):
try:
tree = etree.parse(os.path.expanduser(junit_file))
except Exception as err:
raise Dump2PolarionException("Failed to parse XML file '{}': {}".format(junit_file, err))
return tree.getroot()
def get_testrun_id(args, testrun_id):
"""Returns testrun id."""
if (args.testrun_id and testrun_id and not args.force
and testrun_id != args.testrun_id):
raise Dump2PolarionException(
"The test run id '{}' found in exported data doesn't match '{}'. "
"If you really want to proceed, add '-f'.".format(
testrun_id, args.testrun_id))
found_testrun_id = args.testrun_id or testrun_id
if not found_testrun_id:
raise Dump2PolarionException(
"The testrun id was not specified on command line and not found in the input data."
)
return found_testrun_id
def get_session(credentials, config):
"""Gets requests session."""
session = requests.Session()
session.verify = False
auth_url = config.get("auth_url")
if auth_url:
cookie = session.post(
auth_url,
data={
"j_username": credentials[0],
"j_password": credentials[1],
"submit": "Log In",
"rememberme": "true",
},
headers={"Content-Type": "application/x-www-form-urlencoded"},
)
if not cookie:
raise Dump2PolarionException(
"Cookie was not retrieved from {}.".format(auth_url))
else:
# TODO: can be removed once basic auth is discontinued on prod
session.auth = credentials
return session
def _get_importer(input_file):
"""Select importer based on input file type."""
__, ext = os.path.splitext(input_file)
ext = ext.lower()
if "ostriz" in input_file:
from dump2polarion.results import ostriztools
importer = ostriztools.import_ostriz
elif ext == ".xml":
# expect junit-report from pytest
from dump2polarion.results import junittools
importer = junittools.import_junit
elif ext == ".csv":
from dump2polarion.results import csvtools
importer = csvtools.import_csv
elif ext in dbtools.SQLITE_EXT:
importer = dbtools.import_sqlite
elif ext == ".json":
from dump2polarion.results import jsontools
importer = jsontools.import_json
else:
raise Dump2PolarionException(
"Cannot recognize type of input data, add file extension.")
return importer
def parse(self):
"""Parse log file produced by the Requirements Importer."""
existing_items = []
duplicate_items = []
new_items = []
for line in self.fp:
line = line.strip()
if "Updated requirement" in line:
existing_it = self.get_requirement(line)
if existing_it:
existing_items.append(existing_it)
elif "Found multiple work items with the title" in line:
duplicate_it = self.get_requirement_warn(line)
if duplicate_it:
duplicate_items.append(duplicate_it)
elif "Created requirement" in line:
new_it = self.get_requirement(line)
if new_it:
new_items.append(new_it)
outcome = ParsedLog("requirement", new_items, existing_items, duplicate_items)
if not outcome:
raise Dump2PolarionException(
"No valid data found in the log file '{}'".format(self.log_file)
)
return outcome
def _get_json(location):
"""Reads JSON data from file or URL."""
location = os.path.expanduser(location)
try:
if os.path.isfile(location):
with io.open(location, encoding='utf-8') as json_data:
return json.load(json_data, object_pairs_hook=OrderedDict).get('tests')
elif 'http' in location:
json_data = requests.get(location)
if not json_data:
raise Dump2PolarionException("Failed to download")
return json.loads(json_data.text, object_pairs_hook=OrderedDict).get('tests')
else:
raise Dump2PolarionException("Invalid location")
except Exception as err:
raise Dump2PolarionException(
"Failed to parse JSON from {}: {}".format(location, err))
def get_xml_root_from_str(xml_str):
"""Return XML root from string."""
try:
xml_root = etree.fromstring(xml_str.encode("utf-8"), NO_BLANKS_PARSER)
# pylint: disable=broad-except
except Exception as err:
raise Dump2PolarionException("Failed to parse XML string: {}".format(err))
return xml_root
def get_xml_root(xml_file):
"""Return XML root."""
try:
xml_root = etree.parse(os.path.expanduser(xml_file), NO_BLANKS_PARSER).getroot()
# pylint: disable=broad-except
except Exception as err:
raise Dump2PolarionException("Failed to parse XML file '{}': {}".format(xml_file, err))
return xml_root
def set_lookup_method(xml_root, value):
"""Change lookup method."""
if xml_root.tag == "testsuites":
_set_property(xml_root, "polarion-lookup-method", value)
elif xml_root.tag in ("testcases", "requirements"):
_set_property(xml_root, "lookup-method", value)
else:
raise Dump2PolarionException(_NOT_EXPECTED_FORMAT_MSG)
def _get_xml_root(xml_root, xml_str, xml_file):
if xml_root is not None:
return xml_root
if xml_str:
return utils.get_xml_root_from_str(xml_str)
if xml_file:
return utils.get_xml_root(xml_file)
raise Dump2PolarionException("Failed to submit to Polarion - no data supplied")
def import_junit(junit_file, **kwargs):
"""Reads the content of the junit-results file produced by pytest and returns imported data."""
try:
tree = ElementTree.parse(os.path.expanduser(junit_file))
except Exception as err:
raise Dump2PolarionException(
"Failed to parse XML file '{}': {}".format(junit_file, err))
xml_root = tree.getroot()
results = []
for test_data in xml_root:
if test_data.tag != 'testcase':
continue
verdict = None
verdict_found = False
comment = ''
properties = {}
for element in test_data:
if not verdict_found:
if element.tag == 'error':
verdict = 'failed'
comment = element.get('message')
# continue to see if there's more telling verdict for this record
elif element.tag == 'failure':
verdict = 'failed'
comment = element.get('message')
verdict_found = True
elif element.tag == 'skipped':
verdict = 'skipped'
comment = element.get('message')
verdict_found = True
if element.tag == 'properties':
for prop in element:
properties[prop.get('name')] = prop.get('value')
if not verdict:
verdict = 'passed'
title = test_data.get('name')
classname = test_data.get('classname')
time = test_data.get('time', 0)
filepath = test_data.get('file')
data = [
('title', title),
('classname', classname),
('verdict', verdict),
('comment', comment),
('time', time),
('file', filepath),
]
for key in properties:
data.append((key, properties[key]))
record = OrderedDict(data)
results.append(record)
return exporter.ImportedData(results=results, testrun=None)
def get_xml_root_from_str(xml):
"""Returns XML root from string."""
try:
xml_root = ElementTree.fromstring(xml.encode('utf-8'))
# pylint: disable=broad-except
except Exception as err:
raise Dump2PolarionException(
"Failed to parse XML file: {}".format(err))
return xml_root
def _get_submit_target(xml_root, config):
if xml_root.tag == 'testcases':
target = config.get('testcase_taget')
elif xml_root.tag == 'testsuites':
target = config.get('xunit_target')
else:
raise Dump2PolarionException(
"Failed to submit results to Polarion - submit target not found")
return target
def get_imported_data(csv_file, **kwargs):
"""Read the content of the Polarion exported csv file and return imported data."""
with open(os.path.expanduser(csv_file), encoding="utf-8") as input_file:
reader = _get_csv_reader(input_file)
fieldnames = _get_csv_fieldnames(reader)
if not fieldnames:
raise Dump2PolarionException(
"Cannot find field names in CSV file '{}'".format(csv_file)
)
results = _get_results(reader, fieldnames)
if not results:
raise Dump2PolarionException("No results read from CSV file '{}'".format(csv_file))
testrun = _get_testrun_from_csv(input_file, reader)
return xunit_exporter.ImportedData(results=results, testrun=testrun)
def _check_config(config):
missing = []
for key in ('testcase_taget', 'xunit_target', 'message_bus'):
if not config.get(key):
missing.append(key)
if missing:
raise Dump2PolarionException(
"Failed to find following keys in config file: {}\n"
"Please see https://mojo.redhat.com/docs/DOC-1098563#config".format(', '.join(missing)))
def import_pytest_collect(json_filename):
"""Reads the content of the JSON file produced by pytest-polarion-collect file."""
try:
results = _load_json(json_filename)["results"]
except Exception as err:
raise Dump2PolarionException("Cannot load results from {}: {}".format(
json_filename, err))
return xunit_exporter.ImportedData(results=results, testrun=None)
def _fill_lookup_prop(self, testcases_properties):
"""Fills the polarion-lookup-method property."""
if not self._lookup_prop:
raise Dump2PolarionException(
"Failed to set the 'polarion-lookup-method' property")
etree.SubElement(testcases_properties, "property", {
"name": "lookup-method",
"value": self._lookup_prop
})
def _open_sqlite(db_file):
"""Opens database connection."""
db_file = os.path.expanduser(db_file)
try:
with open(db_file):
# test that the file can be accessed
pass
return sqlite3.connect(db_file, detect_types=sqlite3.PARSE_DECLTYPES)
except (IOError, sqlite3.Error) as err:
raise Dump2PolarionException('{}'.format(err))
def set_dry_run(xml_root, value=True):
"""Set dry-run so records are not updated, only log file is produced."""
value_str = str(value).lower()
assert value_str in ("true", "false")
if xml_root.tag == "testsuites":
_set_property(xml_root, "polarion-dry-run", value_str)
elif xml_root.tag in ("testcases", "requirements"):
_set_property(xml_root, "dry-run", value_str)
else:
raise Dump2PolarionException(_NOT_EXPECTED_FORMAT_MSG)
def get_xml_root(xml_file):
"""Returns XML root."""
try:
xml_tree = ElementTree.parse(os.path.expanduser(xml_file))
xml_root = xml_tree.getroot()
# pylint: disable=broad-except
except Exception as err:
raise Dump2PolarionException(
"Failed to parse XML file '{}': {}".format(xml_file, err))
return xml_root
def _fill_lookup_prop(self, testsuites_properties):
"""Fills the polarion-lookup-method property."""
if not self._lookup_prop:
raise Dump2PolarionException(
"Failed to set the 'polarion-lookup-method' property")
ElementTree.SubElement(testsuites_properties, 'property', {
'name': 'polarion-lookup-method',
'value': self._lookup_prop
})
