def test_load_inventory_with_errors(self):
location = get_test_loc('test_gen/inv4.csv')
base_dir = get_temp_dir()
errors, abouts = gen.load_inventory(location, base_dir)
expected_errors = [
Error(
CRITICAL,
"Field name: 'confirmed copyright' contains illegal name characters: 0 to 9, a to z, A to Z and _."
),
Error(INFO, 'Field resource is a custom field.'),
Error(INFO, 'Field test is a custom field.'),
Error(INFO, 'Field about_resource: Path')
]
# assert [] == errors
for exp, err in zip(expected_errors, errors):
assert exp.severity == err.severity
assert err.message.startswith(exp.message)
expected = (
'about_resource: .\n'
'name: AboutCode\n'
'version: 0.11.0\n'
'description: |\n'
' multi\n'
' line\n'
# 'confirmed copyright: Copyright (c) nexB, Inc.\n'
'resource: this.ABOUT\n'
'test: This is a test\n')
result = [a.dumps() for a in abouts]
assert expected == result[0]
def test_load_inventory(self):
location = get_test_loc('test_gen/inv.csv')
base_dir = get_temp_dir()
errors, abouts = gen.load_inventory(location, base_dir)
expected_errors = [
Error(INFO, 'Field custom1 is a custom field.'),
Error(INFO, 'Field about_resource: Path')
]
for exp, err in zip(expected_errors, errors):
assert exp.severity == err.severity
assert err.message.startswith(exp.message)
expected = ('''about_resource: .
name: AboutCode
version: 0.11.0
description: |
multi
line
custom1: |
multi
line
''')
result = [a.dumps() for a in abouts]
assert expected == result[0]
def test_filter_errors_no_errors(self):
errors = [
Error(INFO, 'msg3'),
Error(DEBUG, 'msg4'),
Error(NOTSET, 'msg4'),
]
assert [] == cmd.filter_errors(errors)
def test_load_inventory_with_mapping(self):
location = get_test_loc('gen/inv4.csv')
base_dir = get_test_loc('inv')
license_notice_text_location = None
use_mapping = True
errors, abouts = gen.load_inventory(location, base_dir,
license_notice_text_location,
use_mapping)
expected_errors = [
Error(
INFO,
'Field test is not a supported field and is not defined in the mapping file. This field is ignored.'
),
Error(INFO, 'Field resource is a custom field')
]
assert sorted(expected_errors) == sorted(errors)
expected = [
u'about_resource: .\n'
u'name: AboutCode\n'
u'version: 0.11.0\n'
u'description: |\n'
u' multi\n'
u' line\n'
u'copyright: Copyright (c) nexB, Inc.\n'
u'resource: this.ABOUT\n'
]
result = [a.dumps(with_absent=False, with_empty=False) for a in abouts]
assert expected == result
def generate(abouts,
license_dict,
min_license_score,
template=None,
variables=None):
"""
Generate an attribution text from an `abouts` list of About objects, a
`template` template text and a `variables` optional dict of extra
variables.
Return a tuple of (error, attribution text) where error is an Error object
or None and attribution text is the generated text or None.
"""
rendered = None
error = None
template_error = check_template(template)
if template_error:
lineno, message = template_error
error = Error(
CRITICAL,
'Template validation error at line: {lineno}: "{message}"'.format(
**locals()))
return error, None
template = jinja2.Template(template)
# Get the current UTC time
utcnow = datetime.datetime.utcnow()
try:
# Convert the field object to dictionary as it's needed for the
# groupby in JINJA2 template
about_dict_list = []
for about in abouts:
about_dict = convert_object_to_dict(about)
about_dict_list.append(about_dict)
rendered = template.render(abouts=about_dict_list,
license_dict=license_dict,
min_license_score=min_license_score,
utcnow=utcnow,
tkversion=__version__,
variables=variables)
except Exception as e:
lineno = getattr(e, 'lineno', '') or ''
if lineno:
lineno = ' at line: {}'.format(lineno)
err = getattr(e, 'message', '') or ''
error = Error(
CRITICAL,
'Template processing error {lineno}: {err}'.format(**locals()),
)
error = Error(
CRITICAL,
'Template processing error:' + str(e),
)
return error, rendered
def test_About_has_errors_when_required_fields_are_empty(self):
test_file = get_test_loc('test_model/parse/empty_required.ABOUT')
a = model.About(test_file)
expected = [
Error(CRITICAL, 'Field about_resource is required and empty'),
Error(CRITICAL, 'Field name is required and empty'),
]
result = a.errors
assert expected == result
def test_ignore_about_resource_path_not_exist_error(self):
input_err = [
Error(ERROR,
'Field about_resource_path: test.tar.gz does not exist'),
Error(ERROR,
'Field about_resource_path: test.tar.gz does not exist')
]
expected_err = []
assert util.ignore_about_resource_path_not_exist_error(
input_err) == expected_err
def request_license_data(api_url, api_key, license_key):
"""
Return a tuple of (dictionary of license data, list of errors) given a
`license_key`. Send a request to `api_url` authenticating with `api_key`.
"""
headers = {
'Authorization': 'Token %s' % api_key,
}
payload = {'api_key': api_key, 'key': license_key, 'format': 'json'}
api_url = api_url.rstrip('/')
payload = urlencode(payload)
full_url = '%(api_url)s/?%(payload)s' % locals()
# handle special characters in URL such as space etc.
quoted_url = quote(full_url, safe="%/:=&?~#+!$,;'@()*[]")
license_data = {}
errors = []
try:
request = Request(quoted_url, headers=headers)
response = urlopen(request)
response_content = response.read().decode('utf-8')
# FIXME: this should be an ordered dict
license_data = json.loads(response_content)
if not license_data['results']:
msg = u"Invalid 'license': %s" % license_key
errors.append(Error(ERROR, msg))
except HTTPError as http_e:
# some auth problem
if http_e.code == 403:
msg = (u"Authorization denied. Invalid '--api_key'. "
u"License generation is skipped.")
errors.append(Error(ERROR, msg))
else:
# Since no api_url/api_key/network status have
# problem detected, it yields 'license' is the cause of
# this exception.
msg = u"Invalid 'license': %s" % license_key
errors.append(Error(ERROR, msg))
except Exception as e:
errors.append(Error(ERROR, str(e)))
finally:
if license_data.get('count') == 1:
license_data = license_data.get('results')[0]
else:
license_data = {}
return license_data, errors
def test_About_has_errors_for_illegal_custom_field_name(self):
test_file = get_test_loc('test_model/parse/illegal_custom_field.about')
a = model.About(test_file)
expected_errors = [
Error(INFO, 'Field hydrate is a custom field.'),
Error(
CRITICAL,
"Internal error with custom field: 'hydrate': 'illegal name'.")
]
assert expected_errors == a.errors
assert not hasattr(getattr(a, 'hydrate'), 'value')
field = list(a.custom_fields.values())[0]
assert 'hydrate' == field.name
assert 'illegal name' == field.value
def transform_csv_to_csv(location, output, transformer):
"""
Read a CSV file at `location` and write a new CSV file at `output`. Apply
transformations using the `transformer` Transformer.
Return a list of Error objects.
"""
if not transformer:
raise ValueError('Cannot transform without Transformer')
rows = read_csv_rows(location)
errors = []
data = iter(rows)
field_names = next(rows)
dupes = check_duplicate_fields(field_names)
if dupes:
msg = u'Duplicated field name: %(name)s'
for name in dupes:
errors.append(Error(CRITICAL, msg % locals()))
return field_names, [], errors
# Convert to dicts
new_data = [OrderedDict(zip_longest(field_names, item)) for item in data]
field_names, updated_data, errors = transform_data(new_data, transformer)
if errors:
return errors
else:
write_csv(output, updated_data, field_names)
return []
def check_duplicated_columns(location):
"""
Return a list of errors for duplicated column names in a CSV file
at location.
"""
location = add_unc(location)
with codecs.open(location, 'rb', encoding='utf-8-sig',
errors='replace') as csvfile:
reader = csv.reader(csvfile)
columns = next(reader)
columns = [col for col in columns]
seen = set()
dupes = OrderedDict()
for col in columns:
c = col.lower()
if c in seen:
if c in dupes:
dupes[c].append(col)
else:
dupes[c] = [col]
seen.add(c.lower())
errors = []
if dupes:
dup_msg = []
for name, names in dupes.items():
names = u', '.join(names)
msg = '%(name)s with %(names)s' % locals()
dup_msg.append(msg)
dup_msg = u', '.join(dup_msg)
msg = ('Duplicated column name(s): %(dup_msg)s\n' % locals() +
'Please correct the input and re-run.')
errors.append(Error(ERROR, msg))
return unique(errors)
def generate_and_save(abouts, output_location, template_loc=None, variables=None):
"""
Generate an attribution text from an `abouts` list of About objects, a
`template_loc` template file location and a `variables` optional
dict of extra variables. Save the generated attribution text in the
`output_location` file.
Return a list of Error objects if any.
"""
errors = []
# Parse license_expression and save to the license list
for about in abouts:
if not about.license_expression.value:
continue
special_char_in_expression, lic_list = parse_license_expression(about.license_expression.value)
if special_char_in_expression:
msg = (u"The following character(s) cannot be in the license_expression: " +
str(special_char_in_expression))
errors.append(Error(ERROR, msg))
rendering_error, rendered = generate_from_file(
abouts,
template_loc=template_loc,
variables=variables
)
if rendering_error:
errors.append(rendering_error)
if rendered:
output_location = add_unc(output_location)
with io.open(output_location, 'w', encoding='utf-8') as of:
of.write(rendered)
return errors
def test_check_file_names_with_invalid_chars_return_errors(self):
paths = [
'locations/file',
'locations/file with space',
'locations/dir1/dir2/file1',
'locations/dir2/file1',
'Accessibilité/ périmètre'
]
import sys
if sys.version_info[0] < 3: # python2
expected = [Error(CRITICAL, b"Invalid characters '\xe9\xe8' in file name at: 'Accessibilit\xe9/ p\xe9rim\xe8tre'")]
else:
expected = [Error(CRITICAL, "Invalid characters 'éè' in file name at: 'Accessibilité/ périmètre'")]
result = util.check_file_names(paths)
assert expected[0].message == result[0].message
assert expected == result
def test_api_request_license_data_without_result(self, mock_data):
response_content = b'{"count":0,"results":[]}'
mock_data.return_value = FakeResponse(response_content)
license_data = api.request_license_data(api_url='http://fake.url/',
api_key='api_key',
license_key='apache-2.0')
expected = ({}, [Error(ERROR, "Invalid 'license': apache-2.0")])
assert expected == license_data
def test_check_file_names_with_dupes_return_errors(self):
paths = ['some/path', 'some/PAth']
result = util.check_file_names(paths)
expected = [
Error(
CRITICAL,
"Duplicate files: 'some/PAth' and 'some/path' have the same case-insensitive file name")
]
assert expected == result
def test_filter_errors_default(self):
errors = [
Error(CRITICAL, 'msg1'),
Error(ERROR, 'msg2'),
Error(INFO, 'msg3'),
Error(WARNING, 'msg4'),
Error(DEBUG, 'msg4'),
Error(NOTSET, 'msg4'),
]
expected = [
Error(CRITICAL, 'msg1'),
Error(ERROR, 'msg2'),
Error(WARNING, 'msg4'),
]
assert expected == cmd.filter_errors(errors)
def test_check_duplicated_columns_handles_lower_upper_case(self):
test_file = get_test_loc('test_util/dup_keys_with_diff_case.csv')
expected = [
Error(
ERROR,
'Duplicated column name(s): copyright with Copyright\nPlease correct the input and re-run.'
)
]
result = util.check_duplicated_columns(test_file)
assert expected == result
def test_check_duplicated_columns(self):
test_file = get_test_loc('test_gen/dup_keys.csv')
expected = [
Error(
ERROR,
'Duplicated column name(s): copyright with copyright\nPlease correct the input and re-run.'
)
]
result = gen.check_duplicated_columns(test_file)
assert expected == result
def test_SingleLineField_has_errors_if_multiline(self):
value = '''line1
line2'''
field_class = model.SingleLineField
expected = value
expected_errors = [
Error(ERROR,
'Field s: Cannot span multiple lines: line1\n line2')
]
self.check_validate(field_class, value, expected, expected_errors)
def check_About_hydrate(self, about, fields):
expected = set([
'name', 'homepage_url', 'download_url', 'version', 'copyright',
'date', 'license_spdx', 'license_text_file', 'notice_file',
'about_resource'
])
expected_errors = [
Error(INFO, 'Field date is a custom field.'),
Error(INFO, 'Field license_spdx is a custom field.'),
Error(INFO, 'Field license_text_file is a custom field.')
]
errors = about.hydrate(fields)
assert expected_errors == errors
result = set([f.name for f in about.all_fields() if f.present])
assert expected == result
def test_About_dumps_all_non_empty_fields(self):
test_file = get_test_loc('test_model/parse/complete2/about.ABOUT')
a = model.About(test_file)
expected_error = [
Error(INFO, 'Field custom1 is a custom field.'),
Error(INFO, 'Field custom2 is a custom field.'),
Error(INFO, 'Field custom2 is present but empty.')
]
assert sorted(expected_error) == sorted(a.errors)
expected = '''about_resource: .
name: AboutCode
version: 0.11.0
custom1: |
multi
line
'''
result = a.dumps()
assert expected == result
def test_About_has_errors_when_about_resource_does_not_exist(self):
test_file = get_test_loc(
'test_gen/parser_tests/missing_about_ref.ABOUT')
file_path = posixpath.join(posixpath.dirname(test_file),
'about_file_missing.c')
a = model.About(test_file)
err_msg = 'Field about_resource: Path %s not found' % file_path
expected = [Error(INFO, err_msg)]
result = a.errors
assert expected == result
def test_collect_inventory_does_not_raise_error_and_maintains_order_on_custom_fields(
self):
test_loc = get_test_loc('test_model/inventory/custom_fields2.ABOUT')
errors, abouts = model.collect_inventory(test_loc)
expected_errors = [
Error(
INFO,
'inventory/custom_fields2.ABOUT: Field resource is a custom field.'
),
Error(
INFO,
'inventory/custom_fields2.ABOUT: Field custom_mapping is a custom field.'
)
]
assert expected_errors == errors
expected = [
u'about_resource: .\nname: test\nresource: .\ncustom_mapping: test\n'
]
assert expected == [a.dumps() for a in abouts]
def test_have_problematic_error():
have_problematic_errors = [
Error(CRITICAL, 'msg1'),
Error(ERROR, 'msg2'),
Error(INFO, 'msg3'),
Error(WARNING, 'msg4'),
Error(DEBUG, 'msg4'),
Error(NOTSET, 'msg4'),
]
no_problematic_errors = [
Error(INFO, 'msg3'),
Error(DEBUG, 'msg4'),
Error(NOTSET, 'msg4'),
]
assert cmd.have_problematic_error(have_problematic_errors)
assert cmd.have_problematic_error(no_problematic_errors) == False
def check_file_names(paths):
"""
Given a sequence of file paths, check that file names are valid and that
there are no case-insensitive duplicates in any given directories.
Return a list of errors.
From spec :
A file name can contain only these US-ASCII characters:
- digits from 0 to 9
- uppercase and lowercase letters from A to Z
- the _ underscore, - dash and . period signs.
From spec:
The case of a file name is not significant. On case-sensitive file
systems (such as Linux), a tool must raise an error if two ABOUT files
stored in the same directory have the same lowercase file name.
"""
# FIXME: this should be a defaultdicts that accumulates all duplicated paths
seen = {}
errors = []
for orig_path in paths:
path = orig_path
invalid = invalid_chars(path)
if invalid:
invalid = ''.join(invalid)
msg = ('Invalid characters %(invalid)r in file name at: '
'%(path)r' % locals())
errors.append(Error(CRITICAL, msg))
path = to_posix(orig_path)
name = resource_name(path).lower()
parent = posixpath.dirname(path)
path = posixpath.join(parent, name)
path = posixpath.normpath(path)
path = posixpath.abspath(path)
existing = seen.get(path)
if existing:
msg = ('Duplicate files: %(orig_path)r and %(existing)r '
'have the same case-insensitive file name' % locals())
errors.append(Error(CRITICAL, msg))
else:
seen[path] = orig_path
return errors
def test_collect_inventory_return_errors(self):
test_loc = get_test_loc('test_model/collect_inventory_errors')
errors, _abouts = model.collect_inventory(test_loc)
file_path1 = posixpath.join(test_loc, 'distribute_setup.py')
file_path2 = posixpath.join(test_loc, 'date_test.py')
err_msg1 = 'non-supported_date_format.ABOUT: Field about_resource: Path %s not found' % file_path1
err_msg2 = 'supported_date_format.ABOUT: Field about_resource: Path %s not found' % file_path2
expected_errors = [
Error(
INFO,
'non-supported_date_format.ABOUT: Field date is a custom field.'
),
Error(
INFO,
'supported_date_format.ABOUT: Field date is a custom field.'),
Error(INFO, err_msg1),
Error(INFO, err_msg2)
]
assert sorted(expected_errors) == sorted(errors)
def test_About_rejects_non_ascii_names_and_accepts_unicode_values(self):
test_file = get_test_loc(
'test_model/parse/non_ascii_field_name_value.about')
a = model.About(test_file)
expected = [
Error(
CRITICAL,
"Field name: 'mat\xedas' contains illegal name characters: 0 to 9, a to z, A to Z and _. (or empty spaces)"
)
]
assert expected == a.errors
def test_About_duplicate_field_names_are_detected_with_different_case(
self):
# This test is failing because the YAML does not keep the order when
# loads the test files. For instance, it treat the 'About_Resource' as the
# first element and therefore the dup key is 'about_resource'.
test_file = get_test_loc('test_model/parse/dupe_field_name.ABOUT')
a = model.About(test_file)
expected = [
Error(
WARNING,
'Field About_Resource is a duplicate. Original value: "." replaced with: "new value"'
),
Error(
WARNING,
'Field Name is a duplicate. Original value: "old" replaced with: "new"'
)
]
result = a.errors
assert sorted(expected) == sorted(result)
def test_About_file_fields_are_empty_if_present_and_path_missing(self):
test_file = get_test_loc(
'test_model/parse/missing_notice_license_files.ABOUT')
a = model.About(test_file)
file_path1 = posixpath.join(posixpath.dirname(test_file),
'test.LICENSE')
file_path2 = posixpath.join(posixpath.dirname(test_file),
'test.NOTICE')
err_msg1 = Error(CRITICAL,
'Field license_file: Path %s not found' % file_path1)
err_msg2 = Error(CRITICAL,
'Field notice_file: Path %s not found' % file_path2)
expected_errors = [err_msg1, err_msg2]
assert expected_errors == a.errors
assert {'test.LICENSE': None} == a.license_file.value
assert {'test.NOTICE': None} == a.notice_file.value
def test_PathField_contains_dict_after_validate(self):
value = 'string'
field_class = model.PathField
expected = OrderedDict([('string', None)])
expected_errors = [
Error(
ERROR,
'Field s: Unable to verify path: string: No base directory provided'
)
]
self.check_validate(field_class, value, expected, expected_errors)
