def test_get_field_names_does_not_return_duplicates_custom_fields(self):
a = model.About()
a.custom_fields['f'] = model.StringField(name='f',
value='1',
present=True)
a.custom_fields['cf'] = model.StringField(name='cf',
value='1',
present=True)
b = model.About()
b.custom_fields['g'] = model.StringField(name='g',
value='1',
present=True)
b.custom_fields['cf'] = model.StringField(name='cf',
value='2',
present=True)
abouts = [a, b]
# ensure all fields (including custom fields) and
# about_resource are collected in the correct order
expected = [
'about_resource',
'name',
'cf',
'f',
'g',
]
result = model.get_field_names(abouts)
assert expected == result
def test_unique_can_handle_About_object(self):
base_dir = 'some_dir'
test = {
'about_resource': '.',
'author': '',
'copyright': 'Copyright (c) 2013-2014 nexB Inc.',
'custom1': 'some custom',
'custom_empty': '',
'description': 'AboutCode is a tool\nfor files.',
'license': 'apache-2.0',
'name': 'AboutCode',
'owner': 'nexB Inc.'
}
a = model.About()
a.load_dict(test, base_dir)
c = model.About()
c.load_dict(test, base_dir)
b = model.About()
test.update(dict(about_resource='asdasdasd'))
b.load_dict(test, base_dir)
abouts = [a, b]
results = util.unique(abouts)
assert [a] == results
def test_About_are_not_equal_with_small_text_differences(self):
test_file = get_test_loc('test_model/equal/complete2/about.ABOUT')
a = model.About(test_file, about_file_path='complete2/about.ABOUT')
test_file2 = get_test_loc('test_model/equal/complete/about.ABOUT')
b = model.About(test_file2, about_file_path='complete/about.ABOUT')
assert a.dumps() != b.dumps()
assert a != b
def test_get_field_names_only_returns_non_empties(self):
a = model.About()
a.custom_fields['f'] = model.StringField(name='f',
value='1',
present=True)
b = model.About()
b.custom_fields['g'] = model.StringField(name='g',
value='1',
present=True)
abouts = [a, b]
# ensure all fields (including custom fields) and
# about_resource are collected in the correct order
expected = [model.About.ABOUT_RESOURCE_ATTR, 'name', 'f', 'g']
result = model.get_field_names(abouts)
assert expected == result
def test_About_dumps(self):
test_file = get_test_loc('test_model/dumps/about.ABOUT')
a = model.About(test_file)
assert [] == a.errors
expected = '''about_resource: .
name: AboutCode
version: 0.11.0
description: |
AboutCode is a tool
to process ABOUT files.
An ABOUT file is a file.
homepage_url: http://dejacode.org
license_expression: apache-2.0
copyright: Copyright (c) 2013-2014 nexB Inc.
notice_file: NOTICE
owner: nexB Inc.
author: Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez
vcs_tool: git
vcs_repository: https://github.com/dejacode/about-code-tool.git
licenses:
- key: apache-2.0
file: apache-2.0.LICENSE
'''
result = a.dumps()
assert expected == result
def test_as_dict_load_dict_ignores_empties(self):
test = {
'about_resource': '.',
'author': '',
'copyright': 'Copyright (c) 2013-2014 nexB Inc.',
'custom1': 'some custom',
'custom_empty': '',
'description': 'AboutCode is a tool\nfor files.',
'license_expression': 'apache-2.0',
'name': 'AboutCode',
'owner': 'nexB Inc.'
}
expected = {
'about_file_path': None,
'about_resource': OrderedDict([('.', None)]),
'copyright': 'Copyright (c) 2013-2014 nexB Inc.',
'custom1': 'some custom',
'description': 'AboutCode is a tool\nfor files.',
'license_expression': 'apache-2.0',
'name': 'AboutCode',
'owner': 'nexB Inc.'
}
a = model.About()
base_dir = 'some_dir'
a.load_dict(test, base_dir)
as_dict = a.as_dict()
# FIXME: why converting back to dict?
assert expected == dict(as_dict)
def test_About_hydrate_normalize_field_names_to_lowercase(self):
test_content = get_test_content(
'test_gen/parser_tests/upper_field_names.ABOUT')
fields = saneyaml.load(test_content).items()
a = model.About()
for _ in range(3):
self.check_About_hydrate(a, fields)
def test_About_contains_about_file_path(self):
test_file = get_test_loc('test_model/serialize/about.ABOUT')
# TODO: I am not sure this override of the about_file_path makes sense
a = model.About(test_file, about_file_path='complete/about.ABOUT')
assert [] == a.errors
expected = 'complete/about.ABOUT'
result = a.about_file_path
assert expected == result
def test_load_has_errors_for_non_unicode(self):
test_file = get_test_loc('test_model/unicode/not-unicode.ABOUT')
a = model.About()
a.load(test_file)
err = a.errors[0]
assert CRITICAL == err.severity
assert 'Cannot load invalid ABOUT file' in err.message
assert 'UnicodeDecodeError' in err.message
def test_About_with_existing_about_resource_has_no_error(self):
test_file = get_test_loc(
'test_gen/parser_tests/about_resource_field.ABOUT')
a = model.About(test_file)
assert [] == a.errors
result = a.about_resource.value['about_resource.c']
# this means we have a location
self.assertNotEqual([], result)
def test_android_module_license(self):
path = 'test_model/android/single_license.c.ABOUT'
test_file = get_test_loc(path)
abouts = model.About(location=test_file, about_file_path=path)
parent_dir = get_temp_dir()
abouts.android_module_license(parent_dir)
assert os.path.exists(
os.path.join(parent_dir, 'MODULE_LICENSE_PUBLIC_DOMAIN'))
def test_About_as_dict_contains_special_paths(self):
test_file = get_test_loc('test_model/special/about.ABOUT')
a = model.About(test_file, about_file_path='complete/about.ABOUT')
expected_errors = []
assert expected_errors == a.errors
as_dict = a.as_dict()
expected = 'complete/about.ABOUT'
result = as_dict[model.About.ABOUT_FILE_PATH_ATTR]
assert expected == result
def test_About_custom_fields_are_not_ignored_and_order_is_preserved(self):
test_file = get_test_loc(
'test_model/custom_fields/custom_fields.about')
a = model.About(test_file)
result = [(n, f.value) for n, f in a.custom_fields.items()]
expected = [(u'single_line', u'README STUFF'),
(u'multi_line', u'line1\nline2'), (u'other', u'sasasas'),
(u'empty', u'')]
assert sorted(expected) == sorted(result)
def test_About_has_errors_when_required_fields_are_empty(self):
test_file = get_test_loc('test_model/parse/empty_required.ABOUT')
a = model.About(test_file)
expected = [
Error(CRITICAL, 'Field about_resource is required and empty'),
Error(CRITICAL, 'Field name is required and empty'),
]
result = a.errors
assert expected == result
def test_About_has_errors_when_about_resource_does_not_exist(self):
test_file = get_test_loc(
'test_gen/parser_tests/missing_about_ref.ABOUT')
file_path = posixpath.join(posixpath.dirname(test_file),
'about_file_missing.c')
a = model.About(test_file)
err_msg = 'Field about_resource: Path %s not found' % file_path
expected = [Error(INFO, err_msg)]
result = a.errors
assert expected == result
def test_About_duplicate_field_names_are_not_reported_if_same_value(self):
# This test is failing because the YAML does not keep the order when
# loads the test files. For instance, it treat the 'About_Resource' as the
# first element and therefore the dup key is 'about_resource'.
test_file = get_test_loc(
'test_model/parse/dupe_field_name_no_new_value.ABOUT')
a = model.About(test_file)
expected = []
result = a.errors
assert sorted(expected) == sorted(result)
def test_About_load_ignores_original_field_order_and_uses_standard_predefined_order(
self):
# fields in this file are not in the standard order
test_file = get_test_loc('test_model/parse/ordered_fields.ABOUT')
a = model.About(test_file)
assert [] == a.errors
expected = ['about_resource', 'name', 'version', 'download_url']
result = [f.name for f in a.all_fields() if f.present]
assert expected == result
def test_write_output_json(self):
path = 'test_model/this.ABOUT'
test_file = get_test_loc(path)
abouts = model.About(location=test_file, about_file_path=path)
result = get_temp_file()
model.write_output([abouts], result, format='json')
expected = get_test_loc('test_model/expected.json')
check_json(expected, result)
def test_write_output_csv_with_multiple_files(self):
path = 'test_model/multiple_files.ABOUT'
test_file = get_test_loc(path)
abouts = model.About(location=test_file, about_file_path=path)
result = get_temp_file()
model.write_output([abouts], result, format='csv')
expected = get_test_loc('test_model/multiple_files_expected.csv')
check_csv(expected, result)
def test_android_module_multi_licenses(self):
path = 'test_model/android/multi_license.c.ABOUT'
test_file = get_test_loc(path)
abouts = model.About(location=test_file, about_file_path=path)
parent_dir = get_temp_dir()
abouts.android_module_license(parent_dir)
assert os.path.exists(
os.path.join(parent_dir, 'MODULE_LICENSE_BSD_NEW'))
assert os.path.exists(
os.path.join(parent_dir, 'MODULE_LICENSE_BSD_SIMPLIFIED'))
def test_About_rejects_non_ascii_names_and_accepts_unicode_values(self):
test_file = get_test_loc(
'test_model/parse/non_ascii_field_name_value.about')
a = model.About(test_file)
expected = [
Error(
CRITICAL,
"Field name: 'mat\xedas' contains illegal name characters: 0 to 9, a to z, A to Z and _. (or empty spaces)"
)
]
assert expected == a.errors
def load_inventory(location,
configuration=None,
scancode=False,
reference_dir=None):
"""
Load the inventory file at `location`
Optionally use `reference_dir` as the directory location of extra reference
license and notice files to reuse.
"""
errors = []
abouts = []
if scancode:
inventory = load_scancode_json(location, configuration)
else:
if location.endswith('.csv'):
dup_cols_err = check_duplicated_columns(location)
if dup_cols_err:
errors.extend(dup_cols_err)
return errors, abouts
inventory = load_csv(location, configuration)
elif location.endswith('.xlsx'):
dup_cols_err, inventory = load_excel(location, configuration)
if dup_cols_err:
errors.extend(dup_cols_err)
return errors, abouts
else:
inventory = load_json(location)
errors = []
for component in inventory:
newline_in_file_err = check_newline_in_file_field(component)
for err in newline_in_file_err:
errors.append(err)
if errors:
return errors, abouts
for component in inventory:
about = model.About()
ld_errors = about.load_dict(
component,
scancode=scancode,
reference_dir=reference_dir,
)
for e in ld_errors:
if not e in errors:
errors.extend(ld_errors)
abouts.append(about)
return unique(errors), abouts
def test_About_has_errors_for_illegal_custom_field_name(self):
test_file = get_test_loc('test_model/parse/illegal_custom_field.about')
a = model.About(test_file)
expected_errors = [
Error(INFO, 'Field hydrate is a custom field.'),
Error(
CRITICAL,
"Internal error with custom field: 'hydrate': 'illegal name'.")
]
assert expected_errors == a.errors
assert not hasattr(getattr(a, 'hydrate'), 'value')
field = list(a.custom_fields.values())[0]
assert 'hydrate' == field.name
assert 'illegal name' == field.value
def test_load_dump_is_idempotent(self):
test_file = get_test_loc('test_model/this.ABOUT')
a = model.About()
a.load(test_file)
dumped_file = get_temp_file('that.ABOUT')
a.dump(dumped_file)
expected = get_unicode_content(test_file).splitlines()
result = get_unicode_content(dumped_file).splitlines()
# Ignore comment and empty line
filtered_result = []
for line in result:
if not line.startswith('#') and not line == '':
filtered_result.append(line)
assert expected == filtered_result
def test_android_notice(self):
path = 'test_model/android/single_license.c.ABOUT'
test_file = get_test_loc(path)
abouts = model.About(location=test_file, about_file_path=path)
parent_dir = get_temp_dir()
notice_path, notice_context = abouts.android_notice(parent_dir)
expected_path = os.path.join(parent_dir, 'NOTICE')
assert os.path.normpath(notice_path) == expected_path
expected_notice = '''Copyright (c) xyz
This component is released to the public domain by the author.
'''
assert notice_context == expected_notice
def test_About_notice_and_license_text_are_loaded_from_file(self):
test_file = get_test_loc(
'test_model/parse/license_file_notice_file.ABOUT')
a = model.About(test_file)
expected = '''Tester holds the copyright for test component. Tester relinquishes copyright of
this software and releases the component to Public Domain.
* Email [email protected] for any questions'''
result = a.license_file.value['license_text.LICENSE']
assert expected == result
expected = '''Test component is released to Public Domain.'''
result = a.notice_file.value['notice_text.NOTICE']
assert expected == result
def test_load_dict_issue_433(self):
package_data = {
'about_resource':
'package1.zip',
'name':
'package',
'version':
'1.0',
'copyright':
'copyright on package',
'license_expression':
'license1 AND license2',
'notice_file':
'package1.zip.NOTICE',
'licenses': [
{
'key': 'license1',
'name': 'License1',
'file': 'license1.LICENSE'
},
{
'key': 'license2',
'name': 'License2',
'file': 'license2.LICENSE'
},
],
}
about = model.About()
about.load_dict(package_data, base_dir='')
as_dict = about.as_dict()
expected = '''about_resource: package1.zip
name: package
version: '1.0'
license_expression: license1 AND license2
copyright: copyright on package
notice_file: package1.zip.NOTICE
licenses:
- key: license1
name: License1
file: license1.LICENSE
- key: license2
name: License2
file: license2.LICENSE
'''
assert about.dumps() == expected
def test_load_can_load_unicode(self):
test_file = get_test_loc('test_model/unicode/nose-selecttests.ABOUT')
a = model.About()
a.load(test_file)
file_path = posixpath.join(posixpath.dirname(test_file),
'nose-selecttests-0.3.zip')
err_msg = 'Field about_resource: Path %s not found' % file_path
errors = [
Error(INFO, 'Field dje_license is a custom field.'),
Error(INFO, 'Field license_text_file is a custom field.'),
Error(INFO, 'Field scm_tool is a custom field.'),
Error(INFO, 'Field scm_repository is a custom field.'),
Error(INFO, 'Field test is a custom field.'),
Error(INFO, err_msg)
]
assert errors == a.errors
assert 'Copyright (c) 2012, Domen Kožar' == a.copyright.value
def test_load_dict_as_dict_is_idempotent_ignoring_special(self):
test = {
'about_resource': ['.'],
'attribute': 'yes',
'author':
'Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez',
'copyright': 'Copyright (c) 2013-2014 nexB Inc.',
'description':
'AboutCode is a tool to process ABOUT files. An ABOUT file is a file.',
'homepage_url': 'http://dejacode.org',
'license_expression': 'apache-2.0',
'name': 'AboutCode',
'owner': 'nexB Inc.',
'vcs_repository':
'https://github.com/dejacode/about-code-tool.git',
'vcs_tool': 'git',
'version': '0.11.0'
}
a = model.About()
base_dir = 'some_dir'
a.load_dict(test, base_dir)
as_dict = a.as_dict()
expected = {
'about_file_path': None,
'about_resource': OrderedDict([('.', None)]),
'attribute': 'yes',
'author':
'Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez',
'copyright': 'Copyright (c) 2013-2014 nexB Inc.',
'description':
'AboutCode is a tool to process ABOUT files. An ABOUT file is a file.',
'homepage_url': 'http://dejacode.org',
'license_expression': 'apache-2.0',
'name': 'AboutCode',
'owner': 'nexB Inc.',
'vcs_repository':
'https://github.com/dejacode/about-code-tool.git',
'vcs_tool': 'git',
'version': '0.11.0'
}
assert expected == dict(as_dict)
def test_About_dumps_all_non_empty_fields(self):
test_file = get_test_loc('test_model/parse/complete2/about.ABOUT')
a = model.About(test_file)
expected_error = [
Error(INFO, 'Field custom1 is a custom field.'),
Error(INFO, 'Field custom2 is a custom field.'),
Error(INFO, 'Field custom2 is present but empty.')
]
assert sorted(expected_error) == sorted(a.errors)
expected = '''about_resource: .
name: AboutCode
version: 0.11.0
custom1: |
multi
line
'''
result = a.dumps()
assert expected == result
