def test_check_duplicated_columns_handles_lower_upper_case(self):
test_file = get_test_loc('gen/dup_keys_with_diff_case.csv')
expected = [
Error(
ERROR,
u'Duplicated column name(s): copyright with Copyright\nPlease correct the input and re-run.'
)
]
result = gen.check_duplicated_columns(test_file)
assert expected == result
def test_generation_with_no_about_resource_reference_no_resource_validation(
self):
location = get_test_loc('gen/inv3.csv')
base_dir = get_temp_dir()
errors, abouts = gen.generate(location, base_dir)
expected = OrderedDict([(u'test.tar.gz', None)])
assert abouts[0].about_resource.value == expected
assert len(errors) == 1
def test_About_load_ignores_original_field_order_and_uses_standard_predefined_order(
self):
# fields in this file are not in the standard order
test_file = get_test_loc('test_model/parse/ordered_fields.ABOUT')
a = model.About(test_file)
assert [] == a.errors
expected = ['about_resource', 'name', 'version', 'download_url']
result = [f.name for f in a.all_fields() if f.present]
assert expected == result
def test_About_rejects_non_ascii_names_and_accepts_unicode_values(self):
test_file = get_test_loc(
'test_model/parse/non_ascii_field_name_value.about')
a = model.About(test_file)
expected = [
Error(
CRITICAL,
"Field name: 'mat\xedas' contains illegal name characters: 0 to 9, a to z, A to Z and _. (or empty spaces)"
)
]
assert expected == a.errors
def test_generation_with_no_about_resource_reference(self):
location = get_test_loc('gen/inv3.csv')
base_dir = get_temp_dir()
errors, abouts = gen.generate(location, base_dir)
expected = [u'test.tar.gz']
assert abouts[0].about_resource.value == expected
assert len(errors) == 1
msg = u'Field about_resource_path'
assert msg in errors[0].message
def test_TextField_loads_file(self):
field = model.FileTextField(name='f',
value='license.LICENSE',
present=True)
base_dir = get_test_loc('test_model/base_dir')
errors = field.validate(base_dir=base_dir)
assert [] == errors
expected = {'license.LICENSE': 'some license text'}
assert expected == field.value
def test_get_about_locations(self):
test_dir = get_test_loc('test_util/about_locations')
expected = sorted([
'file with_spaces.ABOUT',
'dir1/file2.aBout',
'dir1/dir2/file1.about',
])
result = sorted(util.get_about_locations(test_dir))
result = [l.partition('/about_locations/')[-1] for l in result]
assert expected == result
def test_load_json2(self):
test_file = get_test_loc('test_util/json/expected_need_mapping.json')
expected = [dict(OrderedDict([
('about_file', '/load/this.ABOUT'),
('about_resource', '.'),
('version', '0.11.0'),
('name', 'AboutCode'),
])
)]
result = util.load_json(test_file)
assert expected == result
def test_collect_inventory_with_multi_line(self):
test_loc = get_test_loc(
'test_model/parse/multi_line_license_expresion.ABOUT')
errors, abouts = model.collect_inventory(test_loc)
assert [] == errors
expected_lic_url = [
'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit',
'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:apache-2.0'
]
returned_lic_url = abouts[0].license_url.value
assert expected_lic_url == returned_lic_url
def test_load_non_list_json2(self):
test_file = get_test_loc('test_util/json/not_a_list.json')
expected = [OrderedDict([
('about_file_path', '/load/this.ABOUT'),
('version', '0.11.0'),
('about_resource', '.'),
('name', 'AboutCode'),
])
]
result = util.load_json(test_file)
assert expected == result
def test_android_module_multi_licenses(self):
path = 'test_model/android/multi_license.c.ABOUT'
test_file = get_test_loc(path)
abouts = model.About(location=test_file, about_file_path=path)
parent_dir = get_temp_dir()
abouts.android_module_license(parent_dir)
assert os.path.exists(
os.path.join(parent_dir, 'MODULE_LICENSE_BSD_NEW'))
assert os.path.exists(
os.path.join(parent_dir, 'MODULE_LICENSE_BSD_SIMPLIFIED'))
def test_update_about_dictionary_keys(self):
mapping_output = get_test_loc('util/mapping_output')
about_ordered_dict = OrderedDict()
about_ordered_dict['name'] = 'test.c'
about_dict_list = [about_ordered_dict]
expected_output_dict = OrderedDict()
expected_output_dict['Component'] = 'test.c'
expected_dict_list = [expected_output_dict]
result = util.update_about_dictionary_keys(about_dict_list,
mapping_output)
assert expected_dict_list == result
def test_get_locations(self):
test_dir = get_test_loc('locations')
expected = sorted([
'locations/file with_spaces', 'locations/file1', 'locations/file2',
'locations/dir1/file2', 'locations/dir1/dir2/file1',
'locations/dir2/file1'
])
result = sorted(util.get_locations(test_dir))
for i, res in enumerate(result):
expect = expected[i]
assert res.endswith(expect)
def test_load_inventory(self):
location = get_test_loc('gen/inv.csv')
base_dir = get_test_loc('inv')
errors, abouts = gen.load_inventory(location, base_dir)
expected_errors = [
Error(INFO,
u'Field custom1 is not a supported field and is ignored.')
]
assert expected_errors == errors
expected = [
u'about_resource: .\n'
u'name: AboutCode\n'
u'version: 0.11.0\n'
u'description: |\n'
u' multi\n'
u' line\n'
]
result = [a.dumps(with_absent=False, with_empty=False) for a in abouts]
assert expected == result
def test_load_non_list_json(self):
test_file = get_test_loc('test_util/json/not_a_list_need_mapping.json')
# FIXME: why this dict nesting??
expected = [dict(OrderedDict([
('about_resource', '.'),
('name', 'AboutCode'),
('path', '/load/this.ABOUT'),
('version', '0.11.0'),
])
)]
result = util.load_json(test_file)
assert expected == result
def test_PathField_check_location(self):
test_file = 'license.LICENSE'
field = model.PathField(name='f', value=test_file, present=True)
base_dir = get_test_loc('test_model/base_dir')
errors = field.validate(base_dir=base_dir)
expected_errrors = []
assert expected_errrors == errors
result = field.value[test_file]
expected = add_unc(posixpath.join(to_posix(base_dir), test_file))
assert expected == result
def test_collect_inventory_with_no_about_resource_from_directory(self):
location = get_test_loc('test_model/inventory/no_about_resource_key')
result = get_temp_file()
errors, abouts = model.collect_inventory(location)
model.write_output(abouts, result, format='csv')
expected_errors = [
Error(CRITICAL,
'about/about.ABOUT: Field about_resource is required')
]
assert expected_errors == errors
def test_get_about_locations_with_ABOUT_files(self):
test_dir = get_test_loc('about_locations')
expected = sorted([
'locations/file with_spaces.ABOUT',
'locations/dir1/file2.aBout',
'locations/dir1/dir2/file1.about',
])
result = sorted(util.get_about_locations(test_dir))
for i, res in enumerate(result):
expect = expected[i]
assert res.endswith(expect)
def test_get_mapping_key_order_with_mapping_file(self):
import collections
expected = collections.OrderedDict()
expected['about_file_path'] = ''
expected['name'] = ''
expected['version'] = ''
expected['description'] = ''
expected['license_expression'] = ''
expected['copyright'] = ''
expected_keys = expected.keys()
test_mapping_file = get_test_loc('mapping_config/mapping.config')
result = util.get_mapping_key_order(test_mapping_file)
assert expected_keys == result
def test_load_inventory_with_mapping(self):
location = get_test_loc('gen/inv4.csv')
base_dir = get_test_loc('inv')
license_notice_text_location = None
use_mapping = True
errors, abouts = gen.load_inventory(location, base_dir,
license_notice_text_location,
use_mapping)
expected_error_messages = [
'Field about_resource',
'Field test is not a supported field and is not defined in the mapping file. This field is ignored.',
'Field resource is a custom field'
]
assert len(errors) == 3
for e in errors:
# we don't want to check the path value
if e.message.startswith('Field about_resource'):
continue
else:
assert e.message in expected_error_messages
expected = [
u'about_resource: .\n'
u'name: AboutCode\n'
u'version: 0.11.0\n'
u'copyright: Copyright (c) nexB, Inc.\n'
u'resource: this.ABOUT\n'
u'description: |-\n'
u' multi\n'
u' line\n'
]
result = [
a.dumps(use_mapping,
mapping_file=False,
with_absent=False,
with_empty=False) for a in abouts
]
assert expected == result
def test_generation_dir_endswith_space(self):
location = get_test_loc(
'inventory/complex/about_file_path_dir_endswith_space.csv')
base_dir = get_temp_dir()
errors, _abouts = gen.generate(location, base_dir)
expected_errors_msg1 = 'contains directory name ends with spaces which is not allowed. Generation skipped.'
expected_errors_msg2 = 'Field about_resource'
assert errors
assert len(errors) == 2
assert expected_errors_msg1 in errors[
0].message or expected_errors_msg1 in errors[1].message
assert expected_errors_msg2 in errors[
0].message or expected_errors_msg2 in errors[1].message
def test_About_has_errors_for_illegal_custom_field_name(self):
test_file = get_test_loc('test_model/parse/illegal_custom_field.about')
a = model.About(test_file)
expected_errors = [
Error(INFO, 'Field hydrate is a custom field.'),
Error(
CRITICAL,
"Internal error with custom field: 'hydrate': 'illegal name'.")
]
assert expected_errors == a.errors
assert not hasattr(getattr(a, 'hydrate'), 'value')
field = list(a.custom_fields.values())[0]
assert 'hydrate' == field.name
assert 'illegal name' == field.value
def test_PathField_check_missing_location(self):
test_file = 'does.not.exist'
field = model.PathField(name='f', value=test_file, present=True)
base_dir = get_test_loc('test_model/base_dir')
errors = field.validate(base_dir=base_dir)
file_path = posixpath.join(base_dir, test_file)
err_msg = 'Field f: Path %s not found' % file_path
expected_errors = [Error(CRITICAL, err_msg)]
assert expected_errors == errors
result = field.value[test_file]
assert None == result
def test_load_inventory_simple_xlsx(self):
location = get_test_loc('test_util/load/simple_sample.xlsx')
base_dir = get_temp_dir()
errors, abouts = util.load_inventory(location)
assert errors == []
assert abouts[0].name.value == 'cryptohash-sha256'
assert abouts[1].name.value == 'some_component'
assert abouts[0].version.value == 'v 0.11.100.1'
assert abouts[1].version.value == 'v 0.0.1'
assert abouts[0].license_expression.value == 'bsd-new and mit'
assert abouts[1].license_expression.value == 'mit'
def test_boolean_value_not_lost(self):
location = get_test_loc('test_gen/inv6.csv')
base_dir = get_temp_dir()
_errors, abouts = gen.generate(location, base_dir)
in_mem_result = [a.dumps() for a in abouts][0]
expected = (u'about_resource: .\n'
u'name: AboutCode\n'
u'version: 0.11.0\n'
u'redistribute: yes\n'
u'attribute: yes\n'
u'modified: no\n')
assert expected == in_mem_result
def test_generate_multi_lic_issue_444(self):
location = get_test_loc('test_gen/multi_lic_issue_444/test1.csv')
base_dir = get_temp_dir()
errors, abouts = gen.generate(location, base_dir)
result = [a.dumps() for a in abouts][0]
expected = ('''about_resource: test.c
name: test.c
licenses:
- key: License1
file: LIC1.LICENSE, LIC2.LICENSE
''')
assert expected == result
def test_load_inventory(self):
location = get_test_loc('gen/inv.csv')
base_dir = get_test_loc('inv')
errors, abouts = gen.load_inventory(location, base_dir)
expected_error_messages = [
'Field about_resource',
'Field custom1 is not a supported field and is ignored.'
]
# FIXME: this is not used
expected_errors = [
Error(INFO,
u'Field custom1 is not a supported field and is ignored.')
]
assert len(errors) == 2
for e in errors:
# we don't want to check the path value
if e.message.startswith('Field about_resource'):
continue
else:
assert e.message in expected_error_messages
expected = [
u'about_resource: .\n'
u'name: AboutCode\n'
u'version: 0.11.0\n'
u'description: |-\n'
u' multi\n'
u' line\n'
]
result = [
a.dumps(use_mapping=False,
mapping_file=False,
with_absent=False,
with_empty=False) for a in abouts
]
assert expected == result
def test_load_json_from_abc_mgr(self):
test_file = get_test_loc('load/aboutcode_manager_exported.json')
mapping_file = get_test_loc('custom-mapping-file/mapping.config')
expected = [
dict(
OrderedDict([
('license_expression', 'apache-2.0'),
('copyright', 'Copyright (c) 2017 nexB Inc.'),
('licenses', [{
'key': 'apache-2.0'
}]),
('copyrights', [{
'statements': ['Copyright (c) 2017 nexB Inc.']
}]),
('about_file_path', 'ScanCode'),
('review_status', 'Analyzed'),
('name', 'ScanCode'),
('version', '2.2.1'),
('owner', 'nexB Inc.'),
('code_type', 'Source'),
('is_modified', False),
('is_deployed', False),
('feature', ''),
('purpose', ''),
('homepage_url', None),
('download_url', None),
('license_url', None),
('notice_url', None),
('programming_language', 'Python'),
('notes', ''),
('fileId', 8458),
]))
]
result = util.load_json(test_file,
use_mapping=False,
mapping_file=mapping_file)
assert expected == result
def test_generate_with_default_template(self):
test_file = get_test_loc('test_attrib/default_template/simple_sample.csv')
errors, abouts = util.load_inventory(test_file)
assert not errors
lic_dict = {'bsd-new':
{'key': 'bsd-new', 'short_name': 'BSD-3-Clause', 'name': 'BSD-3-Clause',
'category': 'Permissive', 'owner': 'Regents of the University of California',
'homepage_url': 'http://www.opensource.org/licenses/BSD-3-Clause',
'notes': 'Per SPDX.org, this license is OSI certified.',
'spdx_license_key': 'BSD-3-Clause', 'osi_license_key': 'BSD-3',
'text_urls': ['http://www.opensource.org/licenses/BSD-3-Clause'],
'osi_url': 'http://www.opensource.org/licenses/BSD-3-Clause',
'other_urls': ['http://framework.zend.com/license/new-bsd', 'https://opensource.org/licenses/BSD-3-Clause'],
'license_text': 'Redistribution and use in source and binary forms, with or without modification,\nare permitted provided that the following conditions are met:\n\nRedistributions of source code must retain the above copyright notice, this list\nof conditions and the following disclaimer.\n\nRedistributions in binary form must reproduce the above copyright notice, this\nlist of conditions and the following disclaimer in the documentation and/or\nother materials provided with the distribution.\n\nNeither the name of the ORGANIZATION nor the names of its contributors may be\nused to endorse or promote products derived from this software without specific\nprior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\nARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS\nBE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR\nCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE\nGOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\nHOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\nLIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF\nTHE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.'},
'mit': {'key': 'mit', 'short_name': 'MIT License', 'name': 'MIT License',
'category': 'Permissive', 'owner': 'MIT',
'homepage_url': 'http://opensource.org/licenses/mit-license.php',
'notes': 'Per SPDX.org, this license is OSI certified.', 'spdx_license_key': 'MIT',
'text_urls': ['http://opensource.org/licenses/mit-license.php'],
'osi_url': 'http://www.opensource.org/licenses/MIT',
'other_urls': ['https://opensource.com/article/18/3/patent-grant-mit-license', 'https://opensource.com/article/19/4/history-mit-license', 'https://opensource.org/licenses/MIT'],
'license_text': 'Permission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n"Software"), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so, subject to\nthe following conditions:\n\nThe above copyright notice and this permission notice shall be\nincluded in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\nIN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,\nTORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\nSOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.'}
}
error, result = attrib.generate_from_file(abouts, lic_dict, min_license_score=0)
assert not error
expected_file = get_test_loc(
'test_attrib/default_template/expect.html')
with open(expected_file) as exp:
expected = exp.read()
# strip the timestamp: the timestamp is wrapped in italic block
result = remove_timestamp(result)
expected = remove_timestamp(expected)
assert expected == result
def test_copy_license_notice_files(self):
base_dir = get_temp_dir()
reference_dir = get_test_loc('test_util/licenses')
fields = [(u'license_expression', u'mit or public-domain'),
(u'about_resource', u'.'),
(u'name', u'test'),
(u'license_key', [u'mit', u'public-domain']),
(u'license_file', [u'mit.LICENSE, mit2.LICENSE', u'public-domain.LICENSE'])]
util.copy_license_notice_files(fields, base_dir, reference_dir, '')
licenses = ['mit.LICENSE', 'mit2.LICENSE', 'public-domain.LICENSE']
from os import listdir
copied_files = listdir(base_dir)
assert len(licenses) == len(copied_files)
for license in licenses:
assert license in copied_files
