def build_tests(data_set, clazz):
"""
Dynamically build an individual test method for each license texts and spdx
texts in a licenses `data_set` mapping attaching the test method to the
`clazz` test class.
"""
for license_key, license_obj in data_set.items():
text_file = license_obj.text_file
if os.path.exists(text_file):
test_method = make_test_function(text_file, license_key)
# set good function name to display in reports and use in discovery
test_name = ('test_validate_detection_of_text_for_'+ text.python_safe_name(license_key))
test_method.__name__ = test_name
test_method.funcname = test_name
setattr(clazz, test_name, test_method)
if license_obj.spdx_license_key:
text_file = license_obj.spdx_file
if os.path.exists(text_file):
test_method = make_test_function(text_file, license_key)
# set good function name to display in reports and use in discovery
test_name = ('test_validate_detection_of_spdx_text_for_'+ text.python_safe_name(license_key))
test_method.__name__ = test_name
test_method.funcname = test_name
setattr(clazz, test_name, test_method)
def build_tests(data_set, clazz):
"""
Dynamically build an individual test method for each license texts and spdx
texts in a licenses `data_set` mapping attaching the test method to the
`clazz` test class.
"""
for license_key, license_obj in data_set.items():
text_file = license_obj.text_file
if os.path.exists(text_file):
test_method = make_test_function(text_file, license_key)
# set good function name to display in reports and use in discovery
test_name = ('test_validate_detection_of_text_for_' +
text.python_safe_name(license_key))
test_method.__name__ = test_name
test_method.funcname = test_name
setattr(clazz, test_name, test_method)
if license_obj.spdx_license_key:
text_file = license_obj.spdx_file
if os.path.exists(text_file):
test_method = make_test_function(text_file, license_key)
# set good function name to display in reports and use in discovery
test_name = ('test_validate_detection_of_spdx_text_for_' +
text.python_safe_name(license_key))
test_method.__name__ = test_name
test_method.funcname = test_name
setattr(clazz, test_name, test_method)
def build_validation_tests(rules, test_classes, regen=REGEN_TEST_FIXTURES):
"""
Dynamically build an individual test method for each rule texts in a
``rules`` iterable of Rule objects then attach the test methods to the
``test_classes`` lits of test classes.
"""
# TODO: add test to detect the standard notice??
# we split our rules in chunks, one for each extended classes we have
# so we can split tests more or less evenly between them
# the first chunk is an arbitrary 200 length
chunks = [rules[:200]]
extended_rules = rules[200:]
number_of_ext_cls = len(test_classes) - 1
slice_length = int(len(extended_rules) / number_of_ext_cls)
for i in range(0, len(extended_rules), slice_length):
chnk = extended_rules[i:i + slice_length]
chunks.append(chnk)
for chunk, cls in zip(chunks, test_classes):
for rule in chunk:
# we exclude the non-english rules from validation
# as they are not included in the standard indexing
if rule.language != 'en':
continue
if rule.text_file and os.path.exists(rule.text_file):
test_name = ('test_validate_detect_' +
text.python_safe_name(rule.identifier))
test_method = make_validation_test(
rule=rule,
test_name=test_name,
regen=regen,
)
setattr(cls, test_name, test_method)
def build_tests(
test_dir,
test_file_suffix,
clazz,
tested_function,
test_method_prefix,
regen=False,
):
"""
Dynamically build test methods from files in ``test_dir`` ending with
``test_file_suffix`` and attach a test method to the ``clazz`` test class.
For each method:
- run ``tested_function`` with a test file with``test_file_suffix``
``tested_function`` should return a JSON-serializable object.
- set the name prefixed with ``test_method_prefix``.
- check that a test expected file named <test_file_name>-expected.json`
has content matching the results of the ``tested_function`` returned value.
"""
assert issubclass(clazz, PackageTester)
# loop through all items and attach a test method to our test class
for test_file_path in get_test_files(test_dir, test_file_suffix):
test_name = test_method_prefix + text.python_safe_name(test_file_path)
test_file_loc = os.path.join(test_dir, test_file_path)
test_method = create_test_function(
test_file_loc=test_file_loc,
tested_function=tested_function,
test_name=test_name,
regen=regen,
)
setattr(clazz, test_name, test_method)
def get_test_method_name(self, prefix='test_detection_'):
test_file_name = self.test_file_name
test_name = '{prefix}{test_file_name}'.format(**locals())
test_name = text.python_safe_name(test_name)
if not isinstance(test_name, str):
test_name = test_name.decode('utf-8')
return test_name
def build_validation_tests(rules, test_classes):
"""
Dynamically build an individual test method for each rule texts in a `rules`
iterable of Rule objects then attach the test method to the `class_basic` and `class_extended` test
classes for basic and extended respectively.
"""
# TODO: add test to detect the standard notice??
# we split our rules in chunks, one for each extended classes we have
# so we can split tests more or less evenly between them
# the first chunk is an arbitrary 200 length
chunks = [rules[:200]]
extended_rules = rules[200:]
number_of_ext_cls = len(test_classes) - 1
slice_length = int(len(extended_rules) / number_of_ext_cls)
for i in range(0, len(extended_rules), slice_length):
chnk = extended_rules[i:i + slice_length]
chunks.append(chnk)
for chunk, cls in zip(chunks, test_classes):
for rule in chunk:
if rule.text_file and os.path.exists(rule.text_file):
test_name = ('test_validate_detect_' +
text.python_safe_name(rule.identifier))
test_method = make_validation_test(rule=rule,
test_name=test_name)
setattr(cls, test_name, test_method)
def make_test_function(test_name, test_dir, expected_file, regen=False):
"""
Build and return a test function closing on tests arguments and the function
name. Create only a single function for multiple tests (e.g. copyrights and
holders together).
"""
def closure_test_function(*args, **kwargs):
result_file = test_env.get_temp_file('json')
args = ['--license',
'--copyright',
'--info',
'--classify',
'--license-diag',
'--license-clarity-score',
test_dir, '--json', result_file]
run_scan_click(args)
run_scan_click(args)
check_json_scan(
test_env.get_test_loc(expected_file),
result_file,
remove_file_date=True,
regen=regen)
test_name = 'test_license_clarity_score_%(test_name)s' % locals()
test_name = python_safe_name(test_name)
if isinstance(test_name, unicode):
test_name = test_name.encode('utf-8')
closure_test_function.__name__ = test_name
closure_test_function.funcname = test_name
return closure_test_function, test_name
def a_test_generator(self, test_file, expected):
"""
Use this function to generate new tests.
"""
from commoncode.text import python_safe_name
from pprint import pformat
test_loc = self.get_test_loc(test_file)
function = python_safe_name('test_elf_symbols_' + test_file)
testf = '''
def %(function)s(self):
test_file = self.get_test_loc(%(test_file)r)
expected_file = self.get_test_loc('%(test_file)s.expected_elf_symbols')
with codecs.open(expected_file, encoding='utf-8') as expect:
expected = json.load(expect)
elf = Elf(test_file)
for section, expected_values in expected.items():
result = getattr(elf.symbols_section, section)
for i, v in enumerate(sorted(expected_values)):
assert v == sorted(result)[i]''' % locals()
print(testf)
expected_dict = {}
expected_dict['files'] = expected[0]
expected_dict['standard_files'] = expected[1]
expected_dict['local_functions'] = expected[2]
expected_dict['local_objects'] = expected[3]
expected_dict['global_functions'] = expected[4]
expected_dict['global_objects'] = expected[5]
expected_dict['external_libs_functions'] = expected[6]
expected_dict['external_libs_objects'] = expected[7]
expected_dict['standard_functions'] = expected[8]
expected_dict['standard_objects'] = expected[9]
with open(test_loc + '.expected_elf_symbols', 'wb') as out:
json.dump(expected_dict, out, indent=2)
def build_tests(test_dir, clazz, prefix='test_', regen=False):
"""
Dynamically build test methods for each copyright file in `test_dir` and
attach the test method to the `clazz` class.
"""
test_data_dir = path.join(path.dirname(__file__), 'data')
test_dir_loc = path.join(test_data_dir, test_dir)
# loop through all items and attach a test method to our test class
for test_file in relative_walk(test_dir_loc):
test_name = prefix + text.python_safe_name(test_file)
test_loc = path.join(test_dir_loc, test_file)
# create two test methods: one with and one without details
test_method = create_test_function(
test_loc=test_loc,
expected_loc=test_loc + '.expected.yml',
test_name=test_name,
regen=regen,
with_details=False,
)
# attach that method to the class
setattr(clazz, test_name, test_method)
test_method = create_test_function(
test_loc=test_loc,
expected_loc=test_loc + '-detailed.expected.yml',
test_name=test_name,
regen=regen,
with_details=True,
)
# attach that method to the class
setattr(clazz, test_name, test_method)
def build_tests(data_set, clazz):
"""
Dynamically build an individual test method for each license texts and spdx
texts in a licenses `data_set` mapping attaching the test method to the
`clazz` test class.
"""
for license_key, license_obj in data_set.items():
if license_obj.text_file and os.path.exists(license_obj.text_file):
test_name = ('test_validate_detection_of_text_for_' + text.python_safe_name(license_key))
test_method = make_license_test_function(license_key, license_obj.text_file, test_name)
setattr(clazz, test_name, test_method)
if license_obj.spdx_license_key:
if license_obj.spdx_file and os.path.exists(license_obj.spdx_file):
test_name = ('test_validate_detection_of_spdx_text_for_' + text.python_safe_name(license_key))
test_method = make_license_test_function(license_key, license_obj.spdx_file, test_name)
setattr(clazz, test_name, test_method)
def get_test_method_name(self, prefix='test_detection_'):
test_file_name = self.test_file_name
test_name = '{prefix}{test_file_name}'.format(**locals())
test_name = text.python_safe_name(test_name)
if py2 and not isinstance(test_name, bytes):
test_name = test_name.encode('utf-8')
if py3 and not isinstance(test_name, compat.unicode):
test_name = test_name.decode('utf-8')
return test_name
def system_temp_dir():
"""
Return the global temp directory for the current user.
"""
temp_dir = os.getenv('SCANCODE_TMP')
if not temp_dir:
sc = text.python_safe_name('scancode_' + system.username)
temp_dir = os.path.join(tempfile.gettempdir(), sc)
create_dir(temp_dir)
return temp_dir
def system_temp_dir():
"""
Return the global temp directory for the current user.
"""
temp_dir = os.getenv('SCANCODE_TMP')
if not temp_dir:
sc = text.python_safe_name('scancode_' + system.username)
temp_dir = os.path.join(tempfile.gettempdir(), sc)
create_dir(temp_dir)
return temp_dir
def build_license_validation_tests(licenses_by_key, cls):
"""
Dynamically build an individual test method for each license texts in a licenses
`data_set` then mapping attaching the test method to the `cls` test class.
"""
for license_key, license_obj in licenses_by_key.items():
if license_obj.text_file and os.path.exists(license_obj.text_file):
test_name = ('test_validate_self_detection_of_text_for_' + text.python_safe_name(license_key))
# also verify that we are detecting exactly with the license rule itself
test_method = make_license_test_function(
license_key, license_obj.text_file, license_obj.data_file, test_name, detect_negative=True, trace_text=True)
setattr(cls, test_name, test_method)
def build_package_name(input_path):
"""
Return a package name built from an ``input_path`` path.
"""
if input_path:
absinput = absinput = os.path.abspath(input_path)
if os.path.isfile(absinput):
input_path = parent_directory(absinput)
return python_safe_name(file_name(input_path))
return 'scancode-toolkit-analyzed-package'
def check_license(location=None, query_string=None, expected=(), test_data_dir=None):
if query_string:
idx = cache.get_index()
matches = idx.match(location=location, query_string=query_string)
results = functional.flatten(map(unicode, match.rule.licenses) for match in matches)
assert expected == results
else:
test_name = python_safe_name('test_' + location.replace(test_data_dir, ''))
tester = make_license_test_function(
expected_licenses=expected, test_file=location,
test_data_file=None, test_name=test_name,
trace_text=True)
tester()
def build_tests(data_set, clazz):
"""
Dynamically build an individual test method for each license texts and spdx
texts in a licenses `data_set` mapping attaching the test method to the
`clazz` test class.
"""
for license_key, license_obj in data_set.items():
if license_obj.text_file and os.path.exists(license_obj.text_file):
test_name = ('test_validate_detection_of_text_for_' +
text.python_safe_name(license_key))
test_method = make_license_test_function(license_key,
license_obj.text_file,
test_name)
setattr(clazz, test_name, test_method)
if license_obj.spdx_license_key:
if license_obj.spdx_file and os.path.exists(license_obj.spdx_file):
test_name = ('test_validate_detection_of_spdx_text_for_' +
text.python_safe_name(license_key))
test_method = make_license_test_function(
license_key, license_obj.spdx_file, test_name)
setattr(clazz, test_name, test_method)
def build_rule_validation_tests(rules, cls):
"""
Dynamically build an individual test method for each rule texts in a rules
`data_set` then mapping attaching the test method to the `cls` test class.
"""
for rule in rules:
if rule.negative:
continue
expected_identifier = rule.identifier
test_name = ('test_validate_self_detection_of_rule_for_' + text.python_safe_name(expected_identifier))
test_method = make_license_test_function(
rule.licenses, rule.text_file, rule.data_file, test_name, detect_negative=not rule.negative, trace_text=True
)
setattr(cls, test_name, test_method)
def build_tests(test_dir, clazz, prefix='test_rubygems_parse_', regen=False):
"""
Dynamically build test methods for each gem in `test_dir` and
attach the test method to the `clazz` class.
"""
test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
test_dir = os.path.join(test_data_dir, test_dir)
# loop through all items and attach a test method to our test class
for test_file in relative_walk(test_dir):
test_name = prefix + text.python_safe_name(test_file)
test_pom_loc = os.path.join(test_dir, test_file)
test_method = create_test_function(test_pom_loc, test_name, regen=regen)
# attach that method to the class
setattr(clazz, test_name, test_method)
def build_tests(test_dir, clazz, prefix='test_maven2_parse_', check_pom=True, regen=REGEN_TEST_FIXTURES):
"""
Dynamically build test methods for each POMs in `test_dir` and
attach the test method to the `clazz` class.
If check_parse_pom is True, test the POM parsing; otherwise, test Package creation
"""
test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
test_dir = os.path.join(test_data_dir, test_dir)
# loop through all items and attach a test method to our test class
for test_file in relative_walk(test_dir):
test_name = prefix + text.python_safe_name(test_file)
test_pom_loc = os.path.join(test_dir, test_file)
test_method = create_test_function(test_pom_loc, test_name, check_pom=check_pom, regen=regen)
# attach that method to the class
setattr(clazz, test_name, test_method)
def build_tests(test_dir, clazz, prefix='test_maven2_parse_', check_pom=True, regen=False):
"""
Dynamically build test methods for each POMs in `test_dir` and
attach the test method to the `clazz` class.
If check_parse_pom is True, test the POM parsing; otherwise, test Package creation
"""
test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
test_dir = os.path.join(test_data_dir, test_dir)
# loop through all items and attach a test method to our test class
for test_file in relative_walk(test_dir):
test_name = prefix + text.python_safe_name(test_file)
test_pom_loc = os.path.join(test_dir, test_file)
test_method = create_test_function(test_pom_loc, test_name, check_pom=check_pom, regen=regen)
# attach that method to the class
setattr(clazz, test_name, test_method)
def build_tests(license_tests, clazz):
"""
Dynamically build test methods from a sequence of LicenseTest and attach
these method to the clazz test class.
"""
for test in license_tests:
# path relative to the data directory
tfn = 'licenses/'+ test.test_file_name
test_name = 'test_detection_%(tfn)s'% locals()
test_name = text.python_safe_name(test_name)
# closure on the test params
test_method = make_test_function(test.licenses, tfn, test_name, sort=test.sort)
if test.expected_failure:
test_method = expectedFailure(test_method)
# attach that method to our test class
setattr(clazz, test_name, test_method)
def build_license_validation_tests(licenses_by_key, cls):
"""
Dynamically build an individual test method for each license texts in a licenses
`data_set` then mapping attaching the test method to the `cls` test class.
"""
for license_key, license_obj in licenses_by_key.items():
if license_obj.text_file and os.path.exists(license_obj.text_file):
test_name = ('test_validate_self_detection_of_text_for_' +
text.python_safe_name(license_key))
# also verify that we are detecting exactly with the license rule itself
test_method = make_license_test_function(license_key,
license_obj.text_file,
license_obj.data_file,
test_name,
detect_negative=True,
trace_text=True)
setattr(cls, test_name, test_method)
def build_rule_validation_tests(rules, cls):
"""
Dynamically build an individual test method for each rule texts in a rules
`data_set` then mapping attaching the test method to the `cls` test class.
"""
for rule in rules:
expected_identifier = rule.identifier
test_name = ('test_validate_self_detection_of_rule_for_' +
text.python_safe_name(expected_identifier))
test_method = make_license_test_function(
rule.licenses,
rule.text_file,
rule.data_file,
test_name,
detect_negative=not rule.negative(),
trace_text=True)
setattr(cls, test_name, test_method)
def build_tests(license_tests, clazz):
"""
Dynamically build test methods from a sequence of LicenseTest and attach
these method to the clazz test class.
"""
for test in license_tests:
# absolute path
tfn = test.test_file_name
tf = test.test_file
test_name = 'test_detection_%(tfn)s' % locals()
test_name = text.python_safe_name(test_name)
# closure on the test params
test_method = make_license_test_function(test.licenses, tf, test_name)
if test.expected_failure:
test_method = expectedFailure(test_method)
# attach that method to our test class
setattr(clazz, test_name, test_method)
def build_validation_tests(rules, class_basic, class_extended):
"""
Dynamically build an individual test method for each rule texts in a `rules`
iterable of Rule objects then attach the test method to the `class_basic` and `class_extended` test
classes for basic and extended respectively.
"""
# TODO: add test to detect the standard notice??
cls = class_basic
for i, rule in enumerate(rules):
# only push 20 rules in the basic set
if i > 20:
cls = class_extended
if rule.text_file and os.path.exists(rule.text_file):
test_name = ('test_validate_detect_' +
text.python_safe_name(rule.identifier))
test_method = make_validation_test(rule=rule, test_name=test_name)
setattr(cls, test_name, test_method)
def build_tests(license_tests, clazz):
"""
Dynamically build test methods from a sequence of LicenseTest and attach
these method to the clazz test class.
"""
for test in license_tests:
# absolute path
tfn = test.test_file_name
tf = test.test_file
test_name = "test_detection_%(tfn)s" % locals()
test_name = text.python_safe_name(test_name)
# closure on the test params
test_method = make_license_test_function(test.licenses, tf, test_name)
if test.expected_failure:
test_method = expectedFailure(test_method)
# attach that method to our test class
setattr(clazz, test_name, test_method)
def build_spdx_line_tests(clazz, test_dir='spdx/lines', regen=False):
"""
Dynamically build test methods from test files to test SPDX lines collection.
"""
test_dir = os.path.join(TEST_DATA_DIR, test_dir)
for test_file in os.listdir(test_dir):
if test_file.endswith('.json'):
continue
test_loc = os.path.join(test_dir, test_file)
expected_loc = test_loc + '.json'
test_name = 'test_collect_spdx_query_lines_%(test_file)s' % locals()
test_name = text.python_safe_name(test_name)
test_name = str(test_name)
test_method = get_query_spdx_lines_test_method(test_loc, expected_loc, regen)
test_method.__name__ = test_name
# attach that method to our test class
setattr(clazz, test_name, test_method)
def make_filetype_test_functions(
test,
index,
test_data_dir=test_env.test_data_dir,
regen=False,
):
"""
Build and return a test function closing on tests arguments and the function
name.
"""
def closure_test_function(*args, **kwargs):
results = get_type(test_file).to_dict(include_date=False)
if regen:
for key, value in results.items():
setattr(test, key, value)
test.dump()
expected = test.to_dict(filter_empty=False, filter_extra=True)
passing = check_types_equal(expected, results)
# this is done to display slightly eaier to handle error traces
if not passing:
expected['data file'] = 'file://' + data_file
expected['test_file'] = 'file://' + test_file
assert dict(results) == dict(expected)
data_file = test.data_file
test_file = test.test_file
tfn = test_file.replace(test_data_dir, '').strip('\\/\\')
test_name = 'test_%(tfn)s_%(index)s' % locals()
test_name = python_safe_name(test_name)
closure_test_function.__name__ = test_name
if (test.expected_failure is True
or (isinstance(test.expected_failure, str) and
(('windows' in test.expected_failure and on_windows) or
('macos' in test.expected_failure and on_mac)))):
closure_test_function = pytest.mark.xfail(closure_test_function)
return closure_test_function, test_name
def check_license(location=None,
query_string=None,
expected=(),
test_data_dir=None):
if query_string:
idx = cache.get_index()
matches = idx.match(location=location, query_string=query_string)
results = functional.flatten(
map(unicode, match.rule.licenses) for match in matches)
assert expected == results
else:
test_name = python_safe_name('test_' +
location.replace(test_data_dir, ''))
tester = make_license_test_function(expected_licenses=expected,
test_file=location,
test_data_file=None,
test_name=test_name,
trace_text=True)
tester()
def atest_gen(self):
"""
Rename to test xxx to regen tests.
"""
test_dir = self.get_test_loc(u'markup', True)
expected_dir = self.get_test_loc(u'markup_expected')
template = u"""
def test_%(tn)s(self):
test_file = self.get_test_loc(u'markup/%(test_file)s')
expected = self.get_test_loc(u'markup_expected/%(test_file)s')
result = markup.convert_to_text(test_file)
file_cmp(expected, result, ignore_line_endings=True)"""
for test_file in os.listdir(test_dir):
tn = text.python_safe_name(test_file)
location = os.path.join(test_dir, test_file)
result = markup.convert_to_text(location)
expected_file = os.path.join(expected_dir, test_file)
fileutils.copyfile(result, expected_file)
print(template % locals())
def atest_gen(self):
"""
Rename to test xxx to regen tests.
"""
test_dir = self.get_test_loc(u'markup', True)
expected_dir = self.get_test_loc(u'markup_expected')
template = u"""
def test_%(tn)s(self):
test_file = self.get_test_loc(u'markup/%(test_file)s')
expected = self.get_test_loc(u'markup_expected/%(test_file)s')
result = markup.convert_to_text(test_file)
file_cmp(expected, result, ignore_line_endings=True)"""
for test_file in os.listdir(test_dir):
tn = text.python_safe_name(test_file)
location = os.path.join(test_dir, test_file)
result = markup.convert_to_text(location)
expected_file = os.path.join(expected_dir, test_file)
fileutils.copyfile(result, expected_file)
print(template % locals())
def collect_url_tests(location):
# the urls file 'ccurls.t1' is special
ccurl = 'ccurls.t1'
data = open(os.path.join(location, '..', ccurl)).read()
lics, _, urls = data.partition('\n\n')
lics = (e for e in lics.splitlines(False) if not e.endswith('%'))
for i, lic in enumerate(lics):
expected_lic, offsets, _ = lic.split()
start, end = offsets.split(',')
text = urls[int(start):int(end)]
expected_lic = expected_lic.strip()
fn = python_safe_name(expected_lic)
yield Test(location=os.path.join(location, f'url_{fn}_{i}.txt'),
filename=ccurl,
text=text,
license_key=expected_lic,
notes='This is a URL test extracted from ccurls.t1.')
def build_tests(test_dir, clazz, prefix='test_', regen=False):
'''
Dynamically build test methods for each copyright file in `test_dir` and
attach the test method to the `clazz` class.
'''
test_data_dir = path.join(path.dirname(__file__), 'data')
test_dir_loc = path.join(test_data_dir, test_dir)
# loop through all items and attach a test method to our test class
for test_file in relative_walk(test_dir_loc):
test_loc = path.join(test_dir_loc, test_file)
test_name2 = prefix + 'detailed_' + text.python_safe_name(test_file)
test_method = create_test_function(
test_loc=test_loc,
expected_loc=test_loc + '-detailed.expected.yml',
test_name=test_name2,
regen=regen,
simplified=False,
)
# attach that method to the class
setattr(clazz, test_name2, test_method)
def build_tests(test_dir, clazz, prefix='test_', regen=False):
"""
Dynamically build test methods for each copyright file in `test_dir` and
attach the test method to the `clazz` class.
"""
test_data_dir = path.join(path.dirname(__file__), 'data')
test_dir_loc = path.join(test_data_dir, test_dir)
if py2:
return
# loop through all items and attach a test method to our test class
for test_file in relative_walk(test_dir_loc):
test_name = prefix + text.python_safe_name(test_file)
test_loc = path.join(test_dir_loc, test_file)
expected_loc = test_loc + '.expected.json'
test_method = create_test_function(test_loc=test_loc,
expected_loc=expected_loc,
test_name=test_name,
regen=regen)
# attach that method to the class
setattr(clazz, test_name, test_method)
def build_tests(license_tests, clazz):
"""
Dynamically build test methods from a sequence of LicenseTest and attach
these method to the clazz test class.
"""
# TODO: check that we do not have duplicated tests with same data and text
for test in license_tests:
tfn = test.test_file_name
test_name = 'test_detection_%(tfn)s' % locals()
test_name = text.python_safe_name(test_name)
# closure on the test params
test_method = make_license_test_function(
test.licenses, test.test_file, test.data_file,
test_name=test_name,
expected_failure=test.expected_failure,
skip_test=test.skip and 'Skipping long test' or False,
trace_text=TRACE_TEXTS
)
# attach that method to our test class
setattr(clazz, test_name, test_method)
def build_tests(test_dir,
clazz,
prefix='test_jar_manifest',
check_parse=True,
regen=False):
"""
Dynamically build test methods for each MANIFEST.MF in `test_dir` and
attach the test method to the `clazz` class.
If check_parse is True, test the parse_manifest; otherwise, test the package
data normalization.
"""
test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
test_dir = os.path.join(test_data_dir, test_dir)
# loop through all items and attach a test method to our test class
for idx, test_file in enumerate(relative_walk(test_dir)):
test_name = prefix + text.python_safe_name(test_file + str(idx))
test_manifest_loc = os.path.join(test_dir, test_file)
test_method = create_test_function(test_manifest_loc,
test_name,
check_parse=check_parse,
regen=regen)
# attach that method to the class
setattr(clazz, test_name, test_method)
def process_codebase(self, codebase, **kwargs):
# Collect ConsolidatedPackages and ConsolidatedComponents
# TODO: Have a "catch-all" Component for the things that we haven't grouped
consolidations = []
root = codebase.root
if hasattr(root, 'packages') and hasattr(
root, 'copyrights') and hasattr(root, 'licenses'):
consolidations.extend(get_consolidated_packages(codebase))
if hasattr(root, 'copyrights') and hasattr(root, 'licenses'):
consolidations.extend(
get_holders_consolidated_components(codebase))
if not consolidations:
return
# Sort consolidations by holders for consistent ordering before enumeration
consolidations = sorted(
consolidations,
key=lambda c: '_'.join(h.key
for h in c.consolidation.core_holders))
# Add ConsolidatedPackages and ConsolidatedComponents to top-level codebase attributes
codebase.attributes.consolidated_packages = consolidated_packages = []
codebase.attributes.consolidated_components = consolidated_components = []
identifier_counts = Counter()
for index, c in enumerate(consolidations, start=1):
# Skip consolidation if it does not have any Files
if c.consolidation.files_count == 0:
continue
if isinstance(c, ConsolidatedPackage):
# We use the purl as the identifier for ConsolidatedPackages
purl = c.package.purl
identifier_counts[purl] += 1
identifier = python_safe_name('{}_{}'.format(
purl, identifier_counts[purl]))
c.consolidation.identifier = identifier
for resource in c.consolidation.resources:
resource.consolidated_to.append(identifier)
resource.save(codebase)
consolidated_packages.append(c.to_dict())
elif isinstance(c, ConsolidatedComponent):
consolidation_identifier = c.consolidation.identifier
if consolidation_identifier:
# Use existing identifier
identifier_counts[consolidation_identifier] += 1
identifier = python_safe_name('{}_{}'.format(
consolidation_identifier,
identifier_counts[consolidation_identifier]))
else:
# Create identifier if we don't have one
# TODO: Consider adding license expression to be part of name
holders = '_'.join(h.key
for h in c.consolidation.core_holders)
other_holders = '_'.join(
h.key for h in c.consolidation.other_holders)
holders = holders or other_holders
# We do not want the name to be too long
holders = holders[:65]
if holders:
# We use holders as the identifier for ConsolidatedComponents
identifier_counts[holders] += 1
identifier = python_safe_name('{}_{}'.format(
holders, identifier_counts[holders]))
else:
# If we can't use holders, we use the ConsolidatedComponent's position
# in the list of Consolidations
identifier = index
c.consolidation.identifier = identifier
for resource in c.consolidation.resources:
resource.consolidated_to.append(identifier)
resource.save(codebase)
consolidated_components.append(c.to_dict())
# Dedupe and sort names in consolidated_to
for resource in codebase.walk(topdown=True):
resource.consolidated_to = sorted(set(resource.consolidated_to))
resource.save(codebase)
def get_test_method_name(self):
dfn = fileutils.file_base_name(self.data_file.lower())
test_name = f'test_alpine_license_detection_{dfn}'
return text.python_safe_name(test_name)
def test_python_safe_name():
s = "not `\\a /`good` -safe name ??"
assert text.python_safe_name(s) == 'not___a___good___safe_name'
s1 = "string1++or+"
s2 = "string1 +or "
assert text.python_safe_name(s2) == text.python_safe_name(s1)
