class TestMatchGeneric(unittest.TestCase):
def setUp(self):
self.logger = ManifestLogger(False)
self.utils = ManifestUtils(self.logger)
def test_curate_file_license(self):
files = sample_data.transformed_files()
# curate file "bonkey.txt" to be mit
self.utils._curate_file_license(files, "bonkey.txt", "mit")
# find the file bonkey.txt
the_file = None
for f in files['included']:
if f['name'] == "bonkey.txt":
the_file = f
break
# compare bonkey's license with expected
expected_l = 'mit'
actual_l = the_file['scancode_manifestor']['curated_license']
self.assertTrue(expected_l == actual_l)
#expected_l = {'gpl-3.0-or-later', 'gpl-2.0-or-later'}
#assert expected_l == fl
def test_curate_license(self):
files = sample_data.files()
# Make sure, no curations
for f in files['included']:
# No curation yet, so no 'scancode_manifestor'
self.assertFalse('scancode_manifestor' in f)
#print(f['path'] + ": " + str(f['license_expressions']) )
self.utils._curate_license(files, "x11", "mit")
for f in files['included']:
if f['name'] == "readme.txt":
# find the file readme.txt, which is licensed under x11 but curated to mit
# curation, so 'scancode_manifestor' and 'curated_license' in there
self.assertTrue('scancode_manifestor' in f)
self.assertTrue('curated_license' in f['scancode_manifestor'])
# only one original license
self.assertTrue(len(f['license_expressions']) == 1)
# original license should be "x11"
self.assertTrue(f['license_expressions'][0] == "x11")
# curated license should be "mit"
self.assertTrue(
f['scancode_manifestor']['curated_license'] == "mit")
else:
# No curation, so no 'scancode_manifestor'
#print(json.dumps(f, indent=4))
self.assertFalse('scancode_manifestor' in f)
def setUp(self):
self.logger = ManifestLogger(False)
self.utils = ManifestUtils(self.logger)
class TestMiscIs(unittest.TestCase):
def setUp(self):
self.logger = ManifestLogger(False)
self.utils = ManifestUtils(self.logger)
def test_isfile(self):
self.assertTrue(self.utils._isfile(sample_data.file()))
self.assertFalse(self.utils._isfile(sample_data.dir()))
def test_isdir(self):
self.assertFalse(self.utils._isdir(sample_data.file()))
self.assertTrue(self.utils._isdir(sample_data.dir()))
def test_extract_license(self):
lic = self.utils._extract_license(sample_data.file())
self.assertTrue( lic == \
{"gpl-2.0-or-later", "gpl-3.0-or-later"} or \
{"gpl-3.0-or-later", "gpl-2.0-or-later"})
def test_dir_licenses(self):
files = sample_data.files()
dir = sample_data.sample_dir("git/dit")
lic_list = list(self.utils._dir_licenses(files, dir, "included"))
lic_list.sort()
right_list = [
'bsd-new', 'gpl-2.0-or-later', 'gpl-3.0-only', 'gpl-3.0-or-later',
'mit'
]
self.assertTrue(right_list == lic_list)
def test_keep_file(self):
f = sample_data.file()
self.assertTrue(self.utils._keep_file(True, FilterAction.INCLUDE))
self.assertFalse(self.utils._keep_file(True, FilterAction.EXCLUDE))
self.assertFalse(self.utils._keep_file(False, FilterAction.INCLUDE))
self.assertTrue(self.utils._keep_file(False, FilterAction.EXCLUDE))
def test_extract_license(self):
f = sample_data.file()
fl = self.utils._extract_license(f)
expected_l = {'gpl-3.0-or-later', 'gpl-2.0-or-later'}
self.assertTrue(expected_l == fl)
class TestFilterGeneric(unittest.TestCase):
def setUp(self):
self.logger = ManifestLogger(False)
self.utils = ManifestUtils(self.logger)
def test_include_bad_indata(self):
files = sample_data.files()
# None as list
self.assertRaises(
Exception, lambda: self.utils._filter_generic(
None, FilterAttribute.PATH, "onkey", FilterAction.INCLUDE,
FilterModifier.ANY))
# Existing but faulty list
self.assertRaises(
Exception, lambda: self.utils._filter_generic([
], FilterAttribute.PATH, "onkey", FilterAction.INCLUDE,
FilterModifier.ANY))
# No filter
self.assertRaises(
Exception, lambda: self.utils._filter_generic(
files, None, "onkey", FilterAction.INCLUDE, FilterModifier.ANY)
)
# Incorrect filter
self.assertRaises(
Exception, lambda: self.utils._filter_generic(
files, "incrorect indata", "onkey", FilterAction.INCLUDE,
FilterModifier.ANY))
# Incorrect filter action
self.assertRaises(
Exception, lambda: self.utils._filter_generic(
files, FilterAttribute.PATH, "onkey", None, FilterModifier.ANY)
)
# Incorrect filter action
self.assertRaises(
Exception, lambda: self.utils._filter_generic(
files, FilterAttribute.PATH, "onkey", "incorrect data",
FilterModifier.ANY))
# None as modifier
self.assertRaises(
Exception, lambda: self.utils._filter_generic(
files, FilterAttribute.PATH, "onkey", FilterAction.INCLUDE,
None))
# Incorrect modifier
self.assertRaises(
Exception, lambda: self.utils._filter_generic(
files, FilterAttribute.PATH, "onkey", FilterAction.INCLUDE,
"incorrect data"))
def test_include_path(self):
files = sample_data.files()
assert len(files['included']) == 6
assert len(files['excluded']) == 0
self.utils._filter_generic(files, FilterAttribute.PATH, "onkey",
FilterAction.INCLUDE, FilterModifier.ANY)
assert len(files['included']) == 3
assert len(files['excluded']) == 3
# try include same files, no change
self.utils._filter_generic(files, FilterAttribute.PATH, "onkey",
FilterAction.INCLUDE, FilterModifier.ANY)
assert len(files['included']) == 3
assert len(files['excluded']) == 3
def test_exclude_path(self):
files = sample_data.files()
assert len(files['included']) == 6
assert len(files['excluded']) == 0
self.utils._filter_generic(files, FilterAttribute.PATH, "onkey",
FilterAction.EXCLUDE, FilterModifier.ANY)
assert len(files['included']) == 3
assert len(files['excluded']) == 3
# try include same files, no change
self.utils._filter_generic(files, FilterAttribute.PATH, "onkey",
FilterAction.EXCLUDE, FilterModifier.ANY)
assert len(files['included']) == 3
assert len(files['excluded']) == 3
def test_exclude_license(self):
files = sample_data.files()
assert len(files['included']) == 6
assert len(files['excluded']) == 0
self.utils._filter_generic(files, FilterAttribute.LICENSE, "mit",
FilterAction.EXCLUDE, FilterModifier.ANY)
assert len(files['included']) == 5
assert len(files['excluded']) == 1
# try include same files, no change
self.utils._filter_generic(files, FilterAttribute.PATH, "mit",
FilterAction.EXCLUDE, FilterModifier.ANY)
assert len(files['included']) == 5
assert len(files['excluded']) == 1
def test_include_license(self):
files = sample_data.files()
assert len(files['included']) == 6
assert len(files['excluded']) == 0
self.utils._filter_generic(files, FilterAttribute.LICENSE, "mit",
FilterAction.INCLUDE, FilterModifier.ANY)
assert len(files['included']) == 1
assert len(files['excluded']) == 5
# try include same files, no change
self.utils._filter_generic(files, FilterAttribute.LICENSE, "mit",
FilterAction.INCLUDE, FilterModifier.ANY)
assert len(files['included']) == 1
assert len(files['excluded']) == 5
def test_include_only_license(self):
files = sample_data.files()
assert len(files['included']) == 6
assert len(files['excluded']) == 0
self.utils._filter_generic(files, FilterAttribute.LICENSE,
"gpl-2.0-or-later", FilterAction.INCLUDE,
FilterModifier.ANY)
assert len(files['included']) == 2
assert len(files['excluded']) == 4
files = sample_data.files()
# try include same files, no change
self.utils._filter_generic(files, FilterAttribute.LICENSE,
"gpl-2.0-or-later", FilterAction.INCLUDE,
FilterModifier.ONLY)
assert len(files['included']) == 1
assert len(files['excluded']) == 5
class TestMiscIs(unittest.TestCase):
def setUp(self):
self.logger = ManifestLogger(False)
self.utils = ManifestUtils(self.logger)
def test_isfile(self):
assert self.utils._isfile(sample_data.file())
assert not self.utils._isfile(sample_data.dir())
def test_isdir(self):
assert not self.utils._isdir(sample_data.file())
assert self.utils._isdir(sample_data.dir())
def test_extract_license(self):
lic = self.utils._extract_license(sample_data.file())
assert lic == \
{"gpl-2.0-or-later", "gpl-3.0-or-later"} or \
{"gpl-3.0-or-later", "gpl-2.0-or-later"}
def test_dir_licenses(self):
files = sample_data.files()
dir = sample_data.sample_dir("git/dit")
lic_list = list(self.utils._dir_licenses(files, dir, "included"))
lic_list.sort()
right_list = ['bsd-new', 'gpl-2.0-or-later', 'gpl-3.0-only', 'gpl-3.0-or-later', 'mit']
assert right_list == lic_list
def test_keep_file(self):
f = sample_data.file()
assert self.utils._keep_file(True, FilterAction.INCLUDE)
assert not self.utils._keep_file(True, FilterAction.EXCLUDE)
assert not self.utils._keep_file(False, FilterAction.INCLUDE)
assert self.utils._keep_file(False, FilterAction.EXCLUDE)
def test_extract_license(self):
f = sample_data.file()
fl = self.utils._extract_license(f)
expected_l = {'gpl-3.0-or-later', 'gpl-2.0-or-later'}
assert expected_l == fl
def test_curate_file_license(self):
files = sample_data.transformed_files()
# curate file "bonkey.txt" to be mit
self.utils._curate_file_license(files, "bonkey.txt", "mit")
# find the file bonkey.txt
the_file = None
for f in files['included']:
if f['name'] == "bonkey.txt":
the_file = f
break
# compare bonkey's license with expected
expected_l = 'mit'
actual_l = the_file['license_key']
assert expected_l == actual_l
class TestFilterGeneric(unittest.TestCase):
def setUp(self):
self.logger = ManifestLogger(False)
self.utils = ManifestUtils(self.logger)
def test_include_bad_indata(self):
files = sample_data.files()
# None as liste
self.assertRaises(
Exception, lambda: self.utils._filter_generic(
None, FilterAttribute.PATH, "onkey", FilterAction.INCLUDE,
FilterModifier.ANY))
# Existing but faulty list
self.assertRaises(
Exception, lambda: self.utils._filter_generic([
], FilterAttribute.PATH, "onkey", FilterAction.INCLUDE,
FilterModifier.ANY))
# No filter
self.assertRaises(
Exception, lambda: self.utils._filter_generic(
files, None, "onkey", FilterAction.INCLUDE, FilterModifier.ANY)
)
# Incorrect filter
self.assertRaises(
Exception, lambda: self.utils._filter_generic(
files, "incrorect indata", "onkey", FilterAction.INCLUDE,
FilterModifier.ANY))
# Incorrect filter action
self.assertRaises(
Exception, lambda: self.utils._filter_generic(
files, FilterAttribute.PATH, "onkey", None, FilterModifier.ANY)
)
# Incorrect filter action
self.assertRaises(
Exception, lambda: self.utils._filter_generic(
files, FilterAttribute.PATH, "onkey", "incorrect data",
FilterModifier.ANY))
# None as modifier
self.assertRaises(
Exception, lambda: self.utils._filter_generic(
files, FilterAttribute.PATH, "onkey", FilterAction.INCLUDE,
None))
# Incorrect modifier
self.assertRaises(
Exception, lambda: self.utils._filter_generic(
files, FilterAttribute.PATH, "onkey", FilterAction.INCLUDE,
"incorrect data"))
def test_include_path(self):
files = sample_data.files()
self.assertTrue(len(files['included']) == 6)
self.assertTrue(len(files['excluded']) == 0)
self.utils._filter_generic(files, FilterAttribute.PATH, "onkey",
FilterAction.INCLUDE, FilterModifier.ANY)
self.assertTrue(len(files['included']) == 3)
self.assertTrue(len(files['excluded']) == 3)
# try include same files, no change
self.utils._filter_generic(files, FilterAttribute.PATH, "onkey",
FilterAction.INCLUDE, FilterModifier.ANY)
self.assertTrue(len(files['included']) == 3)
self.assertTrue(len(files['excluded']) == 3)
def test_exclude_path(self):
files = sample_data.files()
self.assertTrue(len(files['included']) == 6)
self.assertTrue(len(files['excluded']) == 0)
self.utils._filter_generic(files, FilterAttribute.PATH, "onkey",
FilterAction.EXCLUDE, FilterModifier.ANY)
self.assertTrue(len(files['included']) == 3)
self.assertTrue(len(files['excluded']) == 3)
# try include same files, no change
self.utils._filter_generic(files, FilterAttribute.PATH, "onkey",
FilterAction.EXCLUDE, FilterModifier.ANY)
self.assertTrue(len(files['included']) == 3)
self.assertTrue(len(files['excluded']) == 3)
def test_exclude_license(self):
files = sample_data.files()
self.assertTrue(len(files['included']) == 6)
self.assertTrue(len(files['excluded']) == 0)
self.utils._filter_generic(files, FilterAttribute.LICENSE, "mit",
FilterAction.EXCLUDE, FilterModifier.ANY)
self.assertTrue(len(files['included']) == 5)
self.assertTrue(len(files['excluded']) == 1)
# try include same files, no change
self.utils._filter_generic(files, FilterAttribute.PATH, "mit",
FilterAction.EXCLUDE, FilterModifier.ANY)
self.assertTrue(len(files['included']) == 5)
self.assertTrue(len(files['excluded']) == 1)
def test_include_license(self):
files = sample_data.files()
self.assertTrue(len(files['included']) == 6)
self.assertTrue(len(files['excluded']) == 0)
self.utils._filter_generic(files, FilterAttribute.LICENSE, "mit",
FilterAction.INCLUDE, FilterModifier.ANY)
self.assertTrue(len(files['included']) == 1)
self.assertTrue(len(files['excluded']) == 5)
# try include same files, no change
self.utils._filter_generic(files, FilterAttribute.LICENSE, "mit",
FilterAction.INCLUDE, FilterModifier.ANY)
self.assertTrue(len(files['included']) == 1)
self.assertTrue(len(files['excluded']) == 5)
def test_include_only_license(self):
files = sample_data.files()
self.assertTrue(len(files['included']) == 6)
self.assertTrue(len(files['excluded']) == 0)
self.utils._filter_generic(files, FilterAttribute.LICENSE,
"gpl-2.0-or-later", FilterAction.INCLUDE,
FilterModifier.ANY)
self.assertTrue(len(files['included']) == 2)
self.assertTrue(len(files['excluded']) == 4)
#print("inc: " + str(len(files['included'])))
#for f in files['included']:
# import json
# print(" * " + str(f['path']) + str(f['license_expressions']))
#print("exc: " + str(len(files['excluded'])))
#for f in files['excluded']:
# print(" * " + str(f['path']) + str(f['license_expressions']))
#print("")
# try include ONLY license
self.utils._filter_generic(files, FilterAttribute.LICENSE,
"gpl-2.0-or-later", FilterAction.INCLUDE,
FilterModifier.ONLY)
self.assertTrue(len(files['included']) == 1)
self.assertTrue(len(files['excluded']) == 5)
def main():
commands = ManifestorCommands()
parsed_args = parse(commands)
args = parsed_args.__dict__
#print("command line: " + str(sys.argv))
#print("command line: " + str(parsed_args.mode))
#exit(0)
logger = ManifestLogger(args['verbose'])
utils = ManifestUtils(logger)
manifestor = ScancodeManifestor(commands, logger, utils)
#
# if config file supplied - read up args and merge with those
# supplied on command line
#
if args['config'] != None:
args = manifestor._read_merge_args(args)
#
# if config mode - dump args and leave
#
if args['mode'] == MODE_CONFIG:
keys = manifestor._using_hide_args(args)
if keys != set():
logger.error("Can't save config file if hide options are used. Remove the following options from your command line and try again: " + str(keys))
exit(2)
utils._output_args_to_file(args)
exit(0)
#
# if outputfilters mode - dump filters and leave
#
if args['output_filters']:
manifestor._output_filters(args)
exit(0)
# check inconsistent arguments
if args['mode'] == MODE_INTERACTIVE and args['output'] == None:
logger.error("When in acteractive mode you must specify a file name for the manifest")
exit(3)
#
# SETUP
#
hiders = utils._hiders(args)
# Open scancode report
with open(args['input_file']) as fp:
scancode_report = json.load(fp)
# Setup files map
files = manifestor._setup_files(scancode_report['files'])
#
# filter files
#
#print("reading file file: " + str(args['excluded_file_file']))
manifestor._merge_exclude_files(args)
utils._filter(files, args['included_regexps'], args['excluded_regexps'])
filtered = files
if args['mode'] == MODE_INTERACTIVE:
#print("excluded: " + str(args['excluded_regexps']))
interactor = ManifestorInteractor(commands, utils, logger)
interactor._interact(filtered, args)
#print("excluded: " + str(args['excluded_regexps']))
if args['mode'] == MODE_FILTER:
#
# if filter mode - this is the final step in the pipe
#
if args['verbose_file']:
utils._output_verbose_file(files, args['verbose_file'])
else:
utils._output_filtered(filtered, args['hide_files']==False, args['show_directories'], sys.stdout, args['show_excluded_files'], hiders)
exit(0)
keys = manifestor._using_hide_args(args)
if keys != set():
logger.error("Hide options are only allowed in filter mode. Remove the following options from your command line and try again: " + str(keys))
exit(2)
#
# Continue with pipe, but first verbose files
#
if args['verbose']:
logger.verbose("---- filtered files -----")
utils._output_filtered(filtered, sys.stderr, hiders)
#
# transform to intermediate format
#
utils._transform_files(filtered)
transformed = filtered
if args['verbose']:
logger.verbose("---- transformed files -----")
utils._output_filtered(transformed, sys.stderr, hiders)
#print(json.dumps((transformed)))
#
# Copyright output mode
#
if args['mode'] == MODE_COPYRIGHT:
copyrights = utils.copyrights(filtered)
print("---")
print("legal:")
print(" copyrights:")
prefix = " "
max_line = 75
max_length = max_line - len(prefix)
for c_line in copyrights:
if (len(prefix) + len(c_line)) > max_line:
print(prefix + " - ", end="")
lines = [c_line[i:i+max_length] for i in range(0, len(c_line), max_length)]
first = True
for line in lines:
if first:
print(line)
first = False
else:
print(prefix + " " + line)
else:
print(prefix + " - " + c_line)
exit(0)
#
# License output mode
#
if args['mode'] == MODE_LICENSE:
licenses = utils.license_summary(filtered['included'])
uni_licenses = set()
for lic in licenses:
if lic is None:
pass
else:
for exp in lic.replace("(", " ").replace(")", " " ).split(" "):
if exp.lower() == "and":
pass
elif exp.lower() == "or":
pass
elif exp.lower() == "(":
pass
elif exp.lower() == ")":
pass
else:
#print(" " + str(exp))
uni_licenses.add(exp)
uni_license_list = list(uni_licenses)
uni_license_list.sort()
for lic in uni_license_list:
print(" " + str(lic))
exit(0)
#
# add curations
#
curations = utils._curate(transformed, args['file_curations'], args['license_curations'], args['missing_license_curation'])
curated = transformed
#
# validate
#
report = utils._report(args, scancode_report, transformed)
validation = utils._validate(curated, report, args['outbound_license'])
if validation['errors'] != []:
for err in validation['errors']:
logger.error(err)
exit(9)
#
# if validate mode - this is the final step in the pipe
#
if args['mode'] == MODE_VALIDATE:
print("Curated and validated files:")
utils._output_files(curated)
print("")
print("Excluded files: ")
print(" " + str(len(curated['excluded'])))
print("Included files:")
print(" " + str(len(curated['included'])))
print("Copyright:")
for c in report['conclusion']['copyright']:
print(" " + str(c))
print("License (original):")
print(" " + str(report['conclusion']['license_expression_original']))
print("License (simplified):")
print(" " + str(report['conclusion']['license_expression']))
exit(0)
if args['mode'] == MODE_CREATE:
formatter = None
if args['format'].lower() == OUTPUT_FORMAT_TEXT:
formatter = TextFormatter(args, utils)
elif args['format'].lower() == OUTPUT_FORMAT_JSON:
formatter = JSONFormatter(args, utils)
elif args['format'].lower() == OUTPUT_FORMAT_MARKDOWN:
formatter = MarkdownFormatter(args, utils)
format_report = formatter.format(report)
if args['output'] != None:
with open(args['output'], "w") as manifest_file:
manifest_file.write(format_report)
else:
print(format_report)
class TestMatchGeneric(unittest.TestCase):
def setUp(self):
self.logger = ManifestLogger(False)
self.utils = ManifestUtils(self.logger)
def test_match_path_bad_indata(self):
f = sample_data.file()
self.assertRaises(
Exception, lambda: self.utils.
_match_file(None, FilterAttribute.PATH, "bonkey.txt", FilterAction.
INCLUDE, FilterModifier.ANY))
self.assertRaises(
Exception,
lambda: self.utils._match_file(f, None, "bonkey.txt", FilterAction.
INCLUDE, FilterModifier.ANY))
self.assertRaises(
Exception, lambda: self.utils.
_match_file(f, "incorrect", "bonkey.txt", FilterAction.INCLUDE,
FilterModifier.ANY))
self.assertRaises(
Exception, lambda: self.utils.
_match_file(f, FilterAttribute.PATH, None, FilterAction.INCLUDE,
FilterModifier.ANY))
self.assertRaises(
Exception,
lambda: self.utils._match_file(f, FilterAttribute.PATH, set(
), FilterAction.INCLUDE, FilterModifier.ANY))
self.assertRaises(
Exception, lambda: self.utils.
_match_file(f, FilterAttribute.PATH, "bonkey.txt", None,
FilterModifier.ANY))
self.assertRaises(
Exception, lambda: self.utils.
_match_file(f, FilterAttribute.PATH, "bonkey.txt", "incorrect",
FilterModifier.ANY))
self.assertRaises(
Exception, lambda: self.utils.
_match_file(f, FilterAttribute.PATH, "bonkey.txt", FilterAction.
INCLUDE, None))
self.assertRaises(
Exception, lambda: self.utils.
_match_file(f, FilterAttribute.PATH, "bonkey.txt", FilterAction.
INCLUDE, "incorrect"))
def test_match_path(self):
f = sample_data.file()
assert self.utils._match_file(f, FilterAttribute.PATH, "bonkey.txt",
FilterAction.INCLUDE, FilterModifier.ANY)
assert not self.utils._match_file(f, FilterAttribute.PATH,
"plonkey.txt", FilterAction.INCLUDE,
FilterModifier.ANY)
def test_match_license(self):
f = sample_data.file()
assert self.utils._match_file(f, FilterAttribute.LICENSE,
"gpl-2.0-or-later", FilterAction.INCLUDE,
FilterModifier.ANY)
# file has both gpl3 and gpl2, thus false
assert not self.utils._match_file(
f, FilterAttribute.LICENSE, "gpl-2.0-or-later",
FilterAction.INCLUDE, FilterModifier.ONLY)
assert not self.utils._match_file(f, FilterAttribute.LICENSE,
"bsd-4-Clause", FilterAction.INCLUDE,
FilterModifier.ANY)