def test_collect_inventory_populate_about_file_path(self):
test_loc = get_test_loc('test_model/inventory/complete')
errors, abouts = model.collect_inventory(test_loc)
assert [] == errors
expected = 'about.ABOUT'
result = abouts[0].about_file_path
assert expected == result
def inventory(location, output, mapping, quiet, format, show_all):
"""
Collect a JSON or CSV inventory of components from .ABOUT files.
LOCATION: Path to an .ABOUT file or a directory with .ABOUT files.
OUTPUT: Path to the JSON or CSV inventory file to create.
"""
print_version()
if not exists(os.path.dirname(output)):
# FIXME: there is likely a better way to return an error
click.echo('ERROR: <OUTPUT> path does not exists.')
# FIXME: return error code?
return
click.echo(
'Collecting inventory from: %(location)r and writing output to: %(output)r'
% locals())
# FIXME: do we really want to continue support zip as an input?
if location.lower().endswith('.zip'):
# accept zipped ABOUT files as input
location = extract_zip(location)
errors, abouts = model.collect_inventory(location, use_mapping=mapping)
write_errors = model.write_output(abouts, output, format)
for err in write_errors:
errors.append(err)
finalized_errors = ignore_about_resource_path_not_exist_error(errors)
log_errors(finalized_errors, quiet, show_all, os.path.dirname(output))
sys.exit(0)
def test_collect_inventory_return_no_warnings_and_model_can_uuse_relative_paths(
self):
test_loc = get_test_loc('test_model/rel/allAboutInOneDir')
errors, _abouts = model.collect_inventory(test_loc)
expected_errors = []
result = [(level, e) for level, e in errors if level > INFO]
assert expected_errors == result
def inventory(location, output, format, quiet, verbose): # NOQA
"""
Collect the inventory of .ABOUT file data as CSV or JSON.
LOCATION: Path to an .ABOUT file or a directory with .ABOUT files.
OUTPUT: Path to the JSON or CSV inventory file to create.
"""
if not quiet:
print_version()
click.echo('Collecting inventory from ABOUT files...')
if location.lower().endswith('.zip'):
# accept zipped ABOUT files as input
location = extract_zip(location)
errors, abouts = collect_inventory(location)
write_errors = write_output(abouts=abouts, location=output, format=format)
errors.extend(write_errors)
errors_count = report_errors(errors,
quiet,
verbose,
log_file_loc=output + '-error.log')
if not quiet:
msg = 'Inventory collected in {output}.'.format(**locals())
click.echo(msg)
sys.exit(errors_count)
def attrib(location, output, template, vartext, quiet, verbose):
"""
Generate an attribution document at OUTPUT using .ABOUT files at LOCATION.
LOCATION: Path to a file, directory or .zip archive containing .ABOUT files.
OUTPUT: Path where to write the attribution document.
"""
if not quiet:
print_version()
click.echo('Generating attribution...')
# accept zipped ABOUT files as input
if location.lower().endswith('.zip'):
location = extract_zip(location)
errors, abouts = collect_inventory(location)
attrib_errors = generate_attribution_doc(
abouts=abouts,
output_location=output,
template_loc=template,
variables=vartext,
)
errors.extend(attrib_errors)
errors_count = report_errors(errors,
quiet,
verbose,
log_file_loc=output + '-error.log')
if not quiet:
msg = 'Attribution generated in: {output}'.format(**locals())
click.echo(msg)
sys.exit(errors_count)
def check(location, show_all):
"""
Check and validate .ABOUT file(s) at LOCATION for errors and
print error messages on the terminal.
LOCATION: Path to a .ABOUT file or a directory containing .ABOUT files.
"""
click.echo('Running aboutcode-toolkit version ' + __version__)
click.echo('Checking ABOUT files...')
errors, abouts = model.collect_inventory(location)
msg_format = '%(sever)s: %(message)s'
print_errors = []
for severity, message in errors:
sever = severities[severity]
if show_all:
print_errors.append(msg_format % locals())
elif sever in problematic_errors:
print_errors.append(msg_format % locals())
number_of_errors = len(print_errors)
for err in print_errors:
print(err)
if print_errors:
click.echo('Found {} errors.'.format(number_of_errors))
# FIXME: not sure this is the right way to exit with a retrun code
sys.exit(1)
else:
click.echo('No error found.')
sys.exit(0)
def test_collect_inventory_always_collects_custom_fieldsg(self):
test_loc = get_test_loc('test_model/inventory/custom_fields.ABOUT')
errors, abouts = model.collect_inventory(test_loc)
expected_msg1 = 'Field resource is a custom field'
assert len(errors) == 2
assert expected_msg1 in errors[0].message
# The not supported 'resource' value is collected
assert abouts[0].resource.value
def test_collect_inventory_can_collect_a_single_file(self):
test_loc = get_test_loc(
'test_model/single_file/django_snippets_2413.ABOUT')
_errors, abouts = model.collect_inventory(test_loc)
assert 1 == len(abouts)
expected = ['single_file/django_snippets_2413.ABOUT']
result = [a.about_file_path for a in abouts]
assert expected == result
def test_collect_inventory_with_license_expression(self):
test_loc = get_test_loc(
'test_model/parse/multi_line_license_expresion.ABOUT')
errors, abouts = model.collect_inventory(test_loc)
assert [] == errors
expected_lic = 'mit or apache-2.0'
returned_lic = abouts[0].license_expression.value
assert expected_lic == returned_lic
def test_generate(self):
expected = (u'Apache HTTP Server: 2.4.3\n'
u'resource: httpd-2.4.3.tar.gz\n')
test_file = get_test_loc('attrib_gen/attrib.ABOUT')
with open(get_test_loc('attrib_gen/test.template')) as tmpl:
template = tmpl.read()
_errors, abouts = model.collect_inventory(test_file)
result = attrib.generate(abouts, template)
self.assertEqual(expected, result)
def test_generate_from_file_with_default_template(self):
test_file = get_test_loc('attrib_gen/attrib.ABOUT')
_errors, abouts = model.collect_inventory(test_file)
result = attrib.generate_from_file(abouts)
with open(get_test_loc(
'attrib_gen/expected_default_attrib.html')) as exp:
expected = exp.read()
self.assertEqual([x.rstrip() for x in expected.splitlines()],
[x.rstrip() for x in result.splitlines()])
def test_generate_from_file_with_default_template(self):
test_file = get_test_loc('attrib_gen/attrib.ABOUT')
_errors, abouts = model.collect_inventory(test_file)
result = attrib.generate_from_file(abouts)
with open(get_test_loc('attrib_gen/expected_default_attrib.html')) as exp:
expected = exp.read()
# strip the timestamp: the timestamp is wrapped in italic block
self.assertEqual([x.rstrip() for x in expected.splitlines()],
[x.rstrip() for x in result.splitlines() if not '<i>' in x])
def test_generate(self):
expected = (u'Apache HTTP Server: 2.4.3\n'
u'resource: httpd-2.4.3.tar.gz\n')
test_file = get_test_loc('attrib_gen/attrib.ABOUT')
with open(get_test_loc('attrib_gen/test.template')) as tmpl:
template = tmpl.read()
_errors, abouts = model.collect_inventory(test_file)
result = attrib.generate(abouts, template)
self.assertEqual(expected, result)
def test_collect_inventory_works_with_relative_paths(self):
# FIXME: This test need to be run under src/attributecode/
# or otherwise it will fail as the test depends on the launching
# location
test_loc = get_test_loc('test_model/inventory/relative')
# Use '.' as the indication of the current directory
test_loc1 = test_loc + '/./'
# Use '..' to go back to the parent directory
test_loc2 = test_loc + '/../relative'
errors1, abouts1 = model.collect_inventory(test_loc1)
errors2, abouts2 = model.collect_inventory(test_loc2)
assert [] == errors1
assert [] == errors2
expected = 'about.ABOUT'
result1 = abouts1[0].about_file_path
result2 = abouts2[0].about_file_path
assert expected == result1
assert expected == result2
def test_inventory_filter(self):
test_loc = get_test_loc('basic')
_errors, abouts = model.collect_inventory(test_loc)
filter_dict = {'name': ['simple']}
# The test loc has 2 .about files, only the simple.about is taken after
# the filtering
updated_abouts = util.inventory_filter(abouts, filter_dict)
for about in updated_abouts:
assert about.name.value == 'simple'
def test_inventory_filter(self):
test_loc = get_test_loc('basic')
_errors, abouts = model.collect_inventory(test_loc)
filter_dict = {'name': ['simple']}
# The test loc has 2 .about files, only the simple.about is taken after
# the filtering
updated_abouts = util.inventory_filter(abouts, filter_dict)
for about in updated_abouts:
assert about.name.value == 'simple'
def test_collect_inventory_with_multi_line(self):
test_loc = get_test_loc(
'test_model/parse/multi_line_license_expresion.ABOUT')
errors, abouts = model.collect_inventory(test_loc)
assert [] == errors
expected_lic_url = [
'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit',
'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:apache-2.0'
]
returned_lic_url = abouts[0].license_url.value
assert expected_lic_url == returned_lic_url
def test_collect_inventory_does_not_convert_lf_to_crlf_from_directory(
self):
location = get_test_loc('test_model/crlf/about.ABOUT')
result = get_temp_file()
errors, abouts = model.collect_inventory(location)
errors2 = model.write_output(abouts, result, format='csv')
errors.extend(errors2)
assert all(e.severity == INFO for e in errors)
expected = get_test_loc('test_model/crlf/expected.csv')
check_csv(expected, result, fix_cell_linesep=True, regen=False)
def test_collect_inventory_complex_from_directory(self):
location = get_test_loc('test_model/inventory/complex')
result = get_temp_file()
errors, abouts = model.collect_inventory(location)
model.write_output(abouts, result, format='csv')
assert all(e.severity == INFO for e in errors)
expected = get_test_loc('test_model/inventory/complex/expected.csv')
check_csv(expected, result, fix_cell_linesep=True, regen=False)
def check(location, verbose):
"""
Check .ABOUT file(s) at LOCATION for validity and print error messages.
LOCATION: Path to a file or directory containing .ABOUT files.
"""
print_version()
click.echo('Checking ABOUT files...')
errors, _abouts = collect_inventory(location)
severe_errors_count = report_errors(errors, quiet=False, verbose=verbose)
sys.exit(severe_errors_count)
def test_collect_inventory_basic_from_directory(self):
location = get_test_loc('test_model/inventory/basic')
result = get_temp_file()
errors, abouts = model.collect_inventory(location)
model.write_output(abouts, result, format='csv')
expected_errors = []
assert expected_errors == errors
expected = get_test_loc('test_model/inventory/basic/expected.csv')
check_csv(expected, result)
def test_collect_inventory_with_no_about_resource_from_directory(self):
location = get_test_loc('test_model/inventory/no_about_resource_key')
result = get_temp_file()
errors, abouts = model.collect_inventory(location)
model.write_output(abouts, result, format='csv')
expected_errors = [
Error(CRITICAL,
'about/about.ABOUT: Field about_resource is required')
]
assert expected_errors == errors
def attrib(location, output, template, mapping, mapping_file, inventory,
vartext, quiet, verbose):
"""
Generate an attribution document at OUTPUT using .ABOUT files at LOCATION.
LOCATION: Path to an .ABOUT file, a directory containing .ABOUT files or a .zip archive containing .ABOUT files.
OUTPUT: Path to output file to write the attribution to.
"""
print_version()
click.echo('Generating attribution...')
# accept zipped ABOUT files as input
if location.lower().endswith('.zip'):
location = extract_zip(location)
inv_errors, abouts = model.collect_inventory(location,
use_mapping=mapping,
mapping_file=mapping_file)
no_match_errors = attrib_generate_and_save(abouts=abouts,
output_location=output,
use_mapping=mapping,
mapping_file=mapping_file,
template_loc=template,
inventory_location=inventory,
vartext=vartext)
if not no_match_errors:
# Check for template error
with open(output, 'r') as output_file:
first_line = output_file.readline()
if first_line.startswith('Template'):
click.echo(first_line)
sys.exit(errno.ENOEXEC)
for no_match_error in no_match_errors:
inv_errors.append(no_match_error)
error_count = 0
for e in inv_errors:
# Only count as warning/error if CRITICAL, ERROR and WARNING
if e.severity > 20:
error_count = error_count + 1
log_errors(inv_errors, error_count, quiet, verbose,
os.path.dirname(output))
click.echo(' %(error_count)d errors or warnings detected.' % locals())
click.echo('Finished.')
sys.exit(0)
def test_generate_from_collected_inventory_wih_custom_temaplte(self):
test_file = get_test_loc('test_attrib/gen_simple/attrib.ABOUT')
errors, abouts = model.collect_inventory(test_file)
assert not errors
test_template = get_test_loc('test_attrib/gen_simple/test.template')
with open(test_template) as tmpl:
template = tmpl.read()
expected = ('Apache HTTP Server: 2.4.3\n'
'resource: httpd-2.4.3.tar.gz\n')
error, result = attrib.generate(abouts, template)
assert expected == result
assert not error
def test_lic_key_name_sync(self):
test_file = get_test_loc(
'test_attrib/gen_license_key_name_check/test.ABOUT')
expected = get_test_loc(
'test_attrib/gen_license_key_name_check/expected/expected.html')
template_loc = get_test_loc(
'test_attrib/gen_license_key_name_check/custom.template')
output_file = get_temp_file()
errors, abouts = model.collect_inventory(test_file)
attrib.generate_and_save(abouts, output_file, template_loc)
with open(output_file) as of:
f1 = '\n'.join(of.readlines(False))
with open(expected) as ef:
f2 = '\n'.join(ef.readlines(False))
assert f1 == f2
def test_generate_with_default_template(self):
test_file = get_test_loc(
'test_attrib/gen_default_template/attrib.ABOUT')
errors, abouts = model.collect_inventory(test_file)
assert not errors
error, result = attrib.generate_from_file(abouts)
assert not error
expected_file = get_test_loc(
'test_attrib/gen_default_template/expected_default_attrib.html')
with open(expected_file) as exp:
expected = exp.read()
# strip the timestamp: the timestamp is wrapped in italic block
result = remove_timestamp(result)
expected = remove_timestamp(expected)
assert expected == result
def attrib(location, output, template, mapping, mapping_file, inventory, vartext, quiet, verbose):
"""
Generate an attribution document at OUTPUT using .ABOUT files at LOCATION.
LOCATION: Path to an .ABOUT file, a directory containing .ABOUT files or a .zip archive containing .ABOUT files.
OUTPUT: Path to output file to write the attribution to.
"""
print_version()
click.echo('Generating attribution...')
# accept zipped ABOUT files as input
if location.lower().endswith('.zip'):
location = extract_zip(location)
inv_errors, abouts = model.collect_inventory(location, use_mapping=mapping, mapping_file=mapping_file)
no_match_errors = attrib_generate_and_save(
abouts=abouts, output_location=output,
use_mapping=mapping, mapping_file=mapping_file, template_loc=template,
inventory_location=inventory, vartext=vartext)
if not no_match_errors:
# Check for template error
with open(output, 'r') as output_file:
first_line = output_file.readline()
if first_line.startswith('Template'):
click.echo(first_line)
sys.exit(errno.ENOEXEC)
for no_match_error in no_match_errors:
inv_errors.append(no_match_error)
error_count = 0
for e in inv_errors:
# Only count as warning/error if CRITICAL, ERROR and WARNING
if e.severity > 20:
error_count = error_count + 1
log_errors(inv_errors, error_count, quiet, verbose, os.path.dirname(output))
click.echo(' %(error_count)d errors or warnings detected.' % locals())
click.echo('Finished.')
sys.exit(0)
def test_collect_inventory_does_not_raise_error_and_maintains_order_on_custom_fields(
self):
test_loc = get_test_loc('test_model/inventory/custom_fields2.ABOUT')
errors, abouts = model.collect_inventory(test_loc)
expected_errors = [
Error(
INFO,
'inventory/custom_fields2.ABOUT: Field resource is a custom field.'
),
Error(
INFO,
'inventory/custom_fields2.ABOUT: Field custom_mapping is a custom field.'
)
]
assert expected_errors == errors
expected = [
u'about_resource: .\nname: test\nresource: .\ncustom_mapping: test\n'
]
assert expected == [a.dumps() for a in abouts]
def test_collect_inventory_return_errors(self):
test_loc = get_test_loc('test_model/collect_inventory_errors')
errors, _abouts = model.collect_inventory(test_loc)
file_path1 = posixpath.join(test_loc, 'distribute_setup.py')
file_path2 = posixpath.join(test_loc, 'date_test.py')
err_msg1 = 'non-supported_date_format.ABOUT: Field about_resource: Path %s not found' % file_path1
err_msg2 = 'supported_date_format.ABOUT: Field about_resource: Path %s not found' % file_path2
expected_errors = [
Error(
INFO,
'non-supported_date_format.ABOUT: Field date is a custom field.'
),
Error(
INFO,
'supported_date_format.ABOUT: Field date is a custom field.'),
Error(INFO, err_msg1),
Error(INFO, err_msg2)
]
assert sorted(expected_errors) == sorted(errors)
def test_collect_inventory_with_long_path(self):
test_loc = extract_test_loc('test_model/longpath.zip')
_errors, abouts = model.collect_inventory(test_loc)
assert 2 == len(abouts)
expected_paths = (
'longpath/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1'
'/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1'
'/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1'
'/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1'
'/longpath1/non-supported_date_format.ABOUT',
'longpath/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1'
'/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1'
'/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1'
'/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1'
'/longpath1/supported_date_format.ABOUT')
results = [a.about_file_path for a in abouts]
assert all(r.endswith(expected_paths) for r in results)
expected_name = ['distribute', 'date_test']
result_name = [a.name.value for a in abouts]
assert sorted(expected_name) == sorted(result_name)
def attrib(location, output, template, mapping, inventory, quiet, show_all):
"""
Generate an attribution document at OUTPUT using .ABOUT files at LOCATION.
LOCATION: Path to an .ABOUT file, a directory containing .ABOUT files or a .zip archive containing .ABOUT files.
OUTPUT: Path to output file to write the attribution to.
"""
print_version()
click.echo('Generating attribution...')
# accept zipped ABOUT files as input
if location.lower().endswith('.zip'):
location = extract_zip(location)
inv_errors, abouts = model.collect_inventory(location, use_mapping=mapping)
no_match_errors = attrib_generate_and_save(abouts=abouts,
output_location=output,
use_mapping=mapping,
template_loc=template,
inventory_location=inventory)
if not no_match_errors:
# Check for template error
with open(output, 'r') as output_file:
first_line = output_file.readline()
if first_line.startswith('Template'):
click.echo(first_line)
sys.exit(errno.ENOEXEC)
for no_match_error in no_match_errors:
inv_errors.append(no_match_error)
finalized_errors = ignore_about_resource_path_not_exist_error(inv_errors)
log_errors(finalized_errors, quiet, show_all, os.path.dirname(output))
click.echo('Finished.')
sys.exit(0)
def check(location, verbose):
"""
Check and validate .ABOUT file(s) at LOCATION for errors and
print error messages on the terminal.
LOCATION: Path to a .ABOUT file or a directory containing .ABOUT files.
"""
click.echo('Running aboutcode-toolkit version ' + __version__)
click.echo('Checking ABOUT files...')
errors, abouts = model.collect_inventory(location)
msg_format = '%(sever)s: %(message)s'
print_errors = []
number_of_errors = 0
for severity, message in errors:
sever = severities[severity]
# Only problematic_errors should be counted.
# Others such as INFO should not be counted as error.
if sever in problematic_errors:
number_of_errors = number_of_errors + 1
if verbose:
print_errors.append(msg_format % locals())
elif sever in problematic_errors:
print_errors.append(msg_format % locals())
for err in print_errors:
print(err)
if print_errors:
click.echo('Found {} errors.'.format(number_of_errors))
# FIXME: not sure this is the right way to exit with a return code
sys.exit(1)
else:
click.echo('No error found.')
sys.exit(0)
def inventory(location, output, mapping, mapping_file, mapping_output, filter, quiet, format, verbose): # NOQA
"""
Collect a JSON or CSV inventory of components from .ABOUT files.
LOCATION: Path to an .ABOUT file or a directory with .ABOUT files.
OUTPUT: Path to the JSON or CSV inventory file to create.
"""
print_version()
if not exists(os.path.dirname(output)):
# FIXME: there is likely a better way to return an error
click.echo('ERROR: <OUTPUT> path does not exists.')
# FIXME: return error code?
return
click.echo('Collecting inventory from: %(location)s and writing output to: %(output)s' % locals())
# FIXME: do we really want to continue support zip as an input?
if location.lower().endswith('.zip'):
# accept zipped ABOUT files as input
location = extract_zip(location)
errors, abouts = model.collect_inventory(location, use_mapping=mapping, mapping_file=mapping_file)
updated_abouts = []
if filter:
filter_dict = {}
# Parse the filter and save to the filter dictionary with a list of value
for element in filter:
key = element.partition('=')[0]
value = element.partition('=')[2]
if key in filter_dict:
filter_dict[key].append(value)
else:
value_list = [value]
filter_dict[key] = value_list
updated_abouts = inventory_filter(abouts, filter_dict)
else:
updated_abouts = abouts
# Do not write the output if one of the ABOUT files has duplicated key names
dup_error_msg = u'Duplicated key name(s)'
halt_output = False
for err in errors:
if dup_error_msg in err.message:
halt_output = True
break
if not halt_output:
write_errors = model.write_output(updated_abouts, output, format, mapping_output)
for err in write_errors:
errors.append(err)
else:
msg = u'Duplicated key names are not supported.\n' + \
'Please correct and re-run.'
print(msg)
error_count = 0
for e in errors:
# Only count as warning/error if CRITICAL, ERROR and WARNING
if e.severity > 20:
error_count = error_count + 1
log_errors(errors, error_count, quiet, verbose, os.path.dirname(output))
click.echo(' %(error_count)d errors or warnings detected.' % locals())
sys.exit(0)
def inventory(location, output, mapping, mapping_file, mapping_output, filter,
quiet, format, verbose): # NOQA
"""
Collect a JSON or CSV inventory of components from .ABOUT files.
LOCATION: Path to an .ABOUT file or a directory with .ABOUT files.
OUTPUT: Path to the JSON or CSV inventory file to create.
"""
print_version()
if not exists(os.path.dirname(output)):
# FIXME: there is likely a better way to return an error
click.echo('ERROR: <OUTPUT> path does not exists.')
# FIXME: return error code?
return
click.echo(
'Collecting inventory from: %(location)s and writing output to: %(output)s'
% locals())
# FIXME: do we really want to continue support zip as an input?
if location.lower().endswith('.zip'):
# accept zipped ABOUT files as input
location = extract_zip(location)
errors, abouts = model.collect_inventory(location,
use_mapping=mapping,
mapping_file=mapping_file)
updated_abouts = []
if filter:
filter_dict = {}
# Parse the filter and save to the filter dictionary with a list of value
for element in filter:
key = element.partition('=')[0]
value = element.partition('=')[2]
if key in filter_dict:
filter_dict[key].append(value)
else:
value_list = [value]
filter_dict[key] = value_list
updated_abouts = inventory_filter(abouts, filter_dict)
else:
updated_abouts = abouts
# Do not write the output if one of the ABOUT files has duplicated key names
dup_error_msg = u'Duplicated key name(s)'
halt_output = False
for err in errors:
if dup_error_msg in err.message:
halt_output = True
break
if not halt_output:
write_errors = model.write_output(updated_abouts, output, format,
mapping_output)
for err in write_errors:
errors.append(err)
else:
msg = u'Duplicated key names are not supported.\n' + \
'Please correct and re-run.'
print(msg)
error_count = 0
for e in errors:
# Only count as warning/error if CRITICAL, ERROR and WARNING
if e.severity > 20:
error_count = error_count + 1
log_errors(errors, error_count, quiet, verbose, os.path.dirname(output))
click.echo(' %(error_count)d errors or warnings detected.' % locals())
sys.exit(0)
