def test_write_output_json(self):
path = 'test_model/this.ABOUT'
test_file = get_test_loc(path)
abouts = model.About(location=test_file, about_file_path=path)
result = get_temp_file()
model.write_output([abouts], result, format='json')
expected = get_test_loc('test_model/expected.json')
check_json(expected, result)
def test_write_output_csv_with_multiple_files(self):
path = 'test_model/multiple_files.ABOUT'
test_file = get_test_loc(path)
abouts = model.About(location=test_file, about_file_path=path)
result = get_temp_file()
model.write_output([abouts], result, format='csv')
expected = get_test_loc('test_model/multiple_files_expected.csv')
check_csv(expected, result)
def test_collect_inventory_complex_from_directory(self):
location = get_test_loc('test_model/inventory/complex')
result = get_temp_file()
errors, abouts = model.collect_inventory(location)
model.write_output(abouts, result, format='csv')
assert all(e.severity == INFO for e in errors)
expected = get_test_loc('test_model/inventory/complex/expected.csv')
check_csv(expected, result, fix_cell_linesep=True, regen=False)
def test_collect_inventory_with_no_about_resource_from_directory(self):
location = get_test_loc('test_model/inventory/no_about_resource_key')
result = get_temp_file()
errors, abouts = model.collect_inventory(location)
model.write_output(abouts, result, format='csv')
expected_errors = [
Error(CRITICAL,
'about/about.ABOUT: Field about_resource is required')
]
assert expected_errors == errors
def test_collect_inventory_basic_from_directory(self):
location = get_test_loc('test_model/inventory/basic')
result = get_temp_file()
errors, abouts = model.collect_inventory(location)
model.write_output(abouts, result, format='csv')
expected_errors = []
assert expected_errors == errors
expected = get_test_loc('test_model/inventory/basic/expected.csv')
check_csv(expected, result)
def inventory(location, output, format, quiet, verbose): # NOQA
"""
Collect the inventory of .ABOUT file data as CSV or JSON.
LOCATION: Path to an .ABOUT file or a directory with .ABOUT files.
OUTPUT: Path to the JSON or CSV inventory file to create.
"""
if not quiet:
print_version()
click.echo('Collecting inventory from ABOUT files...')
if location.lower().endswith('.zip'):
# accept zipped ABOUT files as input
location = extract_zip(location)
errors, abouts = collect_inventory(location)
write_errors = write_output(abouts=abouts, location=output, format=format)
errors.extend(write_errors)
errors_count = report_errors(errors,
quiet,
verbose,
log_file_loc=output + '-error.log')
if not quiet:
msg = 'Inventory collected in {output}.'.format(**locals())
click.echo(msg)
sys.exit(errors_count)
def inventory(location, output, mapping, quiet, format, show_all):
"""
Collect a JSON or CSV inventory of components from .ABOUT files.
LOCATION: Path to an .ABOUT file or a directory with .ABOUT files.
OUTPUT: Path to the JSON or CSV inventory file to create.
"""
print_version()
if not exists(os.path.dirname(output)):
# FIXME: there is likely a better way to return an error
click.echo('ERROR: <OUTPUT> path does not exists.')
# FIXME: return error code?
return
click.echo(
'Collecting inventory from: %(location)r and writing output to: %(output)r'
% locals())
# FIXME: do we really want to continue support zip as an input?
if location.lower().endswith('.zip'):
# accept zipped ABOUT files as input
location = extract_zip(location)
errors, abouts = model.collect_inventory(location, use_mapping=mapping)
write_errors = model.write_output(abouts, output, format)
for err in write_errors:
errors.append(err)
finalized_errors = ignore_about_resource_path_not_exist_error(errors)
log_errors(finalized_errors, quiet, show_all, os.path.dirname(output))
sys.exit(0)
def test_collect_inventory_does_not_convert_lf_to_crlf_from_directory(
self):
location = get_test_loc('test_model/crlf/about.ABOUT')
result = get_temp_file()
errors, abouts = model.collect_inventory(location)
errors2 = model.write_output(abouts, result, format='csv')
errors.extend(errors2)
assert all(e.severity == INFO for e in errors)
expected = get_test_loc('test_model/crlf/expected.csv')
check_csv(expected, result, fix_cell_linesep=True, regen=False)
def inventory(location, output, mapping, mapping_file, mapping_output, filter,
quiet, format, verbose): # NOQA
"""
Collect a JSON or CSV inventory of components from .ABOUT files.
LOCATION: Path to an .ABOUT file or a directory with .ABOUT files.
OUTPUT: Path to the JSON or CSV inventory file to create.
"""
print_version()
if not exists(os.path.dirname(output)):
# FIXME: there is likely a better way to return an error
click.echo('ERROR: <OUTPUT> path does not exists.')
# FIXME: return error code?
return
click.echo(
'Collecting inventory from: %(location)s and writing output to: %(output)s'
% locals())
# FIXME: do we really want to continue support zip as an input?
if location.lower().endswith('.zip'):
# accept zipped ABOUT files as input
location = extract_zip(location)
errors, abouts = model.collect_inventory(location,
use_mapping=mapping,
mapping_file=mapping_file)
updated_abouts = []
if filter:
filter_dict = {}
# Parse the filter and save to the filter dictionary with a list of value
for element in filter:
key = element.partition('=')[0]
value = element.partition('=')[2]
if key in filter_dict:
filter_dict[key].append(value)
else:
value_list = [value]
filter_dict[key] = value_list
updated_abouts = inventory_filter(abouts, filter_dict)
else:
updated_abouts = abouts
# Do not write the output if one of the ABOUT files has duplicated key names
dup_error_msg = u'Duplicated key name(s)'
halt_output = False
for err in errors:
if dup_error_msg in err.message:
halt_output = True
break
if not halt_output:
write_errors = model.write_output(updated_abouts, output, format,
mapping_output)
for err in write_errors:
errors.append(err)
else:
msg = u'Duplicated key names are not supported.\n' + \
'Please correct and re-run.'
print(msg)
error_count = 0
for e in errors:
# Only count as warning/error if CRITICAL, ERROR and WARNING
if e.severity > 20:
error_count = error_count + 1
log_errors(errors, error_count, quiet, verbose, os.path.dirname(output))
click.echo(' %(error_count)d errors or warnings detected.' % locals())
sys.exit(0)
def inventory(location, output, mapping, mapping_file, mapping_output, filter, quiet, format, verbose): # NOQA
"""
Collect a JSON or CSV inventory of components from .ABOUT files.
LOCATION: Path to an .ABOUT file or a directory with .ABOUT files.
OUTPUT: Path to the JSON or CSV inventory file to create.
"""
print_version()
if not exists(os.path.dirname(output)):
# FIXME: there is likely a better way to return an error
click.echo('ERROR: <OUTPUT> path does not exists.')
# FIXME: return error code?
return
click.echo('Collecting inventory from: %(location)s and writing output to: %(output)s' % locals())
# FIXME: do we really want to continue support zip as an input?
if location.lower().endswith('.zip'):
# accept zipped ABOUT files as input
location = extract_zip(location)
errors, abouts = model.collect_inventory(location, use_mapping=mapping, mapping_file=mapping_file)
updated_abouts = []
if filter:
filter_dict = {}
# Parse the filter and save to the filter dictionary with a list of value
for element in filter:
key = element.partition('=')[0]
value = element.partition('=')[2]
if key in filter_dict:
filter_dict[key].append(value)
else:
value_list = [value]
filter_dict[key] = value_list
updated_abouts = inventory_filter(abouts, filter_dict)
else:
updated_abouts = abouts
# Do not write the output if one of the ABOUT files has duplicated key names
dup_error_msg = u'Duplicated key name(s)'
halt_output = False
for err in errors:
if dup_error_msg in err.message:
halt_output = True
break
if not halt_output:
write_errors = model.write_output(updated_abouts, output, format, mapping_output)
for err in write_errors:
errors.append(err)
else:
msg = u'Duplicated key names are not supported.\n' + \
'Please correct and re-run.'
print(msg)
error_count = 0
for e in errors:
# Only count as warning/error if CRITICAL, ERROR and WARNING
if e.severity > 20:
error_count = error_count + 1
log_errors(errors, error_count, quiet, verbose, os.path.dirname(output))
click.echo(' %(error_count)d errors or warnings detected.' % locals())
sys.exit(0)