def check_csv(expected, result, regen=False, fix_cell_linesep=False):
"""
Assert that the contents of two CSV files locations `expected` and
`result` are equal.
"""
if regen:
shutil.copyfile(result, expected)
expected = sorted([sorted(d.items()) for d in load_csv(expected)])
result = [d.items() for d in load_csv(result)]
if fix_cell_linesep:
result = [list(fix_crlf(items)) for items in result]
result = sorted(sorted(items) for items in result)
assert expected == result
def test_load_csv_utf_8(self):
test_file = get_test_loc('test_util/csv/test_utf8.csv')
expected = [
OrderedDict([(u'about_resource', u'/myFile'),
(u'name', u'\u540d')])
]
result = util.load_csv(test_file)
assert expected == result
def test_load_csv_does_convert_column_names_to_lowercase(self):
test_file = get_test_loc('test_util/csv/about_key_with_upper_case.csv')
expected = [
OrderedDict([('about_file', 'about.ABOUT'),
('about_resource', '.'), ('name', 'ABOUT tool'),
('version', '0.8.1')])
]
result = util.load_csv(test_file)
assert expected == result
def test_load_csv_load_rows(self):
test_file = get_test_loc('test_util/csv/about.csv')
expected = [
OrderedDict([('about_file', 'about.ABOUT'),
('about_resource', '.'), ('name', 'ABOUT tool'),
('version', '0.8.1')])
]
result = util.load_csv(test_file)
assert expected == result
def test_load_csv_does_not_convert_column_names_to_lowercase(self):
test_file = get_test_loc('util/about_key_with_upper_case.csv')
expected = [OrderedDict(
[('about_file', 'about.ABOUT'),
('about_resource', '.'),
('nAme', 'ABOUT tool'),
('Version', '0.8.1')])
]
result = util.load_csv(test_file)
assert expected == result
def test_load_csv_without_mapping(self):
test_file = get_test_loc('util/about.csv')
expected = [OrderedDict([
('about_file', 'about.ABOUT'),
('about_resource', '.'),
('name', 'ABOUT tool'),
('version', '0.8.1')])
]
result = util.load_csv(test_file)
assert expected == result
def test_load_csv_with_conf(self):
location = get_test_loc('test_util/load/simple_sample.csv')
base_dir = get_temp_dir()
configuration = get_test_loc('test_util/load/key.config')
inventory = util.load_csv(location, configuration)
expected = [
OrderedDict([('name', 'cryptohash-sha256'),
('version', 'v 0.11.100.1'),
('license_expression', 'bsd-new and mit'),
('resource', '/project/cryptohash-sha256')]),
OrderedDict([('name', 'some_component'), ('version', 'v 0.0.1'),
('license_expression', 'mit'),
('resource', '/project/some_component')])
]
assert inventory == expected
def load_inventory(location, base_dir, reference_dir=None):
"""
Load the inventory file at `location` for ABOUT and LICENSE files stored in
the `base_dir`. Return a list of errors and a list of About objects
validated against the `base_dir`.
Optionally use `reference_dir` as the directory location of extra reference
license and notice files to reuse.
"""
errors = []
abouts = []
base_dir = util.to_posix(base_dir)
# FIXME: do not mix up CSV and JSON
if location.endswith('.csv'):
# FIXME: this should not be done here.
dup_cols_err = check_duplicated_columns(location)
if dup_cols_err:
errors.extend(dup_cols_err)
return errors, abouts
inventory = util.load_csv(location)
else:
inventory = util.load_json(location)
try:
# FIXME: this should not be done here.
dup_about_resource_err = check_duplicated_about_resource(inventory)
if dup_about_resource_err:
errors.extend(dup_about_resource_err)
return errors, abouts
newline_in_file = check_newline_in_file_field(inventory)
if newline_in_file:
errors.extend(newline_in_file)
return errors, abouts
except Exception as e:
# TODO: why catch ALL Exception
msg = "The essential field 'about_resource' is not found in the <input>"
errors.append(Error(CRITICAL, msg))
return errors, abouts
for i, fields in enumerate(inventory):
# check does the input contains the required fields
required_fields = model.About.required_fields
for f in required_fields:
if f not in fields:
msg = "Required field: %(f)r not found in the <input>" % locals(
)
errors.append(Error(ERROR, msg))
return errors, abouts
afp = fields.get(model.About.ABOUT_RESOURCE_ATTR)
# FIXME: this should not be a failure condition
if not afp or not afp.strip():
msg = 'Empty column: %(afp)r. Cannot generate .ABOUT file.' % locals(
)
errors.append(Error(ERROR, msg))
continue
else:
afp = util.to_posix(afp)
loc = join(base_dir, afp)
about = model.About(about_file_path=afp)
about.location = loc
# Update value for 'about_resource'
# keep only the filename or '.' if it's a directory
if 'about_resource' in fields:
updated_resource_value = u''
resource_path = fields['about_resource']
if resource_path.endswith(u'/'):
updated_resource_value = u'.'
else:
updated_resource_value = basename(resource_path)
fields['about_resource'] = updated_resource_value
ld_errors = about.load_dict(
fields,
base_dir,
running_inventory=False,
reference_dir=reference_dir,
)
"""
# 'about_resource' field will be generated during the process.
# No error need to be raise for the missing 'about_resource'.
for e in ld_errors:
if e.message == 'Field about_resource is required':
ld_errors.remove(e)
"""
for e in ld_errors:
if not e in errors:
errors.extend(ld_errors)
abouts.append(about)
return unique(errors), abouts
def load_inventory(location, base_dir, license_notice_text_location=None,
use_mapping=False, mapping_file=None):
"""
Load the inventory file at `location` for ABOUT and LICENSE files
stored in the `base_dir`. Return a list of errors and a list of
About objects validated against the base_dir.
Optionally use `license_notice_text_location` as the location of
license and notice texts.
Optionally use mappings for field names if `use_mapping` is True
or a custom mapping_file if provided.
"""
errors = []
abouts = []
base_dir = util.to_posix(base_dir)
if location.endswith('.csv'):
dup_cols_err = check_duplicated_columns(location)
if dup_cols_err:
errors.extend(dup_cols_err)
return errors, abouts
inventory = util.load_csv(location, use_mapping, mapping_file)
else:
inventory = util.load_json(location, use_mapping, mapping_file)
try:
dup_about_paths_err = check_duplicated_about_file_path(inventory)
if dup_about_paths_err:
errors.extend(dup_about_paths_err)
return errors, abouts
except:
msg = (
"The essential field 'about_file_path' is not found.\n"
"Use the --mapping or --mapping-file option to map the "
"input keys and verify the mapping information are correct.\n"
"OR correct the column names in the <input>"
)
errors.append(Error(CRITICAL, msg))
return errors, abouts
for i, fields in enumerate(inventory):
# check does the input contains the required fields
required_fields = model.About.required_fields
for f in required_fields:
if f not in fields:
msg = (
"Required column: %(f)r not found.\n"
"Use the --mapping or --mapping-file option to map the "
"input keys and verify the mapping information are correct.\n"
"OR correct the column names in the <input>"
) % locals()
errors.append(Error(ERROR, msg))
return errors, abouts
afp = fields.get(model.About.about_file_path_attr)
if not afp or not afp.strip():
msg = 'Empty column: %(afp)r. Cannot generate .ABOUT file.' % locals()
errors.append(Error(ERROR, msg))
continue
else:
afp = util.to_posix(afp)
loc = join(base_dir, afp)
about = model.About(about_file_path=afp)
about.location = loc
running_inventory = False
ld_errors = about.load_dict(fields, base_dir, running_inventory,
use_mapping, mapping_file, license_notice_text_location,
with_empty=False)
# 'about_resource' field will be generated during the process.
# No error need to be raise for the missing 'about_resource'.
for e in ld_errors:
if e.message == 'Field about_resource is required':
ld_errors.remove(e)
for e in ld_errors:
if not e in errors:
errors.extend(ld_errors)
abouts.append(about)
return errors, abouts
