def test_load_json_from_abc_mgr(self):
test_file = get_test_loc('test_util/json/aboutcode_manager_exported.json')
expected = [dict(OrderedDict([
('license_expression', 'apache-2.0'),
('copyright', 'Copyright (c) 2017 nexB Inc.'),
('licenses', [{'key':'apache-2.0'}]),
('copyrights', [{'statements':['Copyright (c) 2017 nexB Inc.']}]),
('path', 'ScanCode'),
('review_status', 'Analyzed'),
('name', 'ScanCode'),
('version', '2.2.1'),
('owner', 'nexB Inc.'),
('code_type', 'Source'),
('is_modified', False),
('is_deployed', False),
('feature', ''),
('purpose', ''),
('homepage_url', None),
('download_url', None),
('license_url', None),
('notice_url', None),
('programming_language', 'Python'),
('notes', ''),
('fileId', 8458),
]))]
result = util.load_json(test_file)
assert expected == result
def test_load_json_from_scancode(self):
test_file = get_test_loc('test_util/json/scancode_info.json')
expected = [dict(OrderedDict([
('type', 'file'),
('name', 'Api.java'),
('path', 'Api.java'),
('base_name', 'Api'),
('extension', '.java'),
('size', 5074),
('date', '2017-07-15'),
('sha1', 'c3a48ec7e684a35417241dd59507ec61702c508c'),
('md5', '326fb262bbb9c2ce32179f0450e24601'),
('mime_type', 'text/plain'),
('file_type', 'ASCII text'),
('programming_language', 'Java'),
('is_binary', False),
('is_text', True),
('is_archive', False),
('is_media', False),
('is_source', True),
('is_script', False),
('files_count', 0),
('dirs_count', 0),
('size_count', 0),
('scan_errors', []),
]))]
result = util.load_json(test_file)
assert expected == result
def test_load_json_from_abc_mgr(self):
test_file = get_test_loc('load/aboutcode_manager_exported.json')
mapping_file = get_test_loc('custom-mapping-file/mapping.config')
expected = [dict(OrderedDict(
[('license_expression', 'apache-2.0'),
('copyright', 'Copyright (c) 2017 nexB Inc.'),
('licenses', [{'key':'apache-2.0'}]),
('copyrights', [{'statements':['Copyright (c) 2017 nexB Inc.']}]),
('about_file_path', 'ScanCode'),
('review_status', 'Analyzed'),
('name', 'ScanCode'),
('version', '2.2.1'),
('owner', 'nexB Inc.'),
('code_type', 'Source'),
('is_modified', False),
('is_deployed', False),
('feature', ''),
('purpose', ''),
('homepage_url', None),
('download_url', None),
('license_url', None),
('notice_url', None),
('programming_language', 'Python'),
('notes', ''),
('fileId', 8458),
]
))]
result = util.load_json(test_file, use_mapping=False, mapping_file=mapping_file)
assert expected == result
def test_load_json_from_scancode(self):
test_file = get_test_loc('load/scancode_info.json')
mapping_file = get_test_loc('custom-mapping-file/mapping.config')
expected = [dict(OrderedDict(
[('about_file_path', 'Api.java'),
('type', 'file'),
('name', 'Api.java'),
('base_name', 'Api'),
('extension', '.java'),
('size', 5074),
('date', '2017-07-15'),
('sha1', 'c3a48ec7e684a35417241dd59507ec61702c508c'),
('md5', '326fb262bbb9c2ce32179f0450e24601'),
('mime_type', 'text/plain'),
('file_type', 'ASCII text'),
('programming_language', 'Java'),
('is_binary', False),
('is_text', True),
('is_archive', False),
('is_media', False),
('is_source', True),
('is_script', False),
('files_count', 0),
('dirs_count', 0),
('size_count', 0),
('scan_errors', []),
]
))]
result = util.load_json(test_file, use_mapping=False, mapping_file=mapping_file)
assert expected == result
def test_load_json_without_mapping(self):
test_file = get_test_loc('load/expected.json')
expected = [
OrderedDict([('about_file_path', '/load/this.ABOUT'),
('about_resource', '.'), ('name', 'AboutCode'),
('version', '0.11.0')])
]
result = util.load_json(test_file)
assert expected == result
def test_load_json_without_mapping(self):
test_file = get_test_loc('load/expected.json')
expected = [OrderedDict([
('about_file_path', '/load/this.ABOUT'),
('about_resource', '.'),
('name', 'AboutCode'),
('version', '0.11.0')])
]
result = util.load_json(test_file)
assert expected == result
def test_load_non_list_json2(self):
test_file = get_test_loc('test_util/json/not_a_list.json')
expected = [{
'about_file_path': '/load/this.ABOUT',
'about_resource': '.',
'name': 'AboutCode',
'version': '0.11.0'
}]
result = util.load_json(test_file)
assert expected == result
def test_load_non_list_json2(self):
test_file = get_test_loc('test_util/json/not_a_list.json')
expected = [OrderedDict([
('about_file_path', '/load/this.ABOUT'),
('version', '0.11.0'),
('about_resource', '.'),
('name', 'AboutCode'),
])
]
result = util.load_json(test_file)
assert expected == result
def test_load_json2(self):
test_file = get_test_loc('test_util/json/expected_need_mapping.json')
expected = [dict(OrderedDict([
('about_file', '/load/this.ABOUT'),
('about_resource', '.'),
('version', '0.11.0'),
('name', 'AboutCode'),
])
)]
result = util.load_json(test_file)
assert expected == result
def test_load_non_list_json(self):
test_file = get_test_loc('test_util/json/not_a_list_need_mapping.json')
# FIXME: why this dict nesting??
expected = [dict(OrderedDict([
('about_resource', '.'),
('name', 'AboutCode'),
('path', '/load/this.ABOUT'),
('version', '0.11.0'),
])
)]
result = util.load_json(test_file)
assert expected == result
def test_load_non_list_json_with_mapping(self):
test_file = get_test_loc('load/not_a_list_need_mapping.json')
mapping_file = get_test_loc('custom-mapping-file/mapping.config')
expected = [dict(OrderedDict([
('about_file_path', '/load/this.ABOUT'),
('about_resource', '.'),
('name', 'AboutCode'),
('version', '0.11.0'),
])
)]
result = util.load_json(test_file, use_mapping=False, mapping_file=mapping_file)
assert expected == result
def test_load_non_list_json_with_mapping(self):
test_file = get_test_loc('load/not_a_list_need_mapping.json')
mapping_file = get_test_loc('custom-mapping-file/mapping.config')
expected = [
dict(
OrderedDict([
('about_file_path', '/load/this.ABOUT'),
('about_resource', '.'),
('name', 'AboutCode'),
('version', '0.11.0'),
]))
]
result = util.load_json(test_file,
use_mapping=False,
mapping_file=mapping_file)
assert expected == result
def load_inventory(location, base_dir, reference_dir=None):
"""
Load the inventory file at `location` for ABOUT and LICENSE files stored in
the `base_dir`. Return a list of errors and a list of About objects
validated against the `base_dir`.
Optionally use `reference_dir` as the directory location of extra reference
license and notice files to reuse.
"""
errors = []
abouts = []
base_dir = util.to_posix(base_dir)
# FIXME: do not mix up CSV and JSON
if location.endswith('.csv'):
# FIXME: this should not be done here.
dup_cols_err = check_duplicated_columns(location)
if dup_cols_err:
errors.extend(dup_cols_err)
return errors, abouts
inventory = util.load_csv(location)
else:
inventory = util.load_json(location)
try:
# FIXME: this should not be done here.
dup_about_resource_err = check_duplicated_about_resource(inventory)
if dup_about_resource_err:
errors.extend(dup_about_resource_err)
return errors, abouts
newline_in_file = check_newline_in_file_field(inventory)
if newline_in_file:
errors.extend(newline_in_file)
return errors, abouts
except Exception as e:
# TODO: why catch ALL Exception
msg = "The essential field 'about_resource' is not found in the <input>"
errors.append(Error(CRITICAL, msg))
return errors, abouts
for i, fields in enumerate(inventory):
# check does the input contains the required fields
required_fields = model.About.required_fields
for f in required_fields:
if f not in fields:
msg = "Required field: %(f)r not found in the <input>" % locals(
)
errors.append(Error(ERROR, msg))
return errors, abouts
afp = fields.get(model.About.ABOUT_RESOURCE_ATTR)
# FIXME: this should not be a failure condition
if not afp or not afp.strip():
msg = 'Empty column: %(afp)r. Cannot generate .ABOUT file.' % locals(
)
errors.append(Error(ERROR, msg))
continue
else:
afp = util.to_posix(afp)
loc = join(base_dir, afp)
about = model.About(about_file_path=afp)
about.location = loc
# Update value for 'about_resource'
# keep only the filename or '.' if it's a directory
if 'about_resource' in fields:
updated_resource_value = u''
resource_path = fields['about_resource']
if resource_path.endswith(u'/'):
updated_resource_value = u'.'
else:
updated_resource_value = basename(resource_path)
fields['about_resource'] = updated_resource_value
ld_errors = about.load_dict(
fields,
base_dir,
running_inventory=False,
reference_dir=reference_dir,
)
"""
# 'about_resource' field will be generated during the process.
# No error need to be raise for the missing 'about_resource'.
for e in ld_errors:
if e.message == 'Field about_resource is required':
ld_errors.remove(e)
"""
for e in ld_errors:
if not e in errors:
errors.extend(ld_errors)
abouts.append(about)
return unique(errors), abouts
def load_inventory(location, base_dir, license_notice_text_location=None,
use_mapping=False, mapping_file=None):
"""
Load the inventory file at `location` for ABOUT and LICENSE files
stored in the `base_dir`. Return a list of errors and a list of
About objects validated against the base_dir.
Optionally use `license_notice_text_location` as the location of
license and notice texts.
Optionally use mappings for field names if `use_mapping` is True
or a custom mapping_file if provided.
"""
errors = []
abouts = []
base_dir = util.to_posix(base_dir)
if location.endswith('.csv'):
dup_cols_err = check_duplicated_columns(location)
if dup_cols_err:
errors.extend(dup_cols_err)
return errors, abouts
inventory = util.load_csv(location, use_mapping, mapping_file)
else:
inventory = util.load_json(location, use_mapping, mapping_file)
try:
dup_about_paths_err = check_duplicated_about_file_path(inventory)
if dup_about_paths_err:
errors.extend(dup_about_paths_err)
return errors, abouts
except:
msg = (
"The essential field 'about_file_path' is not found.\n"
"Use the --mapping or --mapping-file option to map the "
"input keys and verify the mapping information are correct.\n"
"OR correct the column names in the <input>"
)
errors.append(Error(CRITICAL, msg))
return errors, abouts
for i, fields in enumerate(inventory):
# check does the input contains the required fields
required_fields = model.About.required_fields
for f in required_fields:
if f not in fields:
msg = (
"Required column: %(f)r not found.\n"
"Use the --mapping or --mapping-file option to map the "
"input keys and verify the mapping information are correct.\n"
"OR correct the column names in the <input>"
) % locals()
errors.append(Error(ERROR, msg))
return errors, abouts
afp = fields.get(model.About.about_file_path_attr)
if not afp or not afp.strip():
msg = 'Empty column: %(afp)r. Cannot generate .ABOUT file.' % locals()
errors.append(Error(ERROR, msg))
continue
else:
afp = util.to_posix(afp)
loc = join(base_dir, afp)
about = model.About(about_file_path=afp)
about.location = loc
running_inventory = False
ld_errors = about.load_dict(fields, base_dir, running_inventory,
use_mapping, mapping_file, license_notice_text_location,
with_empty=False)
# 'about_resource' field will be generated during the process.
# No error need to be raise for the missing 'about_resource'.
for e in ld_errors:
if e.message == 'Field about_resource is required':
ld_errors.remove(e)
for e in ld_errors:
if not e in errors:
errors.extend(ld_errors)
abouts.append(about)
return errors, abouts