def test_VirtualCodebase_can_be_created_from_dict(self):
test_scan = {
"scancode_notice": "Generated with ScanCode and provided on an ....",
"scancode_version": "2.9.7.post137.2e29fe3.dirty.20181120225811",
"scancode_options": {
"input": "han/",
"--json-pp": "-"
},
"scan_start": "2018-11-23T123252.191917",
"files_count": 1,
"files": [
{
"path": "han",
"type": "directory",
"scan_errors": []
},
{
"path": "han/bar.svg",
"type": "file",
"scan_errors": []
}
]
}
codebase= VirtualCodebase(test_scan)
results = sorted(r.name for r in codebase.walk())
expected = ['bar.svg', 'han']
assert expected == results
def test_virtual_codebase__create_resource_can_add_child_to_dir(self):
scan_data = self.get_test_loc('resource/virtual_codebase/resource.json')
virtual_codebase = VirtualCodebase(location=scan_data)
virtual_codebase._create_resource('some child', virtual_codebase.root, is_file=False)
results = list(virtual_codebase.walk())
expected = [('resource', False), (u'some child', False)]
assert expected == [(r.name, r.is_file) for r in results]
def test_virtual_codebase_walk_filtered_with_skip_root_and_single_file_not_filtered(self):
scan_data = self.get_test_loc('resource/virtual_codebase/et131x.h.json')
virtual_codebase = VirtualCodebase(location=scan_data)
results = list(virtual_codebase.walk_filtered(skip_root=True))
expected = [
('et131x.h', True)
]
assert expected == [(r.name, r.is_file) for r in results]
def test_virtual_codebase_can_process_minimal_resources_with_only_path(self):
scan_data = self.get_test_loc('resource/virtual_codebase/only-path.json')
codebase = VirtualCodebase(location=scan_data)
expected = [OrderedDict([
(u'path', u'NOTICE'),
(u'type', u'file'),
(u'scan_errors', [])
])]
assert expected == [r.to_dict() for r in codebase.walk()]
def test_virtual_codebase_walk_skip_root_single_file_with_children(self):
scan_data = self.get_test_loc('resource/virtual_codebase/et131x.h.json')
virtual_codebase = VirtualCodebase(location=scan_data)
c1 = virtual_codebase._create_resource('some child', parent=virtual_codebase.root, is_file=True)
_c2 = virtual_codebase._create_resource('some child2', parent=c1, is_file=False)
results = list(virtual_codebase.walk(skip_root=True))
expected = [
(u'some child', True), (u'some child2', False)
]
assert expected == [(r.name, r.is_file) for r in results]
def test_get_package_resources_on_nested_packages_should_include_manifest(self):
from packagedcode import get_package_instance
from scancode.resource import VirtualCodebase
scan_loc = self.get_test_loc('plugin_consolidate/nested-npm-packages.json')
codebase = VirtualCodebase(scan_loc)
for resource in codebase.walk():
for package_data in resource.packages:
package = get_package_instance(package_data)
package_resources = list(package.get_package_resources(resource, codebase))
assert any(r.name == 'package.json' for r in package_resources), resource.path
def test_virtual_codebase_walk_skip_root_basic(self):
scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json')
virtual_codebase = VirtualCodebase(location=scan_data)
results = list(virtual_codebase.walk(skip_root=True))
expected = [
('abc', True),
('et131x.h', True),
('dir', False),
('that', True),
('this', True),
('other dir', False),
('file', True),
]
assert expected == [(r.name, r.is_file) for r in results]
def test_virtual_codebase_walk_bottomup(self):
test_file = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json')
codebase = VirtualCodebase(location=test_file)
results = list(codebase.walk(topdown=False))
expected = [
('abc', True),
('et131x.h', True),
('that', True),
('this', True),
('dir', False),
('file', True),
('other dir', False),
('codebase', False),
]
assert expected == [(r.name, r.is_file) for r in results]
def test_Package_get_package_resource_works_with_nested_packages_and_ignores(
self):
from packagedcode import get_package_instance
from packagedcode import npm
from scancode.resource import VirtualCodebase
scan_loc = self.get_test_loc('models/nested-npm-packages.json')
codebase = VirtualCodebase(scan_loc)
for resource in codebase.walk():
for package_data in resource.packages:
package = get_package_instance(package_data)
assert isinstance(package, npm.NpmPackage)
package_resources = list(
package.get_package_resources(resource, codebase))
assert any(r.name == 'package.json'
for r in package_resources), resource.path
def test_HtmlOutput_process_codebase_does_not_fail_with_non_ascii_scanned_paths_and_file_opened_in_text_mode_with_utf(
):
test_scan = '''{
"scancode_notice": "Generated with ScanCode...",
"scancode_version": "2.9.7.post137.2e29fe3.dirty.20181120225811",
"scancode_options": {
"input": "han/",
"--json-pp": "-"
},
"scan_start": "2018-11-23T123252.191917",
"files_count": 1,
"files": [
{
"path": "han",
"type": "directory",
"scan_errors": []
},
{
"path": "han/\u636e.svg",
"type": "file",
"scan_errors": []
}
]
}'''
codebase = VirtualCodebase(test_scan)
result_file = test_env.get_temp_file('html')
ho = HtmlOutput()
with io.open(result_file, 'w', encoding='utf-8') as html:
ho.process_codebase(codebase, html)
results = io.open(result_file, encoding='utf-8').read()
assert '<td>han/据.svg</td>' in results
def test_HtmlOutput_process_codebase_fails_with_non_ascii_scanned_paths_and_file_opened_in_binary_mode(
):
test_scan = '''{
"scancode_notice": "Generated with ScanCode...",
"scancode_version": "2.9.7.post137.2e29fe3.dirty.20181120225811",
"scancode_options": {
"input": "han/",
"--json-pp": "-"
},
"scan_start": "2018-11-23T123252.191917",
"files_count": 1,
"files": [
{
"path": "han",
"type": "directory",
"scan_errors": []
},
{
"path": "han/\u636e.svg",
"type": "file",
"scan_errors": []
}
]
}'''
codebase = VirtualCodebase(test_scan)
result_file = test_env.get_temp_file('html')
ho = HtmlOutput()
try:
with open(result_file, 'wb') as html:
ho.process_codebase(codebase, html)
raise Exception('Exception not raised.')
except Exception as e:
assert 'UnicodeEncodeError' in str(e)
def test_virtual_codebase_cache_mixed_two_in_memory(self):
scan_data = self.get_test_loc('resource/virtual_codebase/cache2.json')
virtual_codebase = VirtualCodebase(location=scan_data,
max_in_memory=2)
for rid in virtual_codebase.resource_ids:
if rid == 0:
assert virtual_codebase.root == virtual_codebase.get_resource(rid)
assert virtual_codebase._exists_in_memory(rid)
assert not virtual_codebase._exists_on_disk(rid)
elif rid < 2:
assert virtual_codebase._exists_in_memory(rid)
assert not virtual_codebase._exists_on_disk(rid)
else:
assert not virtual_codebase._exists_in_memory(rid)
assert virtual_codebase._exists_on_disk(rid)
assert len(virtual_codebase.resource_ids) == len(list(virtual_codebase.walk()))
def test_virtual_codebase_walk_filtered_with_filtered_root(self):
scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json')
virtual_codebase = VirtualCodebase(location=scan_data)
virtual_codebase.root.is_filtered = True
virtual_codebase.save_resource(virtual_codebase.root)
results = list(virtual_codebase.walk_filtered())
expected = [
('abc', True),
('et131x.h', True),
('dir', False),
('that', True),
('this', True),
('other dir', False),
('file', True),
]
assert expected == [(r.name, r.is_file) for r in results]
def test_virtual_codebase_can_process_minimal_resources_without_info(self):
scan_data = self.get_test_loc('resource/virtual_codebase/noinfo.json')
codebase = VirtualCodebase(location=scan_data)
expected = [
OrderedDict([
(u'path', u'NOTICE'),
(u'type', u'file'),
(u'copyrights', [
OrderedDict([
(u'statements', [u'Copyright (c) 2017 nexB Inc. and others.']),
(u'holders', [u'nexB Inc. and others.']),
(u'authors', []),
(u'start_line', 4),
(u'end_line', 4)
])
]),
(u'scan_errors', [])
])
]
assert expected == [r.to_dict() for r in codebase.walk()]
def test_virtual_codebase_compute_counts_filtered_all_with_cache(self):
scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json')
virtual_codebase = VirtualCodebase(location=scan_data)
for res in virtual_codebase.walk():
res.is_filtered = True
virtual_codebase.save_resource(res)
results = virtual_codebase.compute_counts(skip_filtered=True)
expected = (0, 0, 0)
assert expected == results
def test_virtual_codebase_walk_filtered_all_skip_root(self):
scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json')
virtual_codebase = VirtualCodebase(location=scan_data)
for res in virtual_codebase.walk():
res.is_filtered = True
virtual_codebase.save_resource(res)
results = list(virtual_codebase.walk_filtered(skip_root=True))
expected = []
assert expected == [(r.name, r.is_file) for r in results]
def test_virtual_codebase_compute_counts_filtered_None_with_size(self):
scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json')
virtual_codebase = VirtualCodebase(location=scan_data)
for res in virtual_codebase.walk():
if res.is_file:
res.size = 10
virtual_codebase.save_resource(res)
results = virtual_codebase.compute_counts(skip_filtered=True)
expected = (5, 3, 50)
assert expected == results
def test_virtual_codebase_cache_default(self):
scan_data = self.get_test_loc('resource/virtual_codebase/cache2.json')
virtual_codebase = VirtualCodebase(location=scan_data)
assert virtual_codebase.temp_dir
assert virtual_codebase.cache_dir
virtual_codebase.cache_dir
root = virtual_codebase.root
cp = virtual_codebase._get_resource_cache_location(root.rid, create=False)
assert not exists(cp)
cp = virtual_codebase._get_resource_cache_location(root.rid, create=True)
assert not exists(cp)
assert exists(parent_directory(cp))
child = virtual_codebase._create_resource('child', root, is_file=True)
child.size = 12
virtual_codebase.save_resource(child)
child_2 = virtual_codebase.get_resource(child.rid)
assert child == child_2
def check_plugin(plugin_class, test_file='reuse/vb.json', binary=False):
# this is the result of this scan:
# ./scancode -clip --summary --license-clarity-score --summary-key-files
# --classify samples/ --json-pp vb.json -n
test_file = test_env.get_test_loc(test_file)
from scancode.resource import VirtualCodebase
cb = VirtualCodebase(test_file)
result_file = test_env.get_temp_file('reuse')
op = plugin_class()
if binary:
with io.open(result_file, 'wb') as out:
op.process_codebase(cb, out)
else:
with io.open(result_file, 'w', encoding='utf-8') as out:
op.process_codebase(cb, out)
with io.open(result_file, 'r', encoding='utf-8') as inp:
assert 'zlib' in inp.read()
def test_virtual_codebase_walk_filtered_dirs(self):
scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json')
virtual_codebase = VirtualCodebase(location=scan_data)
for res in virtual_codebase.walk():
if not res.is_file:
res.is_filtered = True
virtual_codebase.save_resource(res)
results = list(virtual_codebase.walk_filtered(topdown=True))
expected = [
('abc', True),
('et131x.h', True),
('that', True),
('this', True),
('file', True),
]
assert expected == [(r.name, r.is_file) for r in results]
def test_virtual_codebase_compute_counts_filtered_None_with_cache(self):
scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json')
virtual_codebase = VirtualCodebase(location=scan_data)
results = virtual_codebase.compute_counts(skip_filtered=True)
expected = (5, 3, 2228)
assert expected == results
def test_virtual_codebase_has_default_for_plugin_attributes(self):
scan_data = self.get_test_loc('resource/virtual_codebase/only-path.json')
VirtualCodebase(location=scan_data)
def test_lowest_common_parent_on_virtual_codebase(self):
scan_data = self.get_test_loc('resource/virtual_codebase/lcp.json')
virtual_codebase = VirtualCodebase(location=scan_data)
lcp = virtual_codebase.lowest_common_parent()
assert 'lcp/test1' == lcp.path
assert 'test1' == lcp.name
def test_virtual_codebase_get_resource(self):
scan_data = self.get_test_loc('resource/virtual_codebase/resource.json')
virtual_codebase = VirtualCodebase(location=scan_data)
assert not (virtual_codebase.root is virtual_codebase.get_resource(0))
assert virtual_codebase.root == virtual_codebase.get_resource(0)
def test_VirtualCodebase_can_be_created_from_json_file(self):
test_file = self.get_test_loc('resource/virtual_codebase/from_file.json')
codebase= VirtualCodebase(test_file)
results = sorted(r.name for r in codebase.walk())
expected = ['bar.svg', 'han']
assert expected == results