def test__create_resource_can_add_child_to_file(self):
test_codebase = self.get_test_loc('resource/codebase/et131x.h')
codebase = Codebase(test_codebase)
codebase._create_resource('some child', codebase.root, is_file=True)
results = list(codebase.walk())
expected = [('et131x.h', True), (u'some child', True)]
assert expected == [(r.name, r.is_file) for r in results]
def test_display_summary_edge_case_scan_time_zero():
from cStringIO import StringIO
import sys
from scancode.cli import display_summary
from scancode.resource import Codebase
# Set up test codebase
test_codebase = test_env.get_test_loc('resource/client')
codebase = Codebase(test_codebase, strip_root=True)
codebase.timings['scan'] = 0
scan_names = ''
processes = 0
verbose = False
errors = []
# Redirect summary output from `stderr` to `result`
result = StringIO()
sys.stderr = result
# Output from `display_summary` will be in `result`
display_summary(codebase, scan_names, processes, errors, verbose)
# Set `stderr` back
sys.stderr = sys.__stderr__
# No exception should be thrown and this assertion should pass
assert 'Scan Speed: 0.00 files/sec.' in result.getvalue()
def test__create_resource_can_add_child_to_dir(self):
test_codebase = self.get_temp_dir('resource')
codebase = Codebase(test_codebase)
codebase._create_resource('some child', codebase.root, is_file=False)
results = list(codebase.walk())
expected = [('resource', False), (u'some child', False)]
assert expected == [(r.name, r.is_file) for r in results]
def test_compute_counts_when_using_disk_cache(self):
test_codebase = self.get_test_loc('resource/samples')
codebase = Codebase(test_codebase, strip_root=True, max_in_memory=-1)
files_count, dirs_count, size_count = codebase.compute_counts()
assert files_count == 33
assert dirs_count == 11
assert size_count == 0
def test_low_max_in_memory_does_not_raise_exception_when_ignoring_files(self):
def is_ignored(location, ignores):
"""
Return a tuple of (pattern , message) if a file at location is ignored or
False otherwise. `ignores` is a mappings of patterns to a reason.
Taken from scancode/plugin_ignore.py
"""
from commoncode.fileset import match
return match(location, includes=ignores, excludes={})
from functools import partial
test_codebase = self.get_test_loc('resource/client')
codebase = Codebase(test_codebase, strip_root=True, max_in_memory=1)
# Ignore GIFs, code taken from scancode/plugin_ignore.py
ignores = {
'*.gif': 'User ignore: Supplied by --ignore'
}
ignorable = partial(is_ignored, ignores=ignores)
remove_resource = codebase.remove_resource
for resource in codebase.walk(topdown=True):
if ignorable(resource.path):
for child in resource.children(codebase):
remove_resource(child)
remove_resource(resource)
# Walk through the codebase and save each Resource,
# UnknownResource exception should not be raised
save_resource = codebase.save_resource
for resource in codebase.walk(topdown=True):
save_resource(resource)
def test_root_dir(self):
test_file = self.get_test_loc('haxe/tags/haxelib.json')
test_dir = self.get_test_loc('haxe/tags')
codebase = Codebase(test_dir)
manifest_resource = codebase.get_resource_from_path(test_file,
absolute=True)
proot = haxe.HaxePackage.get_package_root(manifest_resource, codebase)
assert test_dir == proot.location
def test_walk_filtered_with_skip_root_and_single_file_not_filtered(self):
test_codebase = self.get_test_loc('resource/codebase/et131x.h')
codebase = Codebase(test_codebase)
results = list(codebase.walk_filtered(skip_root=True))
expected = [
('et131x.h', True)
]
assert expected == [(r.name, r.is_file) for r in results]
def test_walk_skip_root_single_dir(self):
test_codebase = self.get_temp_dir('walk')
codebase = Codebase(test_codebase, strip_root=True)
results = list(codebase.walk(skip_root=True))
expected = [
('walk', False)
]
assert expected == [(r.name, r.is_file) for r in results]
def test_set_classification_flags_is_manifest(self):
test_dir = self.get_test_loc('classify/manifest')
codebase = Codebase(
test_dir, resource_attributes=FileClassifier.resource_attributes)
for res in codebase.walk():
if not res.is_file:
continue
set_classification_flags(res)
assert res.is_manifest
def test_walk_skip_root_single_file_with_children(self):
test_codebase = self.get_test_loc('resource/codebase/et131x.h')
codebase = Codebase(test_codebase, strip_root=True)
c1 = codebase._create_resource('some child', parent=codebase.root, is_file=True)
_c2 = codebase._create_resource('some child2', parent=c1, is_file=False)
results = list(codebase.walk(skip_root=True))
expected = [
(u'some child', True), (u'some child2', False)
]
assert expected == [(r.name, r.is_file) for r in results]
def test_package_root_is_properly_returned_for_metainf_poms(self):
test_dir = self.get_test_loc('maven_misc/package_root')
resource_attributes = dict(packages=attr.ib(default=attr.Factory(list), repr=False))
codebase = Codebase(test_dir, resource_attributes=resource_attributes)
manifest_resource = [r for r in codebase.walk() if r.name == 'pom.xml'][0]
package = maven.MavenPomPackage.recognize(manifest_resource.location)
manifest_resource.packages.append(package.to_dict())
manifest_resource.save(codebase)
proot = maven.MavenPomPackage.get_package_root(manifest_resource, codebase)
assert 'activiti-image-generator-7-201802-EA-sources.jar-extract' == proot.name
def test_root_dir(self):
test_file = self.get_test_loc(
'conda/requests-kerberos-0.8.0-py35_0.tar.bz2-extract/info/recipe.tar-extract/recipe/meta.yaml'
)
test_dir = self.get_test_loc(
'conda/requests-kerberos-0.8.0-py35_0.tar.bz2-extract')
codebase = Codebase(test_dir)
manifest_resource = codebase.get_resource_from_path(test_file,
absolute=True)
proot = conda.CondaPackage.get_package_root(manifest_resource,
codebase)
assert test_dir == proot.location
def test_distance(self):
test_dir = self.get_test_loc('resource/dist')
codebase = Codebase(test_dir)
assert 0 == codebase.root.distance(test_dir)
res = codebase.get_resource(1)
assert 'JGroups' == res.name
assert 1 == res.distance(codebase)
res = codebase.get_resource(10)
assert 'MANIFEST.MF' == res.name
assert 3 == res.distance(codebase)
def test_walk_skip_root_basic(self):
test_codebase = self.get_test_loc('resource/codebase')
codebase = Codebase(test_codebase)
results = list(codebase.walk(skip_root=True))
expected = [
('abc', True),
('et131x.h', True),
('dir', False),
('that', True),
('this', True),
('other dir', False),
('file', True),
]
assert expected == [(r.name, r.is_file) for r in results]
def test_walk_filtered_with_filtered_root(self):
test_codebase = self.get_test_loc('resource/codebase')
codebase = Codebase(test_codebase)
codebase.root.is_filtered = True
codebase.save_resource(codebase.root)
results = list(codebase.walk_filtered())
expected = [
('abc', True),
('et131x.h', True),
('dir', False),
('that', True),
('this', True),
('other dir', False),
('file', True),
]
assert expected == [(r.name, r.is_file) for r in results]
def test_package_root_is_properly_returned_for_metainf_poms(self):
from packagedcode.plugin_package import PackageScanner
test_dir = self.get_test_loc('maven_misc/package_root')
codebase = Codebase(
test_dir, resource_attributes=PackageScanner.resource_attributes)
manifest_resource = [
r for r in codebase.walk() if r.name == 'pom.xml'
][0]
packages = list(
maven.MavenPomPackage.recognize(manifest_resource.location))
assert packages
manifest_resource.packages.append(packages[0].to_dict())
manifest_resource.save(codebase)
proot = maven.MavenPomPackage.get_package_root(manifest_resource,
codebase)
assert 'activiti-image-generator-7-201802-EA-sources.jar-extract' == proot.name
def test_codebase_cache_mixed_two_in_memory(self):
test_codebase = self.get_test_loc('resource/cache2')
codebase = Codebase(test_codebase, max_in_memory=2)
for rid in codebase.resource_ids:
if rid == 0:
assert codebase.root == codebase.get_resource(rid)
assert codebase._exists_in_memory(rid)
assert not codebase._exists_on_disk(rid)
elif rid < 2:
assert codebase._exists_in_memory(rid)
assert not codebase._exists_on_disk(rid)
else:
assert not codebase._exists_in_memory(rid)
assert codebase._exists_on_disk(rid)
assert len(codebase.resource_ids) == len(list(codebase.walk()))
def test_build_get_package_resources(self):
test_loc = self.get_test_loc('build/get_package_resources')
codebase = Codebase(test_loc)
root = codebase.root
expected = [
'get_package_resources',
'get_package_resources/BUCK',
'get_package_resources/file1',
]
results = [r.path for r in build.BaseBuildManifestPackage.get_package_resources(root, codebase)]
assert expected == results
def test_npm_get_package_resources(self):
test_loc = self.get_test_loc('npm/get_package_resources')
codebase = Codebase(test_loc)
root = codebase.root
expected = [
'get_package_resources', 'get_package_resources/package.json',
'get_package_resources/this-should-be-returned'
]
results = [
r.path
for r in npm.NpmPackage.get_package_resources(root, codebase)
]
assert expected == results
def test_get_displayable_summary():
from scancode.cli import get_displayable_summary
from scancode.resource import Codebase
# Set up test codebase
test_codebase = test_env.get_test_loc('resource/client')
codebase = Codebase(test_codebase, strip_root=True)
codebase.timings['scan'] = 0
scan_names = 'foo, bar, baz'
processes = 23
errors = ['failed to scan ABCD']
results = get_displayable_summary(codebase, scan_names, processes, errors)
expected = ([
u'Some files failed to scan properly:', u'failed to scan ABCD'
], [
u'Summary: foo, bar, baz with 23 process(es)',
u'Errors count: 1', u'Scan Speed: 0.00 files/sec. ',
u'Initial counts: 0 resource(s): 0 file(s) and 0 directorie(s) ',
u'Final counts: 0 resource(s): 0 file(s) and 0 directorie(s) ',
u'Timings:', u' scan_start: None', u' scan_end: None'
])
assert expected == results
def test_compute_counts_filtered_all_with_cache(self):
test_codebase = self.get_test_loc('resource/codebase')
codebase = Codebase(test_codebase)
for res in codebase.walk():
res.is_filtered = True
codebase.save_resource(res)
results = codebase.compute_counts(skip_filtered=True)
expected = (0, 0, 0)
assert expected == results
def test_walk_filtered_all_skip_root(self):
test_codebase = self.get_test_loc('resource/codebase')
codebase = Codebase(test_codebase)
for res in codebase.walk():
res.is_filtered = True
codebase.save_resource(res)
results = list(codebase.walk_filtered(skip_root=True))
expected = []
assert expected == [(r.name, r.is_file) for r in results]
def test_display_summary_edge_case_scan_time_zero_should_not_fail():
from io import StringIO
import sys
from scancode.cli import display_summary
from scancode.resource import Codebase
# Set up test codebase
test_codebase = test_env.get_test_loc('resource/client')
codebase = Codebase(test_codebase, strip_root=True)
codebase.timings['scan'] = 0
scan_names = 'foo, bar, baz'
processes = 23
errors = ['failed to scan ABCD']
try:
# Redirect summary output from `stderr` to `result`
result = StringIO()
sys.stderr = result
# Output from `display_summary` will be in `result`
display_summary(codebase, scan_names, processes, errors)
finally:
# Set `stderr` back
sys.stderr = sys.__stderr__
def test_compute_counts_filtered_None_with_size(self):
test_codebase = self.get_test_loc('resource/codebase')
codebase = Codebase(test_codebase)
for res in codebase.walk():
if res.is_file:
res.size = 10
codebase.save_resource(res)
results = codebase.compute_counts(skip_filtered=True)
expected = (5, 3, 50)
assert expected == results
def test_codebase_cache_default(self):
test_codebase = self.get_test_loc('resource/cache2')
codebase = Codebase(test_codebase)
assert codebase.temp_dir
assert codebase.cache_dir
codebase.cache_dir
root = codebase.root
cp = codebase._get_resource_cache_location(root.rid, create=False)
assert not exists(cp)
cp = codebase._get_resource_cache_location(root.rid, create=True)
assert not exists(cp)
assert exists(parent_directory(cp))
child = codebase._create_resource('child', root, is_file=True)
child.size = 12
codebase.save_resource(child)
child_2 = codebase.get_resource(child.rid)
assert child == child_2
def test_walk_filtered_dirs(self):
test_codebase = self.get_test_loc('resource/codebase')
codebase = Codebase(test_codebase)
for res in codebase.walk():
if not res.is_file:
res.is_filtered = True
codebase.save_resource(res)
results = list(codebase.walk_filtered(topdown=True))
expected = [
('abc', True),
('et131x.h', True),
('that', True),
('this', True),
('file', True),
]
assert expected == [(r.name, r.is_file) for r in results]
def test_lowest_common_parent_solo_file_strip(self):
test_codebase = self.get_test_loc('resource/lcp/test1/screenshot.png')
codebase = Codebase(test_codebase, strip_root=True)
lcp = codebase.lowest_common_parent()
assert 'screenshot.png' == lcp.path
assert 'screenshot.png' == lcp.name
def test_lowest_common_parent_deep(self):
test_codebase = self.get_test_loc('resource/lcp/test1/simple/org')
codebase = Codebase(test_codebase)
lcp = codebase.lowest_common_parent()
assert 'org/jvnet/glassfish/comms/sipagent' == lcp.path
assert 'sipagent' == lcp.name
def test_lowest_common_parent_3(self):
test_codebase = self.get_test_loc('resource/lcp/test1/simple')
codebase = Codebase(test_codebase)
lcp = codebase.lowest_common_parent()
assert 'simple' == lcp.path
assert 'simple' == lcp.name
def test_lowest_common_parent_full(self):
test_codebase = self.get_test_loc('resource/lcp/test1')
codebase = Codebase(test_codebase, full_root=True)
lcp = codebase.lowest_common_parent()
assert 'test1' == lcp.name
