def analyze(self):
pkgs_required = self.requirements_txt.packages
advice_list = []
for pkg_req in pkgs_required:
pkg_installed = self.installed_packages.get_by_name(pkg_req.name)
if not pkg_installed:
advice = Advice(
analyzer=self,
severity='error',
message=("Required dependency '%s' is not installed" %
pkg_req.as_display_name()),
)
advice_list.append(advice)
if pkg_installed and not pkg_req.is_satisfied_by(pkg_installed):
advice = Advice(
analyzer=self,
severity='error',
message=
"Required dependency '%s' is not satisfied by installed '%s'"
% (
pkg_req.as_display_name(),
pkg_installed.as_display_name(),
),
)
advice_list.append(advice)
return AdviceList(advice_list=advice_list)
def test_advice_equality():
analyzer1 = RequiredNotInstalledAnalyzer(
requirements_txt=None,
installed_packages=None,
)
adv1 = Advice(analyzer=analyzer1,
severity='error',
message='Package not good')
# different analyzer
# different severity
adv3 = Advice(analyzer=analyzer1,
severity='warn',
message='Package not good')
# different message
adv4 = Advice(analyzer=analyzer1,
severity='error',
message='Package very good')
assert not adv1 == None # noqa
assert adv1 == adv1
assert not adv1 != adv1
assert adv1 != adv3
assert not adv1 == adv3
assert adv1 != adv4
assert not adv1 == adv4
# Test repr
assert repr(adv1) == ("<Advice analyzer='RequiredNotInstalledAnalyzer', "
"severity='error', message='Package not good'>")
def test_advice_list_equality():
advl1 = AdviceList(advice_list=[
Advice(analyzer=None, severity='info', message='It rains'),
Advice(analyzer=None, severity='debug', message='It snows'),
], )
# missing one advice
advl2 = AdviceList(advice_list=[
Advice(analyzer=None, severity='info', message='It rains'),
], )
# severity differs
advl3 = AdviceList(advice_list=[
Advice(analyzer=None, severity='error', message='It rains'),
Advice(analyzer=None, severity='debug', message='It snows'),
], )
assert not advl1 == None # noqa
assert advl1 == advl1
assert not advl1 != advl1
assert advl1 != advl2
assert not advl1 == advl2
assert advl1 != advl3
assert not advl1 == advl3
# Test repr
assert repr(advl1) == ("<AdviceList num_advice=2>")
# XXX improve this
assert advl1.has_problems() is False
assert advl2.has_problems() is False
assert advl3.has_problems() is True
def analyze(self):
advice_list = []
pkgs_installed = self.installed_packages.packages
for pkg_ins in pkgs_installed:
pkg_req = self.requirements_txt.get_by_name(pkg_ins.name)
dependents = self.site_packages.get_package_dependents(pkg_ins.name)
# The installed dependency is not a transitive dependency
if not dependents:
# Check if it's a required dependency
if not pkg_req and pkg_ins.name not in self.VIRTUAL_DEPENDENCIES:
advice = Advice(
analyzer=self,
severity='warn',
message=(
"Installed non-transitive dependency '%s' is not required"
) % (
pkg_ins.as_display_name(),
),
)
advice_list.append(advice)
# The installed dependency is a transitive dependency
elif dependents:
# Check if it's a required dependency
if pkg_req:
# TODO: Here we should pick a dependent which is also a
# requirement, rather than (possibly) getting an untracked
# top level package (which will be reported in the case
# above anyway)
first_dependent = dependents[0]
dependent_req = self.requirements_txt.get_by_name(
first_dependent,
ignore_case=True,
)
# This is displayed as:
# requests==2.18.4' is a transitive dependency of 'datadog==0.16.0'
# and is a bit misleading, because we know it's the package
# by name that is a transitive dependency, not by exact
# version...
advice = Advice(
analyzer=self,
severity='info',
message=(
"Required dependency '%s' is a transitive dependency of '%s'"
) % (
pkg_req.as_display_name(),
(dependent_req.as_display_name() if dependent_req
else first_dependent),
),
)
advice_list.append(advice)
return AdviceList(advice_list=advice_list)
def test_required_installed_analyzer():
pkgs_reqs = RequirementsTxt(packages=[
PackageRequirement(name='coverage', operator='<', version='3.6'),
PackageRequirement(name='parsimonious', operator='<=', version='0.5'),
], )
pkgs_installed = InstalledPackages(
packages=[
InstalledPackage(name='coverage', version='3.5'), # ok
InstalledPackage(name='parsimonious', version='0.5'), # ok
], )
analyzer = RequiredInstalledAnalyzer(
requirements_txt=pkgs_reqs,
installed_packages=pkgs_installed,
)
advice_list = analyzer.analyze()
expected_advice_list = AdviceList(advice_list=[
Advice(
analyzer=analyzer,
severity='debug',
message=("Required dependency 'coverage<3.6' "
"is satisfied by installed 'coverage-3.5'"),
),
Advice(
analyzer=analyzer,
severity='debug',
message=("Required dependency 'parsimonious<=0.5' "
"is satisfied by installed 'parsimonious-0.5'"),
),
], )
assert expected_advice_list == advice_list
def test_advice_format_display_line():
analyzer1 = RequiredNotInstalledAnalyzer(
requirements_txt=None,
installed_packages=None,
)
adv1 = Advice(analyzer=analyzer1,
severity='error',
message='Package not good')
assert adv1.format_display_line() == (
'[RequiredNotInstalled] error: Package not good')
def analyze(self):
advice_list = []
pkgs = [
DistributionStub(pkg) for pkg in self.installed_packages.packages
]
vulns = self.check_func(
packages=pkgs,
key=None,
db_mirror=None,
cached=None,
ignore_ids=[],
)
for vuln in vulns:
message = (
"Installed dependency '%s' has a known vulnerability in '%s'\n %s"
) % (
'%s-%s' % (vuln.name, vuln.version),
'%s%s' % (vuln.name, vuln.spec),
vuln.advisory,
)
advice = Advice(
analyzer=self,
severity='warn',
message=message,
)
advice_list.append(advice)
return AdviceList(advice_list=advice_list)
def test_is_vulnerable_analyzer():
pkgs_installed = InstalledPackages(packages=[
InstalledPackage(name='click', version='1.8'),
InstalledPackage(name='tornado', version='2.2.0'),
], )
analyzer = IsVulnerableAnalyzer(installed_packages=pkgs_installed, )
# mock checker
analyzer.check_func = mocked_check_func
advice_list = analyzer.analyze()
expected_advice_list = AdviceList(advice_list=[
Advice(
analyzer=analyzer,
severity='warn',
message=("Installed dependency 'tornado-2.2.0' "
"has a known vulnerability in 'tornado<2.2.1'\n"
" CRLF injection vulnerability in the "
"tornado.web.RequestHandler.set_header function "
"in Tornado before 2.2.1 allows remote attackers "
"to inject arbitrary HTTP headers and conduct HTTP "
"response splitting attacks via crafted input."),
),
], )
assert expected_advice_list == advice_list
def analyze(self):
pkgs_required = self.requirements_txt.packages
advice_list = []
for pkg_req in pkgs_required:
# Check if the requirement can be upgraded
pkg_releases = self.available_packages.get_by_name(pkg_req.name)
pkg_release = pkg_releases.get_more_recent_than_requirement(
pkg_req)
if pkg_release:
advice = Advice(
analyzer=self,
severity='info',
message="Required dependency '%s' can be upgraded to '%s'"
% (
pkg_req.as_display_name(),
pkg_release.as_display_name_single(),
),
)
advice_list.append(advice)
# Check if the installed version can be updated
pkg_installed = self.installed_packages.get_by_name(pkg_req.name)
if pkg_installed:
pkg_release = pkg_releases.update_installed(
pkg_req, pkg_installed)
if pkg_release:
advice = Advice(
analyzer=self,
severity='info',
message=
"Installed dependency '%s' can be updated to '%s'" % (
pkg_installed.as_display_name(),
pkg_release.as_display_name_single(),
),
)
advice_list.append(advice)
return AdviceList(advice_list=advice_list)
def test_required_not_installed_analyzer():
pkgs_reqs = RequirementsTxt(packages=[
PackageRequirement(name='coverage', operator='<', version='3.6'),
PackageRequirement(name='parsimonious', operator='<=', version='0.5'),
PackageRequirement(name='six', operator='>=', version='0.5'),
], )
pkgs_installed = InstalledPackages(
packages=[
InstalledPackage(name='coverage', version='3.5'), # ok
# parsimonious missing
InstalledPackage(name='six', version='0.4.9'), # version too low
], )
analyzer = RequiredNotInstalledAnalyzer(
requirements_txt=pkgs_reqs,
installed_packages=pkgs_installed,
)
advice_list = analyzer.analyze()
expected_advice_list = AdviceList(advice_list=[
Advice(
analyzer=analyzer,
severity='error',
message=(
"Required dependency 'parsimonious<=0.5' is not installed"),
),
Advice(
analyzer=analyzer,
severity='error',
message=("Required dependency 'six>=0.5' "
"is not satisfied by installed 'six-0.4.9'"),
),
], )
assert expected_advice_list == advice_list
def test_is_unused_analyzer():
pkgs_reqs = RequirementsTxt(packages=[
PackageRequirement(name='click', operator='==', version='1.8'),
PackageRequirement(name='six', operator='==', version='3.5'),
], )
pkgs_installed = InstalledPackages(packages=[
InstalledPackage(name='click', version='1.8'),
InstalledPackage(name='six', version='3.5'),
], )
# mock: the top level of the package is the name of the package
site_packages = SitePackages(
python_path=None,
installed_package_names=[],
)
site_packages.get_package_top_levels = lambda pkg_name: [pkg_name]
# mock: grep says six is used, nothing else is
git_grep = GitGrep(basedir=None)
git_grep.package_is_imported = lambda pkg_name: (True if pkg_name == 'six'
else False)
analyzer = IsUnusedAnalyzer(
requirements_txt=pkgs_reqs,
installed_packages=pkgs_installed,
site_packages=site_packages,
grep=git_grep,
)
advice_list = analyzer.analyze()
expected_advice_list = AdviceList(advice_list=[
Advice(
analyzer=analyzer,
severity='info',
message=("Required dependency 'click==1.8' "
"is never imported (click)"),
),
], )
assert expected_advice_list == advice_list
def analyze(self):
advice_list = []
pkgs_required = self.requirements_txt.packages
for pkg_req in pkgs_required:
# If the package is installed we can discover its top level
# importable names
pkg_ins = self.installed_packages.get_by_name(pkg_req.name)
if pkg_ins:
# If it have top level names we can see if they are ever
# imported in the code
top_levels = self.site_packages.get_package_top_levels(
pkg_ins.name)
if top_levels:
# Check every top level name
is_imported = False
for top_level in top_levels:
if self.grep.package_is_imported(top_level):
is_imported = True
if not is_imported:
advice = Advice(
analyzer=self,
severity='info',
message=
("Required dependency '%s' is never imported (%s)")
%
(pkg_req.as_display_name(), ', '.join(top_levels)),
)
advice_list.append(advice)
return AdviceList(advice_list=advice_list)
def test_advice_ctor():
with pytest.raises(ValueError):
Advice(analyzer=None, severity='extreme', message='OK')
def test_can_be_upgraded_analyzer():
pkgs_available = AvailablePackages(
packages=[
PackageReleases(name='abc', versions=('8.5', '8.6')),
PackageReleases(name='click', versions=('1.8', '1.9')),
PackageReleases(name='coverage', versions=('3.5', '3.6', '4.0', '4.1')),
PackageReleases(name='ply', versions=('0.5', '0.5.1')),
PackageReleases(name='six', versions=('0.8', '0.9')),
],
)
pkgs_reqs = RequirementsTxt(
packages=[
PackageRequirement(name='abc', operator='<=', version='9.0'),
PackageRequirement(name='click', operator=None, version=None),
PackageRequirement(name='coverage', operator='<', version='4.0'), # -> 4.1
PackageRequirement(name='ply', operator='==', version='0.5'), # -> 0.5.1
PackageRequirement(name='six', operator='==', version='0.9'),
],
)
pkgs_installed = InstalledPackages(
packages=[
InstalledPackage(name='abc', version='8.5'), # -> 8.6
InstalledPackage(name='click', version='1.8'), # -> 1.9
InstalledPackage(name='coverage', version='3.5'), # -> 3.6
],
)
analyzer = CanBeUpgradedAnalyzer(
requirements_txt=pkgs_reqs,
installed_packages=pkgs_installed,
available_packages=pkgs_available,
)
advice_list = analyzer.analyze()
expected_advice_list = AdviceList(
advice_list=[
Advice(
analyzer=analyzer,
severity='info',
message=(
"Installed dependency 'abc-8.5' "
"can be updated to 'abc-8.6'"
),
),
Advice(
analyzer=analyzer,
severity='info',
message=(
"Installed dependency 'click-1.8' "
"can be updated to 'click-1.9'"
),
),
Advice(
analyzer=analyzer,
severity='info',
message=(
"Required dependency 'coverage<4.0' "
"can be upgraded to 'coverage-4.1'"
),
),
Advice(
analyzer=analyzer,
severity='info',
message=(
"Installed dependency 'coverage-3.5' "
"can be updated to 'coverage-3.6'"
),
),
Advice(
analyzer=analyzer,
severity='info',
message=(
"Required dependency 'ply==0.5' "
"can be upgraded to 'ply-0.5.1'"
),
),
],
)
assert expected_advice_list == advice_list
def test_can_be_upgraded_analyzer():
pkgs_reqs = RequirementsTxt(
packages=[
PackageRequirement(name='locustio', operator='==',
version='0.8a2'), # top level
PackageRequirement(name='Flask', operator='==',
version='0.12.2'), # one level down
PackageRequirement(name='Jinja2', operator='==',
version='2.9.6'), # two levels down
], )
pkgs_installed = InstalledPackages(
packages=[
# locustio dependency closure
InstalledPackage(name='Flask', version='0.12.2'),
InstalledPackage(name='Jinja2', version='2.9.6'),
InstalledPackage(name='MarkupSafe', version='1.0'),
InstalledPackage(name='Werkzeug', version='0.12.2'),
InstalledPackage(name='certifi', version='2017.7.27.1'),
InstalledPackage(name='chardet', version='3.0.4'),
InstalledPackage(name='click', version='6.7'),
InstalledPackage(name='gevent', version='1.2.2'),
InstalledPackage(name='greenlet', version='0.4.12'),
InstalledPackage(name='idna', version='2.6'),
InstalledPackage(name='itsdangerous', version='0.24'),
InstalledPackage(name='locustio', version='0.8a2'),
InstalledPackage(name='msgpack-python', version='0.4.8'),
InstalledPackage(name='pyzmq', version='15.2.0'),
InstalledPackage(name='requests', version='2.18.4'),
InstalledPackage(name='six', version='1.11.0'),
InstalledPackage(name='urllib3', version='1.22'),
# top level dependencies, but not reported as they are known
# "virtual" dependencies
InstalledPackage(name='pkg-resources', version='0.0.0'),
InstalledPackage(name='setuptools', version='36.5.0'),
# top level dependency that is not required - should be reported
InstalledPackage(name='ipdb', version='0.10.3'),
], )
installed_names = [pkg.name for pkg in pkgs_installed.packages]
site_packages = SitePackages(
python_path=None,
installed_package_names=installed_names,
)
site_packages._query_cache = EXAMPLE_RESULTS_DICT
analyzer = IsTransitiveDepAnalyzer(
requirements_txt=pkgs_reqs,
installed_packages=pkgs_installed,
site_packages=site_packages,
)
advice_list = analyzer.analyze()
expected_advice_list = AdviceList(advice_list=[
Advice(
analyzer=analyzer,
severity='info',
message=("Required dependency 'Flask==0.12.2' "
"is a transitive dependency of 'locustio==0.8a2'"),
),
Advice(
analyzer=analyzer,
severity='info',
message=("Required dependency 'Jinja2==2.9.6' "
"is a transitive dependency of 'locustio==0.8a2'"),
),
Advice(
analyzer=analyzer,
severity='warn',
message=("Installed non-transitive dependency 'ipdb-0.10.3' "
"is not required"),
),
], )
assert expected_advice_list == advice_list