def test_strip_marker_extra_crazy():
marker = get_without_extra(
Marker(
'(os_name == "nt" or sys_platform == "Windows" and extra == "huh") '
'and extra == "sock"', ))
assert str(marker) == 'os_name == "nt" or sys_platform == "Windows"'
def parse_marker_dict(marker_dict):
op = marker_dict["op"]
lhs = marker_dict["lhs"]
rhs = marker_dict["rhs"]
# This is where the spec sets for each side land if we have an "or" operator
side_spec_list = []
side_markers_list = []
finalized_marker = ""
# And if we hit the end of the parse tree we use this format string to make a marker
format_string = "{lhs} {op} {rhs}"
specset = SpecifierSet()
specs = set()
# Essentially we will iterate over each side of the parsed marker if either one is
# A mapping instance (i.e. a dictionary) and recursively parse and reduce the specset
# Union the "and" specs, intersect the "or"s to find the most appropriate range
if any(issubclass(type(side), Mapping) for side in (lhs, rhs)):
for side in (lhs, rhs):
side_specs = set()
side_markers = set()
if issubclass(type(side), Mapping):
merged_side_specs, merged_side_markers = parse_marker_dict(
side)
side_specs.update(merged_side_specs)
side_markers.update(merged_side_markers)
else:
marker = _ensure_marker(side)
marker_parts = getattr(marker, "_markers", [])
if marker_parts[0][0].value == "python_version":
side_specs |= set(get_specset(marker_parts))
else:
side_markers.add(str(marker))
side_spec_list.append(side_specs)
side_markers_list.append(side_markers)
if op == "and":
# When we are "and"-ing things together, it probably makes the most sense
# to reduce them here into a single PySpec instance
specs = reduce(lambda x, y: set(x) | set(y), side_spec_list)
markers = reduce(lambda x, y: set(x) | set(y), side_markers_list)
if not specs and not markers:
return specset, finalized_marker
if markers and isinstance(markers, (tuple, list, Set)):
finalized_marker = Marker(" and ".join(
[m for m in markers if m]))
elif markers:
finalized_marker = str(markers)
specset._specs = frozenset(specs)
return specset, finalized_marker
# Actually when we "or" things as well we can also just turn them into a reduced
# set using this logic now
sides = reduce(lambda x, y: set(x) & set(y), side_spec_list)
finalized_marker = " or ".join(
[normalize_marker_str(m) for m in side_markers_list])
specset._specs = frozenset(sorted(sides))
return specset, finalized_marker
else:
# At the tip of the tree we are dealing with strings all around and they just need
# to be smashed together
specs = set()
if lhs == "python_version":
format_string = "{lhs}{op}{rhs}"
marker = Marker(format_string.format(**marker_dict))
marker_parts = getattr(marker, "_markers", [])
_set = get_specset(marker_parts)
if _set:
specs |= set(_set)
specset._specs = frozenset(specs)
return specset, finalized_marker
def match_markers(self):
if self.markers is not None:
from packaging.markers import Marker
return Marker(self.markers).evaluate()
else:
return True
def test_fails_when_undefined(self):
with pytest.raises(UndefinedComparison):
Marker("'2.7.0' ~= os_name").evaluate()
def updates(ctx, sync_dependencies, include_security_deps, batch_size):
ignore_deps = set(IGNORED_DEPS)
if not include_security_deps:
ignore_deps.update(SECURITY_DEPS)
ignore_deps = {normalize_project_name(d) for d in ignore_deps}
dependencies, errors = read_agent_dependencies()
if errors:
for error in errors:
echo_failure(error)
abort()
api_urls = [
f'https://pypi.org/pypi/{package}/json' for package in dependencies
]
package_data = asyncio.run(scrape_version_data(api_urls))
package_data = {
normalize_project_name(package_name): versions
for package_name, versions in package_data.items()
}
new_dependencies = copy.deepcopy(dependencies)
version_updates = defaultdict(lambda: defaultdict(set))
updated_packages = set()
for name, python_versions in sorted(new_dependencies.items()):
if name in ignore_deps:
continue
elif batch_size is not None and len(updated_packages) >= batch_size:
break
new_python_versions = package_data[name]
dropped_py2 = len(set(new_python_versions.values())) > 1
for python_version, package_version in new_python_versions.items():
dependency_definitions = python_versions[python_version]
if not dependency_definitions or package_version is None:
continue
dependency_definition, checks = dependency_definitions.popitem()
requirement = Requirement(dependency_definition)
requirement.specifier = SpecifierSet(f'=={package_version}')
if dropped_py2 and 'python_version' not in dependency_definition:
python_marker = f'python_version {"<" if python_version == "py2" else ">"} "3.0"'
if not requirement.marker:
requirement.marker = Marker(python_marker)
else:
requirement.marker = Marker(
f'{requirement.marker} and {python_marker}')
new_dependency_definition = get_normalized_dependency(requirement)
dependency_definitions[new_dependency_definition] = checks
if dependency_definition != new_dependency_definition:
version_updates[name][package_version].add(python_version)
updated_packages.add(name)
if sync_dependencies:
if updated_packages:
update_agent_dependencies(new_dependencies)
ctx.invoke(sync)
echo_info(f'Updated {len(updated_packages)} dependencies')
else:
if updated_packages:
echo_failure(
f"{len(updated_packages)} dependencies are out of sync:")
for name, versions in version_updates.items():
for package_version, python_versions in versions.items():
echo_failure(
f'{name} can be updated to version {package_version} '
f'on {" and ".join(sorted(python_versions))}')
abort()
else:
echo_info('All dependencies are up to date')
def test_parses_setuptools_legacy_valid(self, marker_string):
Marker(marker_string)
def test_prefers_pep440(self):
assert Marker('"2.7.9" < "foo"').evaluate(dict(foo="2.7.10"))
(">", Version("2.6")),
(">=", Version("2.7")),
("<", Version("3.6")),
("<", Version("3.7")),
],
),
],
)
def test_get_versions(specset, versions):
assert requirementslib.models.markers.get_versions(specset) == versions
@pytest.mark.parametrize(
"marker, extras",
[
(Marker("extra == 'security' and os_name == 'nt'"), {"security"}),
(Marker("os_name == 'security' and python_version >= '2.7'"), set()),
],
)
def test_get_extras(marker, extras):
assert requirementslib.models.markers.get_contained_extras(
marker) == extras
@pytest.mark.parametrize(
"marker, pyversions",
[
(
Marker(
"os_name == 'nt' and python_version >= '2.7' and python_version <= '3.5'"
),
def tox_configure(config):
"""
For more info, see: https://tox.readthedocs.io/en/latest/plugins.html
For an example, see: https://github.com/tox-dev/tox-travis
"""
sections = config._cfg.sections
base_testenv = sections.get('testenv', {})
# Default to false so:
# 1. we don't affect other projects using tox
# 2. check migrations can happen gradually
if str(base_testenv.get(STYLE_FLAG, 'false')).lower() == 'true':
# Disable flake8 since we already include that
config.envlist[:] = [
env for env in config.envlist if not env.endswith('flake8')
]
make_envconfig = get_make_envconfig()
reader = get_reader(config)
add_style_checker(config, sections, make_envconfig, reader)
add_style_formatter(config, sections, make_envconfig, reader)
# Workaround for https://github.com/tox-dev/tox/issues/1593
#
# Do this only after all dynamic environments have been created
if str(base_testenv.get(FIX_DEFAULT_ENVDIR_FLAG,
'false')).lower() == 'true':
for env_name, env_config in config.envconfigs.items():
if env_config.envdir == config.toxinidir:
env_config.envdir = config.toxworkdir / env_name
env_config.envlogdir = env_config.envdir / 'log'
env_config.envtmpdir = env_config.envdir / 'tmp'
# Conditionally set 'e2e ready' depending on env variables or environment markers
for cfg in config.envconfigs.values():
if E2E_READY_CONDITION not in cfg.description:
continue
data, var = cfg.description.split(' if ')
var = var.strip()
if var in os.environ:
cfg.description = data
else:
from packaging.markers import InvalidMarker, Marker
try:
marker = Marker(var)
except InvalidMarker:
cfg.description = '{} is missing'.format(var)
else:
if marker.evaluate():
cfg.description = data
else:
cfg.description = 'environment does not match: {}'.format(
var)
# Next two sections hack the sequencing of Tox's package installation.
#
# Tox package installation order:
# 1. Install `deps`, which typically looks like:
# deps =
# -e../datadog_checks_base[deps]
# -rrequirements-dev.txt
# 2. Install check package, editable mode
# 3. Execute `commands` which typically looks like:
# commands =
# pip install -r requirements.in
# pytest -v {posargs} --benchmark-skip
# Forcibly remove any dependencies from the `tox.ini` deps that are meant to be unpinned
# This should have the effect of relying on a package's setup.py for installation
# We manually pip install `datadog_checks_base[deps]` to ensure that test dependencies are installed,
# but this should have no effect on the version of the base package.
force_unpinned = os.getenv('TOX_FORCE_UNPINNED', None)
if force_unpinned:
for env in config.envlist:
deps = [
d for d in config.envconfigs[env].deps
if force_unpinned not in d.name
]
config.envconfigs[env].deps = deps
config.envconfigs[env].commands.insert(
0, 'pip install datadog_checks_base[deps]'.split())
# Workaround for tox's `--force-dep` having limited functionality when package is already installed.
# Primarily meant for pinning datadog-checks-base to a specific version, but could be
# applied to other packages in the future. Since we typically install checks base via
# editable mode, we need to force the reinstall to ensure that the PyPI package installs.
force_install = os.getenv('TOX_FORCE_INSTALL', None)
if force_install:
command = f'pip install --force-reinstall {force_install}'.split()
for env in config.envlist:
config.envconfigs[env].commands.insert(0, command)
# This will inherit the configuration of `[testenv]` and if it doesn't support the platform then for some reason
# running a user defined environment and a generated environment at the same time (e.g. `tox -e py38,style`) will
# attempt to use the Python executable in the `.package` environment but will fail because the environment has
# not been created due to the platform mismatch.
if '.package' in config.envconfigs:
config.envconfigs['.package'].platform = 'linux|darwin|win32'
def test_strip_marker_extra_crazy_cancelled():
marker = get_without_extra(
Marker(
'(extra == "foo" or extra == "sock") or '
'(extra == "huh" or extra == "bar")', ))
assert marker is None
def resolve_reqs(self, download_dir, ireq, wheel_cache, setup_requires={}, dist=None):
results = None
setup_requires = {}
dist = None
try:
from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer
except ImportError:
# Pip 9 and below
reqset = RequirementSet(
self.build_dir,
self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
session=self.session,
ignore_installed=True,
ignore_compatibility=False,
wheel_cache=wheel_cache
)
results = reqset._prepare_file(self.finder, ireq, ignore_requires_python=True)
else:
# pip >= 10
preparer_kwargs = {
'build_dir': self.build_dir,
'src_dir': self.source_dir,
'download_dir': download_dir,
'wheel_download_dir': self._wheel_download_dir,
'progress_bar': 'off',
'build_isolation': False
}
resolver_kwargs = {
'finder': self.finder,
'session': self.session,
'upgrade_strategy': "to-satisfy-only",
'force_reinstall': True,
'ignore_dependencies': False,
'ignore_requires_python': True,
'ignore_installed': True,
'isolated': False,
'wheel_cache': wheel_cache,
'use_user_site': False,
'ignore_compatibility': False
}
resolver = None
preparer = None
with RequirementTracker() as req_tracker:
# Pip 18 uses a requirement tracker to prevent fork bombs
if req_tracker:
preparer_kwargs['req_tracker'] = req_tracker
preparer = RequirementPreparer(**preparer_kwargs)
resolver_kwargs['preparer'] = preparer
reqset = RequirementSet()
ireq.is_direct = True
# reqset.add_requirement(ireq)
resolver = PipResolver(**resolver_kwargs)
resolver.require_hashes = False
try:
results = resolver._resolve_one(reqset, ireq)
except InstallationError:
pass
reqset.cleanup_files()
if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)):
# Collect setup_requires info from local eggs.
# Do this after we call the preparer on these reqs to make sure their
# egg info has been created
from pipenv.utils import chdir
with chdir(ireq.setup_py_dir):
try:
from setuptools.dist import distutils
dist = distutils.core.run_setup(ireq.setup_py)
except InstallationError:
ireq.run_egg_info()
except (TypeError, ValueError, AttributeError):
pass
if not dist:
try:
dist = ireq.get_dist()
except (ImportError, ValueError, TypeError, AttributeError):
pass
if ireq.editable and dist:
setup_requires = getattr(dist, "extras_require", None)
if not setup_requires:
setup_requires = {"setup_requires": getattr(dist, "setup_requires", None)}
if not getattr(ireq, 'req', None):
try:
ireq.req = dist.as_requirement() if dist else None
except (ValueError, TypeError) as e:
pass
# Convert setup_requires dict into a somewhat usable form.
if setup_requires:
for section in setup_requires:
python_version = section
not_python = not (section.startswith('[') and ':' in section)
# This is for cleaning up :extras: formatted markers
# by adding them to the results of the resolver
# since any such extra would have been returned as a result anyway
for value in setup_requires[section]:
# This is a marker.
if value.startswith('[') and ':' in value:
python_version = value[1:-1]
not_python = False
# Strip out other extras.
if value.startswith('[') and ':' not in value:
not_python = True
if ':' not in value:
try:
if not not_python:
results.add(InstallRequirement.from_line("{0}{1}".format(value, python_version).replace(':', ';')))
# Anything could go wrong here -- can't be too careful.
except Exception:
pass
# this section properly creates 'python_version' markers for cross-python
# virtualenv creation and for multi-python compatibility.
requires_python = reqset.requires_python if hasattr(reqset, 'requires_python') else resolver.requires_python
if requires_python:
marker_str = ''
# This corrects a logic error from the previous code which said that if
# we Encountered any 'requires_python' attributes, basically only create a
# single result no matter how many we resolved. This should fix
# a majority of the remaining non-deterministic resolution issues.
if any(requires_python.startswith(op) for op in Specifier._operators.keys()):
# We are checking first if we have leading specifier operator
# if not, we can assume we should be doing a == comparison
specifierset = SpecifierSet(requires_python)
# for multiple specifiers, the correct way to represent that in
# a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')`
from passa.internals.specifiers import cleanup_pyspecs
marker_str = str(Marker(" and ".join(dedup([
"python_version {0[0]} '{0[1]}'".format(spec)
for spec in cleanup_pyspecs(specifierset)
]))))
# The best way to add markers to a requirement is to make a separate requirement
# with only markers on it, and then to transfer the object istelf
marker_to_add = Requirement('fakepkg; {0}'.format(marker_str)).marker
if ireq in results:
results.remove(ireq)
print(marker_to_add)
ireq.req.marker = marker_to_add
results = set(results) if results else set()
return results, ireq
def test_strip_marker_extra_paranthesized_cancelled():
marker = get_without_extra(
Marker(
'(extra == "sock") or (extra == "huh") or (sys_platform == "Windows")',
))
assert str(marker) == 'sys_platform == "Windows"'
def test_strip_marker_extra_noop():
marker = get_without_extra(
Marker('os_name == "nt" or sys_platform == "Windows"'), )
assert str(marker) == 'os_name == "nt" or sys_platform == "Windows"'
def test_strip_marker_extra_cancelled():
marker = get_without_extra(Marker('extra == "sock" or extra == "huh"'))
assert marker is None
def test_parses_pep345_valid(self, marker_string):
Marker(marker_string)
def inspect(self, pkg):
if self.setup_path:
stdout = call_setup_py(self.setup_path, ['egg_info'],
env=pkg.fresh_environ(),
stdout=True).decode()
m = re.search(r'writing requirements to (.+?)\n', stdout)
if not m:
log.debug("No requirements")
return
requirements_path = os.path.join(os.path.dirname(self.setup_path),
m.group(1))
for line in open(requirements_path):
# Stop once we get to the "extras".
if line.startswith('['):
break
m = re.match(r'^([\w\.-]+)', line)
if m:
name = m.group(1).lower()
log.debug('%s depends on %s' % (pkg.name, name))
pkg.add_dependency(name=name, url='pypi:%s' % name.lower())
if self.dist_info_dir:
for line in open(os.path.join(self.dist_info_dir, 'METADATA')):
line = line.strip()
if not line:
break # We're at the end of the headers.
key, value = line.split(': ', 1)
key = key.lower()
if key == 'requires-dist':
# Environmental markers look like `FOO; extra == 'BAR'`.
if ';' in value:
value, raw_marker = value.split(';')
value = value.strip()
# We delay the import just in case the bootstrap is borked.
from packaging.markers import Marker
marker = Marker(raw_marker)
if not marker.evaluate({'extra': None}):
continue
m = re.match(r'([\w-]+)(?:\s+\(([^)]+)\))?', value)
if not m:
log.warning(
'Could not parse requires-dist {!r}'.format(value))
continue
dep_name, version_expr = m.groups()
pkg.add_dependency(
name=dep_name,
url='pypi:{}'.format(dep_name),
version=version_expr,
)
def test_evaluate_pep345_markers(self, marker_string, environment,
expected):
args = [] if environment is None else [environment]
assert Marker(marker_string).evaluate(*args) == expected
def test_match_empty(self) -> None:
m = Marker("implementation_name=='cpython'")
e = EnvironmentMarkers.for_python("3.7.5")
self.assertTrue(e.match(m))
def test_evaluate_setuptools_legacy_markers(self):
marker_string = "python_implementation=='Jython'"
args = [{"platform_python_implementation": "CPython"}]
assert Marker(marker_string).evaluate(*args) is False
def test_match_lt_3(self) -> None:
m = Marker("python_version < '3'")
e = EnvironmentMarkers.for_python("2.7.0")
self.assertTrue(e.match(m))
def test_falls_back_to_python(self):
assert Marker('"b" > "a"').evaluate(dict(a="a"))
def test_match_3(self) -> None:
m = Marker("python_version < '3'")
e = EnvironmentMarkers.for_python("3.7.5")
self.assertFalse(e.match(m))
def validate_markers(instance, attr_, value):
from packaging.markers import Marker, InvalidMarker
try:
Marker("{0}{1}".format(attr_.name, value))
except InvalidMarker:
raise ValueError("Invalid Marker {0}{1}".format(attr_, value))
def test_parses_invalid(self, marker_string):
with pytest.raises(InvalidMarker):
Marker(marker_string)
def gen_marker(mkr):
# type: (List[STRING_TYPE]) -> Marker
m = Marker("python_version == '1'")
m._markers.pop()
m._markers.append(mkr)
return m
def test_str_and_repr(self, marker_string, expected):
m = Marker(marker_string)
assert str(m) == expected
assert repr(m) == "<Marker({0!r})>".format(str(m))
def validate_markers(instance, attr_, value):
try:
Marker("{0}{1}".format(attr_.name, value))
except InvalidMarker:
raise ValueError("Invalid Marker {0}{1}".format(attr_, value))
def test_extra_with_no_extra_in_environment(self):
# We can't evaluate an extra if no extra is passed into the environment
m = Marker("extra == 'security'")
with pytest.raises(UndefinedEnvironmentName):
m.evaluate()
def _ensure_marker(marker):
# type: (Union[STRING_TYPE, Marker]) -> Marker
if not is_instance(marker, Marker):
return Marker(str(marker))
return marker
def test_strip_marker_extra_in_front():
marker = get_without_extra(Marker('extra == "sock" or os_name == "nt"'))
assert str(marker) == 'os_name == "nt"'
