def should_include(cls, builder_context: "PipelineBuilderContext") -> Generator[Dict[str, Any], None, None]:
"""Enable this pipeline unit if the adjustment is enabled."""
if builder_context.is_included(cls):
yield from ()
return None
if not builder_context.project.constraints or not builder_context.project.constraints.package_versions:
_LOGGER.debug("No constraints defined, not registering constraint sieve unit")
yield from ()
return None
if builder_context.project.runtime_environment.python_version is None:
_LOGGER.warning(
"No Python version specified in the runtime environment, not registering constraints sieve unit"
)
yield from ()
return None
marker_default_environment = cls.default_environment(builder_context)
for package_version in builder_context.project.constraints.package_versions.values():
if package_version.markers:
marker_instance = Marker(package_version.markers)
marker_evaluation_result = marker_instance.evaluate(environment=marker_default_environment)
if not marker_evaluation_result:
_LOGGER.warning(
"Skipping constraint on %r as marker does not apply to the environment %r",
package_version.name,
package_version.markers,
)
continue
yield {"package_name": package_version.name, "specifier": package_version.version}
def get_dependencies(whl, version):
with zipfile.ZipFile(str(whl), "r") as zip_file:
name = "/".join([
"{}.dist-info".format("-".join(whl.name.split("-")[0:2])),
"METADATA"
])
with zip_file.open(name) as file_handler:
metadata = message_from_string(
file_handler.read().decode("utf-8"))
deps = metadata.get_all("Requires-Dist")
if deps is None:
return
for dep in deps:
req = Requirement(dep)
markers = getattr(req.marker, "_markers", tuple()) or ()
if any(m for m in markers if isinstance(m, tuple) and len(m) == 3
and m[0].value == "extra"):
continue
py_versions = WheelDownloader._marker_at(markers, "python_version")
if py_versions:
marker = Marker('python_version < "1"')
marker._markers = [
markers[ver] for ver in sorted(
list(i for i in set(py_versions)
| {i - 1
for i in py_versions} if i >= 0))
]
matches_python = marker.evaluate({"python_version": version})
if not matches_python:
continue
deleted = 0
for ver in py_versions:
deleted += WheelDownloader._del_marker_at(
markers, ver - deleted)
platforms = []
platform_positions = WheelDownloader._marker_at(
markers, "sys_platform")
deleted = 0
for pos in platform_positions: # can only be ore meaningfully
platform = "{}{}".format(markers[pos][1].value,
markers[pos][2].value)
deleted += WheelDownloader._del_marker_at(
markers, pos - deleted)
platforms.append(platform)
if not platforms:
platforms.append(None)
for platform in platforms:
yield version, platform, req
def evaluated_deps(pkg):
"Returns dependencies that apply to the current environment"
deps = []
for k, dep in pkg.get("dependencies", {}).items():
if "markers" in dep:
marker = Marker(dep["markers"])
# Remove any "extra" clauses from the marker before we evaluate it. If a package is
# in this list, the extra has already been matched.
marker._markers = remove_extra_marker(marker._markers)
try:
if not marker.evaluate():
continue
except UndefinedEnvironmentName as e:
print(f"Failed to evaluate marker for {pkg['name']}: {e}")
raise
deps.append(k)
return deps
def tox_configure(config):
"""
For more info, see: https://tox.readthedocs.io/en/latest/plugins.html
For an example, see: https://github.com/tox-dev/tox-travis
"""
sections = config._cfg.sections
base_testenv = sections.get('testenv', {})
# Default to false so:
# 1. we don't affect other projects using tox
# 2. check migrations can happen gradually
if str(base_testenv.get(STYLE_FLAG, 'false')).lower() == 'true':
# Disable flake8 since we already include that
config.envlist[:] = [
env for env in config.envlist if not env.endswith('flake8')
]
make_envconfig = get_make_envconfig()
reader = get_reader(config)
add_style_checker(config, sections, make_envconfig, reader)
add_style_formatter(config, sections, make_envconfig, reader)
# Workaround for https://github.com/tox-dev/tox/issues/1593
#
# Do this only after all dynamic environments have been created
if str(base_testenv.get(FIX_DEFAULT_ENVDIR_FLAG,
'false')).lower() == 'true':
for env_name, env_config in config.envconfigs.items():
if env_config.envdir == config.toxinidir:
env_config.envdir = config.toxworkdir / env_name
env_config.envlogdir = env_config.envdir / 'log'
env_config.envtmpdir = env_config.envdir / 'tmp'
# Conditionally set 'e2e ready' depending on env variables or environment markers
for cfg in config.envconfigs.values():
if E2E_READY_CONDITION not in cfg.description:
continue
data, var = cfg.description.split(' if ')
var = var.strip()
if var in os.environ:
cfg.description = data
else:
from packaging.markers import InvalidMarker, Marker
try:
marker = Marker(var)
except InvalidMarker:
cfg.description = '{} is missing'.format(var)
else:
if marker.evaluate():
cfg.description = data
else:
cfg.description = 'environment does not match: {}'.format(
var)
# Next two sections hack the sequencing of Tox's package installation.
#
# Tox package installation order:
# 1. Install `deps`, which typically looks like:
# deps =
# -e../datadog_checks_base[deps]
# -rrequirements-dev.txt
# 2. Install check package, editable mode
# 3. Execute `commands` which typically looks like:
# commands =
# pip install -r requirements.in
# pytest -v {posargs} --benchmark-skip
# Forcibly remove any dependencies from the `tox.ini` deps that are meant to be unpinned
# This should have the effect of relying on a package's setup.py for installation
# We manually pip install `datadog_checks_base[deps]` to ensure that test dependencies are installed,
# but this should have no effect on the version of the base package.
force_unpinned = os.getenv('TOX_FORCE_UNPINNED', None)
if force_unpinned:
for env in config.envlist:
deps = [
d for d in config.envconfigs[env].deps
if force_unpinned not in d.name
]
config.envconfigs[env].deps = deps
config.envconfigs[env].commands.insert(
0, 'pip install datadog_checks_base[deps]'.split())
# Workaround for tox's `--force-dep` having limited functionality when package is already installed.
# Primarily meant for pinning datadog-checks-base to a specific version, but could be
# applied to other packages in the future. Since we typically install checks base via
# editable mode, we need to force the reinstall to ensure that the PyPI package installs.
force_install = os.getenv('TOX_FORCE_INSTALL', None)
if force_install:
command = f'pip install --force-reinstall {force_install}'.split()
for env in config.envlist:
config.envconfigs[env].commands.insert(0, command)
def test_extra_with_no_extra_in_environment(self):
# We can't evaluate an extra if no extra is passed into the environment
m = Marker("extra == 'security'")
with pytest.raises(UndefinedEnvironmentName):
m.evaluate()
def test_extra_with_no_extra_in_environment(self):
# We can't evaluate an extra if no extra is passed into the environment
m = Marker("extra == 'security'")
with pytest.raises(UndefinedEnvironmentName):
m.evaluate()
def inspect(self, pkg):
if self.setup_path:
stdout = call_setup_py(self.setup_path, ['egg_info'],
env=pkg.fresh_environ(),
stdout=True).decode()
m = re.search(r'writing requirements to (.+?)\n', stdout)
if not m:
log.debug("No requirements")
return
requirements_path = os.path.join(os.path.dirname(self.setup_path),
m.group(1))
for line in open(requirements_path):
# Stop once we get to the "extras".
if line.startswith('['):
break
m = re.match(r'^([\w\.-]+)', line)
if m:
name = m.group(1).lower()
log.debug('%s depends on %s' % (pkg.name, name))
pkg.add_dependency(name=name, url='pypi:%s' % name.lower())
if self.dist_info_dir:
for line in open(os.path.join(self.dist_info_dir, 'METADATA')):
line = line.strip()
if not line:
break # We're at the end of the headers.
key, value = line.split(': ', 1)
key = key.lower()
if key == 'requires-dist':
# Environmental markers look like `FOO; extra == 'BAR'`.
if ';' in value:
value, raw_marker = value.split(';')
value = value.strip()
# We delay the import just in case the bootstrap is borked.
from packaging.markers import Marker
marker = Marker(raw_marker)
if not marker.evaluate({'extra': None}):
continue
m = re.match(r'([\w-]+)(?:\s+\(([^)]+)\))?', value)
if not m:
log.warning(
'Could not parse requires-dist {!r}'.format(value))
continue
dep_name, version_expr = m.groups()
pkg.add_dependency(
name=dep_name,
url='pypi:{}'.format(dep_name),
version=version_expr,
)
def match(self, marker: Marker, extras: Sequence[str] = ()) -> bool:
env = dict(**asdict(self), extra=Extras(extras))
return bool(marker.evaluate(env))