def test_r_base_cross_solvable():
feedstock_dir = os.path.join(os.path.dirname(__file__), "r-base-feedstock")
solvable, errors, _ = is_recipe_solvable(feedstock_dir)
assert not solvable, pprint.pformat(errors)
solvable, errors, _ = is_recipe_solvable(
feedstock_dir,
build_platform={"osx_arm64": "osx_64"},
)
assert solvable, pprint.pformat(errors)
def test_arrow_solvable(tmp_path):
feedstock_dir = clone_and_checkout_repo(
tmp_path,
"https://github.com/conda-forge/arrow-cpp-feedstock",
ref="master",
)
solvable, errors, solvable_by_variant = is_recipe_solvable(feedstock_dir)
pprint.pprint(solvable_by_variant)
assert solvable, pprint.pformat(errors)
def test_grpcio_solvable(tmp_path):
"""grpcio has a runtime dep on openssl which has strange pinning things in it"""
feedstock_dir = clone_and_checkout_repo(
tmp_path,
"https://github.com/conda-forge/grpcio-feedstock",
ref="master",
)
solvable, errors, solvable_by_variant = is_recipe_solvable(feedstock_dir)
pprint.pprint(solvable_by_variant)
assert solvable, pprint.pformat(errors)
def test_datalad_solvable(tmp_path):
"""has an odd thing where it hangs"""
feedstock_dir = clone_and_checkout_repo(
tmp_path,
"https://github.com/conda-forge/datalad-feedstock",
ref="master",
)
solvable, errors, solvable_by_variant = is_recipe_solvable(feedstock_dir)
pprint.pprint(solvable_by_variant)
assert solvable, pprint.pformat(errors)
def test_guiqwt_solvable(tmp_path):
"""test for run exports as a single string in pyqt"""
feedstock_dir = clone_and_checkout_repo(
tmp_path,
"https://github.com/conda-forge/guiqwt-feedstock",
ref="master",
)
solvable, errors, solvable_by_variant = is_recipe_solvable(feedstock_dir)
pprint.pprint(solvable_by_variant)
assert solvable, pprint.pformat(errors)
def test_unsolvable_for_particular_python(feedstock_dir):
recipe_file = os.path.join(feedstock_dir, "recipe", "meta.yaml")
os.makedirs(os.path.dirname(recipe_file), exist_ok=True)
with open(recipe_file, "w") as fp:
fp.write(
"""\
{% set name = "cf-autotick-bot-test-package" %}
{% set version = "0.9" %}
package:
name: {{ name|lower }}
version: {{ version }}
source:
path: .
build:
number: 8
requirements:
host:
- python
- pip
run:
- python
- galsim
test:
commands:
- echo "works!"
about:
home: https://github.com/regro/cf-scripts
license: BSD-3-Clause
license_family: BSD
license_file: LICENSE
summary: testing feedstock for the regro-cf-autotick-bot
extra:
recipe-maintainers:
- beckermr
- conda-forge/bot
""", )
solvable, errors, solvable_by_variant = is_recipe_solvable(feedstock_dir)
print(solvable_by_variant)
assert not solvable
# we don't have galsim for this variant so this is an expected failure
assert not solvable_by_variant["linux_aarch64_python3.6.____cpython"]
assert not solvable_by_variant["linux_ppc64le_python3.6.____cpython"]
# But we do have this one
assert solvable_by_variant["linux_python3.7.____cpython"]
def test_is_recipe_solvable_notok():
recipe_file = os.path.join(FEEDSTOCK_DIR, "recipe", "meta.yaml")
os.makedirs(os.path.dirname(recipe_file), exist_ok=True)
try:
with open(recipe_file, "w") as fp:
fp.write(
"""\
{% set name = "cf-autotick-bot-test-package" %}
{% set version = "0.9" %}
package:
name: {{ name|lower }}
version: {{ version }}
source:
path: .
build:
number: 8
requirements:
host:
- python >=4.0 # [osx]
- python # [not osx]
- pip
run:
- python
test:
commands:
- echo "works!"
about:
home: https://github.com/regro/cf-scripts
license: BSD-3-Clause
license_family: BSD
license_file: LICENSE
summary: testing feedstock for the regro-cf-autotick-bot
extra:
recipe-maintainers:
- beckermr
- conda-forge/bot
""", )
assert not is_recipe_solvable(FEEDSTOCK_DIR)
finally:
try:
os.remove(recipe_file)
except Exception:
pass
def test_cupy_solvable(tmp_path):
"""grpcio has a runtime dep on openssl which has strange pinning things in it"""
feedstock_dir = clone_and_checkout_repo(
tmp_path,
"https://github.com/conda-forge/cupy-feedstock",
ref="master",
)
subprocess.run(
f"cd {feedstock_dir} && git checkout 72d6c5808ca79c9cd9a3eb4064a72586c73c3430",
shell=True,
check=True,
)
solvable, errors, solvable_by_variant = is_recipe_solvable(feedstock_dir)
pprint.pprint(solvable_by_variant)
assert solvable, pprint.pformat(errors)
def test_arrow_solvable_timeout(tmp_path):
feedstock_dir = clone_and_checkout_repo(
tmp_path,
"https://github.com/conda-forge/arrow-cpp-feedstock",
ref="master",
)
# let's run this over and over again to make sure nothing weird is happening
# with the killed processes
for _ in range(6):
solvable, errors, solvable_by_variant = is_recipe_solvable(
feedstock_dir,
timeout=10,
)
assert solvable
assert errors == []
assert solvable_by_variant == {}
def test_virtual_package(feedstock_dir, tmp_path_factory):
recipe_file = os.path.join(feedstock_dir, "recipe", "meta.yaml")
os.makedirs(os.path.dirname(recipe_file), exist_ok=True)
with FakeRepoData(tmp_path_factory.mktemp("channel")) as repodata:
for pkg in [
FakePackage("fakehostvirtualpkgdep",
depends=frozenset(["__virtual >=10"])),
FakePackage("__virtual", version="10"),
]:
repodata.add_package(pkg)
with open(recipe_file, "w") as fp:
fp.write(
dedent(
"""
package:
name: "cf-autotick-bot-test-package"
version: "0.9"
source:
path: .
build:
number: 8
requirements:
host:
- python
- fakehostvirtualpkgdep
- pip
run:
- python
""", ), )
solvable, err, solve_by_variant = is_recipe_solvable(
feedstock_dir,
additional_channels=[repodata.channel_url],
)
assert solvable
def run(
feedstock_ctx: FeedstockContext,
migrator: Migrator,
protocol: str = "ssh",
pull_request: bool = True,
rerender: bool = True,
fork: bool = True,
**kwargs: typing.Any,
) -> Tuple["MigrationUidTypedDict", dict]:
"""For a given feedstock and migration run the migration
Parameters
----------
feedstock_ctx: FeedstockContext
The node attributes
migrator: Migrator instance
The migrator to run on the feedstock
protocol : str, optional
The git protocol to use, defaults to ``ssh``
pull_request : bool, optional
If true issue pull request, defaults to true
rerender : bool
Whether to rerender
fork : bool
If true create a fork, defaults to true
gh : github3.GitHub instance, optional
Object for communicating with GitHub, if None build from $USERNAME
and $PASSWORD, defaults to None
kwargs: dict
The key word arguments to pass to the migrator
Returns
-------
migrate_return: MigrationUidTypedDict
The migration return dict used for tracking finished migrations
pr_json: dict
The PR json object for recreating the PR as needed
"""
# get the repo
# TODO: stop doing this.
migrator.attrs = feedstock_ctx.attrs # type: ignore
branch_name = migrator.remote_branch(feedstock_ctx) + "_h" + uuid4().hex[0:6]
# TODO: run this in parallel
feedstock_dir, repo = get_repo(
ctx=migrator.ctx.session,
fctx=feedstock_ctx,
branch=branch_name,
feedstock=feedstock_ctx.feedstock_name,
protocol=protocol,
pull_request=pull_request,
fork=fork,
)
recipe_dir = os.path.join(feedstock_dir, "recipe")
# migrate the feedstock
migrator.run_pre_piggyback_migrations(recipe_dir, feedstock_ctx.attrs, **kwargs)
# TODO - make a commit here if the repo changed
migrate_return = migrator.migrate(recipe_dir, feedstock_ctx.attrs, **kwargs)
if not migrate_return:
logger.critical(
"Failed to migrate %s, %s",
feedstock_ctx.package_name,
feedstock_ctx.attrs.get("bad"),
)
eval_cmd(f"rm -rf {feedstock_dir}")
return False, False
# TODO - commit main migration here
migrator.run_post_piggyback_migrations(recipe_dir, feedstock_ctx.attrs, **kwargs)
# TODO commit post migration here
# rerender, maybe
diffed_files: typing.List[str] = []
with indir(feedstock_dir), env.swap(RAISE_SUBPROC_ERROR=False):
msg = migrator.commit_message(feedstock_ctx) # noqa
eval_cmd("git add --all .")
eval_cmd(f"git commit -am '{msg}'")
if rerender:
head_ref = eval_cmd("git rev-parse HEAD") # noqa
logger.info("Rerendering the feedstock")
# In the event we can't rerender, try to update the pinnings,
# then bail if it does not work again
try:
eval_cmd(
"conda smithy rerender -c auto --no-check-uptodate", timeout=300,
)
except SubprocessError:
return False, False
# If we tried to run the MigrationYaml and rerender did nothing (we only
# bumped the build number and dropped a yaml file in migrations) bail
# for instance platform specific migrations
gdiff = eval_cmd(f"git diff --name-only {head_ref}...HEAD")
diffed_files = [
_
for _ in gdiff.split()
if not (
_.startswith("recipe")
or _.startswith("migrators")
or _.startswith("README")
)
]
if (
migrator.check_solvable
or feedstock_ctx.attrs["conda-forge.yml"]
.get("bot", {})
.get("check_solvable", False)
) and not is_recipe_solvable(feedstock_dir):
eval_cmd(f"rm -rf {feedstock_dir}")
return False, False
# TODO: Better annotation here
pr_json: typing.Union[MutableMapping, None, bool]
if (
isinstance(migrator, MigrationYaml)
and not diffed_files
and feedstock_ctx.attrs["name"] != "conda-forge-pinning"
):
# spoof this so it looks like the package is done
pr_json = {
"state": "closed",
"merged_at": "never issued",
"id": str(uuid4()),
}
else:
# push up
try:
pr_json = push_repo(
session_ctx=migrator.ctx.session,
fctx=feedstock_ctx,
feedstock_dir=feedstock_dir,
body=migrator.pr_body(feedstock_ctx),
repo=repo,
title=migrator.pr_title(feedstock_ctx),
head=f"{migrator.ctx.github_username}:{branch_name}",
branch=branch_name,
)
# This shouldn't happen too often any more since we won't double PR
except github3.GitHubError as e:
if e.msg != "Validation Failed":
raise
else:
print(f"Error during push {e}")
# If we just push to the existing PR then do nothing to the json
pr_json = False
ljpr = False
if pr_json:
ljpr = LazyJson(
os.path.join(migrator.ctx.session.prjson_dir, str(pr_json["id"]) + ".json"),
)
ljpr.update(**pr_json)
# from .dynamo_models import PRJson
# PRJson.dump(pr_json)
# If we've gotten this far then the node is good
feedstock_ctx.attrs["bad"] = False
logger.info("Removing feedstock dir")
eval_cmd(f"rm -rf {feedstock_dir}")
return migrate_return, ljpr
def run(
feedstock_ctx: FeedstockContext,
migrator: Migrator,
protocol: str = "ssh",
pull_request: bool = True,
rerender: bool = True,
fork: bool = True,
base_branch: str = "master",
**kwargs: typing.Any,
) -> Tuple["MigrationUidTypedDict", dict]:
"""For a given feedstock and migration run the migration
Parameters
----------
feedstock_ctx: FeedstockContext
The node attributes
migrator: Migrator instance
The migrator to run on the feedstock
protocol : str, optional
The git protocol to use, defaults to ``ssh``
pull_request : bool, optional
If true issue pull request, defaults to true
rerender : bool
Whether to rerender
fork : bool
If true create a fork, defaults to true
base_branch : str, optional
The base branch to which the PR will be targeted. Defaults to "master".
kwargs: dict
The key word arguments to pass to the migrator
Returns
-------
migrate_return: MigrationUidTypedDict
The migration return dict used for tracking finished migrations
pr_json: dict
The PR json object for recreating the PR as needed
"""
# get the repo
# TODO: stop doing this.
migrator.attrs = feedstock_ctx.attrs # type: ignore
branch_name = migrator.remote_branch(
feedstock_ctx) + "_h" + uuid4().hex[0:6]
if hasattr(migrator, "name"):
assert isinstance(migrator.name, str)
migrator_name = migrator.name.lower().replace(" ", "")
else:
migrator_name = migrator.__class__.__name__.lower()
# TODO: run this in parallel
feedstock_dir, repo = get_repo(
ctx=migrator.ctx.session,
fctx=feedstock_ctx,
branch=branch_name,
feedstock=feedstock_ctx.feedstock_name,
protocol=protocol,
pull_request=pull_request,
fork=fork,
base_branch=base_branch,
)
if not feedstock_dir or not repo:
LOGGER.critical(
"Failed to migrate %s, %s",
feedstock_ctx.package_name,
feedstock_ctx.attrs.get("bad"),
)
return False, False
recipe_dir = os.path.join(feedstock_dir, "recipe")
# migrate the feedstock
migrator.run_pre_piggyback_migrations(recipe_dir, feedstock_ctx.attrs,
**kwargs)
# TODO - make a commit here if the repo changed
migrate_return = migrator.migrate(recipe_dir, feedstock_ctx.attrs,
**kwargs)
if not migrate_return:
LOGGER.critical(
"Failed to migrate %s, %s",
feedstock_ctx.package_name,
feedstock_ctx.attrs.get("bad"),
)
eval_cmd(f"rm -rf {feedstock_dir}")
return False, False
# TODO - commit main migration here
migrator.run_post_piggyback_migrations(recipe_dir, feedstock_ctx.attrs,
**kwargs)
# TODO commit post migration here
# rerender, maybe
diffed_files: typing.List[str] = []
with indir(feedstock_dir), env.swap(RAISE_SUBPROC_ERROR=False):
msg = migrator.commit_message(feedstock_ctx) # noqa
try:
eval_cmd("git add --all .")
eval_cmd(f"git commit -am '{msg}'")
except CalledProcessError as e:
LOGGER.info(
"could not commit to feedstock - "
"likely no changes - error is '%s'" % (repr(e)), )
if rerender:
head_ref = eval_cmd("git rev-parse HEAD").strip()
LOGGER.info("Rerendering the feedstock")
try:
eval_cmd(
"conda smithy rerender -c auto --no-check-uptodate",
timeout=300,
)
make_rerender_comment = False
except Exception as e:
# I am trying this bit of code to force these errors
# to be surfaced in the logs at the right time.
print(f"RERENDER ERROR: {e}", flush=True)
if not isinstance(migrator, Version):
raise
else:
# for check solvable or automerge, we always raise rerender errors
if feedstock_ctx.attrs["conda-forge.yml"].get(
"bot", {}).get(
"check_solvable",
False,
) or (feedstock_ctx.attrs["conda-forge.yml"].get(
"bot", {}).get(
"automerge",
False,
)):
raise
else:
make_rerender_comment = True
# If we tried to run the MigrationYaml and rerender did nothing (we only
# bumped the build number and dropped a yaml file in migrations) bail
# for instance platform specific migrations
gdiff = eval_cmd(f"git diff --name-only {head_ref.strip()}...HEAD")
diffed_files = [
_ for _ in gdiff.split()
if not (_.startswith("recipe") or _.startswith("migrators")
or _.startswith("README"))
]
else:
make_rerender_comment = False
if (feedstock_ctx.feedstock_name != "conda-forge-pinning"
and base_branch == "master" and
((
migrator.check_solvable
# we always let stuff in cycles go
and feedstock_ctx.attrs["name"] not in getattr(
migrator, "cycles", set())
# we always let stuff at the top go
and feedstock_ctx.attrs["name"] not in getattr(
migrator, "top_level", set())
# for solveability always assume automerge is on.
and
(feedstock_ctx.attrs["conda-forge.yml"].get("bot", {}).get(
"automerge", True)))
or feedstock_ctx.attrs["conda-forge.yml"].get("bot", {}).get(
"check_solvable",
False,
))):
solvable, errors, _ = is_recipe_solvable(
feedstock_dir,
build_platform=feedstock_ctx.attrs["conda-forge.yml"].get(
"build_platform",
None,
),
)
if not solvable:
_solver_err_str = "not solvable ({}): {}: {}".format(
('<a href="' + os.getenv("CIRCLE_BUILD_URL", "") +
'">bot CI job</a>'),
base_branch,
sorted(set(errors)),
)
if isinstance(migrator, Version):
_new_ver = feedstock_ctx.attrs["new_version"]
if _new_ver in feedstock_ctx.attrs["new_version_errors"]:
feedstock_ctx.attrs["new_version_errors"][
_new_ver] += "\n\nsolver error - {}".format(
_solver_err_str, )
else:
feedstock_ctx.attrs["new_version_errors"][
_new_ver] = _solver_err_str
feedstock_ctx.attrs["new_version_errors"][
_new_ver] = sanitize_string(
feedstock_ctx.attrs["new_version_errors"][_new_ver], )
# remove part of a try for solver errors to make those slightly
# higher priority
feedstock_ctx.attrs["new_version_attempts"][_new_ver] -= 0.8
pre_key = "pre_pr_migrator_status"
if pre_key not in feedstock_ctx.attrs:
feedstock_ctx.attrs[pre_key] = {}
feedstock_ctx.attrs[pre_key][migrator_name] = sanitize_string(
_solver_err_str, )
eval_cmd(f"rm -rf {feedstock_dir}")
return False, False
# TODO: Better annotation here
pr_json: typing.Union[MutableMapping, None, bool]
if (isinstance(migrator, MigrationYaml) and not diffed_files
and feedstock_ctx.attrs["name"] != "conda-forge-pinning"):
# spoof this so it looks like the package is done
pr_json = {
"state": "closed",
"merged_at": "never issued",
"id": str(uuid4()),
}
else:
# push up
try:
# TODO: remove this hack, but for now this is the only way to get
# the feedstock dir into pr_body
feedstock_ctx.feedstock_dir = feedstock_dir
pr_json = push_repo(
session_ctx=migrator.ctx.session,
fctx=feedstock_ctx,
feedstock_dir=feedstock_dir,
body=migrator.pr_body(feedstock_ctx),
repo=repo,
title=migrator.pr_title(feedstock_ctx),
head=f"{migrator.ctx.github_username}:{branch_name}",
branch=branch_name,
base_branch=base_branch,
)
# This shouldn't happen too often any more since we won't double PR
except github3.GitHubError as e:
if e.msg != "Validation Failed":
raise
else:
print(f"Error during push {e}")
# If we just push to the existing PR then do nothing to the json
pr_json = False
ljpr = False
if pr_json and pr_json["state"] != "closed" and make_rerender_comment:
comment_on_pr(
pr_json,
"""\
Hi! This feedstock was not able to be rerendered after the version update changes. I
have pushed the version update changes anyways and am trying to rerender again with this
comment. Hopefully you all can fix this!
@conda-forge-admin rerender""",
repo,
)
if pr_json:
ljpr = LazyJson(
os.path.join(migrator.ctx.session.prjson_dir,
str(pr_json["id"]) + ".json"), )
ljpr.update(**pr_json)
# from .dynamo_models import PRJson
# PRJson.dump(pr_json)
else:
ljpr = False
# If we've gotten this far then the node is good
feedstock_ctx.attrs["bad"] = False
LOGGER.info("Removing feedstock dir")
eval_cmd(f"rm -rf {feedstock_dir}")
return migrate_return, ljpr
def test_pillow_solvable(tmp_path):
"""pillow acted up for python310"""
feedstock_dir = clone_and_checkout_repo(
tmp_path,
"https://github.com/conda-forge/pillow-feedstock",
ref="master",
)
subprocess.run(
f"cd {feedstock_dir} && git checkout 0cae9b1b3450fd8862ac0f48f3389fc349702810",
shell=True,
check=True,
)
with open(
os.path.join(feedstock_dir, ".ci_support", "migrations",
"python310.yaml"),
"w",
) as fp:
fp.write("""\
migrator_ts: 1634137107
__migrator:
migration_number: 1
operation: key_add
primary_key: python
ordering:
python:
- 3.6.* *_cpython
- 3.7.* *_cpython
- 3.8.* *_cpython
- 3.9.* *_cpython
- 3.10.* *_cpython # new entry
- 3.6.* *_73_pypy
- 3.7.* *_73_pypy
paused: false
longterm: True
pr_limit: 40
max_solver_attempts: 10 # this will make the bot retry "not solvable" stuff 10 times
exclude:
# this shouldn't attempt to modify the python feedstocks
- python
- pypy3.6
- pypy-meta
- cross-python
- python_abi
exclude_pinned_pkgs: false
python:
- 3.10.* *_cpython
# additional entries to add for zip_keys
numpy:
- 1.21
python_impl:
- cpython
""", ) # noqa
subprocess.run(
f"cd {feedstock_dir} && conda smithy rerender --no-check-uptodate",
shell=True,
check=True,
)
solvable, errors, solvable_by_variant = is_recipe_solvable(feedstock_dir)
pprint.pprint(solvable_by_variant)
assert solvable, pprint.pformat(errors)
assert any("python3.10" in k for k in solvable_by_variant)
