def initialize_migrators(github_username="", github_password="", github_token=None, dry_run=False): ''' Setup graph, required contexts, and migrators Parameters ---------- github_username: str, optional Username for bot on GitHub github_password: str, optional Password for bot on GitHub github_token: str, optional Token for bot on GitHub dry_run: bool, optional If true, does not submit pull requests on GitHub Returns ------- tuple Migrator session to interact with GitHub and list of migrators. Currently only returns pre-defined migrators. ''' gx = load_graph() smithy_version = eval_cmd("conda smithy --version").strip() pinning_version = json.loads( eval_cmd("conda list conda-forge-pinning --json"))[0]["version"] for m in MIGRATORS: print( f'{getattr(m, "name", m)} graph size: {len(getattr(m, "graph", []))}' ) ctx = MigratorSessionContext( circle_build_url=os.getenv("CIRCLE_BUILD_URL", ""), graph=gx, smithy_version=smithy_version, pinning_version=pinning_version, github_username=github_username, github_password=github_password, github_token=github_token, dry_run=dry_run, ) return ctx, MIGRATORS
def migrate( self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any ) -> "MigrationUidTypedDict": migration_yaml_dict = { "__migrator": {"build_number": 1, "kind": "version", "migration_number": 1}, self.package_name: [self.new_pin_version], "migrator_ts": float(time.time()), } with indir(os.path.join(recipe_dir, "migrations")): mig_fname = "{}{}.yaml".format( self.package_name, self.new_pin_version.replace(".", ""), ) with open(mig_fname, "w") as f: yaml.dump(migration_yaml_dict, f, default_flow_style=False, indent=2) eval_cmd("git add .") return super().migrate(recipe_dir, attrs)
def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> "MigrationUidTypedDict": # if conda-forge-pinning update the pins and close the migration if attrs.get("name", "") == "conda-forge-pinning": # read up the conda build config with indir(recipe_dir), open("conda_build_config.yaml") as f: cbc_contents = f.read() merged_cbc = merge_migrator_cbc(self.yaml_contents, cbc_contents) with indir(os.path.join(recipe_dir, "migrations")): os.remove(f"{self.name}.yaml") # replace the conda build config with the merged one with indir(recipe_dir), open("conda_build_config.yaml", "w") as f: f.write(merged_cbc) # don't need to bump build number once we move to datetime # version numbers for pinning return super().migrate(recipe_dir, attrs) else: # in case the render is old os.makedirs(os.path.join(recipe_dir, "../.ci_support"), exist_ok=True) with indir(os.path.join(recipe_dir, "../.ci_support")): os.makedirs("migrations", exist_ok=True) with indir("migrations"): with open(f"{self.name}.yaml", "w") as f: f.write(self.yaml_contents) eval_cmd("git add .") if self.conda_forge_yml_patches is not None: with indir(os.path.join(recipe_dir, "..")): with open("conda-forge.yml", "r") as fp: cfg = yaml.safe_load(fp.read()) _patch_dict(cfg, self.conda_forge_yml_patches) with open("conda-forge.yml", "w") as fp: yaml.dump(cfg, fp, default_flow_style=False, indent=2) eval_cmd("git add conda-forge.yml") with indir(recipe_dir): self.set_build_number("meta.yaml") return super().migrate(recipe_dir, attrs)
def main(args: "CLIArgs") -> None: # logging if args.debug: setup_logger(logging.getLogger("conda_forge_tick"), level="debug") else: setup_logger(logging.getLogger("conda_forge_tick")) github_username = env.get("USERNAME", "") github_password = env.get("PASSWORD", "") github_token = env.get("GITHUB_TOKEN") mctx, temp, migrators = initialize_migrators( github_username=github_username, github_password=github_password, dry_run=args.dry_run, github_token=github_token, ) # compute the time per migrator print("computing time per migration", flush=True) (num_nodes, time_per_migrator, tot_time_per_migrator) = _compute_time_per_migrator( mctx, migrators, ) for i, migrator in enumerate(migrators): if hasattr(migrator, "name"): extra_name = "-%s" % migrator.name else: extra_name = "" print( " %s%s: %d - gets %f seconds (%f percent)" % ( migrator.__class__.__name__, extra_name, num_nodes[i], time_per_migrator[i], time_per_migrator[i] / max(tot_time_per_migrator, 1) * 100, ), flush=True, ) for mg_ind, migrator in enumerate(migrators): if hasattr(migrator, "name"): assert isinstance(migrator.name, str) migrator_name = migrator.name.lower().replace(" ", "") else: migrator_name = migrator.__class__.__name__.lower() mmctx = MigratorContext(session=mctx, migrator=migrator) migrator.bind_to_ctx(mmctx) good_prs = 0 _mg_start = time.time() effective_graph = mmctx.effective_graph time_per = time_per_migrator[mg_ind] if hasattr(migrator, "name"): extra_name = "-%s" % migrator.name else: extra_name = "" print( "\n========================================" "========================================" "\n" "========================================" "========================================", flush=True, ) print( "Running migrations for %s%s: %d\n" % ( migrator.__class__.__name__, extra_name, len(effective_graph.nodes), ), flush=True, ) possible_nodes = list(migrator.order(effective_graph, mctx.graph)) # version debugging info if isinstance(migrator, Version): LOGGER.info("possible version migrations:") for node_name in possible_nodes: with effective_graph.nodes[node_name]["payload"] as attrs: LOGGER.info( " node|curr|new|attempts: %s|%s|%s|%d", node_name, attrs.get("version"), attrs.get("new_version"), (attrs.get("new_version_attempts", {}).get( attrs.get("new_version", ""), 0, )), ) for node_name in possible_nodes: with mctx.graph.nodes[node_name]["payload"] as attrs: base_branches = migrator.get_possible_feedstock_branches(attrs) orig_branch = attrs.get("branch", "master") # Don't let CI timeout, break ahead of the timeout so we make certain # to write to the repo # TODO: convert these env vars _now = time.time() if ((_now - int(env.get("START_TIME", time.time())) > int( env.get("TIMEOUT", 600))) or good_prs >= migrator.pr_limit or (_now - _mg_start) > time_per): break fctx = FeedstockContext( package_name=node_name, feedstock_name=attrs["feedstock_name"], attrs=attrs, ) try: for base_branch in base_branches: attrs["branch"] = base_branch if migrator.filter(attrs): continue print("\n", flush=True, end="") LOGGER.info( "%s%s IS MIGRATING %s:%s", migrator.__class__.__name__.upper(), extra_name, fctx.package_name, base_branch, ) try: # Don't bother running if we are at zero if mctx.gh_api_requests_left == 0: break migrator_uid, pr_json = run( feedstock_ctx=fctx, migrator=migrator, rerender=migrator.rerender, protocol="https", hash_type=attrs.get("hash_type", "sha256"), base_branch=base_branch, ) # if migration successful if migrator_uid: d = frozen_to_json_friendly(migrator_uid) # if we have the PR already do nothing if d["data"] in [ existing_pr["data"] for existing_pr in attrs.get( "PRed", []) ]: pass else: if not pr_json: pr_json = { "state": "closed", "head": { "ref": "<this_is_not_a_branch>" }, } d["PR"] = pr_json attrs.setdefault("PRed", []).append(d) attrs.update( { "smithy_version": mctx.smithy_version, "pinning_version": mctx.pinning_version, }, ) except github3.GitHubError as e: if e.msg == "Repository was archived so is read-only.": attrs["archived"] = True else: LOGGER.critical( "GITHUB ERROR ON FEEDSTOCK: %s", fctx.feedstock_name, ) if is_github_api_limit_reached(e, mctx.gh): break except URLError as e: LOGGER.exception("URLError ERROR") attrs["bad"] = { "exception": str(e), "traceback": str(traceback.format_exc()).split("\n"), "code": getattr(e, "code"), "url": getattr(e, "url"), } pre_key = "pre_pr_migrator_status" if pre_key not in attrs: attrs[pre_key] = {} attrs[pre_key][migrator_name] = sanitize_string( "bot error (%s): %s: %s" % ( '<a href="' + os.getenv("CIRCLE_BUILD_URL", "") + '">bot CI job</a>', base_branch, str(traceback.format_exc()), ), ) except Exception as e: LOGGER.exception("NON GITHUB ERROR") # we don't set bad for rerendering errors if ("conda smithy rerender -c auto --no-check-uptodate" not in str(e)): attrs["bad"] = { "exception": str(e), "traceback": str(traceback.format_exc()).split("\n", ), } pre_key = "pre_pr_migrator_status" if pre_key not in attrs: attrs[pre_key] = {} attrs[pre_key][migrator_name] = sanitize_string( "bot error (%s): %s: %s" % ( '<a href="' + os.getenv("CIRCLE_BUILD_URL", "") + '">bot CI job</a>', base_branch, str(traceback.format_exc()), ), ) else: if migrator_uid: # On successful PR add to our counter good_prs += 1 finally: # reset branch attrs["branch"] = orig_branch # Write graph partially through if not args.dry_run: dump_graph(mctx.graph) eval_cmd(f"rm -rf {mctx.rever_dir}/*") LOGGER.info(os.getcwd()) for f in glob.glob("/tmp/*"): if f not in temp: try: eval_cmd(f"rm -rf {f}") except Exception: pass if mctx.gh_api_requests_left == 0: break print("\n", flush=True) LOGGER.info("API Calls Remaining: %d", mctx.gh_api_requests_left) LOGGER.info("Done")
def initialize_migrators( github_username: str = "", github_password: str = "", github_token: Optional[str] = None, dry_run: bool = False, ) -> Tuple[MigratorSessionContext, list, MutableSequence[Migrator]]: temp = glob.glob("/tmp/*") gx = load_graph() smithy_version = eval_cmd("conda smithy --version").strip() pinning_version = json.loads( eval_cmd("conda list conda-forge-pinning --json"))[0]["version"] migrators = [] add_arch_migrate(migrators, gx) migration_factory(migrators, gx) add_replacement_migrator( migrators, gx, "matplotlib", "matplotlib-base", ("Unless you need `pyqt`, recipes should depend only on " "`matplotlib-base`."), alt_migrator=MatplotlibBase, ) create_migration_yaml_creator(migrators=migrators, gx=gx) print("rebuild migration graph sizes:", flush=True) for m in migrators: print( f' {getattr(m, "name", m)} graph size: ' f'{len(getattr(m, "graph", []))}', flush=True, ) print(" ", flush=True) mctx = MigratorSessionContext( circle_build_url=os.getenv("CIRCLE_BUILD_URL", ""), graph=gx, smithy_version=smithy_version, pinning_version=pinning_version, github_username=github_username, github_password=github_password, github_token=github_token, dry_run=dry_run, ) print("building package import maps and version migrator", flush=True) python_nodes = { n for n, v in mctx.graph.nodes("payload") if "python" in v.get("req", "") } python_nodes.update([ k for node_name, node in mctx.graph.nodes("payload") for k in node.get("outputs_names", []) if node_name in python_nodes ], ) version_migrator = Version( python_nodes=python_nodes, pr_limit=PR_LIMIT * 4, piggy_back_migrations=[ Jinja2VarsCleanup(), DuplicateLinesCleanup(), PipMigrator(), LicenseMigrator(), CondaForgeYAMLCleanup(), ExtraJinja2KeysCleanup(), Build2HostMigrator(), NoCondaInspectMigrator(), Cos7Config(), ], ) migrators = [version_migrator] + migrators print(" ", flush=True) return mctx, temp, migrators
def run( feedstock_ctx: FeedstockContext, migrator: Migrator, protocol: str = "ssh", pull_request: bool = True, rerender: bool = True, fork: bool = True, base_branch: str = "master", **kwargs: typing.Any, ) -> Tuple["MigrationUidTypedDict", dict]: """For a given feedstock and migration run the migration Parameters ---------- feedstock_ctx: FeedstockContext The node attributes migrator: Migrator instance The migrator to run on the feedstock protocol : str, optional The git protocol to use, defaults to ``ssh`` pull_request : bool, optional If true issue pull request, defaults to true rerender : bool Whether to rerender fork : bool If true create a fork, defaults to true base_branch : str, optional The base branch to which the PR will be targeted. Defaults to "master". kwargs: dict The key word arguments to pass to the migrator Returns ------- migrate_return: MigrationUidTypedDict The migration return dict used for tracking finished migrations pr_json: dict The PR json object for recreating the PR as needed """ # get the repo # TODO: stop doing this. migrator.attrs = feedstock_ctx.attrs # type: ignore branch_name = migrator.remote_branch( feedstock_ctx) + "_h" + uuid4().hex[0:6] if hasattr(migrator, "name"): assert isinstance(migrator.name, str) migrator_name = migrator.name.lower().replace(" ", "") else: migrator_name = migrator.__class__.__name__.lower() # TODO: run this in parallel feedstock_dir, repo = get_repo( ctx=migrator.ctx.session, fctx=feedstock_ctx, branch=branch_name, feedstock=feedstock_ctx.feedstock_name, protocol=protocol, pull_request=pull_request, fork=fork, base_branch=base_branch, ) if not feedstock_dir or not repo: LOGGER.critical( "Failed to migrate %s, %s", feedstock_ctx.package_name, feedstock_ctx.attrs.get("bad"), ) return False, False recipe_dir = os.path.join(feedstock_dir, "recipe") # migrate the feedstock migrator.run_pre_piggyback_migrations(recipe_dir, feedstock_ctx.attrs, **kwargs) # TODO - make a commit here if the repo changed migrate_return = migrator.migrate(recipe_dir, feedstock_ctx.attrs, **kwargs) if not migrate_return: LOGGER.critical( "Failed to migrate %s, %s", feedstock_ctx.package_name, feedstock_ctx.attrs.get("bad"), ) eval_cmd(f"rm -rf {feedstock_dir}") return False, False # TODO - commit main migration here migrator.run_post_piggyback_migrations(recipe_dir, feedstock_ctx.attrs, **kwargs) # TODO commit post migration here # rerender, maybe diffed_files: typing.List[str] = [] with indir(feedstock_dir), env.swap(RAISE_SUBPROC_ERROR=False): msg = migrator.commit_message(feedstock_ctx) # noqa try: eval_cmd("git add --all .") eval_cmd(f"git commit -am '{msg}'") except CalledProcessError as e: LOGGER.info( "could not commit to feedstock - " "likely no changes - error is '%s'" % (repr(e)), ) if rerender: head_ref = eval_cmd("git rev-parse HEAD").strip() LOGGER.info("Rerendering the feedstock") try: eval_cmd( "conda smithy rerender -c auto --no-check-uptodate", timeout=300, ) make_rerender_comment = False except Exception as e: # I am trying this bit of code to force these errors # to be surfaced in the logs at the right time. print(f"RERENDER ERROR: {e}", flush=True) if not isinstance(migrator, Version): raise else: # for check solvable or automerge, we always raise rerender errors if feedstock_ctx.attrs["conda-forge.yml"].get( "bot", {}).get( "check_solvable", False, ) or (feedstock_ctx.attrs["conda-forge.yml"].get( "bot", {}).get( "automerge", False, )): raise else: make_rerender_comment = True # If we tried to run the MigrationYaml and rerender did nothing (we only # bumped the build number and dropped a yaml file in migrations) bail # for instance platform specific migrations gdiff = eval_cmd(f"git diff --name-only {head_ref.strip()}...HEAD") diffed_files = [ _ for _ in gdiff.split() if not (_.startswith("recipe") or _.startswith("migrators") or _.startswith("README")) ] else: make_rerender_comment = False if (feedstock_ctx.feedstock_name != "conda-forge-pinning" and base_branch == "master" and (( migrator.check_solvable # we always let stuff in cycles go and feedstock_ctx.attrs["name"] not in getattr( migrator, "cycles", set()) # we always let stuff at the top go and feedstock_ctx.attrs["name"] not in getattr( migrator, "top_level", set()) # for solveability always assume automerge is on. and (feedstock_ctx.attrs["conda-forge.yml"].get("bot", {}).get( "automerge", True))) or feedstock_ctx.attrs["conda-forge.yml"].get("bot", {}).get( "check_solvable", False, ))): solvable, errors, _ = is_recipe_solvable( feedstock_dir, build_platform=feedstock_ctx.attrs["conda-forge.yml"].get( "build_platform", None, ), ) if not solvable: _solver_err_str = "not solvable ({}): {}: {}".format( ('<a href="' + os.getenv("CIRCLE_BUILD_URL", "") + '">bot CI job</a>'), base_branch, sorted(set(errors)), ) if isinstance(migrator, Version): _new_ver = feedstock_ctx.attrs["new_version"] if _new_ver in feedstock_ctx.attrs["new_version_errors"]: feedstock_ctx.attrs["new_version_errors"][ _new_ver] += "\n\nsolver error - {}".format( _solver_err_str, ) else: feedstock_ctx.attrs["new_version_errors"][ _new_ver] = _solver_err_str feedstock_ctx.attrs["new_version_errors"][ _new_ver] = sanitize_string( feedstock_ctx.attrs["new_version_errors"][_new_ver], ) # remove part of a try for solver errors to make those slightly # higher priority feedstock_ctx.attrs["new_version_attempts"][_new_ver] -= 0.8 pre_key = "pre_pr_migrator_status" if pre_key not in feedstock_ctx.attrs: feedstock_ctx.attrs[pre_key] = {} feedstock_ctx.attrs[pre_key][migrator_name] = sanitize_string( _solver_err_str, ) eval_cmd(f"rm -rf {feedstock_dir}") return False, False # TODO: Better annotation here pr_json: typing.Union[MutableMapping, None, bool] if (isinstance(migrator, MigrationYaml) and not diffed_files and feedstock_ctx.attrs["name"] != "conda-forge-pinning"): # spoof this so it looks like the package is done pr_json = { "state": "closed", "merged_at": "never issued", "id": str(uuid4()), } else: # push up try: # TODO: remove this hack, but for now this is the only way to get # the feedstock dir into pr_body feedstock_ctx.feedstock_dir = feedstock_dir pr_json = push_repo( session_ctx=migrator.ctx.session, fctx=feedstock_ctx, feedstock_dir=feedstock_dir, body=migrator.pr_body(feedstock_ctx), repo=repo, title=migrator.pr_title(feedstock_ctx), head=f"{migrator.ctx.github_username}:{branch_name}", branch=branch_name, base_branch=base_branch, ) # This shouldn't happen too often any more since we won't double PR except github3.GitHubError as e: if e.msg != "Validation Failed": raise else: print(f"Error during push {e}") # If we just push to the existing PR then do nothing to the json pr_json = False ljpr = False if pr_json and pr_json["state"] != "closed" and make_rerender_comment: comment_on_pr( pr_json, """\ Hi! This feedstock was not able to be rerendered after the version update changes. I have pushed the version update changes anyways and am trying to rerender again with this comment. Hopefully you all can fix this! @conda-forge-admin rerender""", repo, ) if pr_json: ljpr = LazyJson( os.path.join(migrator.ctx.session.prjson_dir, str(pr_json["id"]) + ".json"), ) ljpr.update(**pr_json) # from .dynamo_models import PRJson # PRJson.dump(pr_json) else: ljpr = False # If we've gotten this far then the node is good feedstock_ctx.attrs["bad"] = False LOGGER.info("Removing feedstock dir") eval_cmd(f"rm -rf {feedstock_dir}") return migrate_return, ljpr
def auto_tick(dry_run=False, debug=False, fork=False, organization='nsls-ii-forge'): ''' Automatically update package versions and submit pull requests to associated feedstocks Parameters ---------- dry_run: bool, optional Generate version migration yamls but do not run them debug: bool, optional Setup logging to be in debug mode fork: bool, optional Create a fork of the repo from the organization to $GITHUB_USERNAME organization: str, optional GitHub organization that manages feedstock repositories ''' from conda_forge_tick.xonsh_utils import env if debug: setup_logger(logger, level="debug") else: setup_logger(logger) # set Version.pr_body to custom pr_body function Version.pr_body = bot_pr_body # TODO: use ~/.netrc instead github_username = env.get("GITHUB_USERNAME", "") github_password = env.get("GITHUB_TOKEN", "") github_token = env.get("GITHUB_TOKEN") global MIGRATORS print('Initializing migrators...') mctx, MIGRATORS = initialize_migrators( github_username=github_username, github_password=github_password, dry_run=dry_run, github_token=github_token, ) # compute the time per migrator print('Computing time per migrator') (num_nodes, time_per_migrator, tot_time_per_migrator) = _compute_time_per_migrator(mctx, ) for i, migrator in enumerate(MIGRATORS): if hasattr(migrator, "name"): extra_name = "-%s" % migrator.name else: extra_name = "" logger.info( "Total migrations for %s%s: %d - gets %f seconds (%f percent)", migrator.__class__.__name__, extra_name, num_nodes[i], time_per_migrator[i], time_per_migrator[i] / tot_time_per_migrator * 100, ) print('Performing migrations...') for mg_ind, migrator in enumerate(MIGRATORS): mmctx = MigratorContext(session=mctx, migrator=migrator) migrator.bind_to_ctx(mmctx) good_prs = 0 _mg_start = time.time() effective_graph = mmctx.effective_graph time_per = time_per_migrator[mg_ind] if hasattr(migrator, "name"): extra_name = "-%s" % migrator.name else: extra_name = "" logger.info( "Running migrations for %s%s: %d", migrator.__class__.__name__, extra_name, len(effective_graph.nodes), ) possible_nodes = list(migrator.order(effective_graph, mctx.graph)) # version debugging info if isinstance(migrator, Version): logger.info("possible version migrations:") for node_name in possible_nodes: with effective_graph.nodes[node_name]["payload"] as attrs: logger.info( " node|curr|new|attempts: %s|%s|%s|%d", node_name, attrs.get("version"), attrs.get("new_version"), (attrs.get("new_version_attempts", {}).get( attrs.get("new_version", ""), 0, )), ) for node_name in possible_nodes: with mctx.graph.nodes[node_name]["payload"] as attrs: # Don't let CI timeout, break ahead of the timeout so we make certain # to write to the repo # TODO: convert these env vars _now = time.time() if ((_now - int(env.get("START_TIME", time.time())) > int( env.get("TIMEOUT", 600))) or good_prs >= migrator.pr_limit or (_now - _mg_start) > time_per): break fctx = FeedstockContext( package_name=node_name, feedstock_name=attrs["feedstock_name"], attrs=attrs, ) print("\n", flush=True, end="") logger.info( "%s%s IS MIGRATING %s", migrator.__class__.__name__.upper(), extra_name, fctx.package_name, ) try: # Don't bother running if we are at zero if (dry_run or mctx.gh.rate_limit()["resources"]["core"] ["remaining"] == 0): break migrator_uid, pr_json = run(feedstock_ctx=fctx, migrator=migrator, rerender=migrator.rerender, protocol="https", hash_type=attrs.get( "hash_type", "sha256"), fork=fork, organization=organization) # if migration successful if migrator_uid: d = frozen_to_json_friendly(migrator_uid) # if we have the PR already do nothing if d["data"] in [ existing_pr["data"] for existing_pr in attrs.get("PRed", []) ]: pass else: if pr_json is None: pr_json = { "state": "closed", "head": { "ref": "<this_is_not_a_branch>" }, } d["PR"] = pr_json attrs.setdefault("PRed", []).append(d) attrs.update( { "smithy_version": mctx.smithy_version, "pinning_version": mctx.pinning_version, }, ) except github3.GitHubError as e: if e.msg == "Repository was archived so is read-only.": attrs["archived"] = True else: logger.critical( "GITHUB ERROR ON FEEDSTOCK: %s", fctx.feedstock_name, ) if is_github_api_limit_reached(e, mctx.gh): break except URLError as e: logger.exception("URLError ERROR") attrs["bad"] = { "exception": str(e), "traceback": str(traceback.format_exc()).split("\n"), "code": getattr(e, "code"), "url": getattr(e, "url"), } except Exception as e: logger.exception("NON GITHUB ERROR") attrs["bad"] = { "exception": str(e), "traceback": str(traceback.format_exc()).split("\n"), } else: if migrator_uid: # On successful PR add to our counter good_prs += 1 finally: # Write graph partially through if not dry_run: dump_graph(mctx.graph) eval_cmd(f"rm -rf {mctx.rever_dir}/*") logger.info(os.getcwd()) if not dry_run: logger.info( "API Calls Remaining: %d", mctx.gh.rate_limit()["resources"]["core"]["remaining"], ) logger.info("Done")
def run(feedstock_ctx, migrator, protocol='ssh', pull_request=True, rerender=True, fork=False, organization='nsls-ii-forge', **kwargs): """ For a given feedstock and migration run the migration and possibly submit pull request Parameters ---------- feedstock_ctx: FeedstockContext The node attributes of the feedstock migrator: Migrator The migrator to run on the feedstock protocol: str, optional The git protocol to use, defaults to ``ssh`` pull_request: bool, optional If true issue pull request, defaults to true rerender: bool Whether to rerender, defaults to true fork: bool If true create a fork, defaults to false organization: str, optional GitHub organization to get repo from gh: github3.GitHub, optional Object for communicating with GitHub, if None, build from $GITHUB_USERNAME and $GITHUB_PASSWORD, defaults to None kwargs: dict The key word arguments to pass to the migrator Returns ------- migrate_return: MigrationUidTypedDict The migration return dict used for tracking finished migrations pr_json: dict The PR json object for recreating the PR as needed """ # get the repo migrator.attrs = feedstock_ctx.attrs branch_name = migrator.remote_branch( feedstock_ctx) + "_h" + uuid4().hex[:6] # TODO: run this in parallel feedstock_dir, repo = get_repo( ctx=migrator.ctx.session, fctx=feedstock_ctx, branch=branch_name, organization=organization, feedstock=feedstock_ctx.feedstock_name, protocol=protocol, pull_request=pull_request, fork=fork, ) recipe_dir = os.path.join(feedstock_dir, "recipe") # migrate the feedstock migrator.run_pre_piggyback_migrations(recipe_dir, feedstock_ctx.attrs, **kwargs) # TODO - make a commit here if the repo changed migrate_return = migrator.migrate(recipe_dir, feedstock_ctx.attrs, **kwargs) if not migrate_return: logger.critical( "Failed to migrate %s, %s", feedstock_ctx.package_name, feedstock_ctx.attrs.get("bad"), ) eval_cmd(f"rm -rf {feedstock_dir}") return False, False # TODO - commit main migration here migrator.run_post_piggyback_migrations(recipe_dir, feedstock_ctx.attrs, **kwargs) # TODO commit post migration here # rerender, maybe diffed_files = [] with indir(feedstock_dir), env.swap(RAISE_SUBPROC_ERROR=False): msg = migrator.commit_message(feedstock_ctx) # noqa try: eval_cmd("git add --all .") eval_cmd(f"git commit -am '{msg}'") except CalledProcessError as e: logger.info( "could not commit to feedstock - " "likely no changes - error is '%s'" % (repr(e)), ) if rerender: head_ref = eval_cmd("git rev-parse HEAD").strip() logger.info("Rerendering the feedstock") # In the event we can't rerender, try to update the pinnings, # then bail if it does not work again try: eval_cmd( "conda smithy rerender -c auto --no-check-uptodate", timeout=300, ) except SubprocessError: return False, False # If we tried to run the MigrationYaml and rerender did nothing (we only # bumped the build number and dropped a yaml file in migrations) bail # for instance platform specific migrations gdiff = eval_cmd(f"git diff --name-only {head_ref.strip()}...HEAD") diffed_files = [ _ for _ in gdiff.split() if not (_.startswith("recipe") or _.startswith("migrators") or _.startswith("README")) ] if ((migrator.check_solvable and feedstock_ctx.attrs["conda-forge.yml"].get("bot", {}).get("automerge")) or feedstock_ctx.attrs["conda-forge.yml"].get("bot", {}).get( "check_solvable", False)) and not is_recipe_solvable(feedstock_dir): eval_cmd(f"rm -rf {feedstock_dir}") return False, False if (isinstance(migrator, MigrationYaml) and not diffed_files and feedstock_ctx.attrs["name"] != "conda-forge-pinning"): # spoof this so it looks like the package is done pr_json = { "state": "closed", "merged_at": "never issued", "id": str(uuid4()), } else: # push up try: if fork: head = f"{migrator.ctx.github_username}:{branch_name}" else: head = f"{organization}:{branch_name}" pr_json = push_repo(session_ctx=migrator.ctx.session, fctx=feedstock_ctx, feedstock_dir=feedstock_dir, body=migrator.pr_body(feedstock_ctx), repo=repo, title=migrator.pr_title(feedstock_ctx), head=head, branch=branch_name, fork=fork, organization=organization) # This shouldn't happen too often any more since we won't double PR except github3.GitHubError as e: if e.msg != "Validation Failed": raise else: print(f"Error during push {e}") print(f'Errors: {e.errors}') # If we just push to the existing PR then do nothing to the json pr_json = None ljpr = None if pr_json is not None: ljpr = LazyJson( os.path.join(migrator.ctx.session.prjson_dir, str(pr_json["id"]) + ".json"), ) ljpr.update(**pr_json) else: ljpr = None # If we've gotten this far then the node is good feedstock_ctx.attrs["bad"] = False logger.info("Removing feedstock dir") eval_cmd(f"rm -rf {feedstock_dir}") return migrate_return, ljpr
def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> None: # r- recipes have a special syntax here if (attrs.get("feedstock_name", "").startswith("r-") or attrs.get( "name", "").startswith("r-")) and "r-base" in attrs["raw_meta_yaml"]: if attrs.get("feedstock_name", None) is not None: if attrs.get("feedstock_name", None).endswith("-feedstock"): name = attrs.get("feedstock_name")[:-len("-feedstock")] else: name = attrs.get("feedstock_name") else: name = attrs.get("name", None) _do_r_license_munging(name, recipe_dir) return try: cb_work_dir = _get_source_code(recipe_dir) except Exception: return if cb_work_dir is None: return with indir(cb_work_dir): # look for a license file license_files = [ s for s in os.listdir(".") if any(s.lower().startswith(k) for k in ["license", "copying", "copyright"]) ] eval_cmd(f"rm -r {cb_work_dir}") # if there is a license file in tarball update things if license_files: with indir(recipe_dir): """BSD 3-Clause License Copyright (c) 2017, Anthony Scopatz Copyright (c) 2018, The Regro Developers All rights reserved.""" with open("meta.yaml") as f: raw = f.read() lines = raw.splitlines() ptn = re.compile(r"(\s*?)" + "license:") for i, line in enumerate(lines): m = ptn.match(line) if m is not None: break # TODO: Sketchy type assertion assert m is not None ws = m.group(1) if len(license_files) == 1: replace_in_file( line, line + "\n" + ws + f"license_file: {list(license_files)[0]}", "meta.yaml", ) else: # note that this white space is not perfect but works for # most of the situations replace_in_file( line, line + "\n" + ws + "license_file: \n" + "".join(f"{ws*2}- {z} \n" for z in license_files), "meta.yaml", )
def _run_migrator(migrator, mctx, temp, time_per, dry_run): if hasattr(migrator, "name"): assert isinstance(migrator.name, str) migrator_name = migrator.name.lower().replace(" ", "") else: migrator_name = migrator.__class__.__name__.lower() mmctx = MigratorContext(session=mctx, migrator=migrator) migrator.bind_to_ctx(mmctx) good_prs = 0 _mg_start = time.time() effective_graph = mmctx.effective_graph if hasattr(migrator, "name"): extra_name = "-%s" % migrator.name else: extra_name = "" print( "Running migrations for %s%s: %d\n" % ( migrator.__class__.__name__, extra_name, len(effective_graph.nodes), ), flush=True, ) possible_nodes = list(migrator.order(effective_graph, mctx.graph)) # version debugging info if isinstance(migrator, Version): LOGGER.info("possible version migrations:") for node_name in possible_nodes: with effective_graph.nodes[node_name]["payload"] as attrs: LOGGER.info( " node|curr|new|attempts: %s|%s|%s|%f", node_name, attrs.get("version"), attrs.get("new_version"), ( attrs.get("new_version_attempts", {}).get( attrs.get("new_version", ""), 0, ) ), ) for node_name in possible_nodes: with mctx.graph.nodes[node_name]["payload"] as attrs: # Don't let CI timeout, break ahead of the timeout so we make certain # to write to the repo # TODO: convert these env vars _now = time.time() if ( ( _now - int(env.get("START_TIME", time.time())) > int(env.get("TIMEOUT", 600)) ) or good_prs >= migrator.pr_limit or (_now - _mg_start) > time_per ): break base_branches = migrator.get_possible_feedstock_branches(attrs) if "branch" in attrs: has_attrs_branch = True orig_branch = attrs.get("branch") else: has_attrs_branch = False orig_branch = None fctx = FeedstockContext( package_name=node_name, feedstock_name=attrs["feedstock_name"], attrs=attrs, ) # map main to current default branch base_branches = [ br if br != "main" else fctx.default_branch for br in base_branches ] try: for base_branch in base_branches: attrs["branch"] = base_branch if migrator.filter(attrs): continue print("\n", flush=True, end="") sys.stderr.flush() sys.stdout.flush() LOGGER.info( "%s%s IS MIGRATING %s:%s", migrator.__class__.__name__.upper(), extra_name, fctx.package_name, base_branch, ) try: # Don't bother running if we are at zero if mctx.gh_api_requests_left == 0: break migrator_uid, pr_json = run( feedstock_ctx=fctx, migrator=migrator, rerender=migrator.rerender, protocol="https", hash_type=attrs.get("hash_type", "sha256"), base_branch=base_branch, ) # if migration successful if migrator_uid: d = frozen_to_json_friendly(migrator_uid) # if we have the PR already do nothing if d["data"] in [ existing_pr["data"] for existing_pr in attrs.get("PRed", []) ]: pass else: if not pr_json: pr_json = { "state": "closed", "head": {"ref": "<this_is_not_a_branch>"}, } d["PR"] = pr_json attrs.setdefault("PRed", []).append(d) attrs.update( { "smithy_version": mctx.smithy_version, "pinning_version": mctx.pinning_version, }, ) except github3.GitHubError as e: if e.msg == "Repository was archived so is read-only.": attrs["archived"] = True else: LOGGER.critical( "GITHUB ERROR ON FEEDSTOCK: %s", fctx.feedstock_name, ) if is_github_api_limit_reached(e, mctx.gh): break except URLError as e: LOGGER.exception("URLError ERROR") attrs["bad"] = { "exception": str(e), "traceback": str(traceback.format_exc()).split("\n"), "code": getattr(e, "code"), "url": getattr(e, "url"), } _set_pre_pr_migrator_fields( attrs, migrator_name, sanitize_string( "bot error (%s): %s: %s" % ( '<a href="' + os.getenv("CIRCLE_BUILD_URL", "") + '">bot CI job</a>', base_branch, str(traceback.format_exc()), ), ), ) except Exception as e: LOGGER.exception("NON GITHUB ERROR") # we don't set bad for rerendering errors if ( "conda smithy rerender -c auto --no-check-uptodate" not in str(e) ): attrs["bad"] = { "exception": str(e), "traceback": str(traceback.format_exc()).split( "\n", ), } _set_pre_pr_migrator_fields( attrs, migrator_name, sanitize_string( "bot error (%s): %s: %s" % ( '<a href="' + os.getenv("CIRCLE_BUILD_URL", "") + '">bot CI job</a>', base_branch, str(traceback.format_exc()), ), ), ) else: if migrator_uid: # On successful PR add to our counter good_prs += 1 finally: # reset branch if has_attrs_branch: attrs["branch"] = orig_branch # do this but it is crazy gc.collect() # Write graph partially through if not dry_run: dump_graph(mctx.graph) eval_cmd(f"rm -rf {mctx.rever_dir}/*") LOGGER.info(os.getcwd()) for f in glob.glob("/tmp/*"): if f not in temp: try: eval_cmd(f"rm -rf {f}") except Exception: pass if mctx.gh_api_requests_left == 0: break return good_prs