def main(args: Any = None) -> None: if CONDA_FORGE_TICK_DEBUG: setup_logger(logger, level="debug") else: setup_logger(logger) logger.info("Reading graph") gx = load_graph() update_upstream_versions(gx) logger.info("writing out file") dump_graph(gx)
def main(args): gx = load_graph() ctx = MigratorSessionContext("", "", "") start_time = time.time() os.makedirs("audits", exist_ok=True) for k in AUDIT_REGISTRY: os.makedirs(os.path.join("audits", k), exist_ok=True) # TODO: generalize for cran skeleton # limit graph to things that depend on python python_des = nx.descendants(gx, "pypy-meta") for node in sorted( python_des, key=lambda x: (len(nx.descendants(gx, x)), x), reverse=True, ): if time.time() - int(env.get("START_TIME", start_time)) > int( env.get("TIMEOUT", 60 * 30), ): break # depfinder only work on python at the moment so only work on things # with python as runtime dep payload = gx.nodes[node]["payload"] for k, v in AUDIT_REGISTRY.items(): version = payload.get("version", None) ext = v["ext"] if (not payload.get("archived", False) and version and "python" in payload["requirements"]["run"] and f"{node}_{version}.{ext}" not in os.listdir(f"audits/{k}")): print(node) fctx = FeedstockContext( package_name=node, feedstock_name=payload["name"], attrs=payload, ) try: deps = v["run"](fctx, ctx) except Exception as e: deps = { "exception": str(e), "traceback": str(traceback.format_exc()).split("\n"), } if "dumper" in v: deps = v["dumper"](deps) finally: with open(f"audits/{k}/{node}_{version}.{ext}", "w") as f: v["writer"](deps, f) compare_grayskull_audits(gx) compare_depfinder_audits(gx)
def main(args: Any = None) -> None: if CONDA_FORGE_TICK_DEBUG: setup_logger(logger, level="debug") else: setup_logger(logger) logger.info("Reading graph") # Graph enabled for inspection gx = load_graph() # call update to_update = new_update_upstream_versions(gx) logger.info("writing out file") with open("new_version.json", "w") as outfile: json.dump(to_update, outfile)
def initialize_migrators(github_username="", github_password="", github_token=None, dry_run=False): ''' Setup graph, required contexts, and migrators Parameters ---------- github_username: str, optional Username for bot on GitHub github_password: str, optional Password for bot on GitHub github_token: str, optional Token for bot on GitHub dry_run: bool, optional If true, does not submit pull requests on GitHub Returns ------- tuple Migrator session to interact with GitHub and list of migrators. Currently only returns pre-defined migrators. ''' gx = load_graph() smithy_version = eval_cmd("conda smithy --version").strip() pinning_version = json.loads( eval_cmd("conda list conda-forge-pinning --json"))[0]["version"] for m in MIGRATORS: print( f'{getattr(m, "name", m)} graph size: {len(getattr(m, "graph", []))}' ) ctx = MigratorSessionContext( circle_build_url=os.getenv("CIRCLE_BUILD_URL", ""), graph=gx, smithy_version=smithy_version, pinning_version=pinning_version, github_username=github_username, github_password=github_password, github_token=github_token, dry_run=dry_run, ) return ctx, MIGRATORS
def main(args: Any = None) -> None: sources = (PyPI(), CRAN(), NPM(), ROSDistro(), RawURL(), Github()) if CONDA_FORGE_TICK_DEBUG: setup_logger(logger, level="debug") else: setup_logger(logger) logger.info("Reading graph") # Graph enabled for inspection gx = load_graph() # call update new_update_upstream_versions(gx) logger.info("writing out file") with open("new_version.json", "w") as outfile: json.dump(to_update, outfile)
def _query_graph_handle_args(args): from conda_forge_tick.utils import load_graph if args.filepath != 'graph.json': copyfile(args.filepath, 'graph.json') gx = load_graph() if args.query == 'depends_on': dependencies = list_dependencies_on(gx, args.package) print( f'The following packages require {args.package} to be installed:') for dep in dependencies: print(dep) print(f'Total: {len(dependencies)}') elif args.query == 'depends_of': dependencies = list_dependencies_of(gx, args.package) print( f'{args.package} requires the following packages to be installed:') for dep in dependencies: print(dep) print(f'Total: {len(dependencies)}') else: print(f'Unknown query type: {args.query}')
def _make_graph_handle_args(args): from conda_forge_tick.utils import load_graph, dump_graph from conda_forge_tick.make_graph import update_nodes_with_bot_rerun # get a list of all feedstocks from nsls-ii-forge global DEBUG DEBUG = args.debug global MAX_WORKERS MAX_WORKERS = args.max_workers organization = args.organization names = get_all_feedstocks(cached=args.cached, filepath=args.filepath, organization=organization) if os.path.exists("graph.json"): gx = load_graph() else: gx = None print(f'Using {MAX_WORKERS} workers in process pool') gx = make_graph(names, organization, gx=gx) print("nodes w/o payload:", [k for k, v in gx.nodes.items() if "payload" not in v]) update_nodes_with_bot_rerun(gx) print('Saving graph to graph.json') dump_graph(gx)
def main(args): gx = load_graph() ctx = MigratorSessionContext("", "", "") start_time = time.time() # limit graph to things that depend on python python_des = nx.descendants(gx, "pypy-meta") for node in sorted( python_des, key=lambda x: (len(nx.descendants(gx, x)), x), reverse=True, ): if time.time() - int(env.get("START_TIME", start_time)) > int( env.get("TIMEOUT", 60 * 30)): break # depfinder only work on python at the moment so only work on things # with python as runtime dep os.makedirs("audits", exist_ok=True) with gx.nodes[node]["payload"] as payload: version = payload.get('version', None) if (not payload.get("archived", False) and version and "python" in payload["requirements"]["run"] and f'{node}_{version}.json' not in os.listdir("audits")): print(node) fctx = FeedstockContext(package_name=node, feedstock_name=payload["name"], attrs=payload) try: deps = audit_feedstock(fctx, ctx) except Exception as e: deps = { "exception": str(e), "traceback": str(traceback.format_exc()).split("\n"), } finally: with open(f"audits/{node}_{version}.json", "w") as f: dump(deps, f)
def main(args): gx = load_graph() ctx = MigratorSessionContext("", "", "") start_time = time.time() os.makedirs("audits", exist_ok=True) for k, v in AUDIT_REGISTRY.items(): audit_dir = os.path.join("audits", k) version_path = os.path.join(audit_dir, "_version.json") audit_version = "_".join([v["version"], v["creation_version"]]) if os.path.exists(version_path): version = load(open(version_path)) # if the version of the code generating the audits is different from our current audit data # clear out the audit data so we always use the latest version if version != audit_version: shutil.rmtree(audit_dir) os.makedirs(audit_dir, exist_ok=True) dump(audit_version, open(version_path, "w")) # TODO: generalize for cran skeleton # limit graph to things that depend on python python_des = nx.descendants(gx, "python") for node in sorted( python_des, key=lambda x: (len(nx.descendants(gx, x)), x), reverse=True, ): if time.time() - int(env.get("START_TIME", start_time)) > int( env.get("TIMEOUT", 60 * RUNTIME_MINUTES), ): break # depfinder only work on python at the moment so only work on things # with python as runtime dep payload = gx.nodes[node]["payload"] for k, v in AUDIT_REGISTRY.items(): version = payload.get("version", None) ext = v["ext"] if ( not payload.get("archived", False) and not payload.get("bad", False) and version and "python" in payload["requirements"]["run"] and f"{node}_{version}.{ext}" not in os.listdir(f"audits/{k}") ): fctx = FeedstockContext( package_name=node, feedstock_name=payload["feedstock_name"], attrs=payload, ) try: deps = v["run"](fctx, ctx) except Exception as e: deps = { "exception": str(e), "traceback": str(traceback.format_exc()).split("\n"), } if "dumper" in v: deps = v["dumper"](deps) finally: if deps: with open(f"audits/{k}/{node}_{version}.{ext}", "w") as f: v["writer"](deps, f) # grayskull_audit_outcome = compare_grayskull_audits(gx) # compute_grayskull_accuracy(grayskull_audit_outcome) depfinder_audit_outcome = compare_depfinder_audits(gx) compute_depfinder_accuracy(depfinder_audit_outcome)
from conda_forge_tick.utils import load_graph, dump_graph_json gx = load_graph() for node in gx.nodes: with gx.nodes[node]["payload"] as node_attrs: prs = node_attrs.get("PRed", []) for i, pr in enumerate(prs): if "bot_rerun" in pr: print("fixing:", node) pr["data"]["bot_rerun"] = pr["bot_rerun"] del pr["bot_rerun"] dump_graph_json(gx, "graph.json")
def initialize_migrators( github_username: str = "", github_password: str = "", github_token: Optional[str] = None, dry_run: bool = False, ) -> Tuple[MigratorSessionContext, list, MutableSequence[Migrator]]: temp = glob.glob("/tmp/*") gx = load_graph() smithy_version = eval_cmd("conda smithy --version").strip() pinning_version = json.loads( eval_cmd("conda list conda-forge-pinning --json"))[0]["version"] migrators = [] add_arch_migrate(migrators, gx) migration_factory(migrators, gx) add_replacement_migrator( migrators, gx, "matplotlib", "matplotlib-base", ("Unless you need `pyqt`, recipes should depend only on " "`matplotlib-base`."), alt_migrator=MatplotlibBase, ) create_migration_yaml_creator(migrators=migrators, gx=gx) print("rebuild migration graph sizes:", flush=True) for m in migrators: print( f' {getattr(m, "name", m)} graph size: ' f'{len(getattr(m, "graph", []))}', flush=True, ) print(" ", flush=True) mctx = MigratorSessionContext( circle_build_url=os.getenv("CIRCLE_BUILD_URL", ""), graph=gx, smithy_version=smithy_version, pinning_version=pinning_version, github_username=github_username, github_password=github_password, github_token=github_token, dry_run=dry_run, ) print("building package import maps and version migrator", flush=True) python_nodes = { n for n, v in mctx.graph.nodes("payload") if "python" in v.get("req", "") } python_nodes.update([ k for node_name, node in mctx.graph.nodes("payload") for k in node.get("outputs_names", []) if node_name in python_nodes ], ) version_migrator = Version( python_nodes=python_nodes, pr_limit=PR_LIMIT * 4, piggy_back_migrations=[ Jinja2VarsCleanup(), DuplicateLinesCleanup(), PipMigrator(), LicenseMigrator(), CondaForgeYAMLCleanup(), ExtraJinja2KeysCleanup(), Build2HostMigrator(), NoCondaInspectMigrator(), Cos7Config(), ], ) migrators = [version_migrator] + migrators print(" ", flush=True) return mctx, temp, migrators
def _update_handle_args(args): from conda_forge_tick.utils import load_graph if args.filepath != 'graph.json': copyfile(args.filepath, 'graph.json') gx = load_graph() update_versions_in_graph(gx)