def collect_temp_requirements(source_dir) -> Set[str]: click.echo("[Temporary project] Starting to collect requirements...") requirements = set() for directory in PathIterator(root=source_dir, rule=is_item_dir, as_path=True): item_requirements = get_item_yaml_requirements(directory) for item_requirement in item_requirements: requirements.add(item_requirement) requirements_txt = directory / "requirements.txt" if requirements_txt.exists(): with open(requirements_txt, "r") as f: lines = f.read().split("\n") for line in filter(lambda l: l, lines): requirements.add(line) parsed = set() for req in requirements: parsed.add(req.split("==")[0]) if _verbose: click.echo( f"[Temporary project] Done requirements ({', '.join(requirements)})" ) return requirements
def discover(self, path: Union[str, Path]) -> List[str]: path = Path(path) testables = [] # Handle single test directory if (path / "item.yaml").exists(): testables.append(str((path / "item.yaml").resolve())) if testables: click.echo("Found testable directory...") # Handle multiple directories else: item_iterator = PathIterator(root=path, rule=is_item_dir, as_path=True) for item_dir in item_iterator: testables.append(str((item_dir / "item.yaml").resolve())) click.echo(f"Found {len(testables)} testable items...") if not testables: click.echo( "No tests found, make sure your test file names are structures as 'test_*.py')" ) exit(0) return testables
def build_temp_project(source_dir, temp_root): click.echo("[Temporary project] Starting to build project...") if _verbose: click.echo(f"Source dir: {source_dir}") click.echo(f"Temp root: {temp_root}") item_count = 0 for directory in PathIterator(root=source_dir, rule=is_item_dir, as_path=True): if _verbose: item_count += 1 click.echo( f"[Temporary project] Now processing: {directory / 'item.yaml'}" ) with open(directory / "item.yaml", "r") as f: item = yaml.full_load(f) filename = item.get("spec")["filename"] temp_dir = temp_root / directory.name temp_dir.mkdir(parents=True, exist_ok=True) (temp_dir / "__init__.py").touch() if filename: py_file = directory / filename temp_file = temp_dir / py_file.name shutil.copy(py_file, temp_file) if _verbose: click.echo( f"[Temporary project] Done project (item count: {item_count})")
def discover(self, path: Union[str, Path]) -> List[str]: path = Path(path) testables = [] # Handle single test directory if (path / "item.yaml").exists(): for inner_file in path.iterdir(): if self.is_test_ipynb(inner_file): testables.append(str(inner_file.resolve())) if testables: click.echo("Found testable directory...") # Handle multiple directories else: item_iterator = PathIterator(root=path, rule=is_item_dir, as_path=True) for inner_dir in item_iterator: # Iterate individual files in each directory for inner_file in inner_dir.iterdir(): #click.echo("test inner file"+str(inner_file)) if self.is_test_ipynb(inner_file): #click.echo("adding "+str(inner_file)) testables.append(str(inner_dir.resolve())) click.echo(f"Found {len(testables)} testable items...") if not testables: click.echo( "No tests found, make sure your test file names are structures as 'test_*.py')" ) exit(0) testables.sort() click.echo("tests list " + str(testables)) return testables
def function_to_item(path: str): path = Path(path) if not path.exists(): click.echo(f"{path} not found") exit(1) # If this is a directory, its either project root or a specific directory in the project if path.is_dir(): function_path = path / "function.yaml" # If its a function directory if function_path.exists(): item_path = path / "item.yaml" item = function_yaml_to_item(function_path) with open(item_path, "w") as f: yaml.dump(item, f) # Otherwise its a root directory to be iterated else: function_iterator = PathIterator(root=path, rule=is_function_dir, as_path=True) for function in function_iterator: function_path = function / "function.yaml" item_path = function / "item.yaml" item = function_yaml_to_item(function_path) with open(item_path, "w") as f: yaml.dump(item, f) # Otherwise its a path to function.yaml else: path_dir = path.parent item_path = path_dir / "item.yaml" item = function_yaml_to_item(path) with open(item_path, "w") as f: yaml.dump(item, f) exit(0)
def item_to_function( item_path: str, output_path: Optional[str] = None, code_output: bool = False, format_code: bool = True, ): item_path = Path(item_path) if item_path.is_dir(): item_path = item_path / "item.yaml" # That means we are in a specific item directory if item_path.exists(): _output_path = output_path or item_path.parent / "function.yaml" create_function_yaml(item_path, _output_path, code_output, format_code) # That means we need to search for items inside this direcotry else: for inner_dir in PathIterator( root=item_path.parent, rule=is_item_dir, as_path=True, ): try: _output_path = output_path or (inner_dir / "function.yaml") create_function_yaml(inner_dir, _output_path, code_output, format_code) except Exception as e: print(e) click.echo( f"{inner_dir.name}: Failed to generate function.yaml")
def update_functions_yaml(root_directory: str, version: str, mlrun_version: str, platform_version: str, date_time: str): if not root_directory: click.echo("-r/--root-directory is required") exit(1) item_iterator = PathIterator(root=root_directory, rule=is_item_dir, as_path=True) for inner_dir in item_iterator: item_yaml = "item.yaml" if (inner_dir / item_yaml).exists(): path = str(inner_dir)+"/"+item_yaml stream = open(path, 'r') data = yaml.load(stream) if version: data['version'] = version if mlrun_version: data['mlrunVersion'] = mlrun_version if platform_version: data['platformVersion'] = platform_version if date_time: data['generationDate'] = datetime.datetime.now().strftime('%Y-%m-%d:%H-%M') print(data) with open(path, 'w') as yaml_file: yaml_file.write(yaml.dump(data, default_flow_style=False))
def patch_temp_docs(source_dir, temp_docs): click.echo("Patching temporary docs...") for directory in PathIterator(root=source_dir, rule=is_item_dir): directory = Path(directory) with open(directory / "item.yaml", "r") as f: item = yaml.full_load(f) example_file = directory / item["example"] if example_file: example_file = Path(example_file) shutil.copy( example_file, temp_docs / f"{directory.name}_example{example_file.suffix}", )
def update_or_create_items(source_dir, marketplace_dir, temp_docs, change_log): click.echo("Creating items...") for item_dir in PathIterator(root=source_dir, rule=is_item_dir, as_path=True): update_or_create_item(item_dir, marketplace_dir, temp_docs, change_log)
def create_legacy_catalog(root_dir: Union[str, Path]): root_dir = Path(root_dir) if not root_dir.is_dir(): raise RuntimeError("Root directory must be a directory") catalog = {} file_list = Path(root_dir).glob("**/*.yaml") for file in sorted(file_list, key=lambda f: str(f)): file = file.resolve() click.echo(f"Now inspecting file: {file}") if file.is_file(): try: fn = import_function(str(file)) except Exception as e: click.echo(f"failed to load func {file}, {e}") continue if not fn.kind or fn.kind in ["", "local", "handler"]: click.echo( f"illegal function or kind in {file}, kind={fn.kind}") continue if fn.metadata.name in catalog: entry = catalog[fn.metadata.name] else: file_dir = file.parent notebook_iterator = PathIterator( root=file_dir, rule=lambda p: p.name.endswith(".ipynb"), as_path=True, ) notebooks = list(notebook_iterator) doc_file = file_dir if not notebooks else file_dir / notebooks[ 0] entry = { "description": fn.spec.description, "categories": fn.metadata.categories, "kind": fn.kind, "docfile": str(doc_file.resolve()), "versions": {}, } entry["versions"][fn.metadata.tag or "latest"] = str(file) print(fn.metadata.name, entry) catalog[fn.metadata.name] = entry with open("catalog.yaml", "w") as fp: yaml.dump(catalog, fp) with open("catalog.json", "w") as fp: json.dump(catalog, fp) mdheader = """# Functions hub This functions hub is intended to be a centralized location for open source contributions of function components. These are functions expected to be run as independent mlrun pipeline compnents, and as public contributions, it is expected that contributors follow certain guidelines/protocols (please chip-in). ## Functions """ with open(root_dir / "README.md", "w") as fp: fp.write(mdheader) rows = [] for k, v in catalog.items(): kind = v["kind"] if kind == "remote": kind = "nuclio" row = [ f"[{k}]({v['docfile']})", kind, v["description"], ", ".join(v["categories"] or []), ] rows.append(row) text = gen_md_table(["function", "kind", "description", "categories"], rows) fp.write(text)