def collect_temp_requirements(source_dir) -> Set[str]:
click.echo("[Temporary project] Starting to collect requirements...")
requirements = set()
for directory in PathIterator(root=source_dir,
rule=is_item_dir,
as_path=True):
item_requirements = get_item_yaml_requirements(directory)
for item_requirement in item_requirements:
requirements.add(item_requirement)
requirements_txt = directory / "requirements.txt"
if requirements_txt.exists():
with open(requirements_txt, "r") as f:
lines = f.read().split("\n")
for line in filter(lambda l: l, lines):
requirements.add(line)
parsed = set()
for req in requirements:
parsed.add(req.split("==")[0])
if _verbose:
click.echo(
f"[Temporary project] Done requirements ({', '.join(requirements)})"
)
return requirements
def discover(self, path: Union[str, Path]) -> List[str]:
path = Path(path)
testables = []
# Handle single test directory
if (path / "item.yaml").exists():
testables.append(str((path / "item.yaml").resolve()))
if testables:
click.echo("Found testable directory...")
# Handle multiple directories
else:
item_iterator = PathIterator(root=path,
rule=is_item_dir,
as_path=True)
for item_dir in item_iterator:
testables.append(str((item_dir / "item.yaml").resolve()))
click.echo(f"Found {len(testables)} testable items...")
if not testables:
click.echo(
"No tests found, make sure your test file names are structures as 'test_*.py')"
)
exit(0)
return testables
def build_temp_project(source_dir, temp_root):
click.echo("[Temporary project] Starting to build project...")
if _verbose:
click.echo(f"Source dir: {source_dir}")
click.echo(f"Temp root: {temp_root}")
item_count = 0
for directory in PathIterator(root=source_dir,
rule=is_item_dir,
as_path=True):
if _verbose:
item_count += 1
click.echo(
f"[Temporary project] Now processing: {directory / 'item.yaml'}"
)
with open(directory / "item.yaml", "r") as f:
item = yaml.full_load(f)
filename = item.get("spec")["filename"]
temp_dir = temp_root / directory.name
temp_dir.mkdir(parents=True, exist_ok=True)
(temp_dir / "__init__.py").touch()
if filename:
py_file = directory / filename
temp_file = temp_dir / py_file.name
shutil.copy(py_file, temp_file)
if _verbose:
click.echo(
f"[Temporary project] Done project (item count: {item_count})")
def discover(self, path: Union[str, Path]) -> List[str]:
path = Path(path)
testables = []
# Handle single test directory
if (path / "item.yaml").exists():
for inner_file in path.iterdir():
if self.is_test_ipynb(inner_file):
testables.append(str(inner_file.resolve()))
if testables:
click.echo("Found testable directory...")
# Handle multiple directories
else:
item_iterator = PathIterator(root=path,
rule=is_item_dir,
as_path=True)
for inner_dir in item_iterator:
# Iterate individual files in each directory
for inner_file in inner_dir.iterdir():
#click.echo("test inner file"+str(inner_file))
if self.is_test_ipynb(inner_file):
#click.echo("adding "+str(inner_file))
testables.append(str(inner_dir.resolve()))
click.echo(f"Found {len(testables)} testable items...")
if not testables:
click.echo(
"No tests found, make sure your test file names are structures as 'test_*.py')"
)
exit(0)
testables.sort()
click.echo("tests list " + str(testables))
return testables
def function_to_item(path: str):
path = Path(path)
if not path.exists():
click.echo(f"{path} not found")
exit(1)
# If this is a directory, its either project root or a specific directory in the project
if path.is_dir():
function_path = path / "function.yaml"
# If its a function directory
if function_path.exists():
item_path = path / "item.yaml"
item = function_yaml_to_item(function_path)
with open(item_path, "w") as f:
yaml.dump(item, f)
# Otherwise its a root directory to be iterated
else:
function_iterator = PathIterator(root=path,
rule=is_function_dir,
as_path=True)
for function in function_iterator:
function_path = function / "function.yaml"
item_path = function / "item.yaml"
item = function_yaml_to_item(function_path)
with open(item_path, "w") as f:
yaml.dump(item, f)
# Otherwise its a path to function.yaml
else:
path_dir = path.parent
item_path = path_dir / "item.yaml"
item = function_yaml_to_item(path)
with open(item_path, "w") as f:
yaml.dump(item, f)
exit(0)
def item_to_function(
item_path: str,
output_path: Optional[str] = None,
code_output: bool = False,
format_code: bool = True,
):
item_path = Path(item_path)
if item_path.is_dir():
item_path = item_path / "item.yaml"
# That means we are in a specific item directory
if item_path.exists():
_output_path = output_path or item_path.parent / "function.yaml"
create_function_yaml(item_path, _output_path, code_output, format_code)
# That means we need to search for items inside this direcotry
else:
for inner_dir in PathIterator(
root=item_path.parent,
rule=is_item_dir,
as_path=True,
):
try:
_output_path = output_path or (inner_dir / "function.yaml")
create_function_yaml(inner_dir, _output_path, code_output,
format_code)
except Exception as e:
print(e)
click.echo(
f"{inner_dir.name}: Failed to generate function.yaml")
def update_functions_yaml(root_directory: str,
version: str,
mlrun_version: str,
platform_version: str,
date_time: str):
if not root_directory:
click.echo("-r/--root-directory is required")
exit(1)
item_iterator = PathIterator(root=root_directory, rule=is_item_dir, as_path=True)
for inner_dir in item_iterator:
item_yaml = "item.yaml"
if (inner_dir / item_yaml).exists():
path = str(inner_dir)+"/"+item_yaml
stream = open(path, 'r')
data = yaml.load(stream)
if version:
data['version'] = version
if mlrun_version:
data['mlrunVersion'] = mlrun_version
if platform_version:
data['platformVersion'] = platform_version
if date_time:
data['generationDate'] = datetime.datetime.now().strftime('%Y-%m-%d:%H-%M')
print(data)
with open(path, 'w') as yaml_file:
yaml_file.write(yaml.dump(data, default_flow_style=False))
def patch_temp_docs(source_dir, temp_docs):
click.echo("Patching temporary docs...")
for directory in PathIterator(root=source_dir, rule=is_item_dir):
directory = Path(directory)
with open(directory / "item.yaml", "r") as f:
item = yaml.full_load(f)
example_file = directory / item["example"]
if example_file:
example_file = Path(example_file)
shutil.copy(
example_file,
temp_docs / f"{directory.name}_example{example_file.suffix}",
)
def update_or_create_items(source_dir, marketplace_dir, temp_docs, change_log):
click.echo("Creating items...")
for item_dir in PathIterator(root=source_dir,
rule=is_item_dir,
as_path=True):
update_or_create_item(item_dir, marketplace_dir, temp_docs, change_log)
def create_legacy_catalog(root_dir: Union[str, Path]):
root_dir = Path(root_dir)
if not root_dir.is_dir():
raise RuntimeError("Root directory must be a directory")
catalog = {}
file_list = Path(root_dir).glob("**/*.yaml")
for file in sorted(file_list, key=lambda f: str(f)):
file = file.resolve()
click.echo(f"Now inspecting file: {file}")
if file.is_file():
try:
fn = import_function(str(file))
except Exception as e:
click.echo(f"failed to load func {file}, {e}")
continue
if not fn.kind or fn.kind in ["", "local", "handler"]:
click.echo(
f"illegal function or kind in {file}, kind={fn.kind}")
continue
if fn.metadata.name in catalog:
entry = catalog[fn.metadata.name]
else:
file_dir = file.parent
notebook_iterator = PathIterator(
root=file_dir,
rule=lambda p: p.name.endswith(".ipynb"),
as_path=True,
)
notebooks = list(notebook_iterator)
doc_file = file_dir if not notebooks else file_dir / notebooks[
0]
entry = {
"description": fn.spec.description,
"categories": fn.metadata.categories,
"kind": fn.kind,
"docfile": str(doc_file.resolve()),
"versions": {},
}
entry["versions"][fn.metadata.tag or "latest"] = str(file)
print(fn.metadata.name, entry)
catalog[fn.metadata.name] = entry
with open("catalog.yaml", "w") as fp:
yaml.dump(catalog, fp)
with open("catalog.json", "w") as fp:
json.dump(catalog, fp)
mdheader = """# Functions hub
This functions hub is intended to be a centralized location for open source contributions of function components.
These are functions expected to be run as independent mlrun pipeline compnents, and as public contributions,
it is expected that contributors follow certain guidelines/protocols (please chip-in).
## Functions
"""
with open(root_dir / "README.md", "w") as fp:
fp.write(mdheader)
rows = []
for k, v in catalog.items():
kind = v["kind"]
if kind == "remote":
kind = "nuclio"
row = [
f"[{k}]({v['docfile']})",
kind,
v["description"],
", ".join(v["categories"] or []),
]
rows.append(row)
text = gen_md_table(["function", "kind", "description", "categories"],
rows)
fp.write(text)
