async def check_rust(self, repo):
async with self.octx.parent(DATAFLOW) as octx:
async for _, results in octx.run(
[Input(value=repo, definition=self.parent.op.inputs["repo"])]):
if results[has_package_cargo.op.outputs["result"].name]:
return {"rust": True}
pypi_package_url,
pypi_package_contents,
cleanup_pypi_package,
safety_check,
run_bandit,
GetSingle,
)
# Seed inputs are added to each executing context. The following Input tells the
# GetSingle output operation that we want the output of the network to include
# data matching the "issues" output of the safety_check operation, and the
# "report" output of the run_bandit operation, for each context.
DATAFLOW.seed.append(
Input(
value=[
safety_check.op.outputs["issues"].name,
run_bandit.op.outputs["report"].name,
],
definition=GetSingle.op.inputs["spec"],
))
class Install(CMD):
arg_packages = Arg("packages",
nargs="+",
help="Package to check if we should install")
async def run(self):
# Run all the operations, Each iteration of this loop happens
# when all inputs are exhausted for a context, the output
# operations are then run and their results are yielded
"result": Definition(name="has_package_cargo_result", primitive="bool")
},
)
def has_package_cargo(repo: clone_git_repo.op.outputs["repo"].spec):
return {
"result":
pathlib.Path(repo.directory, "cargo.toml").is_file()
or pathlib.Path(repo.directory, "Cargo.toml").is_file()
}
DATAFLOW = DataFlow.auto(has_package_cargo, GetSingle)
DATAFLOW.seed.append(
Input(
value=[
has_package_cargo.op.outputs["result"].name,
],
definition=GetSingle.op.inputs["spec"],
))
@op(
inputs={"repo": clone_git_repo.op.outputs["repo"]},
outputs={"rust": Definition(name="repo_is_rust", primitive="string")},
)
async def check_rust(self, repo):
async with self.octx.parent(DATAFLOW) as octx:
async for _, results in octx.run(
[Input(value=repo, definition=self.parent.op.inputs["repo"])]):
if results[has_package_cargo.op.outputs["result"].name]:
return {"rust": True}
from dffml import DataFlow, Input, GetSingle
from .operations import convert_to_gif
DATAFLOW = DataFlow.auto(convert_to_gif, GetSingle)
DATAFLOW.seed = [
Input(
value=[convert_to_gif.op.outputs["output_file"].name],
definition=GetSingle.op.inputs["spec"],
),
Input(value=480, definition=convert_to_gif.op.inputs["resolution"]),
]
from dffml import op, DataFlow, Input, GetSingle
from dffml_feature_git.feature.operations import clone_git_repo
from .cargo_audit import run_cargo_audit
from .check import check_rust
from ..types import SA_RESULTS
DATAFLOW = DataFlow.auto(run_cargo_audit, GetSingle,)
DATAFLOW.seed.append(
Input(
value=[run_cargo_audit.op.outputs["report"].name,],
definition=GetSingle.op.inputs["spec"],
)
)
@op(
inputs={"repo": clone_git_repo.op.outputs["repo"]},
outputs={"result": SA_RESULTS},
conditions=[check_rust.op.outputs["rust"]],
)
async def analyze_rust(self, repo):
"""
Run Rust static analysis
"""
async with self.octx.parent(DATAFLOW) as octx:
async for _, results in octx.run(
[
Input(
# Link inputs and outputs together according to their definitions
DATAFLOW = DataFlow.auto(
clone_git_repo,
check_python,
analyze_python,
check_javascript,
analyze_javascript,
check_rust,
analyze_rust,
cleanup_git_repo,
GetMulti,
)
DATAFLOW.seed.append(
Input(value=[SA_RESULTS.name,], definition=GetMulti.op.inputs["spec"],)
)
# Allow for directory to be provided by user instead of the result of cloning a
# Git repo to a temporary directory on disk
for opimp in [
check_python,
analyze_python,
check_javascript,
analyze_javascript,
check_rust,
analyze_rust,
]:
DATAFLOW.flow[opimp.op.name].inputs["repo"].append("seed")
DATAFLOW.update()
async def check_python(self, repo):
async with self.octx.parent(DATAFLOW_ID_PYTHON) as octx:
async for _, results in octx.run(
[Input(value=repo, definition=self.parent.op.inputs["repo"])]):
if results[has_setup_py.op.outputs["result"].name]:
return {"python": True}
@op(
inputs={"repo": clone_git_repo.op.outputs["repo"]},
outputs={
"result": Definition(name="has_setup_py_result", primitive="bool")
},
)
def has_setup_py(repo: clone_git_repo.op.outputs["repo"].spec):
return {"result": pathlib.Path(repo.directory, "setup.py").is_file()}
DATAFLOW_ID_PYTHON = DataFlow.auto(has_setup_py, GetSingle)
DATAFLOW_ID_PYTHON.seed.append(
Input(
value=[
has_setup_py.op.outputs["result"].name,
],
definition=GetSingle.op.inputs["spec"],
))
@op(
inputs={"repo": clone_git_repo.op.outputs["repo"]},
outputs={"python": Definition(name="repo_is_python", primitive="string")},
)
async def check_python(self, repo):
async with self.octx.parent(DATAFLOW_ID_PYTHON) as octx:
async for _, results in octx.run(
[Input(value=repo, definition=self.parent.op.inputs["repo"])]):
if results[has_setup_py.op.outputs["result"].name]:
return {"python": True}
SLEEP_TIME = Definition(name="sleep_time", primitive="int")
INTEGER = Definition(name="integer", primitive="int")
@op(inputs={"obj": OBJ, "sleep_for": SLEEP_TIME, "i": INTEGER})
async def run_me(obj: dict, sleep_for: int, i: int) -> None:
obj["i"] = i
await asyncio.sleep(sleep_for)
print(f"set i = {i}, got i = {obj['i']}")
print("Running dataflow without locked object")
for ctx, result in run(
DataFlow(run_me),
[
Input(value={}, definition=OBJ),
Input(value=0.1, definition=SLEEP_TIME),
Input(value=0.2, definition=SLEEP_TIME),
Input(value=1, definition=INTEGER),
Input(value=2, definition=INTEGER),
],
):
pass
print("Running dataflow with locked object")
run_me.op = run_me.op._replace(inputs={
"obj": LOCKED_OBJ,
"sleep_for": SLEEP_TIME,
"i": INTEGER
})
for ctx, result in run(
from .types import SA_RESULTS
# Link inputs and outputs together according to their definitions
DATAFLOW = DataFlow.auto(
clone_git_repo,
check_python,
analyze_python,
check_javascript,
analyze_javascript,
cleanup_git_repo,
GetSingle,
)
DATAFLOW.seed.append(
Input(
value=[
SA_RESULTS.name,
],
definition=GetSingle.op.inputs["spec"],
))
# Allow for directory to be provided by user instead of the result of cloning a
# Git repo to a temporary directory on disk
for opimp in [
check_python,
analyze_python,
check_javascript,
analyze_javascript,
]:
DATAFLOW.flow[opimp.op.name].inputs["repo"].append("seed")
DATAFLOW.update()
