async def test_load_name_given(self):
self.assertEqual(add.op, Operation.load("add"))
self.assertEqual(mult.op, Operation.load("mult"))
self.assertEqual(
parse_line.op._replace(instance_name="parse_line"),
Operation.load("parse_line")._replace(instance_name="parse_line"),
)
async def test_run(self):
passwords = [str(random.random()) for _ in range(0, 20)]
# Orchestrate the running of these operations
async with MemoryOrchestrator.basic_config(*OPIMPS) as orchestrator:
definitions = Operation.definitions(*OPERATIONS)
passwords = [
Input(value=password,
definition=definitions['UnhashedPassword'],
parents=None) for password in passwords
]
output_spec = Input(value=['ScryptPassword'],
definition=definitions['get_single_spec'],
parents=None)
async with orchestrator() as octx:
# Add our inputs to the input network with the context being the URL
for password in passwords:
await octx.ictx.add(
MemoryInputSet(
MemoryInputSetConfig(
ctx=StringInputSetContext(password.value),
inputs=[password, output_spec])))
try:
async for _ctx, results in octx.run_operations(
strict=True):
self.assertTrue(results)
except AttributeError as error:
if "module 'hashlib' has no attribute 'scrypt'" \
in str(error):
return
raise
async def test_dataflow_usage_example(self):
# Write out shouldi dataflow
orig = self.mktempfile() + ".json"
pathlib.Path(orig).write_text(json.dumps(self.DATAFLOW.export()))
# Import from feature/git
transform_to_repo = Operation.load("dffml.mapping.create")
lines_of_code_by_language, lines_of_code_to_comments = list(
load(
"dffml_feature_git.feature.operations:lines_of_code_by_language",
"dffml_feature_git.feature.operations:lines_of_code_to_comments",
relative=relative_path("..", "..", "feature", "git"),
))
# Create new dataflow
override = DataFlow.auto(
transform_to_repo,
lines_of_code_by_language,
lines_of_code_to_comments,
)
# TODO Modify and compare against yaml in docs example
# Write out override dataflow
created = self.mktempfile() + ".json"
pathlib.Path(created).write_text(json.dumps(override.export()))
# Merge the two
with contextlib.redirect_stdout(self.stdout):
await CLI.cli("dataflow", "merge", orig, created)
DataFlow._fromdict(**json.loads(self.stdout.getvalue()))
async def test_run(self):
self.required_plugins("dffml-config-yaml", "dffml-model-scratch")
# Load get_single and model_predict
get_single = Operation.load("get_single")
model_predict = list(load("dffml.operation.model:model_predict"))[0]
# Create new dataflow from operations
dataflow = DataFlow.auto(get_single, model_predict)
# Add the seed inputs
dataflow.seed.append(
Input(
value=[
definition.name
for definition in model_predict.op.outputs.values()
],
definition=get_single.inputs["spec"],
))
# Write out the dataflow
dataflow_yaml = pathlib.Path(self.mktempfile() + ".yaml")
async with BaseConfigLoader.load("yaml").withconfig(
{}) as configloader:
async with configloader() as loader:
dataflow_yaml.write_bytes(await loader.dumpb(
dataflow.export(linked=True)))
# TODO Figure out how nested model config options will work
# print(dataflow_yaml.read_text())
return
async def test_run(self):
dataflow = DataFlow.auto(*OPIMPS)
passwords = [str(random.random()) for _ in range(0, 20)]
# Orchestrate the running of these operations
async with MemoryOrchestrator.withconfig({}) as orchestrator:
definitions = Operation.definitions(*OPERATIONS)
passwords = [
Input(
value=password,
definition=definitions["UnhashedPassword"],
parents=None,
) for password in passwords
]
output_spec = Input(
value=["ScryptPassword"],
definition=definitions["get_single_spec"],
parents=None,
)
async with orchestrator(dataflow) as octx:
try:
async for _ctx, results in octx.run({
password.value: [password, output_spec]
for password in passwords
}):
self.assertTrue(results)
except AttributeError as error:
raise
async def test_load(self):
loaded = Operation.load()
self.assertIn(add.op, loaded)
self.assertIn(mult.op, loaded)
try:
self.assertIn(parse_line.op, loaded)
except:
self.assertIn(parse_line.op._replace(instance_name="parse_line"),
loaded)
async def test_run(self):
repos = [
"http://pkg.freebsd.org/FreeBSD:13:amd64/latest/All/ImageMagick7-7.0.8.48.txz",
"https://download.clearlinux.org/releases/10540/clear/x86_64/os/Packages/sudo-setuid-1.8.17p1-34.x86_64.rpm",
"https://rpmfind.net/linux/fedora/linux/updates/29/Everything/x86_64/Packages/g/gzip-1.9-9.fc29.x86_64.rpm",
"https://archives.fedoraproject.org/pub/archive/fedora/linux/releases/20/Everything/x86_64/os/Packages/c/curl-7.32.0-3.fc20.x86_64.rpm",
]
dataflow = DataFlow.auto(
URLToURLBytes,
files_in_rpm,
urlbytes_to_rpmfile,
urlbytes_to_tarfile,
is_binary_pie,
Associate,
cleanup_rpm,
)
async with MemoryOrchestrator.withconfig({}) as orchestrator:
definitions = Operation.definitions(*OPERATIONS)
async with orchestrator(dataflow) as octx:
async for ctx, results in octx.run(
{
URL: [
Input(value=URL, definition=definitions["URL"]),
Input(
value=["rpm_filename", "binary_is_PIE"],
definition=definitions["associate_spec"],
),
]
for URL in repos
},
strict=True,
):
self.assertTrue(results)
async def test_load(self):
loaded = Operation.load()
self.assertIn(add.op, loaded)
self.assertIn(mult.op, loaded)
self.assertIn(parse_line.imp.op, loaded)
import hashlib
import asyncio
import concurrent.futures
from typing import Dict, Any
from dffml.df.types import Operation
from dffml.df.base import OperationImplementationContext, \
OperationImplementation
# pylint: disable=no-name-in-module
from .definitions import UnhashedPassword, \
ScryptPassword
scrypt = Operation(name='scrypt',
inputs={
'password': UnhashedPassword,
},
outputs={'password': ScryptPassword},
conditions=[])
class ScryptContext(OperationImplementationContext):
@staticmethod
def hash_password(password):
# ---- BEGIN Python hashlib docs ----
# The function provides scrypt password-based key derivation function as
# defined in RFC 7914.
# password and salt must be bytes-like objects. Applications and
# libraries should limit password to a sensible length (e.g. 1024). salt
# should be about 16 or more bytes from a proper source, e.g.
async def test_load_name_given(self):
self.assertEqual(add.op, Operation.load("add"))
self.assertEqual(mult.op, Operation.load("mult"))
self.assertEqual(parse_line.op, Operation.load("parse_line"))
from dffml.df.types import Operation, Definition
from dffml.df.base import (
op,
OperationImplementationContext,
OperationImplementation,
)
# Definitions
UserInput = Definition(name="UserInput", primitive="str")
DataToPrint = Definition(name="DataToPrint", primitive="generic")
AcceptUserInput = Operation(
name="AcceptUserInput",
inputs={},
outputs={"InputData": UserInput},
conditions=[],
)
class AcceptUserInputContext(OperationImplementationContext):
@staticmethod
def receive_input():
print("Enter the value: ", end="")
return input()
async def run(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
user_input = await self.parent.loop.run_in_executor(
self.parent.pool, self.receive_input
)
return {"InputData": user_input}
OperationImplementation,
)
from .log import LOGGER
# pylint: disable=no-name-in-module
from .definitions import (
URL,
URLBytes,
RPMObject,
rpm_filename,
binary_is_PIE,
)
url_to_urlbytes = Operation(
name="url_to_urlbytes",
inputs={"URL": URL},
outputs={"download": URLBytes},
conditions=[],
)
class URLBytesObject(NamedTuple):
URL: str
body: bytes
def __repr__(self):
return "%s(URL=%s, body=%s...)" % (
self.__class__.__qualname__,
self.URL,
self.body[:10],
)
import asyncio
import concurrent.futures
from typing import Dict, Any
from dffml.df.types import Operation
from dffml.df.base import (
OperationImplementationContext,
OperationImplementation,
)
# pylint: disable=no-name-in-module
from .definitions import UnhashedPassword, ScryptPassword
scrypt = Operation(
name="scrypt",
inputs={"password": UnhashedPassword},
outputs={"password": ScryptPassword},
conditions=[],
)
class ScryptContext(OperationImplementationContext):
@staticmethod
def hash_password(password):
# ---- BEGIN Python hashlib docs ----
# The function provides scrypt password-based key derivation function as
# defined in RFC 7914.
# password and salt must be bytes-like objects. Applications and
# libraries should limit password to a sensible length (e.g. 1024). salt
# should be about 16 or more bytes from a proper source, e.g.