def test_args(self):
self.assertEqual(
FileSource.args({}),
{
"source": {
"arg": None,
"config": {
"file": {
"arg": None,
"config": {
"filename": {
"arg": Arg(type=str),
"config": {},
},
"readonly": {
"arg":
Arg(
type=bool,
action="store_true",
default=False,
),
"config": {},
},
"label": {
"arg": Arg(type=str, default="unlabeled"),
"config": {},
},
},
}
},
}
},
)
def args(cls, args, *above) -> Dict[str, Arg]:
cls.config_set(args, above, "host", Arg(default="127.0.0.1"))
cls.config_set(args, above, "port", Arg(type=int, default=3306))
cls.config_set(args, above, "user", Arg(default="user"))
cls.config_set(args, above, "password", Arg(default="pass"))
cls.config_set(args, above, "db", Arg(default="db"))
return args
def args(cls, args, *above) -> Dict[str, Arg]:
cls.config_set(
args,
above,
"directory",
Arg(
default=os.path.join(
os.path.expanduser("~"),
".cache",
"dffml",
f"scikit-{entry_point_name}",
),
help="Directory where state should be saved",
),
)
cls.config_set(
args,
above,
"predict",
Arg(type=str, help="Label or the value to be predicted"),
)
for param in inspect.signature(cls.SCIKIT_MODEL).parameters.values():
# TODO if param.default is an array then Args needs to get a
# nargs="+"
cls.config_set(
args,
above,
param.name,
Arg(
type=cls.type_for(param),
default=NoDefaultValue
if param.default == inspect._empty else param.default,
),
)
return args
def args(cls, args, *above) -> Dict[str, Arg]:
cls.config_set(
args,
above,
"directory",
Arg(
default=os.path.join(os.path.expanduser("~"), ".cache",
"dffml", "scratch"),
help="Directory where state should be saved",
),
)
cls.config_set(
args,
above,
"predict",
Arg(type=str, help="Label or the value to be predicted"),
)
cls.config_set(
args,
above,
"features",
Arg(
nargs="+",
required=True,
type=Feature.load,
action=list_action(Features),
help="Features to train on",
),
)
return args
def test_args(self):
self.assertEqual(
MemoryRedundancyChecker.args({}),
{
"rchecker": {
"arg": None,
"config": {
"memory": {
"arg": None,
"config": {
"kvstore": {
"arg":
Arg(
type=BaseKeyValueStore.load,
default=MemoryKeyValueStore,
),
"config": {
"withargs": {
"arg": None,
"config": {
"filename": {
"arg": Arg(type=str),
"config": {},
}
},
}
},
}
},
}
},
}
},
)
def test_modify(self):
arg = Arg("-test", key="value")
first = arg.modify(name="-first")
second = arg.modify(key="new_value")
self.assertEqual(arg.name, "-test")
self.assertEqual(first.name, "-first")
self.assertEqual(second.name, "-test")
self.assertEqual(second["key"], "new_value")
def args(cls, args, *above) -> Dict[str, Arg]:
cls.config_set(
args,
above,
"directory",
Arg(
default=os.path.join(os.path.expanduser("~"), ".cache",
"dffml", "tensorflow"),
help="Directory where state should be saved",
),
)
cls.config_set(
args,
above,
"steps",
Arg(
type=int,
default=3000,
help="Number of steps to train the model",
),
)
cls.config_set(
args,
above,
"epochs",
Arg(
type=int,
default=30,
help="Number of iterations to pass over all repos in a source",
),
)
cls.config_set(
args,
above,
"hidden",
Arg(
type=int,
nargs="+",
default=[12, 40, 15],
help=
"List length is the number of hidden layers in the network. Each entry in the list is the number of nodes in that hidden layer",
),
)
cls.config_set(
args,
above,
"predict",
Arg(help="Feature name holding truth value"),
)
return args
def args(cls, args, *above) -> Dict[str, Arg]:
cls.config_set(args, above, "directory", Arg())
cls.config_set(
args,
above,
"features",
Arg(
nargs="+",
required=True,
type=Feature.load,
action=list_action(Features),
),
)
return args
def args(cls, args, *above) -> Dict[str, Arg]:
cls.config_set(args, above, "host", Arg(default="127.0.0.1"))
cls.config_set(args, above, "port", Arg(type=int, default=3306))
cls.config_set(args, above, "user", Arg())
cls.config_set(args, above, "password", Arg())
cls.config_set(args, above, "db", Arg())
cls.config_set(
args,
above,
"repos-query",
Arg(
type=str,
help=
"SELECT `key` as key, data_1 as feature_1, data_2 as feature_2 FROM repo_data",
),
)
cls.config_set(
args,
above,
"repo-query",
Arg(
type=str,
help=
"SELECT `key` as key, data_1 as feature_1, data_2 as feature_2 FROM repo_data WHERE `key`=%s",
),
)
cls.config_set(
args,
above,
"update-query",
Arg(
type=str,
help=
"INSERT INTO repo_data (`key`, data_1, data_2) VALUES(%s, %s, %s) ON DUPLICATE KEY UPDATE data_1 = %s, data_2=%s",
),
)
cls.config_set(
args,
above,
"model-columns",
Arg(type=str, nargs="+", help="Order of Columns in table"),
)
cls.config_set(
args,
above,
"ca",
Arg(type=str, help="Path to server TLS certificate", default=None),
)
return args
class MultiCommCMD(CMD):
arg_mc_config = Arg(
"-mc-config",
dest="mc_config",
default=None,
help="MultiComm config directory",
)
arg_mc_atomic = Arg(
"-mc-atomic",
dest="mc_atomic",
action="store_true",
default=False,
help="No routes other than dataflows registered at startup",
)
def test_args(self):
self.assertDictEqual(
MemoryRedundancyChecker.args({}),
{
"rchecker": {
"plugin": None,
"config": {
"memory": {
"plugin": None,
"config": {
"kvstore": {
"plugin":
Arg(
type=load_kvstore_with_args,
help="Key value store to use",
default=MemoryKeyValueStore(),
),
"config": {},
}
},
}
},
}
},
)
class CreateTLSServer(TLSCMD):
"""
Used to generate server key and cert
"""
arg_bits = Arg("-bits",
help="Number of bits to use for key",
default=4096,
type=int)
async def run(self):
subprocess.call([
"openssl",
"req",
"-x509",
"-newkey",
f"rsa:{self.bits}",
"-keyout",
self.key,
"-out",
self.cert,
"-days",
"365",
"-nodes",
"-sha256",
"-subj",
"/C=US/ST=Oregon/L=Portland/O=Feedface/OU=Org/CN=127.0.0.1",
])
def test_00_args(self):
self.maxDiff = 99999
self.assertEqual(
FakeTesting.args({}),
{
"test": {
"plugin": None,
"config": {
"fake": {
"plugin": None,
"config": {
"num": {
"plugin": Arg(type=float),
"config": {},
},
"files": {
"plugin": Arg(type=str, nargs="+"),
"config": {},
},
"features": {
"plugin": Arg(
type=Feature.load,
nargs="+",
action=list_action(Features),
),
"config": {},
},
"name": {
"plugin": Arg(
type=str, help="Name of FakeTesting"
),
"config": {},
},
"readonly": {
"plugin": Arg(
type=bool,
action="store_true",
default=False,
),
"config": {},
},
"label": {
"plugin": Arg(
type=str, default="unlabeled"
),
"config": {},
},
"source": {
"plugin": Arg(
type=BaseSource.load,
default=JSONSource,
),
"config": {},
},
},
}
},
}
},
)
def args(cls, args, *above) -> Dict[str, Arg]:
cls.config_set(
args,
above,
"directory",
Arg(
default=os.path.join(os.path.expanduser("~"), ".cache",
"dffml", "scratch"),
help="Directory where state should be saved",
),
)
cls.config_set(
args,
above,
"predict",
Arg(type=str, help="Label or the value to be predicted"),
)
return args
def test_args(self):
self.assertEqual(
FileSource.args({}),
{
"source": {
"plugin": None,
"config": {
"file": {
"plugin": None,
"config": {
"filename": {
"plugin": Arg(type=str),
"config": {},
},
"readwrite": {
"plugin":
Arg(
type=bool,
action="store_true",
default=False,
),
"config": {},
},
"allowempty": {
"plugin":
Arg(
type=bool,
action="store_true",
default=False,
),
"config": {},
},
"tag": {
"plugin": Arg(type=str,
default="untagged"),
"config": {},
},
},
}
},
}
},
)
class Install(CMD):
arg_packages = Arg("packages",
nargs="+",
help="Package to check if we should install")
async def run(self):
# Create an Orchestrator which will manage the running of our operations
async with MemoryOrchestrator.basic_config(*OPIMPS) as orchestrator:
# Create a orchestrator context, everything in DFFML follows this
# one-two context entry pattern
async with orchestrator() as octx:
for package_name in self.packages:
# For each package add a new input set to the network of
# inputs (ictx). Operations run under a context, the context
# here is the package_name to evaluate (the first argument).
# The next arguments are all the inputs we're seeding the
# network with for that context. We give the package name
# because pypi_latest_package_version needs it to find the
# version, which safety will then use. We also give an input
# to the output operation GetSingle, which takes a list of
# data type definitions we want to select as our results.
await octx.ictx.sadd(
package_name,
Input(
value=package_name,
definition=pypi_package_json.op.inputs["package"],
),
Input(
value=[
safety_check.op.outputs["issues"].name,
run_bandit.op.outputs["report"].name,
],
definition=GetSingle.op.inputs["spec"],
),
)
# Run all the operations, Each iteration of this loop happens
# when all inputs are exhausted for a context, the output
# operations are then run and their results are yielded
async for ctx, results in octx.run_operations():
# The context for this data flow was the package name
package_name = (await ctx.handle()).as_string()
# Get the results of the GetSingle output operation
results = results[GetSingle.op.name]
# Check if any of the values of the operations evaluate to
# true, so if the number of issues found by safety is
# non-zero then this will be true
any_issues = list(results.values())
if (any_issues[0] > 0 or
any_issues[1]["CONFIDENCE.HIGH_AND_SEVERITY.HIGH"]
> 5):
print(f"Do not install {package_name}! {results!r}")
else:
print(f"{package_name} is okay to install")
class MiscService(CMD):
"""
Description of the DFFML related command
"""
arg_integer = Arg(
"-integer",
type=int,
help=f"Port to do nothing with",
default=0,
required=True,
)
async def run(self):
print(f"Your integer was: {self.integer}")
class Install(CMD):
arg_packages = Arg("packages",
nargs="+",
help="Package to check if we should install")
async def run(self):
# Create an Orchestrator which will manage the running of our operations
async with MemoryOrchestrator.withconfig({}) as orchestrator:
# Create a orchestrator context, everything in DFFML follows this
# one-two context entry pattern
async with orchestrator(DATAFLOW) as octx:
# Run all the operations, Each iteration of this loop happens
# when all inputs are exhausted for a context, the output
# operations are then run and their results are yielded
async for package_name, results in octx.run({
# For each package add a new input set to the input network
# The context operations execute under is the package name
# to evaluate. Contexts ensure that data pertaining to
# package A doesn't mingle with data pertaining to package B
package_name:
[
# The only input to the operations is the package name.
Input(
value=package_name,
definition=pypi_package_json.op.inputs["package"],
)
]
for package_name in self.packages
}):
# Grab the number of safety issues and the bandit report
# from the results dict
safety_issues = results[
safety_check.op.outputs["issues"].name]
bandit_report = results[
run_bandit.op.outputs["report"].name]
# Decide if those numbers mean we should stop ship or not
if (safety_issues > 0 or
bandit_report["CONFIDENCE.HIGH_AND_SEVERITY.HIGH"]
> 5):
print(f"Do not install {package_name}!")
for definition_name, result in results.items():
print(f" {definition_name}: {result}")
else:
print(f"{package_name} is okay to install")
def test_init(self):
arg = Arg("-test", key="value")
self.assertEqual(arg.name, "-test")
self.assertIn("key", arg)
self.assertEqual(arg["key"], "value")
def args(cls, args, *above) -> Dict[str, Arg]:
cls.config_set(args, above, "directory", Arg())
return args
def args(cls, args, *above) -> Dict[str, Arg]:
cls.config_set(
args,
above,
"directory",
Arg(
default=os.path.join(os.path.expanduser("~"), ".cache",
"dffml", "tensorflow"),
help="Directory where state should be saved",
),
)
cls.config_set(
args,
above,
"steps",
Arg(
type=int,
default=3000,
help="Number of steps to train the model",
),
)
cls.config_set(
args,
above,
"epochs",
Arg(
type=int,
default=30,
help="Number of iterations to pass over all repos in a source",
),
)
cls.config_set(
args,
above,
"hidden",
Arg(
type=int,
nargs="+",
default=[12, 40, 15],
help=
"List length is the number of hidden layers in the network. Each entry in the list is the number of nodes in that hidden layer",
),
)
cls.config_set(
args,
above,
"classification",
Arg(help="Feature name holding classification value"),
)
cls.config_set(
args,
above,
"classifications",
Arg(nargs="+", help="Options for value of classification"),
)
cls.config_set(
args,
above,
"clstype",
Arg(
type=pydoc.locate,
default=str,
help="Data type of classifications values (default: str)",
),
)
cls.config_set(
args,
above,
"features",
Arg(
nargs="+",
required=True,
type=Feature.load,
action=list_action(Features),
help="Features to train on",
),
)
return args
def args(cls, args, *above):
cls.config_set(args, above, "filename", Arg(type=str))
return args
class CMDTest(CMD):
arg_nope_present = Arg("nope", default=False)
arg_ignored = Arg("ignored")
class FakeSubCMD(CMD):
arg_test = Arg("-test")
def args(cls, args, *above) -> Dict[str, Arg]:
cls.config_set(args, above, "filename", Arg())
return args
class Server(TLSCMD, MultiCommCMD, Routes):
"""
HTTP server providing access to DFFML APIs
"""
# Used for testing
RUN_YIELD_START = False
RUN_YIELD_FINISH = False
INSECURE_NO_TLS = False
arg_port = Arg("-port", help="Port to bind to", type=int, default=8080)
arg_addr = Arg("-addr", help="Address to bind to", default="127.0.0.1")
arg_upload_dir = Arg(
"-upload-dir",
help="Directory to store uploaded files in",
default=None,
)
arg_static = Arg("-static",
help="Directory to serve static content from",
default=None)
arg_js = Arg(
"-js",
help="Serve JavaScript API file at /api.js",
default=False,
action="store_true",
)
arg_insecure = Arg(
"-insecure",
help="Start without TLS encryption",
action="store_true",
default=False,
)
arg_cors_domains = Arg(
"-cors-domains",
help=
"Domains to allow CORS for (see keys in defaults dict for aiohttp_cors.setup)",
nargs="+",
default=[],
)
arg_models = Arg(
"-models",
help="Models configured on start",
nargs="+",
default=AsyncContextManagerList(),
type=Model.load_labeled,
action=list_action(AsyncContextManagerList),
)
arg_sources = Arg(
"-sources",
help="Sources configured on start",
nargs="+",
default=Sources(),
type=BaseSource.load_labeled,
action=list_action(Sources),
)
async def start(self):
if self.insecure:
self.site = web.TCPSite(self.runner,
host=self.addr,
port=self.port)
else:
ssl_context = ssl.create_default_context(
purpose=ssl.Purpose.SERVER_AUTH, cafile=self.cert)
ssl_context.load_cert_chain(self.cert, self.key)
self.site = web.TCPSite(
self.runner,
host=self.addr,
port=self.port,
ssl_context=ssl_context,
)
await self.site.start()
self.port = self.site._server.sockets[0].getsockname()[1]
self.logger.info(f"Serving on {self.addr}:{self.port}")
async def run(self):
"""
Binds to port and starts HTTP server
"""
# Create dictionaries to hold configured sources and models
await self.setup()
await self.start()
# Load
if self.mc_config is not None:
# Restore atomic after config is set, allow setting for now
atomic = self.mc_atomic
self.mc_atomic = False
await self.register_directory(self.mc_config)
self.mc_atomic = atomic
try:
# If we are testing then RUN_YIELD will be an asyncio.Event
if self.RUN_YIELD_START is not False:
await self.RUN_YIELD_START.put(self)
await self.RUN_YIELD_FINISH.wait()
else: # pragma: no cov
# Wait for ctrl-c
while True:
await asyncio.sleep(60)
finally:
await self.app.cleanup()
await self.site.stop()
class TLSCMD(CMD):
arg_key = Arg("-key", help="Path to key file", default="server.key")
arg_cert = Arg("-cert", help="Path to cert file", default="server.pem")
class CreateTLSClient(CMD):
"""
Create TLS client key and cert (used to authenticate to HTTP API server).
curl \\
--cacert server.pem \\
--cert client.pem \\
--key client.key \\
-vvvvv \\
https://127.0.0.1:5000/
"""
CLI_FORMATTER_CLASS = argparse.RawDescriptionHelpFormatter
arg_bits = Arg("-bits",
help="Number of bits to use for key",
default=4096,
type=int)
arg_key = Arg("-key", help="Path to client key file", default="client.key")
arg_cert = Arg("-cert",
help="Path to client cert file",
default="client.pem")
arg_csr = Arg("-csr", help="Path to client csr file", default="client.csr")
arg_server_key = Arg("-server-key",
help="Path to server key file",
default="server.key")
arg_server_cert = Arg("-server-cert",
help="Path to server cert file",
default="server.pem")
async def run(self):
subprocess.check_call([
"openssl",
"req",
"-newkey",
f"rsa:{self.bits}",
"-keyout",
self.key,
"-out",
self.csr,
"-nodes",
"-sha256",
"-subj",
"/CN=RealUser",
])
subprocess.check_call([
"openssl",
"x509",
"-req",
"-in",
self.csr,
"-CA",
self.server_cert,
"-CAkey",
self.server_key,
"-out",
self.cert,
"-set_serial",
"01",
"-days",
"365",
])
def args(cls, args, *above) -> Dict[str, Arg]:
cls.config_set(args, above, "keys", Arg(type=str,
nargs="+",
default=[]))
return args
class Server(TLSCMD, MultiCommCMD, Routes):
"""
HTTP server providing access to DFFML APIs
"""
# Used for testing
RUN_YIELD_START = False
RUN_YIELD_FINISH = False
INSECURE_NO_TLS = False
arg_port = Arg("-port", help="Port to bind to", type=int, default=8080)
arg_addr = Arg("-addr", help="Address to bind to", default="127.0.0.1")
arg_upload_dir = Arg(
"-upload-dir",
help="Directory to store uploaded files in",
default=None,
)
arg_insecure = Arg(
"-insecure",
help="Start without TLS encryption",
action="store_true",
default=False,
)
arg_cors_domains = Arg(
"-cors-domains",
help=
"Domains to allow CORS for (see keys in defaults dict for aiohttp_cors.setup)",
nargs="+",
default=[],
)
def __init__(self, *args, **kwargs):
self._port: int = 0
self.site = None
super().__init__(*args, **kwargs)
@property
def port(self):
if self.site is None or self.site._server is None:
return self._port
return self.site._server.sockets[0].getsockname()[1]
@port.setter
def port(self, value):
self._port = value
async def start(self):
if self.insecure:
self.site = web.TCPSite(self.runner,
host=self.addr,
port=self.port)
else:
ssl_context = ssl.create_default_context(
purpose=ssl.Purpose.SERVER_AUTH, cafile=self.cert)
ssl_context.load_cert_chain(self.cert, self.key)
self.site = web.TCPSite(
self.runner,
host=self.addr,
port=self.port,
ssl_context=ssl_context,
)
await self.site.start()
self.logger.info(f"Serving on {self.addr}:{self.port}")
async def run(self):
"""
Binds to port and starts HTTP server
"""
# Create dictionaries to hold configured sources and models
await self.setup()
await self.start()
# Load
if self.mc_config is not None:
# Restore atomic after config is set, allow setting for now
atomic = self.mc_atomic
self.mc_atomic = False
await self.register_directory(self.mc_config)
self.mc_atomic = atomic
try:
# If we are testing then RUN_YIELD will be an asyncio.Event
if self.RUN_YIELD_START is not False:
await self.RUN_YIELD_START.put(self)
await self.RUN_YIELD_FINISH.wait()
else: # pragma: no cov
# Wait for ctrl-c
while True:
await asyncio.sleep(60)
finally:
await self.app.cleanup()
await self.site.stop()
