def test_args(self): self.assertEqual( FileSource.args({}), { "source": { "arg": None, "config": { "file": { "arg": None, "config": { "filename": { "arg": Arg(type=str), "config": {}, }, "readonly": { "arg": Arg( type=bool, action="store_true", default=False, ), "config": {}, }, "label": { "arg": Arg(type=str, default="unlabeled"), "config": {}, }, }, } }, } }, )
def args(cls, args, *above) -> Dict[str, Arg]: cls.config_set(args, above, "host", Arg(default="127.0.0.1")) cls.config_set(args, above, "port", Arg(type=int, default=3306)) cls.config_set(args, above, "user", Arg(default="user")) cls.config_set(args, above, "password", Arg(default="pass")) cls.config_set(args, above, "db", Arg(default="db")) return args
def args(cls, args, *above) -> Dict[str, Arg]: cls.config_set( args, above, "directory", Arg( default=os.path.join( os.path.expanduser("~"), ".cache", "dffml", f"scikit-{entry_point_name}", ), help="Directory where state should be saved", ), ) cls.config_set( args, above, "predict", Arg(type=str, help="Label or the value to be predicted"), ) for param in inspect.signature(cls.SCIKIT_MODEL).parameters.values(): # TODO if param.default is an array then Args needs to get a # nargs="+" cls.config_set( args, above, param.name, Arg( type=cls.type_for(param), default=NoDefaultValue if param.default == inspect._empty else param.default, ), ) return args
def args(cls, args, *above) -> Dict[str, Arg]: cls.config_set( args, above, "directory", Arg( default=os.path.join(os.path.expanduser("~"), ".cache", "dffml", "scratch"), help="Directory where state should be saved", ), ) cls.config_set( args, above, "predict", Arg(type=str, help="Label or the value to be predicted"), ) cls.config_set( args, above, "features", Arg( nargs="+", required=True, type=Feature.load, action=list_action(Features), help="Features to train on", ), ) return args
def test_args(self): self.assertEqual( MemoryRedundancyChecker.args({}), { "rchecker": { "arg": None, "config": { "memory": { "arg": None, "config": { "kvstore": { "arg": Arg( type=BaseKeyValueStore.load, default=MemoryKeyValueStore, ), "config": { "withargs": { "arg": None, "config": { "filename": { "arg": Arg(type=str), "config": {}, } }, } }, } }, } }, } }, )
def test_modify(self): arg = Arg("-test", key="value") first = arg.modify(name="-first") second = arg.modify(key="new_value") self.assertEqual(arg.name, "-test") self.assertEqual(first.name, "-first") self.assertEqual(second.name, "-test") self.assertEqual(second["key"], "new_value")
def args(cls, args, *above) -> Dict[str, Arg]: cls.config_set( args, above, "directory", Arg( default=os.path.join(os.path.expanduser("~"), ".cache", "dffml", "tensorflow"), help="Directory where state should be saved", ), ) cls.config_set( args, above, "steps", Arg( type=int, default=3000, help="Number of steps to train the model", ), ) cls.config_set( args, above, "epochs", Arg( type=int, default=30, help="Number of iterations to pass over all repos in a source", ), ) cls.config_set( args, above, "hidden", Arg( type=int, nargs="+", default=[12, 40, 15], help= "List length is the number of hidden layers in the network. Each entry in the list is the number of nodes in that hidden layer", ), ) cls.config_set( args, above, "predict", Arg(help="Feature name holding truth value"), ) return args
def args(cls, args, *above) -> Dict[str, Arg]: cls.config_set(args, above, "directory", Arg()) cls.config_set( args, above, "features", Arg( nargs="+", required=True, type=Feature.load, action=list_action(Features), ), ) return args
def args(cls, args, *above) -> Dict[str, Arg]: cls.config_set(args, above, "host", Arg(default="127.0.0.1")) cls.config_set(args, above, "port", Arg(type=int, default=3306)) cls.config_set(args, above, "user", Arg()) cls.config_set(args, above, "password", Arg()) cls.config_set(args, above, "db", Arg()) cls.config_set( args, above, "repos-query", Arg( type=str, help= "SELECT `key` as key, data_1 as feature_1, data_2 as feature_2 FROM repo_data", ), ) cls.config_set( args, above, "repo-query", Arg( type=str, help= "SELECT `key` as key, data_1 as feature_1, data_2 as feature_2 FROM repo_data WHERE `key`=%s", ), ) cls.config_set( args, above, "update-query", Arg( type=str, help= "INSERT INTO repo_data (`key`, data_1, data_2) VALUES(%s, %s, %s) ON DUPLICATE KEY UPDATE data_1 = %s, data_2=%s", ), ) cls.config_set( args, above, "model-columns", Arg(type=str, nargs="+", help="Order of Columns in table"), ) cls.config_set( args, above, "ca", Arg(type=str, help="Path to server TLS certificate", default=None), ) return args
class MultiCommCMD(CMD): arg_mc_config = Arg( "-mc-config", dest="mc_config", default=None, help="MultiComm config directory", ) arg_mc_atomic = Arg( "-mc-atomic", dest="mc_atomic", action="store_true", default=False, help="No routes other than dataflows registered at startup", )
def test_args(self): self.assertDictEqual( MemoryRedundancyChecker.args({}), { "rchecker": { "plugin": None, "config": { "memory": { "plugin": None, "config": { "kvstore": { "plugin": Arg( type=load_kvstore_with_args, help="Key value store to use", default=MemoryKeyValueStore(), ), "config": {}, } }, } }, } }, )
class CreateTLSServer(TLSCMD): """ Used to generate server key and cert """ arg_bits = Arg("-bits", help="Number of bits to use for key", default=4096, type=int) async def run(self): subprocess.call([ "openssl", "req", "-x509", "-newkey", f"rsa:{self.bits}", "-keyout", self.key, "-out", self.cert, "-days", "365", "-nodes", "-sha256", "-subj", "/C=US/ST=Oregon/L=Portland/O=Feedface/OU=Org/CN=127.0.0.1", ])
def test_00_args(self): self.maxDiff = 99999 self.assertEqual( FakeTesting.args({}), { "test": { "plugin": None, "config": { "fake": { "plugin": None, "config": { "num": { "plugin": Arg(type=float), "config": {}, }, "files": { "plugin": Arg(type=str, nargs="+"), "config": {}, }, "features": { "plugin": Arg( type=Feature.load, nargs="+", action=list_action(Features), ), "config": {}, }, "name": { "plugin": Arg( type=str, help="Name of FakeTesting" ), "config": {}, }, "readonly": { "plugin": Arg( type=bool, action="store_true", default=False, ), "config": {}, }, "label": { "plugin": Arg( type=str, default="unlabeled" ), "config": {}, }, "source": { "plugin": Arg( type=BaseSource.load, default=JSONSource, ), "config": {}, }, }, } }, } }, )
def args(cls, args, *above) -> Dict[str, Arg]: cls.config_set( args, above, "directory", Arg( default=os.path.join(os.path.expanduser("~"), ".cache", "dffml", "scratch"), help="Directory where state should be saved", ), ) cls.config_set( args, above, "predict", Arg(type=str, help="Label or the value to be predicted"), ) return args
def test_args(self): self.assertEqual( FileSource.args({}), { "source": { "plugin": None, "config": { "file": { "plugin": None, "config": { "filename": { "plugin": Arg(type=str), "config": {}, }, "readwrite": { "plugin": Arg( type=bool, action="store_true", default=False, ), "config": {}, }, "allowempty": { "plugin": Arg( type=bool, action="store_true", default=False, ), "config": {}, }, "tag": { "plugin": Arg(type=str, default="untagged"), "config": {}, }, }, } }, } }, )
class Install(CMD): arg_packages = Arg("packages", nargs="+", help="Package to check if we should install") async def run(self): # Create an Orchestrator which will manage the running of our operations async with MemoryOrchestrator.basic_config(*OPIMPS) as orchestrator: # Create a orchestrator context, everything in DFFML follows this # one-two context entry pattern async with orchestrator() as octx: for package_name in self.packages: # For each package add a new input set to the network of # inputs (ictx). Operations run under a context, the context # here is the package_name to evaluate (the first argument). # The next arguments are all the inputs we're seeding the # network with for that context. We give the package name # because pypi_latest_package_version needs it to find the # version, which safety will then use. We also give an input # to the output operation GetSingle, which takes a list of # data type definitions we want to select as our results. await octx.ictx.sadd( package_name, Input( value=package_name, definition=pypi_package_json.op.inputs["package"], ), Input( value=[ safety_check.op.outputs["issues"].name, run_bandit.op.outputs["report"].name, ], definition=GetSingle.op.inputs["spec"], ), ) # Run all the operations, Each iteration of this loop happens # when all inputs are exhausted for a context, the output # operations are then run and their results are yielded async for ctx, results in octx.run_operations(): # The context for this data flow was the package name package_name = (await ctx.handle()).as_string() # Get the results of the GetSingle output operation results = results[GetSingle.op.name] # Check if any of the values of the operations evaluate to # true, so if the number of issues found by safety is # non-zero then this will be true any_issues = list(results.values()) if (any_issues[0] > 0 or any_issues[1]["CONFIDENCE.HIGH_AND_SEVERITY.HIGH"] > 5): print(f"Do not install {package_name}! {results!r}") else: print(f"{package_name} is okay to install")
class MiscService(CMD): """ Description of the DFFML related command """ arg_integer = Arg( "-integer", type=int, help=f"Port to do nothing with", default=0, required=True, ) async def run(self): print(f"Your integer was: {self.integer}")
class Install(CMD): arg_packages = Arg("packages", nargs="+", help="Package to check if we should install") async def run(self): # Create an Orchestrator which will manage the running of our operations async with MemoryOrchestrator.withconfig({}) as orchestrator: # Create a orchestrator context, everything in DFFML follows this # one-two context entry pattern async with orchestrator(DATAFLOW) as octx: # Run all the operations, Each iteration of this loop happens # when all inputs are exhausted for a context, the output # operations are then run and their results are yielded async for package_name, results in octx.run({ # For each package add a new input set to the input network # The context operations execute under is the package name # to evaluate. Contexts ensure that data pertaining to # package A doesn't mingle with data pertaining to package B package_name: [ # The only input to the operations is the package name. Input( value=package_name, definition=pypi_package_json.op.inputs["package"], ) ] for package_name in self.packages }): # Grab the number of safety issues and the bandit report # from the results dict safety_issues = results[ safety_check.op.outputs["issues"].name] bandit_report = results[ run_bandit.op.outputs["report"].name] # Decide if those numbers mean we should stop ship or not if (safety_issues > 0 or bandit_report["CONFIDENCE.HIGH_AND_SEVERITY.HIGH"] > 5): print(f"Do not install {package_name}!") for definition_name, result in results.items(): print(f" {definition_name}: {result}") else: print(f"{package_name} is okay to install")
def test_init(self): arg = Arg("-test", key="value") self.assertEqual(arg.name, "-test") self.assertIn("key", arg) self.assertEqual(arg["key"], "value")
def args(cls, args, *above) -> Dict[str, Arg]: cls.config_set(args, above, "directory", Arg()) return args
def args(cls, args, *above) -> Dict[str, Arg]: cls.config_set( args, above, "directory", Arg( default=os.path.join(os.path.expanduser("~"), ".cache", "dffml", "tensorflow"), help="Directory where state should be saved", ), ) cls.config_set( args, above, "steps", Arg( type=int, default=3000, help="Number of steps to train the model", ), ) cls.config_set( args, above, "epochs", Arg( type=int, default=30, help="Number of iterations to pass over all repos in a source", ), ) cls.config_set( args, above, "hidden", Arg( type=int, nargs="+", default=[12, 40, 15], help= "List length is the number of hidden layers in the network. Each entry in the list is the number of nodes in that hidden layer", ), ) cls.config_set( args, above, "classification", Arg(help="Feature name holding classification value"), ) cls.config_set( args, above, "classifications", Arg(nargs="+", help="Options for value of classification"), ) cls.config_set( args, above, "clstype", Arg( type=pydoc.locate, default=str, help="Data type of classifications values (default: str)", ), ) cls.config_set( args, above, "features", Arg( nargs="+", required=True, type=Feature.load, action=list_action(Features), help="Features to train on", ), ) return args
def args(cls, args, *above): cls.config_set(args, above, "filename", Arg(type=str)) return args
class CMDTest(CMD): arg_nope_present = Arg("nope", default=False) arg_ignored = Arg("ignored")
class FakeSubCMD(CMD): arg_test = Arg("-test")
def args(cls, args, *above) -> Dict[str, Arg]: cls.config_set(args, above, "filename", Arg()) return args
class Server(TLSCMD, MultiCommCMD, Routes): """ HTTP server providing access to DFFML APIs """ # Used for testing RUN_YIELD_START = False RUN_YIELD_FINISH = False INSECURE_NO_TLS = False arg_port = Arg("-port", help="Port to bind to", type=int, default=8080) arg_addr = Arg("-addr", help="Address to bind to", default="127.0.0.1") arg_upload_dir = Arg( "-upload-dir", help="Directory to store uploaded files in", default=None, ) arg_static = Arg("-static", help="Directory to serve static content from", default=None) arg_js = Arg( "-js", help="Serve JavaScript API file at /api.js", default=False, action="store_true", ) arg_insecure = Arg( "-insecure", help="Start without TLS encryption", action="store_true", default=False, ) arg_cors_domains = Arg( "-cors-domains", help= "Domains to allow CORS for (see keys in defaults dict for aiohttp_cors.setup)", nargs="+", default=[], ) arg_models = Arg( "-models", help="Models configured on start", nargs="+", default=AsyncContextManagerList(), type=Model.load_labeled, action=list_action(AsyncContextManagerList), ) arg_sources = Arg( "-sources", help="Sources configured on start", nargs="+", default=Sources(), type=BaseSource.load_labeled, action=list_action(Sources), ) async def start(self): if self.insecure: self.site = web.TCPSite(self.runner, host=self.addr, port=self.port) else: ssl_context = ssl.create_default_context( purpose=ssl.Purpose.SERVER_AUTH, cafile=self.cert) ssl_context.load_cert_chain(self.cert, self.key) self.site = web.TCPSite( self.runner, host=self.addr, port=self.port, ssl_context=ssl_context, ) await self.site.start() self.port = self.site._server.sockets[0].getsockname()[1] self.logger.info(f"Serving on {self.addr}:{self.port}") async def run(self): """ Binds to port and starts HTTP server """ # Create dictionaries to hold configured sources and models await self.setup() await self.start() # Load if self.mc_config is not None: # Restore atomic after config is set, allow setting for now atomic = self.mc_atomic self.mc_atomic = False await self.register_directory(self.mc_config) self.mc_atomic = atomic try: # If we are testing then RUN_YIELD will be an asyncio.Event if self.RUN_YIELD_START is not False: await self.RUN_YIELD_START.put(self) await self.RUN_YIELD_FINISH.wait() else: # pragma: no cov # Wait for ctrl-c while True: await asyncio.sleep(60) finally: await self.app.cleanup() await self.site.stop()
class TLSCMD(CMD): arg_key = Arg("-key", help="Path to key file", default="server.key") arg_cert = Arg("-cert", help="Path to cert file", default="server.pem")
class CreateTLSClient(CMD): """ Create TLS client key and cert (used to authenticate to HTTP API server). curl \\ --cacert server.pem \\ --cert client.pem \\ --key client.key \\ -vvvvv \\ https://127.0.0.1:5000/ """ CLI_FORMATTER_CLASS = argparse.RawDescriptionHelpFormatter arg_bits = Arg("-bits", help="Number of bits to use for key", default=4096, type=int) arg_key = Arg("-key", help="Path to client key file", default="client.key") arg_cert = Arg("-cert", help="Path to client cert file", default="client.pem") arg_csr = Arg("-csr", help="Path to client csr file", default="client.csr") arg_server_key = Arg("-server-key", help="Path to server key file", default="server.key") arg_server_cert = Arg("-server-cert", help="Path to server cert file", default="server.pem") async def run(self): subprocess.check_call([ "openssl", "req", "-newkey", f"rsa:{self.bits}", "-keyout", self.key, "-out", self.csr, "-nodes", "-sha256", "-subj", "/CN=RealUser", ]) subprocess.check_call([ "openssl", "x509", "-req", "-in", self.csr, "-CA", self.server_cert, "-CAkey", self.server_key, "-out", self.cert, "-set_serial", "01", "-days", "365", ])
def args(cls, args, *above) -> Dict[str, Arg]: cls.config_set(args, above, "keys", Arg(type=str, nargs="+", default=[])) return args
class Server(TLSCMD, MultiCommCMD, Routes): """ HTTP server providing access to DFFML APIs """ # Used for testing RUN_YIELD_START = False RUN_YIELD_FINISH = False INSECURE_NO_TLS = False arg_port = Arg("-port", help="Port to bind to", type=int, default=8080) arg_addr = Arg("-addr", help="Address to bind to", default="127.0.0.1") arg_upload_dir = Arg( "-upload-dir", help="Directory to store uploaded files in", default=None, ) arg_insecure = Arg( "-insecure", help="Start without TLS encryption", action="store_true", default=False, ) arg_cors_domains = Arg( "-cors-domains", help= "Domains to allow CORS for (see keys in defaults dict for aiohttp_cors.setup)", nargs="+", default=[], ) def __init__(self, *args, **kwargs): self._port: int = 0 self.site = None super().__init__(*args, **kwargs) @property def port(self): if self.site is None or self.site._server is None: return self._port return self.site._server.sockets[0].getsockname()[1] @port.setter def port(self, value): self._port = value async def start(self): if self.insecure: self.site = web.TCPSite(self.runner, host=self.addr, port=self.port) else: ssl_context = ssl.create_default_context( purpose=ssl.Purpose.SERVER_AUTH, cafile=self.cert) ssl_context.load_cert_chain(self.cert, self.key) self.site = web.TCPSite( self.runner, host=self.addr, port=self.port, ssl_context=ssl_context, ) await self.site.start() self.logger.info(f"Serving on {self.addr}:{self.port}") async def run(self): """ Binds to port and starts HTTP server """ # Create dictionaries to hold configured sources and models await self.setup() await self.start() # Load if self.mc_config is not None: # Restore atomic after config is set, allow setting for now atomic = self.mc_atomic self.mc_atomic = False await self.register_directory(self.mc_config) self.mc_atomic = atomic try: # If we are testing then RUN_YIELD will be an asyncio.Event if self.RUN_YIELD_START is not False: await self.RUN_YIELD_START.put(self) await self.RUN_YIELD_FINISH.wait() else: # pragma: no cov # Wait for ctrl-c while True: await asyncio.sleep(60) finally: await self.app.cleanup() await self.site.stop()