async def test_sources(self):
with tempfile.TemporaryDirectory() as tempdir:
# Source the HTTP API will pre-load
source = JSONSource(
filename=str(pathlib.Path(tempdir, "source.json")),
allowempty=True,
readwrite=True,
)
# Record the source will have in it
myrecord = Record("myrecord", data={"features": {"f1": 0}})
await save(source, myrecord)
async with ServerRunner.patch(HTTPService.server) as tserver:
cli = await tserver.start(
HTTPService.server.cli(
"-insecure",
"-port",
"0",
"-sources",
"mysource=json",
"-source-mysource-filename",
source.config.filename,
)
)
async with self.get(
cli, "/source/mysource/record/myrecord"
) as r:
self.assertEqual(await r.json(), myrecord.export())
async def test_models(self):
with tempfile.TemporaryDirectory() as tempdir:
# Model the HTTP API will pre-load
model = SLRModel(
features=Features(Feature("f1", float, 1)),
predict=Feature("ans", int, 1),
directory=tempdir,
)
# y = m * x + b for equation SLR is solving for
m = 5
b = 3
# Train the model
await train(model, *[{
"f1": x,
"ans": m * x + b
} for x in range(0, 10)])
await accuracy(
model, *[{
"f1": x,
"ans": m * x + b
} for x in range(10, 20)])
async with ServerRunner.patch(HTTPService.server) as tserver:
cli = await tserver.start(
HTTPService.server.cli(
"-insecure",
"-port",
"0",
"-models",
"mymodel=slr",
"-model-mymodel-directory",
tempdir,
"-model-mymodel-features",
"f1:float:1",
"-model-mymodel-predict",
"ans:int:1",
))
async with self.post(
cli,
f"/model/mymodel/predict/0",
json={
f"record_{x}": {
"features": {
"f1": x
}
}
for x in range(20, 30)
},
) as response:
response = await response.json()
records = response["records"]
self.assertEqual(len(records), 10)
for record in records.values():
should_be = m * record["features"]["f1"] + b
prediction = record["prediction"]["ans"]["value"]
percent_error = abs(should_be - prediction) / should_be
self.assertLess(percent_error, 0.2)
async def test_redirect(self):
with tempfile.TemporaryDirectory() as tempdir:
pathlib.Path(tempdir, "index.html").write_text("Hello World")
pathlib.Path(tempdir, "mysignup").write_text("MySignUp")
async with ServerRunner.patch(HTTPService.server) as tserver:
cli = await tserver.start(
HTTPService.server.cli(
"-insecure",
"-port",
"0",
"-static",
tempdir,
"-redirect",
"GET",
"/",
"/index.html",
"GET",
"/signup",
"/mysignup",
)
)
async with self.get(cli, "/") as r:
self.assertEqual(await r.text(), "Hello World")
async with self.get(cli, "/signup") as r:
self.assertEqual(await r.text(), "MySignUp")
async def test_redirect_format_error(self):
with self.assertRaises(RedirectFormatError):
async with ServerRunner.patch(HTTPService.server) as tserver:
await tserver.start(
# Missing METHOD
HTTPService.server.cli(
"-insecure",
"-port",
"0",
"-redirect",
"/",
"/index.html",
))
async def test_portfile(self):
with tempfile.TemporaryDirectory() as tempdir:
portfile_path = pathlib.Path(tempdir, "portfile.int")
async with ServerRunner.patch(HTTPService.server) as tserver:
cli = await tserver.start(
HTTPService.server.cli(
"-insecure",
"-port",
"0",
"-portfile",
str(portfile_path),
))
self.assertTrue(portfile_path.is_file())
self.assertEqual(cli.port, int(portfile_path.read_text()))
async def test_http(self):
# Read in command to start HTTP server
server_cmd = pathlib.Path(sh_filepath("model_start_http.sh"))
server_cmd = server_cmd.read_text()
server_cmd = server_cmd.replace("\n", "")
server_cmd = server_cmd.replace("\\", "")
# Remove `dffml service http server`
server_cmd = server_cmd.replace("dffml service http server", "")
# Replace port
server_cmd = server_cmd.replace("8080", "0")
server_cmd = shlex.split(server_cmd)
# Read in the curl command
curl_cmd = pathlib.Path(sh_filepath("model_curl_http.sh"))
curl_cmd = curl_cmd.read_text()
# Modify the curl command to use the correct version of python
curl_cmd = curl_cmd.replace("python", sys.executable)
# Create a temporary directory for new curl command
with directory_with_csv_files() as tempdir:
# Run training
subprocess.check_output(["sh", sh_filepath("train.sh")])
async with ServerRunner.patch(HTTPService.server) as tserver:
# Start the HTTP server
cli = await tserver.start(HTTPService.server.cli(*server_cmd))
# Modify the curl command to use the correct port
curl_cmd = curl_cmd.replace("8080", str(cli.port))
# Write out the modified curl command
pathlib.Path("curl.sh").write_text(curl_cmd)
# Make the prediction
proc = await asyncio.create_subprocess_exec(
"sh",
"curl.sh",
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await proc.communicate()
if proc.returncode != 0:
raise Exception(stderr.decode())
response = json.loads(stdout)
# Check the result
records = response["records"]
self.assertEqual(len(records), 1)
for record in records.values():
# Correct value should be 90
should_be = 90
prediction = record["prediction"]["Salary"]["value"]
# Check prediction within 20% of correct value
percent_error = abs(should_be - prediction) / should_be
self.assertLess(percent_error, 0.2)
async def test_start(self):
with tempfile.TemporaryDirectory() as tempdir:
await HTTPService.createtls.server.cli(
"-bits",
"2048",
"-key",
os.path.join(tempdir, "server.key"),
"-cert",
os.path.join(tempdir, "server.pem"),
)
async with ServerRunner.patch(HTTPService.server) as tserver:
await tserver.start(
HTTPService.server.cli(
"-port",
"0",
"-key",
os.path.join(tempdir, "server.key"),
"-cert",
os.path.join(tempdir, "server.pem"),
))
async def test_mc_config(self):
with tempfile.TemporaryDirectory() as tempdir:
# URLs for endpoints
hello_world_url: str = "/hello/world"
hello_blank_url: str = "/hello/blank"
# Create the required directory structure
# Create directories for multicomm, dataflow, and dataflow overrides
pathlib.Path(tempdir, "mc").mkdir()
pathlib.Path(tempdir, "mc", "http").mkdir()
pathlib.Path(tempdir, "df").mkdir()
# TODO split config part of dataflow into seperate directory
pathlib.Path(tempdir, "config").mkdir()
# Write out multicomm configs
pathlib.Path(tempdir, "mc", "http", "hello_world.json").write_text(
json.dumps(
{
"path": hello_world_url,
"presentation": "json",
"asynchronous": False,
},
sort_keys=True,
indent=4,
)
)
pathlib.Path(tempdir, "mc", "http", "hello_blank.json").write_text(
json.dumps(
{
"path": hello_blank_url,
"presentation": "json",
"asynchronous": False,
},
sort_keys=True,
indent=4,
)
)
# Write out dataflow configs
pathlib.Path(tempdir, "df", "hello_world.json").write_text(
json.dumps(
HELLO_WORLD_DATAFLOW.export(), sort_keys=True, indent=4
)
)
pathlib.Path(tempdir, "df", "hello_blank.json").write_text(
json.dumps(
HELLO_BLANK_DATAFLOW.export(), sort_keys=True, indent=4
)
)
# Start the server
async with ServerRunner.patch(HTTPService.server) as tserver:
cli = await tserver.start(
HTTPService.server.cli(
"-port",
"0",
"-insecure",
"-mc-config",
tempdir,
"-mc-atomic",
)
)
self.assertEqual(cli.mc_config, tempdir)
# Verify routes were registered and preform as expected
message: str = "Hello World"
with self.subTest(test=message):
# Check that hello world works
async with self.get(cli, hello_world_url) as response:
self.assertEqual(
{"response": message},
list((await response.json()).values())[0],
)
# Check that hello blank works
message: str = "Hello Feedface"
with self.subTest(test=message):
async with self.post(
cli,
hello_blank_url,
json={
"Feedface": [
{
"value": "Feedface",
"definition": formatter.op.inputs[
"data"
].name,
}
]
},
) as response:
self.assertEqual(
{"Feedface": {"response": message}},
await response.json(),
)
async def test_start_insecure(self):
async with ServerRunner.patch(HTTPService.server) as tserver:
await tserver.start(
HTTPService.server.cli("-port", "0", "-insecure")
)
async def test_scorer(self):
with tempfile.TemporaryDirectory() as tempdir:
model = SLRModel(
features=Features(Feature("f1", float, 1)),
predict=Feature("ans", int, 1),
location=tempdir,
)
# y = m * x + b for equation SLR is solving for
m = 5
b = 3
# Train the model
await train(model, *[{
"f1": x,
"ans": m * x + b
} for x in range(0, 10)])
source = JSONSource(
filename=pathlib.Path(tempdir, "source.json"),
allowempty=True,
readwrite=True,
)
# Record the source will have in it
await save(
source,
*[
Record(
str(i),
data={"features": {
"f1": x,
"ans": (m * x) + b
}},
) for i, x in enumerate(range(10, 20))
],
)
async with ServerRunner.patch(HTTPService.server) as tserver:
cli = await tserver.start(
HTTPService.server.cli(
"-insecure",
"-port",
"0",
"-models",
"mymodel=slr",
"-model-mymodel-location",
tempdir,
"-model-mymodel-features",
"f1:float:1",
"-model-mymodel-predict",
"ans:int:1",
"-features",
"ans:int:1",
"-sources",
"mysource=json",
"-source-mysource-filename",
str(source.config.filename),
"-scorers",
"myscorer=mse",
))
async with self.post(cli,
"/scorer/myscorer/mymodel/score",
json=["mysource"]) as r:
self.assertEqual(await r.json(), {"accuracy": 0.0})