def start_predictions():
"""start_predictions
Using environment variables, create an AntiNex AI Client.
You can also use command line args if you want.
This can train a new deep neural network if it does not
exist or it can use an existing pre-trained deep neural
network within the AntiNex Core to make new predictions.
"""
parser = argparse.ArgumentParser(
description=("Python client to make Predictions "
"using a Pre-trained Deep Neural Network "
"with AntiNex Django Rest Framework"))
parser.add_argument("-f",
help=("file to use default ./examples/"
"predict-rows-scaler-full-django.json"),
required=False,
dest="datafile")
parser.add_argument("-m",
help="send mock data",
required=False,
dest="use_fake_rows",
action="store_true")
parser.add_argument("-b",
help=("optional - path to CA bundle directory for "
"client encryption over HTTP"),
required=False,
dest="ca_dir")
parser.add_argument("-c",
help=("optional - path to x509 certificate for "
"client encryption over HTTP"),
required=False,
dest="cert_file")
parser.add_argument("-k",
help=("optional - path to x509 key file for "
"client encryption over HTTP"),
required=False,
dest="key_file")
parser.add_argument("-s",
help="silent",
required=False,
dest="silent",
action="store_true")
parser.add_argument("-d",
help="debug",
required=False,
dest="debug",
action="store_true")
args = parser.parse_args()
datafile = ev("DATAFILE",
"./examples/predict-rows-scaler-full-django.json")
ca_dir = os.getenv("API_CA_BUNDLE_DIR", None)
cert_file = os.getenv("API_CERT_FILE", None)
key_file = os.getenv("API_KEY_FILE", None)
verbose = bool(str(ev("API_CLIENT_VERBOSE", "1")).lower() == "1")
debug = bool(str(ev("API_CLIENT_DEBUG", "0")).lower() == "1")
use_fake_rows = False
if args.use_fake_rows:
use_fake_rows = True
if args.datafile:
datafile = args.datafile
if args.ca_dir:
ca_dir = args.ca_dir
if args.cert_file:
cert_file = args.cert_file
if args.key_file:
key_file = args.key_file
if args.silent:
verbose = False
if args.debug:
debug = True
if verbose:
log.info("creating client")
client = build_ai_client_from_env(ca_dir=ca_dir,
cert_file=cert_file,
key_file=key_file,
verbose=verbose,
debug=debug)
if verbose:
log.info(("loading request in datafile={}").format(datafile))
# pass in full or partial prediction record dictionaries
# the generate_ai_request will fill in gaps with defaults
fake_rows_for_predicting = [{
"tcp_seq": 1
}, {
"tcp_seq": 2
}, {
"tcp_seq": 3
}, {
"tcp_seq": 4
}]
res_gen = None
if use_fake_rows:
res_gen = generate_ai_request(predict_rows=fake_rows_for_predicting)
else:
req_with_org_rows = None
with open(datafile, "r") as f:
req_with_org_rows = json.loads(f.read())
res_gen = generate_ai_request(
predict_rows=req_with_org_rows["predict_rows"])
# end of sending mock data from this file or a file on disk
if res_gen["status"] != SUCCESS:
log.error(("failed generate_ai_request with error={}").format(
res_gen["error"]))
sys.exit(1)
req_body = res_gen["data"]
if verbose:
log.info("running job")
job_was_started = False
response = client.run_job(body=req_body)
if response["status"] == SUCCESS:
log.info(("job started with response={}").format(response["data"]))
job_was_started = True
elif response["status"] == FAILED:
log.error(("job failed with error='{}' with response={}").format(
response["error"], response["data"]))
elif response["status"] == ERROR:
log.error(("job had an error='{}' with response={}").format(
response["error"], response["data"]))
elif response["status"] == LOGIN_FAILED:
log.error(("job reported user was not able to log in "
"with an error='{}' with response={}").format(
response["error"], response["data"]))
if not job_was_started:
sys.exit(1)
if debug:
log.info(("parsing response data={}").format(response["data"]))
else:
if verbose:
log.info("parsing data")
res_data = response["data"]
job_data = res_data.get("job", None)
result_data = res_data.get("results", None)
if not job_data:
log.error(("missing job dictionary in response data={}").format(
response["data"]))
sys.exit(1)
if not result_data:
log.error(("missing results dictionary in response data={}").format(
response["data"]))
sys.exit(1)
job_id = job_data.get("id", None)
job_status = job_data.get("status", None)
result_id = result_data.get("id", None)
result_status = result_data.get("status", None)
log.info(
("started job.id={} job.status={} with "
"result.id={} result.status={}").format(job_id, job_status, result_id,
result_status))
job_results = client.wait_for_job_to_finish(job_id=job_id)
if job_results["status"] != SUCCESS:
log.error(
("failed waiting for job.id={} to finish error={} data={}").format(
job_id, job_results["error"], job_results["data"]))
sys.exit(1)
final_job = job_results["data"]["job"]
final_result = job_results["data"]["result"]
log.info(("job={}").format(ppj(final_job)))
log.info(("result={}").format(ppj(final_result)))
log.info(("job.id={} is done").format(job_id))
predictions = final_result["predictions_json"].get("predictions", [])
log.info(("loading predictions={} into pandas dataframe").format(
len(predictions)))
df = pd.DataFrame(predictions)
log.info(("dataframe={}").format(df))
def get_ml_job():
"""get_ml_job
Get an ``MLJob`` by database id.
"""
parser = argparse.ArgumentParser(
description=("Python client get AI Job by ID"))
parser.add_argument("-u", help="username", required=False, dest="user")
parser.add_argument("-p",
help="user password",
required=False,
dest="password")
parser.add_argument("-e", help="user email", required=False, dest="email")
parser.add_argument("-a",
help="url endpoint with default http://localhost:8010",
required=False,
dest="url")
parser.add_argument("-i",
help="User's MLJob.id to look up",
required=False,
dest="job_id")
parser.add_argument("-b",
help=("optional - path to CA bundle directory for "
"client encryption over HTTP"),
required=False,
dest="ca_dir")
parser.add_argument("-c",
help=("optional - path to x509 certificate for "
"client encryption over HTTP"),
required=False,
dest="cert_file")
parser.add_argument("-k",
help=("optional - path to x509 key file for "
"client encryption over HTTP"),
required=False,
dest="key_file")
parser.add_argument("-s",
help="silent",
required=False,
dest="silent",
action="store_true")
parser.add_argument("-d",
help="debug",
required=False,
dest="debug",
action="store_true")
args = parser.parse_args()
user = ev("API_USER", "user-not-set")
password = ev("API_PASSWORD", "password-not-set")
email = ev("API_EMAIL", "email-not-set")
url = ev("API_URL", "http://localhost:8010")
job_id = ev("JOB_ID", "job_id-not-set")
ca_dir = os.getenv("API_CA_BUNDLE_DIR", None)
cert_file = os.getenv("API_CERT_FILE", None)
key_file = os.getenv("API_KEY_FILE", None)
verbose = bool(str(ev("API_VERBOSE", "true")).lower() == "true")
debug = bool(str(ev("API_DEBUG", "false")).lower() == "true")
if args.user:
user = args.user
if args.password:
password = args.password
if args.email:
email = args.email
if args.url:
url = args.url
if args.job_id:
job_id = args.job_id
if args.ca_dir:
ca_dir = args.ca_dir
if args.cert_file:
cert_file = args.cert_file
if args.key_file:
key_file = args.key_file
if args.silent:
verbose = False
if args.debug:
debug = True
usage = ("Please run with "
"-u <username> "
"-p <password> "
"-a <AntiNex URL http://localhost:8010> "
"-i <job_id> "
"-b <optional - path to CA bundle directory> "
"-c <optional - path to x509 ssl certificate file> "
"-k <optional - path to x509 ssl key file>")
valid = True
if not user or user == "user-not-set":
log.error("missing user")
valid = False
if not password or password == "password-not-set":
log.error("missing password")
valid = False
if not job_id or job_id == "job_id-not-set":
log.error("missing job_id")
valid = False
else:
try:
job_id = int(job_id)
except Exception:
log.error("please use -i <job_id with an integer>")
valid = False
if not valid:
log.error(usage)
sys.exit(1)
if verbose:
log.info(("creating client user={} url={} job_id={} "
"ca_dir={} cert_file={} key_file={}").format(
user, url, job_id, ca_dir, cert_file, key_file))
client = AIClient(user=user,
email=email,
password=password,
url=url,
ca_dir=ca_dir,
cert_file=cert_file,
key_file=key_file,
verbose=verbose,
debug=debug)
if verbose:
log.info(("loading request in job_id={}").format(job_id))
response = client.get_job_by_id(job_id=job_id)
if response["status"] == SUCCESS:
if debug:
log.info(("got a job response={}").format(response["data"]))
elif response["status"] == FAILED:
log.error(("job failed with error='{}' with response={}").format(
response["error"], response["data"]))
sys.exit(1)
elif response["status"] == ERROR:
if "missing " in response["error"]:
log.error(("Did not find a job with id={} for user={}").format(
job_id, user))
else:
log.error(("job had an error='{}' with response={}").format(
response["error"], response["data"]))
sys.exit(1)
elif response["status"] == LOGIN_FAILED:
log.error(("job reported user was not able to log in "
"with an error='{}' with response={}").format(
response["error"], response["data"]))
sys.exit(1)
job_data = response["data"]
if len(job_data) == 0:
log.error(
("Did not find a job with id={} for user={}").format(job_id, user))
sys.exit(1)
job_id = job_data.get("id", None)
job_status = job_data.get("status", None)
log.info(("job={}").format(ppj(job_data)))
log.info(("done getting job.id={} status={}").format(job_id, job_status))
def prepare_new_dataset():
"""prepare_new_dataset
Prepare a new ``MLPrepare`` record and dataset files on disk.
"""
parser = argparse.ArgumentParser(
description=("Python client to Prepare a dataset"))
parser.add_argument("-u", help="username", required=False, dest="user")
parser.add_argument("-p",
help="user password",
required=False,
dest="password")
parser.add_argument("-e", help="user email", required=False, dest="email")
parser.add_argument("-a",
help="url endpoint with default http://localhost:8010",
required=False,
dest="url")
parser.add_argument(
"-f",
help="file to use default ./examples/test-keras-dnn.json",
required=False,
dest="prepare_file")
parser.add_argument("-b",
help=("optional - path to CA bundle directory for "
"client encryption over HTTP"),
required=False,
dest="ca_dir")
parser.add_argument("-c",
help=("optional - path to x509 certificate for "
"client encryption over HTTP"),
required=False,
dest="cert_file")
parser.add_argument("-k",
help=("optional - path to x509 key file for "
"client encryption over HTTP"),
required=False,
dest="key_file")
parser.add_argument("-s",
help="silent",
required=False,
dest="silent",
action="store_true")
parser.add_argument("-d",
help="debug",
required=False,
dest="debug",
action="store_true")
args = parser.parse_args()
user = ev("API_USER", "user-not-set")
password = ev("API_PASSWORD", "password-not-set")
email = ev("API_EMAIL", "email-not-set")
url = ev("API_URL", "http://localhost:8010")
prepare_file = ev("DATAFILE", "prepare_file-not-set")
ca_dir = os.getenv("API_CA_BUNDLE_DIR", None)
cert_file = os.getenv("API_CERT_FILE", None)
key_file = os.getenv("API_KEY_FILE", None)
verbose = bool(str(ev("API_VERBOSE", "true")).lower() == "true")
debug = bool(str(ev("API_DEBUG", "false")).lower() == "true")
if args.user:
user = args.user
if args.password:
password = args.password
if args.email:
email = args.email
if args.url:
url = args.url
if args.prepare_file:
prepare_file = args.prepare_file
if args.ca_dir:
ca_dir = args.ca_dir
if args.cert_file:
cert_file = args.cert_file
if args.key_file:
key_file = args.key_file
if args.silent:
verbose = False
if args.debug:
debug = True
usage = ("Please run with "
"-u <username> "
"-p <password> "
"-a <AntiNex URL http://localhost:8010> "
"-f <path to prepare file> "
"-b <optional - path to CA bundle directory> "
"-c <optional - path to x509 ssl certificate file> "
"-k <optional - path to x509 ssl key file>")
valid = True
if not user or user == "user-not-set":
log.error("missing user")
valid = False
if not password or password == "password-not-set":
log.error("missing password")
valid = False
if not prepare_file or prepare_file == "prepare_file-not-set":
log.error("missing prepare_file")
valid = False
else:
if not os.path.exists(prepare_file):
log.error(
("did not find prepare_file={} on disk").format(prepare_file))
valid = False
if not valid:
log.error(usage)
sys.exit(1)
if verbose:
log.info(("creating client user={} url={} file={}").format(
user, url, prepare_file))
client = AIClient(user=user,
email=email,
password=password,
url=url,
ca_dir=ca_dir,
cert_file=cert_file,
key_file=key_file,
verbose=verbose,
debug=debug)
if verbose:
log.info(("loading request in prepare_file={}").format(prepare_file))
req_body = None
with open(prepare_file, "r") as f:
req_body = json.loads(f.read())
if verbose:
log.info("running prepare")
prepare_was_started = False
response = client.run_prepare(body=req_body)
if response["status"] == SUCCESS:
log.info(("prepare started with response={}").format(response["data"]))
prepare_was_started = True
elif response["status"] == FAILED:
log.error(("prepare failed with error='{}' with response={}").format(
response["error"], response["data"]))
elif response["status"] == ERROR:
log.error(("prepare had an error='{}' with response={}").format(
response["error"], response["data"]))
elif response["status"] == LOGIN_FAILED:
log.error(("prepare reported user was not able to log in "
"with an error='{}' with response={}").format(
response["error"], response["data"]))
if not prepare_was_started:
sys.exit(1)
if debug:
log.info(("parsing response data={}").format(response["data"]))
else:
if verbose:
log.info("parsing data")
prepare_data = response["data"]
if not prepare_data:
log.error(("missing prepare dictionary in response data={}").format(
response["data"]))
sys.exit(1)
prepare_id = prepare_data.get("id", None)
prepare_status = prepare_data.get("status", None)
log.info(("started prepare.id={} prepare.status={}").format(
prepare_id, prepare_status))
prepare_results = client.wait_for_prepare_to_finish(prepare_id=prepare_id)
if prepare_results["status"] != SUCCESS:
log.error(
("failed waiting for prepare.id={} to finish "
"error={} data={}").format(prepare_id, prepare_results["error"],
prepare_results["data"]))
sys.exit(1)
final_prepare = prepare_results["data"]
log.info(("prepare={}").format(ppj(final_prepare)))
log.info(("prepare.id={} is done").format(prepare_id))
LOGIN_FAILED = 2
SUCCESS = 0
FAILED = 1
ERROR = 2
NOT_SET = 3
DISABLED = 4
# AntiNex REST API flags, endpoint and credentials
# https://github.com/jay-johnson/train-ai-with-django-swagger-jwt
#
# AntiNex is running using compose.yml file:
# noqa https://github.com/jay-johnson/train-ai-with-django-swagger-jwt/blob/master/compose.yml
#
# AntiNex python client:
# https://github.com/jay-johnson/antinex-client
ANTINEX_PUBLISH_ENABLED = bool(ev("ANTINEX_PUBLISH_ENABLED", "1") == "1")
ANTINEX_URL = ev("ANTINEX_URL", "http://*****:*****@test.com")
ANTINEX_PASSWORD = ev("ANTINEX_PASSWORD", "123321")
# provide a template request publish file like:
# noqa https://github.com/jay-johnson/antinex-client/blob/master/examples/predict-rows-scaler-full-django.json
ANTINEX_PUBLISH_REQUEST_FILE = ev("ANTINEX_PUBLISH_REQUEST_FILE",
("/opt/antinex/client/examples/"
"predict-rows-scaler-full-django.json"))
# comma-separated list
ANTINEX_FEATURES_TO_PROCESS_STR = os.getenv("ANTINEX_FEATURES_TO_PROCESS",
None)
def train_new_deep_neural_network():
"""train_new_deep_neural_network
Train a new deep neural network and store the results as a new:
``MLJob`` and ``MLJobResult`` database records.
"""
parser = argparse.ArgumentParser(
description=("Python client to Train a Deep Neural Network "
"with AntiNex Django Rest Framework"))
parser.add_argument("-u", help="username", required=False, dest="user")
parser.add_argument("-p",
help="user password",
required=False,
dest="password")
parser.add_argument("-e", help="user email", required=False, dest="email")
parser.add_argument("-a",
help="url endpoint with default http://localhost:8010",
required=False,
dest="url")
parser.add_argument(
"-f",
help="file to use default ./examples/test-keras-dnn.json",
required=False,
dest="datafile")
parser.add_argument("-b",
help=("optional - path to CA bundle directory for "
"client encryption over HTTP"),
required=False,
dest="ca_dir")
parser.add_argument("-c",
help=("optional - path to x509 certificate for "
"client encryption over HTTP"),
required=False,
dest="cert_file")
parser.add_argument("-k",
help=("optional - path to x509 key file for "
"client encryption over HTTP"),
required=False,
dest="key_file")
parser.add_argument("-s",
help="silent",
required=False,
dest="silent",
action="store_true")
parser.add_argument("-d",
help="debug",
required=False,
dest="debug",
action="store_true")
args = parser.parse_args()
user = ev("API_USER", "user-not-set")
password = ev("API_PASSWORD", "password-not-set")
email = ev("API_EMAIL", "email-not-set")
url = ev("API_URL", "http://localhost:8010")
datafile = ev("DATAFILE", "datafile-not-set")
ca_dir = os.getenv("API_CA_BUNDLE_DIR", None)
cert_file = os.getenv("API_CERT_FILE", None)
key_file = os.getenv("API_KEY_FILE", None)
verbose = bool(str(ev("API_VERBOSE", "true")).lower() == "true")
debug = bool(str(ev("API_DEBUG", "false")).lower() == "true")
if args.user:
user = args.user
if args.password:
password = args.password
if args.email:
email = args.email
if args.url:
url = args.url
if args.datafile:
datafile = args.datafile
if args.ca_dir:
ca_dir = args.ca_dir
if args.cert_file:
cert_file = args.cert_file
if args.key_file:
key_file = args.key_file
if args.silent:
verbose = False
if args.debug:
debug = True
usage = ("Please run with "
"-u <username> "
"-p <password> "
"-a <AntiNex URL http://localhost:8010> "
"-f <path to json file> "
"-b <optional - path to CA bundle directory> "
"-c <optional - path to x509 ssl certificate file> "
"-k <optional - path to x509 ssl key file>")
valid = True
if not user or user == "user-not-set":
log.error("missing user")
valid = False
if not password or password == "password-not-set":
log.error("missing password")
valid = False
if not datafile or datafile == "datafile-not-set":
log.error("missing datafile")
valid = False
else:
if not os.path.exists(datafile):
log.error(("did not find datafile={} on disk").format(datafile))
valid = False
if not valid:
log.error(usage)
sys.exit(1)
if verbose:
log.info(("creating client user={} url={} file={} "
"ca_dir={} cert_file={} key_file={}").format(
user, url, datafile, ca_dir, cert_file, key_file))
client = AIClient(user=user,
email=email,
password=password,
url=url,
ca_dir=ca_dir,
cert_file=cert_file,
key_file=key_file,
verbose=verbose,
debug=debug)
if verbose:
log.info(("loading request in datafile={}").format(datafile))
req_body = None
with open(datafile, "r") as f:
req_body = json.loads(f.read())
if verbose:
log.info("running job")
job_was_started = False
response = client.run_job(body=req_body)
if response["status"] == SUCCESS:
log.info(("job started with response={}").format(response["data"]))
job_was_started = True
elif response["status"] == FAILED:
log.error(("job failed with error='{}' with response={}").format(
response["error"], response["data"]))
elif response["status"] == ERROR:
log.error(("job had an error='{}' with response={}").format(
response["error"], response["data"]))
elif response["status"] == LOGIN_FAILED:
log.error(("job reported user was not able to log in "
"with an error='{}' with response={}").format(
response["error"], response["data"]))
if not job_was_started:
sys.exit(1)
if debug:
log.info(("parsing response data={}").format(response["data"]))
else:
if verbose:
log.info("parsing data")
res_data = response["data"]
job_data = res_data.get("job", None)
result_data = res_data.get("results", None)
if not job_data:
log.error(("missing job dictionary in response data={}").format(
response["data"]))
sys.exit(1)
if not result_data:
log.error(("missing results dictionary in response data={}").format(
response["data"]))
sys.exit(1)
job_id = job_data.get("id", None)
job_status = job_data.get("status", None)
result_id = result_data.get("id", None)
result_status = result_data.get("status", None)
log.info(
("started job.id={} job.status={} with "
"result.id={} result.status={}").format(job_id, job_status, result_id,
result_status))
job_results = client.wait_for_job_to_finish(job_id=job_id)
if job_results["status"] != SUCCESS:
log.error(
("failed waiting for job.id={} to finish error={} data={}").format(
job_id, job_results["error"], job_results["data"]))
sys.exit(1)
final_job = job_results["data"]["job"]
final_result = job_results["data"]["result"]
if verbose:
log.info(("job={}").format(ppj(final_job)))
else:
log.info(("job={}").format(str(final_job)[0:10]))
if verbose:
log.info(("result={}").format(ppj(final_result)))
else:
log.info(("result={}").format(str(final_result)[0:10]))
log.info(("job.id={} is done").format(job_id))
predictions = final_result["predictions_json"].get("predictions", [])
log.info(("loading predictions={} into pandas dataframe").format(
len(predictions)))
df = pd.DataFrame(predictions)
if verbose:
log.info(("dataframe={}").format(df))
def __init__(self,
user=ev("API_USER", "user-not-set"),
password=ev("API_PASSWORD", "password-not-set"),
url=ev("API_URL", "http://localhost:8010"),
email=ev("API_EMAIL", "email-not-set"),
verbose=True,
ca_dir=None,
cert_file=None,
key_file=None,
debug=False):
"""__init__
:param user: username
:param email: email address
:param password: password for the user
:param url: url running the django rest framework
:param verbose: turn off setup_logging
:param ca_dir: optional path to CA bundle dir
:param cert_file: optional path to x509 ssl cert file
:param key_file: optional path to x509 ssl private key
:param debug: turn on debugging - this will print passwords to stdout
"""
self.user = user
self.email = email
self.password = password
self.url = url
self.verbose = verbose
self.ca_dir = ca_dir
self.cert_file = cert_file
self.key_file = key_file
self.cert = None
self.use_verify = False
if self.ca_dir:
self.use_verify = self.ca_dir
elif self.cert_file:
self.use_verify = self.cert_file
if self.cert_file and self.key_file:
self.cert = (self.cert_file, self.key_file)
self.debug = debug
if self.debug:
self.verbose = True
self.api_urls = {
"login": "******".format(self.url),
"job": "{}/ml/".format(self.url),
"prepare": "{}/mlprepare/".format(self.url),
"results": "{}/mlresults/".format(self.url),
"create_user": "******".format(self.url)
}
self.token = "not-logged-in-no-token"
self.login_status = LOGIN_NOT_ATTEMPTED
self.user_id = None
self.max_retries = 10
self.login_retry_wait_time = 0.1 # in seconds
self.all_prepares = {}
self.all_jobs = {}
self.all_results = {}
import json
import requests
import logging
import time
from spylunking.log.setup_logging import console_logger
from antinex_client.utils import ev
from antinex_client.consts import LOGIN_SUCCESS
from antinex_client.consts import LOGIN_NOT_ATTEMPTED
from antinex_client.consts import LOGIN_FAILED
from antinex_client.consts import SUCCESS
from antinex_client.consts import FAILED
from antinex_client.consts import ERROR
from antinex_client.consts import NOT_SET
log_level = logging.INFO
log_level_str = ev("AI_CLIENT_LEVEL", "info").lower()
if log_level_str == "info":
log_level = logging.INFO
elif log_level_str == "debug":
log_level = logging.DEBUG
elif log_level_str == "silent":
log_level = logging.CRITICAL
elif log_level_str == "critical":
log_level = logging.CRITICAL
elif log_level_str == "error":
log_level = logging.ERROR
log = console_logger(name='ai_client', log_level=log_level)
class AIClient: