def test_ml_predict_helper_works(self): """ Test ML Predict helper works """ test_label = str("mlpredict_{}").format( str(uuid.uuid4())) use_csv = "/tmp/cleaned_attack_scans.csv" model_desc = self.build_model_desc() model_and_weights = self.build_model_and_weights() prediction_manifest = self.build_prediction_manifest() predict_rows = self.build_prediction_rows( data_file=use_csv) self.maxDiff = None data = { "label": test_label, "predict_rows": predict_rows, "manifest": prediction_manifest, "model_desc": model_desc, "model_json": model_and_weights["model"], "weights_json": model_and_weights["weights"] } res = make_predictions(data) self.assertEqual( res["err"], "") self.assertEqual( res["status"], SUCCESS) self.assertTrue(len(res["data"]) > 0)
def test_ml_predict_missing_predictions(self): """ Test ML Predict helper fails if no predictions are provided """ test_label = str("mlpredict_{}").format( str(uuid.uuid4())) model_desc = self.build_model_desc() model_and_weights = self.build_model_and_weights() prediction_manifest = self.build_prediction_manifest() predict_rows = None del prediction_manifest["csv_file"] data = { "label": test_label, "predict_rows": predict_rows, "manifest": prediction_manifest, "model_desc": model_desc, "model_json": model_and_weights["model"], "weights_json": model_and_weights["weights"], "weights_file": "/tmp/{}".format(test_label) } res = make_predictions(data) self.assertEqual( res["status"], ERR) self.assertEqual( res["data"], None)
def test_regression(self): req = self.build_regression_request() res = make_predictions(req) self.assertEqual( res["status"], SUCCESS) self.assertTrue( res["data"]["model"])
def test_regression_deep_dnn(self): req = self.build_regression_request( model_desc_file="./tests/model_desc/deep_dnn.json") res = make_predictions(req) self.assertEqual( res["status"], SUCCESS) self.assertTrue( res["data"]["model"])
def test_regression_wide_dnn_with_auto_scaler(self): req = self.build_regression_request( model_desc_file="./tests/model_desc/wide_dnn.json") req["apply_scaler"] = True req["scaler_cast_type"] = "float32" res = make_predictions(req) self.assertEqual( res["status"], SUCCESS) self.assertTrue( res["data"]["model"])
def test_dataset_regression_using_scaler(self): req = self.build_dataset_regression_request() req["apply_scaler"] = True req["scaler_cast_type"] = "float32" res = make_predictions(req) self.assertEqual( res["status"], SUCCESS) self.assertTrue( res["data"]["model"]) self.assertEqual( len(res["data"]["model"].layers), 3) predictions = res["data"]["sample_predictions"] self.assertTrue( len(predictions), 18)
def test_ml_predict_missing_manifest(self): """ Test ML Predict helper fails missing data """ test_label = str("mlpredict_{}").format(str(uuid.uuid4())) model_desc = self.build_model_desc() model_and_weights = self.build_model_and_weights() data = { "label": test_label, "model_desc": model_desc, "model_json": model_and_weights["model"], "weights_json": model_and_weights["weights"], "weights_file": "/tmp/{}".format(test_label) } res = make_predictions(data) self.assertEqual(res["status"], ERR) self.assertEqual(res["data"], None)
def task_ml_job(self=None, req_node=None): """task_ml_job :param self: parent task object for bind=True :param req_node: job utils dictionary for passing a dictionary """ log.info(("task - {} - start " "req_node={}").format(req_node["task_name"], ppj(req_node))) user_data = req_node["data"].get("user_data", None) ml_job = req_node["data"].get("ml_job_data", None) ml_result = req_node["data"].get("ml_result_data", None) model_desc = req_node["data"].get("model_desc", None) label_rules = req_node["data"].get("label_rules", None) predict_rows = req_node["data"].get("predict_rows", None) user_res = db_lookup_user(user_id=user_data["id"]) user_obj = user_res.get("user_obj", None) ml_job_id = None ml_result_id = None ml_job_obj = None found_predictions = [] found_accuracy = None if req_node["use_cache"]: ml_job_obj = MLJob.objects.select_related().filter( Q(id=int(ml_job["id"])) & Q(user=user_obj)).cache().first() else: ml_job_obj = MLJob.objects.select_related().filter( Q(id=int(ml_job["id"])) & Q(user=user_obj)).first() # end of finding the MLJob record ml_result_obj = None if req_node["use_cache"]: ml_result_obj = MLJobResult.objects.select_related().filter( Q(id=int(ml_result["id"])) & Q(user=user_obj)).cache().first() else: ml_result_obj = MLJobResult.objects.select_related().filter( Q(id=int(ml_result["id"])) & Q(user=user_obj)).first() # end of finding the MLJobResult record res = build_task_response(use_cache=req_node["use_cache"], celery_enabled=req_node["celery_enabled"], cache_key=req_node["cache_key"]) last_step = "not started" data = {} data["job"] = {} data["results"] = {} try: res["status"] = ERR res["error"] = "" predict_manifest = ml_job_obj.predict_manifest csv_file = predict_manifest.get("csv_file", None) meta_file = predict_manifest.get("meta_file", None) epochs = int(predict_manifest.get("epochs", "5")) test_size = float(predict_manifest.get("test_size", "0.2")) batch_size = int(predict_manifest.get("batch_size", "32")) verbose = int(predict_manifest.get("verbose", "1")) # use pre-trained models in memory by label use_model_name = ml_job_obj.predict_manifest.get( "use_model_name", None) dataset = ml_job_obj.predict_manifest.get("dataset", None) predict_rows = ml_job_obj.predict_manifest.get("predict_rows", None) predict_feature = ml_job_obj.predict_manifest.get( "predict_feature", None) features_to_process = ml_job_obj.predict_manifest.get( "features_to_process", None) ignore_features = ml_job_obj.predict_manifest.get( "ignore_features", None) publish_to_core = ml_job_obj.predict_manifest.get( "publish_to_core", None) apply_scaler = ml_job_obj.predict_manifest.get("apply_scaler", True) sort_values = ml_job_obj.predict_manifest.get("sort_values", None) max_records = int( ml_job_obj.predict_manifest.get("max_records", "100000")) loss = ml_job_obj.predict_manifest.get("loss", "binary_crossentropy") metrics = ml_job_obj.predict_manifest.get("metrics", ["accuracy"]) optimizer = ml_job_obj.predict_manifest.get("optimizer", "adam") histories = ml_job_obj.predict_manifest.get( "histories", ["val_loss", "val_acc", "loss", "acc"]) needs_local_builder = True if ((dataset or predict_rows) and features_to_process): log.info(("using antinex builder dataset={} predict_rows={} " "features_to_process={}").format(dataset, predict_rows, features_to_process)) needs_local_builder = False # flag for bypassing build inside django instead of antinex-utils image_file = ml_result_obj.acc_image_file version = ml_job_obj.version ml_job_id = ml_job_obj.id ml_result_id = ml_result_obj.id last_step = ("starting user={} " "job.id={} result.id={} predict={} " "model_desc={} " "csv={} meta={}").format(ml_job_obj.user.id, ml_job_id, ml_result_id, ml_job_obj.predict_feature, model_desc, csv_file, meta_file) log.info(last_step) ml_job_obj.status = "analyzing" ml_job_obj.save() if needs_local_builder: log.info("starting local build_training_request") ml_req = build_training_request( csv_file=csv_file, meta_file=meta_file, predict_feature=ml_job_obj.predict_feature, test_size=test_size) if ml_req["status"] != VALID: last_step = ("Stopping for status={} " "errors: {}").format(ml_req["status"], ml_req["err"]) log.error(last_step) ml_job_obj.status = "error" ml_job_obj.control_state = "error" log.info(("saving job={}").format(ml_job_id)) ml_job_obj.save() data["job"] = ml_job_obj.get_public() error_data = {"status": ml_req["status"], "err": ml_req["err"]} data["results"] = error_data res["status"] = ERR res["error"] = last_step res["data"] = data return res else: predict_manifest["ignore_features"] = \ ml_req.get("ignore_features", []) predict_manifest["features_to_process"] = \ ml_req.get("features_to_process", []) if label_rules: predict_manifest["label_rules"] = \ label_rules else: predict_manifest["label_rules"] = \ ml_req["meta_data"]["label_rules"] predict_manifest["post_proc_rules"] = \ ml_req["meta_data"]["post_proc_rules"] predict_manifest["version"] = version last_step = ("job.id={} built_training_request={} " "predict={} features={} ignore={} " "label_rules={} post_proc={}").format( ml_job_obj.id, ml_req["status"], predict_manifest["predict_feature"], predict_manifest["features_to_process"], predict_manifest["ignore_features"], predict_manifest["label_rules"], predict_manifest["post_proc_rules"]) log.info(last_step) if ml_job_obj.ml_type == "regression": log.info(("using Keras - regression - " "sequential model ml_type={}").format( ml_job_obj.ml_type)) loss = "mse" metrics = ["mse", "mae", "mape", "cosine"] histories = [ "mean_squared_error", "mean_absolute_error", "mean_absolute_percentage_error", "cosine_proximity" ] else: log.info(("using Keras - sequential model " "ml_type={}").format(ml_job_obj.ml_type)) # end of classification vs regression ml_job_obj.predict_manifest["epochs"] = epochs ml_job_obj.predict_manifest["batch_size"] = batch_size ml_job_obj.predict_manifest["verbose"] = verbose ml_job_obj.predict_manifest["loss"] = loss ml_job_obj.predict_manifest["metrics"] = metrics ml_job_obj.predict_manifest["optimizer"] = optimizer ml_job_obj.predict_manifest["histories"] = histories ml_job_obj.predict_manifest = predict_manifest # end of updating without antinex-utils # end of if needs_local_builder: ml_job_obj.status = "started" ml_job_obj.save() scores = None prediction_req = { "label": "job_{}_result_{}".format(ml_job_id, ml_result_id), "manifest": ml_job_obj.predict_manifest, "model_json": ml_result_obj.model_json, "model_desc": model_desc, "weights_json": ml_result_obj.model_weights, } if dataset: prediction_req["dataset"] = dataset if max_records: prediction_req["max_records"] = max_records if predict_rows: prediction_req["predict_rows"] = json.dumps(predict_rows) if features_to_process: prediction_req["features_to_process"] = features_to_process if ignore_features: prediction_req["ignore_features"] = ignore_features if apply_scaler: prediction_req["apply_scaler"] = apply_scaler if sort_values: prediction_req["sort_values"] = sort_values if loss: prediction_req["loss"] = loss if metrics: prediction_req["metrics"] = metrics if optimizer: prediction_req["optimizer"] = optimizer if histories: prediction_req["histories"] = histories if predict_feature: prediction_req["predict_feature"] = predict_feature if csv_file: prediction_req["csv_file"] = csv_file if meta_file: prediction_req["meta_file"] = meta_file already_predicted = False # if you just want to use the core without django training: if publish_to_core or settings.ANTINEX_WORKER_ONLY: log.info(("model_name={} only publish={} worker={}").format( use_model_name, publish_to_core, settings.ANTINEX_WORKER_ONLY)) ml_job_obj.status = "launched" ml_job_obj.control_state = "launched" ml_job_obj.save() ml_result_obj.status = "launched" ml_result_obj.control_state = "launched" ml_result_obj.save() else: log.info( ("start make_predictions req={}").format(ppj(prediction_req))) prediction_res = make_predictions(req=prediction_req) if prediction_res["status"] != SUCCESS: last_step = ("Stopping for prediction_status={} " "errors: {}").format(prediction_res["status"], prediction_res["err"]) log.error(last_step) ml_job_obj.status = "error" ml_job_obj.control_state = "error" log.info(("saving job={}").format(ml_job_id)) ml_job_obj.save() data["job"] = ml_job_obj.get_public() error_data = { "status": prediction_res["status"], "err": prediction_res["err"] } data["results"] = error_data res["status"] = ERR res["error"] = last_step res["data"] = data return res already_predicted = True res_data = prediction_res["data"] model = res_data["model"] model_weights = res_data["weights"] scores = res_data["scores"] acc_data = res_data["acc"] error_data = res_data["err"] predictions_json = { "predictions": json.loads( pd.Series(res_data["sample_predictions"]).to_json( orient="records")) } found_predictions = res_data["sample_predictions"] found_accuracy = acc_data.get("accuracy", None) last_step = ("job={} accuracy={}").format(ml_job_id, scores[1] * 100) log.info(last_step) ml_job_obj.status = "finished" ml_job_obj.control_state = "finished" ml_job_obj.save() log.info(("saved job={}").format(ml_job_id)) data["job"] = ml_job_obj.get_public() acc_data = {"accuracy": scores[1] * 100} error_data = None log.info(("converting job={} model to json").format(ml_job_id)) model_json = json.loads(model.to_json()) log.info(("saving job={} weights_file={}").format( ml_job_id, ml_result_obj.model_weights_file)) log.info(("building job={} results").format(ml_job_id)) ml_result_obj.status = "finished" ml_result_obj.acc_data = acc_data ml_result_obj.error_data = error_data ml_result_obj.model_json = model_json ml_result_obj.model_weights = model_weights ml_result_obj.acc_image_file = image_file ml_result_obj.predictions_json = predictions_json ml_result_obj.version = version # end of handing off to core worker without a database connection log.info(("saving job={} results").format(ml_job_id)) # OpenShift 9.6 Postgres container killed the worker # here. Interested to see if this is a jsonb/jsonfield problem # 2018-05-20 try: ml_result_obj.save() except Exception as e: res["error"] = ("Failed saving model job.id={} with ex={}").format( ml_job_id, e) res["status"] = ERR res["data"] = data log.error(res["error"]) return res # end try/ex log.info(("done saving job={} results").format(ml_job_id)) data["job"] = ml_job_obj.get_public() data["results"] = ml_result_obj.get_public() res["status"] = SUCCESS res["error"] = "" res["data"] = data if settings.ANTINEX_WORKER_ENABLED and not already_predicted: if use_model_name: prediction_req["label"] = use_model_name log.info(("publishing to core use_model_name={} " "worker={} already_predicted={}").format( use_model_name, settings.ANTINEX_WORKER_ENABLED, already_predicted)) publish_req = {"body": prediction_req} if settings.CELERY_ENABLED: task_publish_to_core.delay(publish_node=publish_req) else: task_publish_to_core(publish_node=publish_req) else: log.info(("skip - worker={} already_predicted={}").format( settings.ANTINEX_WORKER_ENABLED, already_predicted)) # send to core except Exception as e: res["status"] = ERR res["err"] = ("Failed task={} with " "ex={}").format(req_node["task_name"], e) if ml_job_obj: data["job"] = ml_job_obj.get_public() else: data["job"] = None if ml_result_obj: data["results"] = ml_result_obj.get_public() else: data["results"] = None log.error(res["err"]) # end of try/ex log.info( ("task - {} - done - " "ml_job.id={} ml_result.id={} " "accuracy={} predictions={}").format(req_node["task_name"], ml_job_id, ml_result_id, found_accuracy, len(found_predictions))) return res