def parse(self, key, dest_key=None, timeoutSecs=300, retryDelaySecs=0.2, initialDelaySecs=None, pollTimeoutSecs=180, noise=None, benchmarkLogging=None, noPoll=False, **kwargs): # # Call ParseSetup?source_frames=[keys] . . . # if benchmarkLogging: cloudPerfH2O.get_log_save(initOnly=True) # TODO: multiple keys parse_setup_params = { 'source_frames': '["' + key + '"]' # NOTE: quote key names } # h2o_test_utils.check_params_update_kwargs(params_dict, kwargs, 'parse_setup', print_params=H2O.verbose) setup_result = self.__do_json_request(jsonRequest="/3/ParseSetup", cmd='post', timeout=timeoutSecs, postData=parse_setup_params) H2O.verboseprint("ParseSetup result:", h2o_test_utils.dump_json(setup_result)) # # and then Parse?source_frames=<keys list> and params from the ParseSetup result # Parse?source_frames=[nfs://Users/rpeck/Source/h2o2/smalldata/logreg/prostate.csv]&destination_frame=prostate.hex&parse_type=CSV&separator=44&number_columns=9&check_header=0&single_quotes=false&column_names=['ID',CAPSULE','AGE','RACE','DPROS','DCAPS','PSA','VOL','GLEASON] # parse_params = { 'source_frames': '["' + setup_result['source_frames'][0]['name'] + '"]', # TODO: cons up the whole list 'destination_frame': dest_key if dest_key else setup_result['destination_frame'], 'parse_type': setup_result['parse_type'], 'separator': setup_result['separator'], 'single_quotes': setup_result['single_quotes'], 'check_header': setup_result['check_header'], 'number_columns': setup_result['number_columns'], 'column_names': setup_result['column_names'], # gets stringified inside __do_json_request() 'column_types': setup_result['column_types'], # gets stringified inside __do_json_request() 'na_strings': setup_result['na_strings'], 'chunk_size': setup_result['chunk_size'], } H2O.verboseprint("parse_params: " + repr(parse_params)) h2o_test_utils.check_params_update_kwargs(parse_params, kwargs, 'parse', print_params=H2O.verbose) parse_result = self.__do_json_request(jsonRequest="/3/Parse", cmd='post', timeout=timeoutSecs, postData=parse_params, **kwargs) H2O.verboseprint("Parse result:", h2o_test_utils.dump_json(parse_result)) # print("Parse result:", repr(parse_result)) job_key = parse_result['job']['key']['name'] # TODO: dislike having different shapes for noPoll and poll if noPoll: return this.jobs(job_key) job_json = self.poll_job(job_key, timeoutSecs=timeoutSecs) if job_json: dest_key = job_json['jobs'][0]['dest']['name'] return self.frames(dest_key) return None
def interaction(self, timeoutSecs=180, **kwargs): a = self.__do_json_request('/3/Interaction', cmd="post", timeout=timeoutSecs, postData=kwargs ) H2O.verboseprint("\ninteraction result:", h2o_test_utils.dump_json(a)) return a
def import_files(self, path, timeoutSecs=180): a = self.__do_json_request('/3/ImportFiles', timeout=timeoutSecs, params={"path": path} ) H2O.verboseprint("\nimport_files result:", h2o_test_utils.dump_json(a)) return a
def create_frame(self, timeoutSecs=180, **kwargs): a = self.__do_json_request('3/CreateFrame', cmd="post", timeout=timeoutSecs, params=kwargs ) H2O.verboseprint("\ncreate_frame result:", h2o_test_utils.dump_json(a)) return a
def split_frame(self, timeoutSecs=180, **kwargs): a = self.__do_json_request('/3/SplitFrame', cmd="post", timeout=timeoutSecs, postData=kwargs ) job_json = self.poll_job(a["key"]["name"], timeoutSecs=timeoutSecs) H2O.verboseprint("\nsplit_frame result:", h2o_test_utils.dump_json(a)) return a
def split_frame(self, raiseIfNon200=True, timeoutSecs=180, **kwargs): a = self.__do_json_request('/'+str(self.rest_version)+'/SplitFrame', cmd="post", timeout=timeoutSecs,raiseIfNon200=raiseIfNon200, postData=kwargs ) if raiseIfNon200: job_json = self.poll_job(a["key"]["name"], timeoutSecs=timeoutSecs) H2O.verboseprint("\nsplit_frame result:", h2o_test_utils.dump_json(a)) return a
if raiseIfNon200 and not r: raise Exception("Maybe bad url? no r in __do_json_request in %s:" % inspect.stack()[1][3]) # this is used to open a browser on results, or to redo the operation in the browser # we don't' have that may urls flying around, so let's keep them all H2O.json_url_history.append(r.url) # if r.json(): # raise Exception("Maybe bad url? no r.json in __do_json_request in %s:" % inspect.stack()[1][3]) rjson = None if returnFast: return try: rjson = r.json() except: print h2o_test_utils.dump_json(r.text) if not isinstance(r, (list, dict)): raise Exception("h2o json responses should always be lists or dicts, see previous for text") raise Exception("Could not decode any json from the request.") # TODO # TODO # TODO # TODO: we should really only look in the response object. This check # prevents us from having a field called "error" (e.g., for a scoring result). for e in ['error', 'Error', 'errors', 'Errors']: # error can be null (python None). This happens in exec2 if e in rjson and rjson[e]: H2O.verboseprint("rjson:" + h2o_test_utils.dump_json(rjson)) emsg = 'rjson %s in %s: %s' % (e, inspect.stack()[1][3], rjson[e])
def test(a_node, pp): if h2o_test_utils.isVerbose(): print "Testing ModelMetrics. . ." ################################### # Compute and check ModelMetrics for 'deeplearning_prostate_binomial' mm = a_node.compute_model_metrics(model="deeplearning_prostate_binomial", frame="prostate_binomial") assert mm is not None, ( "FAIL: Got a null result for scoring: " + "deeplearning_prostate_binomial" + " on: " + "prostate_binomial" ) assert "model_category" in mm, ( "FAIL: ModelMetrics for scoring: " + "deeplearning_prostate_binomial" + " on: " + "prostate_binomial" + " does not contain a model_category." ) assert "Binomial" == mm["model_category"], ( "FAIL: ModelMetrics for scoring: " + "deeplearning_prostate_binomial" + " on: " + "prostate_binomial" + " model_category is not Binomial, it is: " + str(mm["model_category"]) ) assert "AUC" in mm, ( "FAIL: ModelMetrics for scoring: " + "deeplearning_prostate_binomial" + " on: " + "prostate_binomial" + " does not contain an AUC element: " + h2o_test_utils.dump_json(mm) ) assert type(mm["AUC"]) is float, ( "FAIL: ModelMetrics for scoring: " + "deeplearning_prostate_binomial" + " on: " + "prostate_binomial" + " AUC element is not a float: " + h2o_test_utils.dump_json(mm) ) assert "thresholds_and_metric_scores" in mm, ( "FAIL: ModelMetrics for scoring: " + "deeplearning_prostate_binomial" + " on: " + "prostate_binomial" + " does not contain a thresholds_and_metric_scores element: " + h2o_test_utils.dump_json(mm) ) assert type(mm["thresholds_and_metric_scores"]) is dict, ( "FAIL: ModelMetrics for scoring: " + "deeplearning_prostate_binomial" + " on: " + "prostate_binomial" + " confusion_matrices element is not a dict: " + h2o_test_utils.dump_json(mm) ) # print h2o_test_utils.dump_json(mm) h2o.H2O.verboseprint( "ModelMetrics for scoring: ", "deeplearning_prostate_binomial", " on: ", "prostate_binomial", ": ", repr(mm) ) ################################### # Check for ModelMetrics for 'deeplearning_prostate_binomial' in full list mms = a_node.model_metrics() # fetch all assert "model_metrics" in mms, "FAIL: Failed to find model_metrics in result of /3/ModelMetrics." found_mm = False found_a_frame = False for mm in mms["model_metrics"]: assert "model" in mm, "FAIL: mm does not contain a model element: " + repr(mm) assert "name" in mm["model"], "FAIL: mm[model] isn't a key with a name: " + repr(mm) assert "type" in mm["model"], "FAIL: mm[model] does not contain a type: " + repr(mm) assert "Key<Model>" == mm["model"]["type"], "FAIL: mm[model] type is not Key<Model>: " + repr( mm["model"]["type"] ) assert "frame" in mm, "FAIL: mm does not contain a frame element: " + repr(mm) if mm["frame"] is not None: # temp metrics can have null frames assert "name" in mm["frame"], "FAIL: mm[frame] does not contain a name: " + repr(mm) assert "type" in mm["frame"], "FAIL: mm[frame] does not contain a type: " + repr(mm) assert "Key<Frame>" == mm["frame"]["type"], "FAIL: mm[frame] type is not Key<Frame>: " + repr(mm) model_key = mm["model"]["name"] frame_key = mm["frame"]["name"] # TODO: should match if model_key == "deeplearning_prostate_binomial" and frame_key == "prostate_binomial": found_mm = True else: found_a_frame = True assert found_mm, ( "FAIL: Failed to find ModelMetrics object for model: " + "deeplearning_prostate_binomial" + " and frame: " + "prostate_binomial" ) assert found_a_frame, "FAIL: Failed to find a frame in any ModelMetrics object." # test delete_model_metrics mms = a_node.model_metrics("deeplearning_prostate_binomial", "prostate_binomial") assert len(mms["model_metrics"]) == 1, "FAIL: expected 1 ModelMetrics, found: " + str(len(mms["model_metrics"])) a_node.delete_model_metrics("deeplearning_prostate_binomial", "prostate_binomial") mms = a_node.model_metrics("deeplearning_prostate_binomial", "prostate_binomial") assert len(mms["model_metrics"]) == 0, "FAIL: expected 0 ModelMetrics, found: " + str(len(mms["model_metrics"])) ################################### # Predict and check ModelMetrics for 'deeplearning_prostate_binomial' p = a_node.predict( model="deeplearning_prostate_binomial", frame="prostate_binomial", predictions_frame="deeplearning_prostate_binomial_predictions", ) h2o_test_utils.validate_predictions( a_node, p, "deeplearning_prostate_binomial", "prostate_binomial", 380, predictions_frame="deeplearning_prostate_binomial_predictions", ) h2o_test_utils.validate_frame_exists(a_node, "deeplearning_prostate_binomial_predictions") h2o.H2O.verboseprint( "Predictions for scoring: ", "deeplearning_prostate_binomial", " on: ", "prostate_binomial", ": ", repr(p) ) ################################### # Predict and check ModelMetrics for 'gbm_prostate_binomial' p = a_node.predict(model="gbm_prostate_binomial", frame="prostate_binomial") h2o_test_utils.validate_predictions(a_node, p, "gbm_prostate_binomial", "prostate_binomial", 380) h2o.H2O.verboseprint( "Predictions for scoring: ", "gbm_prostate_binomial", " on: ", "prostate_binomial", ": ", repr(p) ) ################################### # Predict and check ModelMetrics for 'deeplearning_prostate_regression' p = a_node.predict(model="deeplearning_prostate_regression", frame="prostate_regression") h2o_test_utils.validate_predictions(a_node, p, "deeplearning_prostate_regression", "prostate_regression", 380) h2o.H2O.verboseprint( "Predictions for scoring: ", "deeplearning_prostate_regression", " on: ", "prostate_regression", ": ", repr(p) ) ################################### # Predict and check ModelMetrics for 'gbm_prostate_regression' p = a_node.predict(model="gbm_prostate_regression", frame="prostate_regression") h2o_test_utils.validate_predictions(a_node, p, "gbm_prostate_regression", "prostate_regression", 380) h2o.H2O.verboseprint( "Predictions for scoring: ", "gbm_prostate_regression", " on: ", "prostate_regression", ": ", repr(p) ) ################################### # Predict and check ModelMetrics (empty now except for predictions frame) for 'kmeans_prostate' p = a_node.predict(model="kmeans_prostate", frame="prostate_regression") h2o_test_utils.validate_predictions(a_node, p, "kmeans_prostate", "prostate_regression", 380) h2o.H2O.verboseprint("Predictions for scoring: ", "kmeans_prostate", " on: ", "prostate_regression", ": ", repr(p))
def test(a_node, pp): ################################### # Compute and check ModelMetrics for 'deeplearning_prostate_binomial' mm = a_node.compute_model_metrics(model='deeplearning_prostate_binomial', frame='prostate_binomial') assert mm is not None, "FAIL: Got a null result for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' assert 'model_category' in mm, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " does not contain a model_category." assert 'Binomial' == mm[ 'model_category'], "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " model_category is not Binomial, it is: " + str( mm['model_category']) assert 'AUC' in mm, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " does not contain an AUC element: " + h2o_test_utils.dump_json( mm) assert type( mm['AUC'] ) is float, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " AUC element is not a float: " + h2o_test_utils.dump_json( mm) assert 'confusion_matrices' in mm, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " does not contain a confusion_matrices element: " + h2o_test_utils.dump_json( mm) assert type( mm['confusion_matrices'] ) is list, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " confusion_matrices element is not a list: " + h2o_test_utils.dump_json( mm) # print h2o_test_utils.dump_json(mm) h2o.H2O.verboseprint("ModelMetrics for scoring: ", 'deeplearning_prostate_binomial', " on: ", 'prostate_binomial', ": ", repr(mm)) ################################### # Check for ModelMetrics for 'deeplearning_prostate_binomial' in full list mms = a_node.model_metrics() # fetch all assert 'model_metrics' in mms, 'FAIL: Failed to find model_metrics in result of /3/ModelMetrics.' found_mm = False for mm in mms['model_metrics']: assert 'model' in mm, "FAIL: mm does not contain a model element: " + repr( mm) assert 'name' in mm[ 'model'], "FAIL: mm[model] isn't a key with a name: " + repr(mm) assert 'type' in mm[ 'model'], "FAIL: mm[model] does not contain a type: " + repr(mm) assert 'Key<Model>' == mm['model'][ 'type'], "FAIL: mm[model] type is not Key<Model>: " + repr( mm['model']['type']) assert 'frame' in mm, "FAIL: mm does not contain a frame element: " + repr( mm) assert 'name' in mm[ 'frame'], "FAIL: mm[frame] does not contain a name: " + repr(mm) assert 'type' in mm[ 'frame'], "FAIL: mm[frame] does not contain a type: " + repr(mm) assert 'Key<Frame>' == mm['frame'][ 'type'], "FAIL: mm[frame] type is not Key<Frame>: " + repr(mm) model_key = mm['model']['name'] frame_key = mm['frame']['name'] # TODO: should match if model_key == 'deeplearning_prostate_binomial' and frame_key == 'prostate_binomial': found_mm = True assert found_mm, "FAIL: Failed to find ModelMetrics object for model: " + 'deeplearning_prostate_binomial' + " and frame: " + 'prostate_binomial' # test delete_model_metrics mms = a_node.model_metrics('deeplearning_prostate_binomial', 'prostate_binomial') assert len(mms['model_metrics'] ) == 1, "FAIL: expected 1 ModelMetrics, found: " + str( len(mms['model_metrics'])) a_node.delete_model_metrics('deeplearning_prostate_binomial', 'prostate_binomial') mms = a_node.model_metrics('deeplearning_prostate_binomial', 'prostate_binomial') assert len(mms['model_metrics'] ) == 0, "FAIL: expected 0 ModelMetrics, found: " + str( len(mms['model_metrics'])) ################################### # Predict and check ModelMetrics for 'deeplearning_prostate_binomial' p = a_node.predict( model='deeplearning_prostate_binomial', frame='prostate_binomial', predictions_frame='deeplearning_prostate_binomial_predictions') h2o_test_utils.validate_predictions( a_node, p, 'deeplearning_prostate_binomial', 'prostate_binomial', 380, predictions_frame='deeplearning_prostate_binomial_predictions') h2o_test_utils.validate_frame_exists( a_node, 'deeplearning_prostate_binomial_predictions') h2o.H2O.verboseprint("Predictions for scoring: ", 'deeplearning_prostate_binomial', " on: ", 'prostate_binomial', ": ", repr(p)) ################################### # Predict and check ModelMetrics for 'deeplearning_prostate_regression' p = a_node.predict(model='deeplearning_prostate_regression', frame='prostate_binomial') h2o_test_utils.validate_predictions(a_node, p, 'deeplearning_prostate_regression', 'prostate_binomial', 380) h2o.H2O.verboseprint("Predictions for scoring: ", 'deeplearning_prostate_regression', " on: ", 'prostate_binomial', ": ", repr(p)) ################################### # Predict and check ModelMetrics for 'gbm_prostate_binomial' p = a_node.predict(model='gbm_prostate_binomial', frame='prostate_binomial') h2o_test_utils.validate_predictions(a_node, p, 'gbm_prostate_binomial', 'prostate_binomial', 380) h2o.H2O.verboseprint("Predictions for scoring: ", 'gbm_prostate_binomial', " on: ", 'prostate_binomial', ": ", repr(p)) ################################### # Predict and check ModelMetrics for 'gbm_prostate_regression' p = a_node.predict(model='gbm_prostate_regression', frame='prostate_binomial') h2o_test_utils.validate_predictions(a_node, p, 'gbm_prostate_regression', 'prostate_binomial', 380) h2o.H2O.verboseprint("Predictions for scoring: ", 'gbm_prostate_regression', " on: ", 'prostate_binomial', ": ", repr(p)) ################################### # Predict and check ModelMetrics (empty now except for predictions frame) for 'kmeans_prostate' p = a_node.predict(model='kmeans_prostate', frame='prostate_binomial') h2o_test_utils.validate_predictions(a_node, p, 'kmeans_prostate', 'prostate_binomial', 380) h2o.H2O.verboseprint("Predictions for scoring: ", 'kmeans_prostate', " on: ", 'prostate_binomial', ": ", repr(p))
def export(self, key, raiseIfNon200=None,timeoutSecs=10, **kwargs): result = self.__do_json_request('/'+str(self.rest_version)+'/Frames/' + key + '/export', cmd="post", timeout=timeoutSecs, raiseIfNon200=raiseIfNon200, params=kwargs) H2O.verboseprint("\ncreate_frame result:", h2o_test_utils.dump_json(result)) return result;
def __do_json_request(self, jsonRequest=None, fullUrl=None, timeout=10, params=None, postData=None, returnFast=False, cmd='get', extraComment=None, ignoreH2oError=False, noExtraErrorCheck=False, raiseIfNon200=True, suppressErrorMsg=False, **kwargs): H2O.verboseprint("__do_json_request, timeout: " + str(timeout)) # if url param is used, use it as full url. otherwise crate from the jsonRequest if fullUrl: url = fullUrl else: url = self.__url(jsonRequest) # remove any params that are 'None' # need to copy dictionary, since can't delete while iterating if params is not None: params_serialized = params.copy() for k in params_serialized: if params_serialized[k] is None: del params[k] paramsStr = '?' + '&'.join(['%s=%s' % (k, v) for (k, v) in params.items()]) else: paramsStr = '' # The requests package takes array parameters and explodes them: ['f00', 'b4r'] becomes "f00,b4r". # NOTE: this handles 1D arrays only; if we need ND this needs to be recursive. # NOTE: we currently don't need to do this for GET, so that's not implemented. if postData is not None: munged_postData = {} for k, v in postData.iteritems(): if type(v) is list: if len(v) == 0: munged_postData[k] = '[]' else: first = True array_str = '[' for val in v: if not first: array_str += ', ' if val is None: array_str += 'null' elif isinstance(val, basestring): array_str += "\"" + str(val) + "\"" else: array_str += str(val) first = False array_str += ']' munged_postData[k] = array_str elif type(v) is dict: if len(v) == 0: munged_postData[k] = '{}' else: first = True map_str = '{' for key, val in v.iteritems(): if not first: map_str += ', ' if val is None: map_str += "\"" + key + "\"" + ': null' elif isinstance(val, basestring): map_str += "\"" + str(key) + "\"" + ":" + "\"" + str(val) + "\"" else: map_str += "\"" + key + "\"" + ':' + str(val) first = False map_str += '}' munged_postData[k] = map_str else: # not list: munged_postData[k] = v else: # None munged_postData = postData # print("munged_postData: " + repr(munged_postData)) if extraComment: log('Start ' + url + paramsStr, comment=extraComment) else: log('Start ' + url + paramsStr) log_rest("") log_rest("----------------------------------------------------------------------\n") if extraComment: log_rest("# Extra comment info about this request: " + extraComment) if cmd == 'get': log_rest("GET") else: log_rest("POST") log_rest(url + paramsStr) # file get passed thru kwargs here try: if 'post' == cmd: # NOTE == cmd: for now, since we don't have deserialization from JSON in h2o-dev, we use form-encoded POST. # This is temporary. # # This following does application/json (aka, posting JSON in the body): # r = requests.post(url, timeout=timeout, params=params, data=json.dumps(munged_postData), **kwargs) # # This does form-encoded, which doesn't allow POST of nested structures r = requests.post(url, timeout=timeout, params=params, data=munged_postData, **kwargs) elif 'delete' == cmd: r = requests.delete(url, timeout=timeout, params=params, **kwargs) elif 'get' == cmd: r = requests.get(url, timeout=timeout, params=params, **kwargs) else: raise ValueError("Unknown HTTP command (expected 'get', 'post' or 'delete'): " + cmd) except Exception as e: # rethrow the exception after we've checked for stack trace from h2o # out of memory errors maybe don't show up right away? so we should wait for h2o # to get it out to h2o stdout. We don't want to rely on cloud teardown to check # because there's no delay, and we don't want to delay all cloud teardowns by waiting. # (this is new/experimental) exc_info = sys.exc_info() # use this to ignore the initial connection errors during build cloud when h2o is coming up if not noExtraErrorCheck: h2p.red_print( "ERROR: got exception on %s to h2o. \nGoing to check sandbox, then rethrow.." % (url + paramsStr)) time.sleep(2) H2O.check_sandbox_for_errors(python_test_name=H2O.python_test_name); log_rest("") log_rest("EXCEPTION CAUGHT DOING REQUEST: " + str(e.message)) raise (exc_info[1], None, exc_info[2]) H2O.verboseprint("r: " + repr(r)) if 200 != r.status_code: pp = pprint.PrettyPrinter(indent=4) msg = "JSON call returned non-200 status: " + url json = r.json() if None != json and 'dev_msg' in json: msg += "\ndev_msg: " msg += str(json['dev_msg']) msg += "\nr.status_code: " + str(r.status_code) msg += "\nr.headers: " + repr(r.headers) if None == json: msg += '\nERROR: the error output from H2O is not JSON!' msg += "\nr.text: " + r.text else: msg += "\nr.json: " msg += pp.pformat(json) if raiseIfNon200: pass # we'll pass msg up with the exception elif not suppressErrorMsg: print(msg) log_rest(msg) log_rest("") try: if r is None: log_rest("r is None") else: log_rest("HTTP status code: " + str(r.status_code)) # The following accesses to r.text were taking most of the runtime: log_text = False if log_text: if hasattr(r, 'text'): if r.text is None: log_rest("r.text is None") else: log_rest(r.text) else: log_rest("r does not have attr text") except Exception as e: # Paranoid exception catch. # Ignore logging exceptions in the case that the above error checking isn't sufficient. print("Caught exception from result logging: ", e, "; result: ", repr(r)) # fatal if no response if raiseIfNon200 and not r: raise Exception("Maybe bad url? no r in __do_json_request in %s:" % inspect.stack()[1][3] + "\n\n" + msg) # this is used to open a browser on results, or to redo the operation in the browser # we don't' have that may urls flying around, so let's keep them all H2O.json_url_history.append(r.url) # if r.json(): # raise Exception("Maybe bad url? no r.json in __do_json_request in %s:" % inspect.stack()[1][3]) rjson = None if returnFast: return try: rjson = r.json() except: print(h2o_test_utils.dump_json(r.text)) if not isinstance(r, (list, dict)): raise Exception("h2o json responses should always be lists or dicts, see previous for text") raise Exception("Could not decode any json from the request.") # TODO # TODO # TODO # TODO: we should really only look in the response object. This check # prevents us from having a field called "error" (e.g., for a scoring result). for e in ['error', 'Error', 'errors', 'Errors']: # error can be null (python None). This happens in exec2 if e in rjson and rjson[e]: H2O.verboseprint("rjson:" + h2o_test_utils.dump_json(rjson)) emsg = 'rjson %s in %s: %s' % (e, inspect.stack()[1][3], rjson[e]) if ignoreH2oError: # well, we print it..so not totally ignore. test can look at rjson returned print(emsg) else: print(emsg) raise Exception(emsg) for w in ['warning', 'Warning', 'warnings', 'Warnings']: # warning can be null (python None). if w in rjson and rjson[w]: H2O.verboseprint(dump_json(rjson)) print('rjson %s in %s: %s' % (w, inspect.stack()[1][3], rjson[w])) # Allow the caller to check things like __http_request.status_code. # The response object is not JSON-serializable, so we capture the fields we want here: response = {} # response['headers'] = r.headers response['url'] = r.url response['status_code'] = r.status_code response['text'] = r.text rjson['__http_response'] = response return rjson
def test(a_node, pp): ################################### # Compute and check ModelMetrics for 'deeplearning_prostate_binomial' mm = a_node.compute_model_metrics(model='deeplearning_prostate_binomial', frame='prostate_binomial') assert mm is not None, "FAIL: Got a null result for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' assert 'model_category' in mm, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " does not contain a model_category." assert 'Binomial' == mm['model_category'], "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " model_category is not Binomial, it is: " + str(mm['model_category']) assert 'AUC' in mm, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " does not contain an AUC element: " + h2o_test_utils.dump_json(mm) assert type(mm['AUC']) is float, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " AUC element is not a float: " + h2o_test_utils.dump_json(mm) assert 'confusion_matrices' in mm, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " does not contain a confusion_matrices element: " + h2o_test_utils.dump_json(mm) assert type(mm['confusion_matrices']) is list, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " confusion_matrices element is not a list: " + h2o_test_utils.dump_json(mm) # print h2o_test_utils.dump_json(mm) h2o.H2O.verboseprint("ModelMetrics for scoring: ", 'deeplearning_prostate_binomial', " on: ", 'prostate_binomial', ": ", repr(mm)) ################################### # Check for ModelMetrics for 'deeplearning_prostate_binomial' in full list mms = a_node.model_metrics() # fetch all assert 'model_metrics' in mms, 'FAIL: Failed to find model_metrics in result of /3/ModelMetrics.' found_mm = False for mm in mms['model_metrics']: assert 'model' in mm, "FAIL: mm does not contain a model element: " + repr(mm) assert 'name' in mm['model'], "FAIL: mm[model] isn't a key with a name: " + repr(mm) assert 'type' in mm['model'], "FAIL: mm[model] does not contain a type: " + repr(mm) assert 'Key<Model>' == mm['model']['type'], "FAIL: mm[model] type is not Key<Model>: " + repr(mm['model']['type']) assert 'frame' in mm, "FAIL: mm does not contain a frame element: " + repr(mm) assert 'name' in mm['frame'], "FAIL: mm[frame] does not contain a name: " + repr(mm) assert 'type' in mm['frame'], "FAIL: mm[frame] does not contain a type: " + repr(mm) assert 'Key<Frame>' == mm['frame']['type'], "FAIL: mm[frame] type is not Key<Frame>: " + repr(mm) model_key = mm['model']['name'] frame_key = mm['frame']['name'] # TODO: should match if model_key == 'deeplearning_prostate_binomial' and frame_key == 'prostate_binomial': found_mm = True assert found_mm, "FAIL: Failed to find ModelMetrics object for model: " + 'deeplearning_prostate_binomial' + " and frame: " + 'prostate_binomial' # test delete_model_metrics mms = a_node.model_metrics('deeplearning_prostate_binomial', 'prostate_binomial') assert len(mms['model_metrics']) == 1, "FAIL: expected 1 ModelMetrics, found: " + str(len(mms['model_metrics'])) a_node.delete_model_metrics('deeplearning_prostate_binomial', 'prostate_binomial') mms = a_node.model_metrics('deeplearning_prostate_binomial', 'prostate_binomial') assert len(mms['model_metrics']) == 0, "FAIL: expected 0 ModelMetrics, found: " + str(len(mms['model_metrics'])) ################################### # Predict and check ModelMetrics for 'deeplearning_prostate_binomial' p = a_node.predict(model='deeplearning_prostate_binomial', frame='prostate_binomial', predictions_frame='deeplearning_prostate_binomial_predictions') h2o_test_utils.validate_predictions(a_node, p, 'deeplearning_prostate_binomial', 'prostate_binomial', 380, predictions_frame='deeplearning_prostate_binomial_predictions') h2o_test_utils.validate_frame_exists(a_node, 'deeplearning_prostate_binomial_predictions') h2o.H2O.verboseprint("Predictions for scoring: ", 'deeplearning_prostate_binomial', " on: ", 'prostate_binomial', ": ", repr(p)) ################################### # Predict and check ModelMetrics for 'deeplearning_prostate_regression' p = a_node.predict(model='deeplearning_prostate_regression', frame='prostate_binomial') h2o_test_utils.validate_predictions(a_node, p, 'deeplearning_prostate_regression', 'prostate_binomial', 380) h2o.H2O.verboseprint("Predictions for scoring: ", 'deeplearning_prostate_regression', " on: ", 'prostate_binomial', ": ", repr(p)) ################################### # Predict and check ModelMetrics for 'gbm_prostate_binomial' p = a_node.predict(model='gbm_prostate_binomial', frame='prostate_binomial') h2o_test_utils.validate_predictions(a_node, p, 'gbm_prostate_binomial', 'prostate_binomial', 380) h2o.H2O.verboseprint("Predictions for scoring: ", 'gbm_prostate_binomial', " on: ", 'prostate_binomial', ": ", repr(p)) ################################### # Predict and check ModelMetrics for 'gbm_prostate_regression' p = a_node.predict(model='gbm_prostate_regression', frame='prostate_binomial') h2o_test_utils.validate_predictions(a_node, p, 'gbm_prostate_regression', 'prostate_binomial', 380) h2o.H2O.verboseprint("Predictions for scoring: ", 'gbm_prostate_regression', " on: ", 'prostate_binomial', ": ", repr(p)) ################################### # Predict and check ModelMetrics (empty now except for predictions frame) for 'kmeans_prostate' p = a_node.predict(model='kmeans_prostate', frame='prostate_binomial') h2o_test_utils.validate_predictions(a_node, p, 'kmeans_prostate', 'prostate_binomial', 380) h2o.H2O.verboseprint("Predictions for scoring: ", 'kmeans_prostate', " on: ", 'prostate_binomial', ": ", repr(p))
def test(a_node, pp): if h2o_test_utils.isVerbose(): print('Testing ModelMetrics. . .') ################################### # Compute and check ModelMetrics for 'deeplearning_prostate_binomial' mm = a_node.compute_model_metrics(model='deeplearning_prostate_binomial', frame='prostate_binomial') assert mm is not None, "FAIL: Got a null result for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' assert 'model_category' in mm, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " does not contain a model_category." assert 'Binomial' == mm['model_category'], "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " model_category is not Binomial, it is: " + str(mm['model_category']) assert 'AUC' in mm, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " does not contain an AUC element: " + h2o_test_utils.dump_json(mm) assert type(mm['AUC']) is float, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " AUC element is not a float: " + h2o_test_utils.dump_json(mm) assert 'thresholds_and_metric_scores' in mm, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " does not contain a thresholds_and_metric_scores element: " + h2o_test_utils.dump_json(mm) assert type(mm['thresholds_and_metric_scores']) is dict, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " confusion_matrices element is not a dict: " + h2o_test_utils.dump_json(mm) # print(h2o_test_utils.dump_json(mm)) h2o.H2O.verboseprint("ModelMetrics for scoring: ", 'deeplearning_prostate_binomial', " on: ", 'prostate_binomial', ": ", repr(mm)) ################################### # Check for ModelMetrics for 'deeplearning_prostate_binomial' in full list mms = a_node.model_metrics() # fetch all assert 'model_metrics' in mms, 'FAIL: Failed to find model_metrics in result of /3/ModelMetrics.' found_mm = False found_a_frame = False for mm in mms['model_metrics']: assert 'model' in mm, "FAIL: mm does not contain a model element: " + repr(mm) assert 'name' in mm['model'], "FAIL: mm[model] isn't a key with a name: " + repr(mm) assert 'type' in mm['model'], "FAIL: mm[model] does not contain a type: " + repr(mm) assert 'Key<Model>' == mm['model']['type'], "FAIL: mm[model] type is not Key<Model>: " + repr(mm['model']['type']) assert 'frame' in mm, "FAIL: mm does not contain a frame element: " + repr(mm) if mm['frame'] is not None: # temp metrics can have null frames assert 'name' in mm['frame'], "FAIL: mm[frame] does not contain a name: " + repr(mm) assert 'type' in mm['frame'], "FAIL: mm[frame] does not contain a type: " + repr(mm) assert 'Key<Frame>' == mm['frame']['type'], "FAIL: mm[frame] type is not Key<Frame>: " + repr(mm) model_key = mm['model']['name'] frame_key = mm['frame']['name'] if model_key == 'deeplearning_prostate_binomial' and frame_key == 'prostate_binomial': found_mm = True else: found_a_frame = True assert found_mm, "FAIL: Failed to find ModelMetrics object for model: " + 'deeplearning_prostate_binomial' + " and frame: " + 'prostate_binomial' assert found_a_frame, "FAIL: Failed to find a frame in any ModelMetrics object." # test delete_model_metrics mms = a_node.model_metrics('deeplearning_prostate_binomial', 'prostate_binomial') assert len(mms['model_metrics']) == 1, "FAIL: expected 1 ModelMetrics, found: " + str(len(mms['model_metrics'])) a_node.delete_model_metrics('deeplearning_prostate_binomial', 'prostate_binomial') mms = a_node.model_metrics('deeplearning_prostate_binomial', 'prostate_binomial') assert len(mms['model_metrics']) == 0, "FAIL: expected 0 ModelMetrics, found: " + str(len(mms['model_metrics'])) ################################### # Predict and check ModelMetrics for 'deeplearning_prostate_binomial' p = a_node.predict(model='deeplearning_prostate_binomial', frame='prostate_binomial', predictions_frame='deeplearning_prostate_binomial_predictions') h2o_test_utils.validate_predictions(a_node, p, 'deeplearning_prostate_binomial', 'prostate_binomial', 380, predictions_frame='deeplearning_prostate_binomial_predictions') h2o_test_utils.validate_frame_exists(a_node, 'deeplearning_prostate_binomial_predictions') h2o.H2O.verboseprint("Predictions for scoring: ", 'deeplearning_prostate_binomial', " on: ", 'prostate_binomial', ": ", repr(p)) ################################### # Predict and check ModelMetrics for 'gbm_prostate_binomial' p = a_node.predict(model='gbm_prostate_binomial', frame='prostate_binomial') h2o_test_utils.validate_predictions(a_node, p, 'gbm_prostate_binomial', 'prostate_binomial', 380) h2o.H2O.verboseprint("Predictions for scoring: ", 'gbm_prostate_binomial', " on: ", 'prostate_binomial', ": ", repr(p)) ################################### # Predict and check ModelMetrics for 'deeplearning_prostate_regression' p = a_node.predict(model='deeplearning_prostate_regression', frame='prostate_regression') h2o_test_utils.validate_predictions(a_node, p, 'deeplearning_prostate_regression', 'prostate_regression', 380) h2o.H2O.verboseprint("Predictions for scoring: ", 'deeplearning_prostate_regression', " on: ", 'prostate_regression', ": ", repr(p)) ################################### # Predict and check ModelMetrics for 'gbm_prostate_regression' p = a_node.predict(model='gbm_prostate_regression', frame='prostate_regression') h2o_test_utils.validate_predictions(a_node, p, 'gbm_prostate_regression', 'prostate_regression', 380) h2o.H2O.verboseprint("Predictions for scoring: ", 'gbm_prostate_regression', " on: ", 'prostate_regression', ": ", repr(p)) ################################### # Predict and check ModelMetrics (empty now except for predictions frame) for 'kmeans_prostate' p = a_node.predict(model='kmeans_prostate', frame='prostate_regression') h2o_test_utils.validate_predictions(a_node, p, 'kmeans_prostate', 'prostate_regression', 380) h2o.H2O.verboseprint("Predictions for scoring: ", 'kmeans_prostate', " on: ", 'prostate_regression', ": ", repr(p)) ################################### # Predict with reversed keys (should get an H2OErrorV1): try: p = a_node.predict(frame='kmeans_prostate', model='prostate_regression') assert 1 == 0, "FAIL: expected a ValueError exception looking for Model 'prostate_regression'" except Exception as e: pass