def parse(self, key, dest_key=None,
timeoutSecs=300, retryDelaySecs=0.2, initialDelaySecs=None, pollTimeoutSecs=180,
noise=None, benchmarkLogging=None, noPoll=False, **kwargs):
#
# Call ParseSetup?source_frames=[keys] . . .
#
if benchmarkLogging:
cloudPerfH2O.get_log_save(initOnly=True)
# TODO: multiple keys
parse_setup_params = {
'source_frames': '["' + key + '"]' # NOTE: quote key names
}
# h2o_test_utils.check_params_update_kwargs(params_dict, kwargs, 'parse_setup', print_params=H2O.verbose)
setup_result = self.__do_json_request(jsonRequest="/3/ParseSetup", cmd='post', timeout=timeoutSecs, postData=parse_setup_params)
H2O.verboseprint("ParseSetup result:", h2o_test_utils.dump_json(setup_result))
#
# and then Parse?source_frames=<keys list> and params from the ParseSetup result
# Parse?source_frames=[nfs://Users/rpeck/Source/h2o2/smalldata/logreg/prostate.csv]&destination_frame=prostate.hex&parse_type=CSV&separator=44&number_columns=9&check_header=0&single_quotes=false&column_names=['ID',CAPSULE','AGE','RACE','DPROS','DCAPS','PSA','VOL','GLEASON]
#
parse_params = {
'source_frames': '["' + setup_result['source_frames'][0]['name'] + '"]', # TODO: cons up the whole list
'destination_frame': dest_key if dest_key else setup_result['destination_frame'],
'parse_type': setup_result['parse_type'],
'separator': setup_result['separator'],
'single_quotes': setup_result['single_quotes'],
'check_header': setup_result['check_header'],
'number_columns': setup_result['number_columns'],
'column_names': setup_result['column_names'], # gets stringified inside __do_json_request()
'column_types': setup_result['column_types'], # gets stringified inside __do_json_request()
'na_strings': setup_result['na_strings'],
'chunk_size': setup_result['chunk_size'],
}
H2O.verboseprint("parse_params: " + repr(parse_params))
h2o_test_utils.check_params_update_kwargs(parse_params, kwargs, 'parse', print_params=H2O.verbose)
parse_result = self.__do_json_request(jsonRequest="/3/Parse", cmd='post', timeout=timeoutSecs, postData=parse_params, **kwargs)
H2O.verboseprint("Parse result:", h2o_test_utils.dump_json(parse_result))
# print("Parse result:", repr(parse_result))
job_key = parse_result['job']['key']['name']
# TODO: dislike having different shapes for noPoll and poll
if noPoll:
return this.jobs(job_key)
job_json = self.poll_job(job_key, timeoutSecs=timeoutSecs)
if job_json:
dest_key = job_json['jobs'][0]['dest']['name']
return self.frames(dest_key)
return None
def interaction(self, timeoutSecs=180, **kwargs):
a = self.__do_json_request('/3/Interaction', cmd="post",
timeout=timeoutSecs,
postData=kwargs
)
H2O.verboseprint("\ninteraction result:", h2o_test_utils.dump_json(a))
return a
def import_files(self, path, timeoutSecs=180):
a = self.__do_json_request('/3/ImportFiles',
timeout=timeoutSecs,
params={"path": path}
)
H2O.verboseprint("\nimport_files result:", h2o_test_utils.dump_json(a))
return a
def create_frame(self, timeoutSecs=180, **kwargs):
a = self.__do_json_request('3/CreateFrame', cmd="post",
timeout=timeoutSecs,
params=kwargs
)
H2O.verboseprint("\ncreate_frame result:", h2o_test_utils.dump_json(a))
return a
def split_frame(self, timeoutSecs=180, **kwargs):
a = self.__do_json_request('/3/SplitFrame', cmd="post",
timeout=timeoutSecs,
postData=kwargs
)
job_json = self.poll_job(a["key"]["name"], timeoutSecs=timeoutSecs)
H2O.verboseprint("\nsplit_frame result:", h2o_test_utils.dump_json(a))
return a
def split_frame(self, raiseIfNon200=True, timeoutSecs=180, **kwargs):
a = self.__do_json_request('/'+str(self.rest_version)+'/SplitFrame', cmd="post",
timeout=timeoutSecs,raiseIfNon200=raiseIfNon200,
postData=kwargs
)
if raiseIfNon200:
job_json = self.poll_job(a["key"]["name"], timeoutSecs=timeoutSecs)
H2O.verboseprint("\nsplit_frame result:", h2o_test_utils.dump_json(a))
return a
if raiseIfNon200 and not r:
raise Exception("Maybe bad url? no r in __do_json_request in %s:" % inspect.stack()[1][3])
# this is used to open a browser on results, or to redo the operation in the browser
# we don't' have that may urls flying around, so let's keep them all
H2O.json_url_history.append(r.url)
# if r.json():
# raise Exception("Maybe bad url? no r.json in __do_json_request in %s:" % inspect.stack()[1][3])
rjson = None
if returnFast:
return
try:
rjson = r.json()
except:
print h2o_test_utils.dump_json(r.text)
if not isinstance(r, (list, dict)):
raise Exception("h2o json responses should always be lists or dicts, see previous for text")
raise Exception("Could not decode any json from the request.")
# TODO
# TODO
# TODO
# TODO: we should really only look in the response object. This check
# prevents us from having a field called "error" (e.g., for a scoring result).
for e in ['error', 'Error', 'errors', 'Errors']:
# error can be null (python None). This happens in exec2
if e in rjson and rjson[e]:
H2O.verboseprint("rjson:" + h2o_test_utils.dump_json(rjson))
emsg = 'rjson %s in %s: %s' % (e, inspect.stack()[1][3], rjson[e])
def test(a_node, pp):
if h2o_test_utils.isVerbose():
print "Testing ModelMetrics. . ."
###################################
# Compute and check ModelMetrics for 'deeplearning_prostate_binomial'
mm = a_node.compute_model_metrics(model="deeplearning_prostate_binomial", frame="prostate_binomial")
assert mm is not None, (
"FAIL: Got a null result for scoring: " + "deeplearning_prostate_binomial" + " on: " + "prostate_binomial"
)
assert "model_category" in mm, (
"FAIL: ModelMetrics for scoring: "
+ "deeplearning_prostate_binomial"
+ " on: "
+ "prostate_binomial"
+ " does not contain a model_category."
)
assert "Binomial" == mm["model_category"], (
"FAIL: ModelMetrics for scoring: "
+ "deeplearning_prostate_binomial"
+ " on: "
+ "prostate_binomial"
+ " model_category is not Binomial, it is: "
+ str(mm["model_category"])
)
assert "AUC" in mm, (
"FAIL: ModelMetrics for scoring: "
+ "deeplearning_prostate_binomial"
+ " on: "
+ "prostate_binomial"
+ " does not contain an AUC element: "
+ h2o_test_utils.dump_json(mm)
)
assert type(mm["AUC"]) is float, (
"FAIL: ModelMetrics for scoring: "
+ "deeplearning_prostate_binomial"
+ " on: "
+ "prostate_binomial"
+ " AUC element is not a float: "
+ h2o_test_utils.dump_json(mm)
)
assert "thresholds_and_metric_scores" in mm, (
"FAIL: ModelMetrics for scoring: "
+ "deeplearning_prostate_binomial"
+ " on: "
+ "prostate_binomial"
+ " does not contain a thresholds_and_metric_scores element: "
+ h2o_test_utils.dump_json(mm)
)
assert type(mm["thresholds_and_metric_scores"]) is dict, (
"FAIL: ModelMetrics for scoring: "
+ "deeplearning_prostate_binomial"
+ " on: "
+ "prostate_binomial"
+ " confusion_matrices element is not a dict: "
+ h2o_test_utils.dump_json(mm)
)
# print h2o_test_utils.dump_json(mm)
h2o.H2O.verboseprint(
"ModelMetrics for scoring: ", "deeplearning_prostate_binomial", " on: ", "prostate_binomial", ": ", repr(mm)
)
###################################
# Check for ModelMetrics for 'deeplearning_prostate_binomial' in full list
mms = a_node.model_metrics() # fetch all
assert "model_metrics" in mms, "FAIL: Failed to find model_metrics in result of /3/ModelMetrics."
found_mm = False
found_a_frame = False
for mm in mms["model_metrics"]:
assert "model" in mm, "FAIL: mm does not contain a model element: " + repr(mm)
assert "name" in mm["model"], "FAIL: mm[model] isn't a key with a name: " + repr(mm)
assert "type" in mm["model"], "FAIL: mm[model] does not contain a type: " + repr(mm)
assert "Key<Model>" == mm["model"]["type"], "FAIL: mm[model] type is not Key<Model>: " + repr(
mm["model"]["type"]
)
assert "frame" in mm, "FAIL: mm does not contain a frame element: " + repr(mm)
if mm["frame"] is not None: # temp metrics can have null frames
assert "name" in mm["frame"], "FAIL: mm[frame] does not contain a name: " + repr(mm)
assert "type" in mm["frame"], "FAIL: mm[frame] does not contain a type: " + repr(mm)
assert "Key<Frame>" == mm["frame"]["type"], "FAIL: mm[frame] type is not Key<Frame>: " + repr(mm)
model_key = mm["model"]["name"]
frame_key = mm["frame"]["name"] # TODO: should match
if model_key == "deeplearning_prostate_binomial" and frame_key == "prostate_binomial":
found_mm = True
else:
found_a_frame = True
assert found_mm, (
"FAIL: Failed to find ModelMetrics object for model: "
+ "deeplearning_prostate_binomial"
+ " and frame: "
+ "prostate_binomial"
)
assert found_a_frame, "FAIL: Failed to find a frame in any ModelMetrics object."
# test delete_model_metrics
mms = a_node.model_metrics("deeplearning_prostate_binomial", "prostate_binomial")
assert len(mms["model_metrics"]) == 1, "FAIL: expected 1 ModelMetrics, found: " + str(len(mms["model_metrics"]))
a_node.delete_model_metrics("deeplearning_prostate_binomial", "prostate_binomial")
mms = a_node.model_metrics("deeplearning_prostate_binomial", "prostate_binomial")
assert len(mms["model_metrics"]) == 0, "FAIL: expected 0 ModelMetrics, found: " + str(len(mms["model_metrics"]))
###################################
# Predict and check ModelMetrics for 'deeplearning_prostate_binomial'
p = a_node.predict(
model="deeplearning_prostate_binomial",
frame="prostate_binomial",
predictions_frame="deeplearning_prostate_binomial_predictions",
)
h2o_test_utils.validate_predictions(
a_node,
p,
"deeplearning_prostate_binomial",
"prostate_binomial",
380,
predictions_frame="deeplearning_prostate_binomial_predictions",
)
h2o_test_utils.validate_frame_exists(a_node, "deeplearning_prostate_binomial_predictions")
h2o.H2O.verboseprint(
"Predictions for scoring: ", "deeplearning_prostate_binomial", " on: ", "prostate_binomial", ": ", repr(p)
)
###################################
# Predict and check ModelMetrics for 'gbm_prostate_binomial'
p = a_node.predict(model="gbm_prostate_binomial", frame="prostate_binomial")
h2o_test_utils.validate_predictions(a_node, p, "gbm_prostate_binomial", "prostate_binomial", 380)
h2o.H2O.verboseprint(
"Predictions for scoring: ", "gbm_prostate_binomial", " on: ", "prostate_binomial", ": ", repr(p)
)
###################################
# Predict and check ModelMetrics for 'deeplearning_prostate_regression'
p = a_node.predict(model="deeplearning_prostate_regression", frame="prostate_regression")
h2o_test_utils.validate_predictions(a_node, p, "deeplearning_prostate_regression", "prostate_regression", 380)
h2o.H2O.verboseprint(
"Predictions for scoring: ", "deeplearning_prostate_regression", " on: ", "prostate_regression", ": ", repr(p)
)
###################################
# Predict and check ModelMetrics for 'gbm_prostate_regression'
p = a_node.predict(model="gbm_prostate_regression", frame="prostate_regression")
h2o_test_utils.validate_predictions(a_node, p, "gbm_prostate_regression", "prostate_regression", 380)
h2o.H2O.verboseprint(
"Predictions for scoring: ", "gbm_prostate_regression", " on: ", "prostate_regression", ": ", repr(p)
)
###################################
# Predict and check ModelMetrics (empty now except for predictions frame) for 'kmeans_prostate'
p = a_node.predict(model="kmeans_prostate", frame="prostate_regression")
h2o_test_utils.validate_predictions(a_node, p, "kmeans_prostate", "prostate_regression", 380)
h2o.H2O.verboseprint("Predictions for scoring: ", "kmeans_prostate", " on: ", "prostate_regression", ": ", repr(p))
def test(a_node, pp):
###################################
# Compute and check ModelMetrics for 'deeplearning_prostate_binomial'
mm = a_node.compute_model_metrics(model='deeplearning_prostate_binomial',
frame='prostate_binomial')
assert mm is not None, "FAIL: Got a null result for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial'
assert 'model_category' in mm, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " does not contain a model_category."
assert 'Binomial' == mm[
'model_category'], "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " model_category is not Binomial, it is: " + str(
mm['model_category'])
assert 'AUC' in mm, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " does not contain an AUC element: " + h2o_test_utils.dump_json(
mm)
assert type(
mm['AUC']
) is float, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " AUC element is not a float: " + h2o_test_utils.dump_json(
mm)
assert 'confusion_matrices' in mm, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " does not contain a confusion_matrices element: " + h2o_test_utils.dump_json(
mm)
assert type(
mm['confusion_matrices']
) is list, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " confusion_matrices element is not a list: " + h2o_test_utils.dump_json(
mm)
# print h2o_test_utils.dump_json(mm)
h2o.H2O.verboseprint("ModelMetrics for scoring: ",
'deeplearning_prostate_binomial', " on: ",
'prostate_binomial', ": ", repr(mm))
###################################
# Check for ModelMetrics for 'deeplearning_prostate_binomial' in full list
mms = a_node.model_metrics() # fetch all
assert 'model_metrics' in mms, 'FAIL: Failed to find model_metrics in result of /3/ModelMetrics.'
found_mm = False
for mm in mms['model_metrics']:
assert 'model' in mm, "FAIL: mm does not contain a model element: " + repr(
mm)
assert 'name' in mm[
'model'], "FAIL: mm[model] isn't a key with a name: " + repr(mm)
assert 'type' in mm[
'model'], "FAIL: mm[model] does not contain a type: " + repr(mm)
assert 'Key<Model>' == mm['model'][
'type'], "FAIL: mm[model] type is not Key<Model>: " + repr(
mm['model']['type'])
assert 'frame' in mm, "FAIL: mm does not contain a frame element: " + repr(
mm)
assert 'name' in mm[
'frame'], "FAIL: mm[frame] does not contain a name: " + repr(mm)
assert 'type' in mm[
'frame'], "FAIL: mm[frame] does not contain a type: " + repr(mm)
assert 'Key<Frame>' == mm['frame'][
'type'], "FAIL: mm[frame] type is not Key<Frame>: " + repr(mm)
model_key = mm['model']['name']
frame_key = mm['frame']['name'] # TODO: should match
if model_key == 'deeplearning_prostate_binomial' and frame_key == 'prostate_binomial':
found_mm = True
assert found_mm, "FAIL: Failed to find ModelMetrics object for model: " + 'deeplearning_prostate_binomial' + " and frame: " + 'prostate_binomial'
# test delete_model_metrics
mms = a_node.model_metrics('deeplearning_prostate_binomial',
'prostate_binomial')
assert len(mms['model_metrics']
) == 1, "FAIL: expected 1 ModelMetrics, found: " + str(
len(mms['model_metrics']))
a_node.delete_model_metrics('deeplearning_prostate_binomial',
'prostate_binomial')
mms = a_node.model_metrics('deeplearning_prostate_binomial',
'prostate_binomial')
assert len(mms['model_metrics']
) == 0, "FAIL: expected 0 ModelMetrics, found: " + str(
len(mms['model_metrics']))
###################################
# Predict and check ModelMetrics for 'deeplearning_prostate_binomial'
p = a_node.predict(
model='deeplearning_prostate_binomial',
frame='prostate_binomial',
predictions_frame='deeplearning_prostate_binomial_predictions')
h2o_test_utils.validate_predictions(
a_node,
p,
'deeplearning_prostate_binomial',
'prostate_binomial',
380,
predictions_frame='deeplearning_prostate_binomial_predictions')
h2o_test_utils.validate_frame_exists(
a_node, 'deeplearning_prostate_binomial_predictions')
h2o.H2O.verboseprint("Predictions for scoring: ",
'deeplearning_prostate_binomial', " on: ",
'prostate_binomial', ": ", repr(p))
###################################
# Predict and check ModelMetrics for 'deeplearning_prostate_regression'
p = a_node.predict(model='deeplearning_prostate_regression',
frame='prostate_binomial')
h2o_test_utils.validate_predictions(a_node, p,
'deeplearning_prostate_regression',
'prostate_binomial', 380)
h2o.H2O.verboseprint("Predictions for scoring: ",
'deeplearning_prostate_regression', " on: ",
'prostate_binomial', ": ", repr(p))
###################################
# Predict and check ModelMetrics for 'gbm_prostate_binomial'
p = a_node.predict(model='gbm_prostate_binomial',
frame='prostate_binomial')
h2o_test_utils.validate_predictions(a_node, p, 'gbm_prostate_binomial',
'prostate_binomial', 380)
h2o.H2O.verboseprint("Predictions for scoring: ", 'gbm_prostate_binomial',
" on: ", 'prostate_binomial', ": ", repr(p))
###################################
# Predict and check ModelMetrics for 'gbm_prostate_regression'
p = a_node.predict(model='gbm_prostate_regression',
frame='prostate_binomial')
h2o_test_utils.validate_predictions(a_node, p, 'gbm_prostate_regression',
'prostate_binomial', 380)
h2o.H2O.verboseprint("Predictions for scoring: ",
'gbm_prostate_regression', " on: ",
'prostate_binomial', ": ", repr(p))
###################################
# Predict and check ModelMetrics (empty now except for predictions frame) for 'kmeans_prostate'
p = a_node.predict(model='kmeans_prostate', frame='prostate_binomial')
h2o_test_utils.validate_predictions(a_node, p, 'kmeans_prostate',
'prostate_binomial', 380)
h2o.H2O.verboseprint("Predictions for scoring: ", 'kmeans_prostate',
" on: ", 'prostate_binomial', ": ", repr(p))
def export(self, key, raiseIfNon200=None,timeoutSecs=10, **kwargs):
result = self.__do_json_request('/'+str(self.rest_version)+'/Frames/' + key + '/export', cmd="post",
timeout=timeoutSecs, raiseIfNon200=raiseIfNon200,
params=kwargs)
H2O.verboseprint("\ncreate_frame result:", h2o_test_utils.dump_json(result))
return result;
def __do_json_request(self, jsonRequest=None, fullUrl=None, timeout=10, params=None, postData=None, returnFast=False,
cmd='get', extraComment=None, ignoreH2oError=False, noExtraErrorCheck=False, raiseIfNon200=True, suppressErrorMsg=False, **kwargs):
H2O.verboseprint("__do_json_request, timeout: " + str(timeout))
# if url param is used, use it as full url. otherwise crate from the jsonRequest
if fullUrl:
url = fullUrl
else:
url = self.__url(jsonRequest)
# remove any params that are 'None'
# need to copy dictionary, since can't delete while iterating
if params is not None:
params_serialized = params.copy()
for k in params_serialized:
if params_serialized[k] is None:
del params[k]
paramsStr = '?' + '&'.join(['%s=%s' % (k, v) for (k, v) in params.items()])
else:
paramsStr = ''
# The requests package takes array parameters and explodes them: ['f00', 'b4r'] becomes "f00,b4r".
# NOTE: this handles 1D arrays only; if we need ND this needs to be recursive.
# NOTE: we currently don't need to do this for GET, so that's not implemented.
if postData is not None:
munged_postData = {}
for k, v in postData.iteritems():
if type(v) is list:
if len(v) == 0:
munged_postData[k] = '[]'
else:
first = True
array_str = '['
for val in v:
if not first: array_str += ', '
if val is None:
array_str += 'null'
elif isinstance(val, basestring):
array_str += "\"" + str(val) + "\""
else:
array_str += str(val)
first = False
array_str += ']'
munged_postData[k] = array_str
elif type(v) is dict:
if len(v) == 0:
munged_postData[k] = '{}'
else:
first = True
map_str = '{'
for key, val in v.iteritems():
if not first: map_str += ', '
if val is None:
map_str += "\"" + key + "\"" + ': null'
elif isinstance(val, basestring):
map_str += "\"" + str(key) + "\"" + ":" + "\"" + str(val) + "\""
else:
map_str += "\"" + key + "\"" + ':' + str(val)
first = False
map_str += '}'
munged_postData[k] = map_str
else:
# not list:
munged_postData[k] = v
else:
# None
munged_postData = postData
# print("munged_postData: " + repr(munged_postData))
if extraComment:
log('Start ' + url + paramsStr, comment=extraComment)
else:
log('Start ' + url + paramsStr)
log_rest("")
log_rest("----------------------------------------------------------------------\n")
if extraComment:
log_rest("# Extra comment info about this request: " + extraComment)
if cmd == 'get':
log_rest("GET")
else:
log_rest("POST")
log_rest(url + paramsStr)
# file get passed thru kwargs here
try:
if 'post' == cmd:
# NOTE == cmd: for now, since we don't have deserialization from JSON in h2o-dev, we use form-encoded POST.
# This is temporary.
#
# This following does application/json (aka, posting JSON in the body):
# r = requests.post(url, timeout=timeout, params=params, data=json.dumps(munged_postData), **kwargs)
#
# This does form-encoded, which doesn't allow POST of nested structures
r = requests.post(url, timeout=timeout, params=params, data=munged_postData, **kwargs)
elif 'delete' == cmd:
r = requests.delete(url, timeout=timeout, params=params, **kwargs)
elif 'get' == cmd:
r = requests.get(url, timeout=timeout, params=params, **kwargs)
else:
raise ValueError("Unknown HTTP command (expected 'get', 'post' or 'delete'): " + cmd)
except Exception as e:
# rethrow the exception after we've checked for stack trace from h2o
# out of memory errors maybe don't show up right away? so we should wait for h2o
# to get it out to h2o stdout. We don't want to rely on cloud teardown to check
# because there's no delay, and we don't want to delay all cloud teardowns by waiting.
# (this is new/experimental)
exc_info = sys.exc_info()
# use this to ignore the initial connection errors during build cloud when h2o is coming up
if not noExtraErrorCheck:
h2p.red_print(
"ERROR: got exception on %s to h2o. \nGoing to check sandbox, then rethrow.." % (url + paramsStr))
time.sleep(2)
H2O.check_sandbox_for_errors(python_test_name=H2O.python_test_name);
log_rest("")
log_rest("EXCEPTION CAUGHT DOING REQUEST: " + str(e.message))
raise (exc_info[1], None, exc_info[2])
H2O.verboseprint("r: " + repr(r))
if 200 != r.status_code:
pp = pprint.PrettyPrinter(indent=4)
msg = "JSON call returned non-200 status: " + url
json = r.json()
if None != json and 'dev_msg' in json:
msg += "\ndev_msg: "
msg += str(json['dev_msg'])
msg += "\nr.status_code: " + str(r.status_code)
msg += "\nr.headers: " + repr(r.headers)
if None == json:
msg += '\nERROR: the error output from H2O is not JSON!'
msg += "\nr.text: " + r.text
else:
msg += "\nr.json: "
msg += pp.pformat(json)
if raiseIfNon200:
pass # we'll pass msg up with the exception
elif not suppressErrorMsg:
print(msg)
log_rest(msg)
log_rest("")
try:
if r is None:
log_rest("r is None")
else:
log_rest("HTTP status code: " + str(r.status_code))
# The following accesses to r.text were taking most of the runtime:
log_text = False
if log_text:
if hasattr(r, 'text'):
if r.text is None:
log_rest("r.text is None")
else:
log_rest(r.text)
else:
log_rest("r does not have attr text")
except Exception as e:
# Paranoid exception catch.
# Ignore logging exceptions in the case that the above error checking isn't sufficient.
print("Caught exception from result logging: ", e, "; result: ", repr(r))
# fatal if no response
if raiseIfNon200 and not r:
raise Exception("Maybe bad url? no r in __do_json_request in %s:" % inspect.stack()[1][3] + "\n\n" + msg)
# this is used to open a browser on results, or to redo the operation in the browser
# we don't' have that may urls flying around, so let's keep them all
H2O.json_url_history.append(r.url)
# if r.json():
# raise Exception("Maybe bad url? no r.json in __do_json_request in %s:" % inspect.stack()[1][3])
rjson = None
if returnFast:
return
try:
rjson = r.json()
except:
print(h2o_test_utils.dump_json(r.text))
if not isinstance(r, (list, dict)):
raise Exception("h2o json responses should always be lists or dicts, see previous for text")
raise Exception("Could not decode any json from the request.")
# TODO
# TODO
# TODO
# TODO: we should really only look in the response object. This check
# prevents us from having a field called "error" (e.g., for a scoring result).
for e in ['error', 'Error', 'errors', 'Errors']:
# error can be null (python None). This happens in exec2
if e in rjson and rjson[e]:
H2O.verboseprint("rjson:" + h2o_test_utils.dump_json(rjson))
emsg = 'rjson %s in %s: %s' % (e, inspect.stack()[1][3], rjson[e])
if ignoreH2oError:
# well, we print it..so not totally ignore. test can look at rjson returned
print(emsg)
else:
print(emsg)
raise Exception(emsg)
for w in ['warning', 'Warning', 'warnings', 'Warnings']:
# warning can be null (python None).
if w in rjson and rjson[w]:
H2O.verboseprint(dump_json(rjson))
print('rjson %s in %s: %s' % (w, inspect.stack()[1][3], rjson[w]))
# Allow the caller to check things like __http_request.status_code.
# The response object is not JSON-serializable, so we capture the fields we want here:
response = {}
# response['headers'] = r.headers
response['url'] = r.url
response['status_code'] = r.status_code
response['text'] = r.text
rjson['__http_response'] = response
return rjson
def test(a_node, pp):
###################################
# Compute and check ModelMetrics for 'deeplearning_prostate_binomial'
mm = a_node.compute_model_metrics(model='deeplearning_prostate_binomial', frame='prostate_binomial')
assert mm is not None, "FAIL: Got a null result for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial'
assert 'model_category' in mm, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " does not contain a model_category."
assert 'Binomial' == mm['model_category'], "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " model_category is not Binomial, it is: " + str(mm['model_category'])
assert 'AUC' in mm, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " does not contain an AUC element: " + h2o_test_utils.dump_json(mm)
assert type(mm['AUC']) is float, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " AUC element is not a float: " + h2o_test_utils.dump_json(mm)
assert 'confusion_matrices' in mm, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " does not contain a confusion_matrices element: " + h2o_test_utils.dump_json(mm)
assert type(mm['confusion_matrices']) is list, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " confusion_matrices element is not a list: " + h2o_test_utils.dump_json(mm)
# print h2o_test_utils.dump_json(mm)
h2o.H2O.verboseprint("ModelMetrics for scoring: ", 'deeplearning_prostate_binomial', " on: ", 'prostate_binomial', ": ", repr(mm))
###################################
# Check for ModelMetrics for 'deeplearning_prostate_binomial' in full list
mms = a_node.model_metrics() # fetch all
assert 'model_metrics' in mms, 'FAIL: Failed to find model_metrics in result of /3/ModelMetrics.'
found_mm = False
for mm in mms['model_metrics']:
assert 'model' in mm, "FAIL: mm does not contain a model element: " + repr(mm)
assert 'name' in mm['model'], "FAIL: mm[model] isn't a key with a name: " + repr(mm)
assert 'type' in mm['model'], "FAIL: mm[model] does not contain a type: " + repr(mm)
assert 'Key<Model>' == mm['model']['type'], "FAIL: mm[model] type is not Key<Model>: " + repr(mm['model']['type'])
assert 'frame' in mm, "FAIL: mm does not contain a frame element: " + repr(mm)
assert 'name' in mm['frame'], "FAIL: mm[frame] does not contain a name: " + repr(mm)
assert 'type' in mm['frame'], "FAIL: mm[frame] does not contain a type: " + repr(mm)
assert 'Key<Frame>' == mm['frame']['type'], "FAIL: mm[frame] type is not Key<Frame>: " + repr(mm)
model_key = mm['model']['name']
frame_key = mm['frame']['name'] # TODO: should match
if model_key == 'deeplearning_prostate_binomial' and frame_key == 'prostate_binomial':
found_mm = True
assert found_mm, "FAIL: Failed to find ModelMetrics object for model: " + 'deeplearning_prostate_binomial' + " and frame: " + 'prostate_binomial'
# test delete_model_metrics
mms = a_node.model_metrics('deeplearning_prostate_binomial', 'prostate_binomial')
assert len(mms['model_metrics']) == 1, "FAIL: expected 1 ModelMetrics, found: " + str(len(mms['model_metrics']))
a_node.delete_model_metrics('deeplearning_prostate_binomial', 'prostate_binomial')
mms = a_node.model_metrics('deeplearning_prostate_binomial', 'prostate_binomial')
assert len(mms['model_metrics']) == 0, "FAIL: expected 0 ModelMetrics, found: " + str(len(mms['model_metrics']))
###################################
# Predict and check ModelMetrics for 'deeplearning_prostate_binomial'
p = a_node.predict(model='deeplearning_prostate_binomial', frame='prostate_binomial', predictions_frame='deeplearning_prostate_binomial_predictions')
h2o_test_utils.validate_predictions(a_node, p, 'deeplearning_prostate_binomial', 'prostate_binomial', 380, predictions_frame='deeplearning_prostate_binomial_predictions')
h2o_test_utils.validate_frame_exists(a_node, 'deeplearning_prostate_binomial_predictions')
h2o.H2O.verboseprint("Predictions for scoring: ", 'deeplearning_prostate_binomial', " on: ", 'prostate_binomial', ": ", repr(p))
###################################
# Predict and check ModelMetrics for 'deeplearning_prostate_regression'
p = a_node.predict(model='deeplearning_prostate_regression', frame='prostate_binomial')
h2o_test_utils.validate_predictions(a_node, p, 'deeplearning_prostate_regression', 'prostate_binomial', 380)
h2o.H2O.verboseprint("Predictions for scoring: ", 'deeplearning_prostate_regression', " on: ", 'prostate_binomial', ": ", repr(p))
###################################
# Predict and check ModelMetrics for 'gbm_prostate_binomial'
p = a_node.predict(model='gbm_prostate_binomial', frame='prostate_binomial')
h2o_test_utils.validate_predictions(a_node, p, 'gbm_prostate_binomial', 'prostate_binomial', 380)
h2o.H2O.verboseprint("Predictions for scoring: ", 'gbm_prostate_binomial', " on: ", 'prostate_binomial', ": ", repr(p))
###################################
# Predict and check ModelMetrics for 'gbm_prostate_regression'
p = a_node.predict(model='gbm_prostate_regression', frame='prostate_binomial')
h2o_test_utils.validate_predictions(a_node, p, 'gbm_prostate_regression', 'prostate_binomial', 380)
h2o.H2O.verboseprint("Predictions for scoring: ", 'gbm_prostate_regression', " on: ", 'prostate_binomial', ": ", repr(p))
###################################
# Predict and check ModelMetrics (empty now except for predictions frame) for 'kmeans_prostate'
p = a_node.predict(model='kmeans_prostate', frame='prostate_binomial')
h2o_test_utils.validate_predictions(a_node, p, 'kmeans_prostate', 'prostate_binomial', 380)
h2o.H2O.verboseprint("Predictions for scoring: ", 'kmeans_prostate', " on: ", 'prostate_binomial', ": ", repr(p))
def test(a_node, pp):
if h2o_test_utils.isVerbose(): print('Testing ModelMetrics. . .')
###################################
# Compute and check ModelMetrics for 'deeplearning_prostate_binomial'
mm = a_node.compute_model_metrics(model='deeplearning_prostate_binomial', frame='prostate_binomial')
assert mm is not None, "FAIL: Got a null result for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial'
assert 'model_category' in mm, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " does not contain a model_category."
assert 'Binomial' == mm['model_category'], "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " model_category is not Binomial, it is: " + str(mm['model_category'])
assert 'AUC' in mm, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " does not contain an AUC element: " + h2o_test_utils.dump_json(mm)
assert type(mm['AUC']) is float, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " AUC element is not a float: " + h2o_test_utils.dump_json(mm)
assert 'thresholds_and_metric_scores' in mm, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " does not contain a thresholds_and_metric_scores element: " + h2o_test_utils.dump_json(mm)
assert type(mm['thresholds_and_metric_scores']) is dict, "FAIL: ModelMetrics for scoring: " + 'deeplearning_prostate_binomial' + " on: " + 'prostate_binomial' + " confusion_matrices element is not a dict: " + h2o_test_utils.dump_json(mm)
# print(h2o_test_utils.dump_json(mm))
h2o.H2O.verboseprint("ModelMetrics for scoring: ", 'deeplearning_prostate_binomial', " on: ", 'prostate_binomial', ": ", repr(mm))
###################################
# Check for ModelMetrics for 'deeplearning_prostate_binomial' in full list
mms = a_node.model_metrics() # fetch all
assert 'model_metrics' in mms, 'FAIL: Failed to find model_metrics in result of /3/ModelMetrics.'
found_mm = False
found_a_frame = False
for mm in mms['model_metrics']:
assert 'model' in mm, "FAIL: mm does not contain a model element: " + repr(mm)
assert 'name' in mm['model'], "FAIL: mm[model] isn't a key with a name: " + repr(mm)
assert 'type' in mm['model'], "FAIL: mm[model] does not contain a type: " + repr(mm)
assert 'Key<Model>' == mm['model']['type'], "FAIL: mm[model] type is not Key<Model>: " + repr(mm['model']['type'])
assert 'frame' in mm, "FAIL: mm does not contain a frame element: " + repr(mm)
if mm['frame'] is not None: # temp metrics can have null frames
assert 'name' in mm['frame'], "FAIL: mm[frame] does not contain a name: " + repr(mm)
assert 'type' in mm['frame'], "FAIL: mm[frame] does not contain a type: " + repr(mm)
assert 'Key<Frame>' == mm['frame']['type'], "FAIL: mm[frame] type is not Key<Frame>: " + repr(mm)
model_key = mm['model']['name']
frame_key = mm['frame']['name']
if model_key == 'deeplearning_prostate_binomial' and frame_key == 'prostate_binomial':
found_mm = True
else:
found_a_frame = True
assert found_mm, "FAIL: Failed to find ModelMetrics object for model: " + 'deeplearning_prostate_binomial' + " and frame: " + 'prostate_binomial'
assert found_a_frame, "FAIL: Failed to find a frame in any ModelMetrics object."
# test delete_model_metrics
mms = a_node.model_metrics('deeplearning_prostate_binomial', 'prostate_binomial')
assert len(mms['model_metrics']) == 1, "FAIL: expected 1 ModelMetrics, found: " + str(len(mms['model_metrics']))
a_node.delete_model_metrics('deeplearning_prostate_binomial', 'prostate_binomial')
mms = a_node.model_metrics('deeplearning_prostate_binomial', 'prostate_binomial')
assert len(mms['model_metrics']) == 0, "FAIL: expected 0 ModelMetrics, found: " + str(len(mms['model_metrics']))
###################################
# Predict and check ModelMetrics for 'deeplearning_prostate_binomial'
p = a_node.predict(model='deeplearning_prostate_binomial', frame='prostate_binomial', predictions_frame='deeplearning_prostate_binomial_predictions')
h2o_test_utils.validate_predictions(a_node, p, 'deeplearning_prostate_binomial', 'prostate_binomial', 380, predictions_frame='deeplearning_prostate_binomial_predictions')
h2o_test_utils.validate_frame_exists(a_node, 'deeplearning_prostate_binomial_predictions')
h2o.H2O.verboseprint("Predictions for scoring: ", 'deeplearning_prostate_binomial', " on: ", 'prostate_binomial', ": ", repr(p))
###################################
# Predict and check ModelMetrics for 'gbm_prostate_binomial'
p = a_node.predict(model='gbm_prostate_binomial', frame='prostate_binomial')
h2o_test_utils.validate_predictions(a_node, p, 'gbm_prostate_binomial', 'prostate_binomial', 380)
h2o.H2O.verboseprint("Predictions for scoring: ", 'gbm_prostate_binomial', " on: ", 'prostate_binomial', ": ", repr(p))
###################################
# Predict and check ModelMetrics for 'deeplearning_prostate_regression'
p = a_node.predict(model='deeplearning_prostate_regression', frame='prostate_regression')
h2o_test_utils.validate_predictions(a_node, p, 'deeplearning_prostate_regression', 'prostate_regression', 380)
h2o.H2O.verboseprint("Predictions for scoring: ", 'deeplearning_prostate_regression', " on: ", 'prostate_regression', ": ", repr(p))
###################################
# Predict and check ModelMetrics for 'gbm_prostate_regression'
p = a_node.predict(model='gbm_prostate_regression', frame='prostate_regression')
h2o_test_utils.validate_predictions(a_node, p, 'gbm_prostate_regression', 'prostate_regression', 380)
h2o.H2O.verboseprint("Predictions for scoring: ", 'gbm_prostate_regression', " on: ", 'prostate_regression', ": ", repr(p))
###################################
# Predict and check ModelMetrics (empty now except for predictions frame) for 'kmeans_prostate'
p = a_node.predict(model='kmeans_prostate', frame='prostate_regression')
h2o_test_utils.validate_predictions(a_node, p, 'kmeans_prostate', 'prostate_regression', 380)
h2o.H2O.verboseprint("Predictions for scoring: ", 'kmeans_prostate', " on: ", 'prostate_regression', ": ", repr(p))
###################################
# Predict with reversed keys (should get an H2OErrorV1):
try:
p = a_node.predict(frame='kmeans_prostate', model='prostate_regression')
assert 1 == 0, "FAIL: expected a ValueError exception looking for Model 'prostate_regression'"
except Exception as e:
pass
def __do_json_request(self, jsonRequest=None, fullUrl=None, timeout=10, params=None, postData=None, returnFast=False,
cmd='get', extraComment=None, ignoreH2oError=False, noExtraErrorCheck=False, raiseIfNon200=True, suppressErrorMsg=False, **kwargs):
H2O.verboseprint("__do_json_request, timeout: " + str(timeout))
# if url param is used, use it as full url. otherwise crate from the jsonRequest
if fullUrl:
url = fullUrl
else:
url = self.__url(jsonRequest)
# remove any params that are 'None'
# need to copy dictionary, since can't delete while iterating
if params is not None:
params_serialized = params.copy()
for k in params_serialized:
if params_serialized[k] is None:
del params[k]
paramsStr = '?' + '&'.join(['%s=%s' % (k, v) for (k, v) in params.items()])
else:
paramsStr = ''
# The requests package takes array parameters and explodes them: ['f00', 'b4r'] becomes "f00,b4r".
# NOTE: this handles 1D arrays only; if we need ND this needs to be recursive.
# NOTE: we currently don't need to do this for GET, so that's not implemented.
if postData is not None:
munged_postData = {}
for k, v in postData.iteritems():
if type(v) is list:
if len(v) == 0:
munged_postData[k] = '[]'
else:
first = True
array_str = '['
for val in v:
if not first: array_str += ', '
if val is None:
array_str += 'null'
elif isinstance(val, basestring):
array_str += "\"" + str(val) + "\""
else:
array_str += str(val)
first = False
array_str += ']'
munged_postData[k] = array_str
elif type(v) is dict:
if len(v) == 0:
munged_postData[k] = '{}'
else:
first = True
map_str = '{'
for key, val in v.iteritems():
if not first: map_str += ', '
if val is None:
map_str += "\"" + key + "\"" + ': null'
elif isinstance(val, basestring):
map_str += "\"" + str(key) + "\"" + ":" + "\"" + str(val) + "\""
else:
map_str += "\"" + key + "\"" + ':' + str(val)
first = False
map_str += '}'
munged_postData[k] = map_str
else:
# not list:
munged_postData[k] = v
else:
# None
munged_postData = postData
# print("munged_postData: " + repr(munged_postData))
if extraComment:
log('Start ' + url + paramsStr, comment=extraComment)
else:
log('Start ' + url + paramsStr)
log_rest("")
log_rest("----------------------------------------------------------------------\n")
if extraComment:
log_rest("# Extra comment info about this request: " + extraComment)
if cmd == 'get':
log_rest("GET")
else:
log_rest("POST")
log_rest(url + paramsStr)
# file get passed thru kwargs here
try:
if 'post' == cmd:
# NOTE == cmd: for now, since we don't have deserialization from JSON in h2o-dev, we use form-encoded POST.
# This is temporary.
#
# This following does application/json (aka, posting JSON in the body):
# r = requests.post(url, timeout=timeout, params=params, data=json.dumps(munged_postData), **kwargs)
#
# This does form-encoded, which doesn't allow POST of nested structures
r = requests.post(url, timeout=timeout, params=params, data=munged_postData, **kwargs)
elif 'delete' == cmd:
r = requests.delete(url, timeout=timeout, params=params, **kwargs)
elif 'get' == cmd:
r = requests.get(url, timeout=timeout, params=params, **kwargs)
else:
raise ValueError("Unknown HTTP command (expected 'get', 'post' or 'delete'): " + cmd)
except Exception as e:
# rethrow the exception after we've checked for stack trace from h2o
# out of memory errors maybe don't show up right away? so we should wait for h2o
# to get it out to h2o stdout. We don't want to rely on cloud teardown to check
# because there's no delay, and we don't want to delay all cloud teardowns by waiting.
# (this is new/experimental)
exc_info = sys.exc_info()
# use this to ignore the initial connection errors during build cloud when h2o is coming up
if not noExtraErrorCheck:
h2p.red_print(
"ERROR: got exception on %s to h2o. \nGoing to check sandbox, then rethrow.." % (url + paramsStr))
time.sleep(2)
H2O.check_sandbox_for_errors(python_test_name=H2O.python_test_name);
log_rest("")
log_rest("EXCEPTION CAUGHT DOING REQUEST: " + str(e.message))
raise (exc_info[1], None, exc_info[2])
H2O.verboseprint("r: " + repr(r))
if 200 != r.status_code:
pp = pprint.PrettyPrinter(indent=4)
msg = "JSON call returned non-200 status: " + url
json = r.json()
if None != json and 'dev_msg' in json:
msg += "\ndev_msg: "
msg += str(json['dev_msg'])
msg += "\nr.status_code: " + str(r.status_code)
msg += "\nr.headers: " + repr(r.headers)
if None == json:
msg += '\nERROR: the error output from H2O is not JSON!'
msg += "\nr.text: " + r.text
else:
msg += "\nr.json: "
msg += pp.pformat(json)
if raiseIfNon200:
pass # we'll pass msg up with the exception
elif not suppressErrorMsg:
print(msg)
log_rest(msg)
log_rest("")
try:
if r is None:
log_rest("r is None")
else:
log_rest("HTTP status code: " + str(r.status_code))
# The following accesses to r.text were taking most of the runtime:
log_text = False
if log_text:
if hasattr(r, 'text'):
if r.text is None:
log_rest("r.text is None")
else:
log_rest(r.text)
else:
log_rest("r does not have attr text")
except Exception as e:
# Paranoid exception catch.
# Ignore logging exceptions in the case that the above error checking isn't sufficient.
print("Caught exception from result logging: ", e, "; result: ", repr(r))
# fatal if no response
if raiseIfNon200 and not r:
raise Exception("Maybe bad url? no r in __do_json_request in %s:" % inspect.stack()[1][3] + "\n\n" + msg)
# this is used to open a browser on results, or to redo the operation in the browser
# we don't' have that may urls flying around, so let's keep them all
H2O.json_url_history.append(r.url)
# if r.json():
# raise Exception("Maybe bad url? no r.json in __do_json_request in %s:" % inspect.stack()[1][3])
rjson = None
if returnFast:
return
try:
rjson = r.json()
except:
print(h2o_test_utils.dump_json(r.text))
if not isinstance(r, (list, dict)):
raise Exception("h2o json responses should always be lists or dicts, see previous for text")
raise Exception("Could not decode any json from the request.")
# TODO
# TODO
# TODO
# TODO: we should really only look in the response object. This check
# prevents us from having a field called "error" (e.g., for a scoring result).
for e in ['error', 'Error', 'errors', 'Errors']:
# error can be null (python None). This happens in exec2
if e in rjson and rjson[e]:
H2O.verboseprint("rjson:" + h2o_test_utils.dump_json(rjson))
emsg = 'rjson %s in %s: %s' % (e, inspect.stack()[1][3], rjson[e])
if ignoreH2oError:
# well, we print it..so not totally ignore. test can look at rjson returned
print(emsg)
else:
print(emsg)
raise Exception(emsg)
for w in ['warning', 'Warning', 'warnings', 'Warnings']:
# warning can be null (python None).
if w in rjson and rjson[w]:
H2O.verboseprint(dump_json(rjson))
print('rjson %s in %s: %s' % (w, inspect.stack()[1][3], rjson[w]))
# Allow the caller to check things like __http_request.status_code.
# The response object is not JSON-serializable, so we capture the fields we want here:
response = {}
# response['headers'] = r.headers
response['url'] = r.url
response['status_code'] = r.status_code
response['text'] = r.text
rjson['__http_response'] = response
return rjson
