def redirect_relative():
if os.environ.get('JOB_NAME') and os.environ.get('JOB_NAME').startswith(
"h2o-3-kerberos-smoke-pipeline/"):
h2o.log_and_echo(
"Skipping test 'redirect_relative' on Kerberos pipeline (it is not configured with form_auth)"
)
return
conn = h2o.connection()
# get default requests arguments
req_args = conn._request_args()
headers = req_args["headers"]
headers["User-Agent"] = "Mozilla/pyunit"
# invalidate authentication
req_args["auth"] = None
req_args["headers"] = headers
response_flow = requests.request("GET",
conn._base_url + "/flow/index.html",
allow_redirects=False,
**req_args)
print(response_flow)
assert response_flow.status_code in [302, 303]
assert response_flow.headers["location"].startswith("/login")
def check_story(story_name, paragraphs):
h2o.remove_all()
h2o.log_and_echo("------------------------------------------------------------")
h2o.log_and_echo("")
h2o.log_and_echo("CHECKING: {0}".format(story_name))
h2o.log_and_echo("")
h2o.log_and_echo("------------------------------------------------------------")
# 1. Combine the related, individual code paragraphs into a single, coherent python story
story = []
for p in paragraphs:
with open(p, "r") as f: story = story + f.readlines()
# 2. Execute the story
# first, remove any h2o.init calls
remove_lines = []
for idx, l in enumerate(story):
if "h2o.init" in l: remove_lines.append(idx)
story = [i for j, i in enumerate(story) if j not in remove_lines]
# write the story that will be executed to the results directory for future reference
story_file = os.path.join(results_dir(), test_name()+"."+story_name+".code")
with open(story_file, 'w') as f: f.writelines(story)
# run it
with open(story_file, "r") as s: booklet = s.read()
booklet_c = compile(booklet, '<string>', 'exec')
p = {}
exec(booklet_c, p)
def check_story(story_name, paragraphs):
h2o.remove_all()
h2o.log_and_echo("------------------------------------------------------------")
h2o.log_and_echo("")
h2o.log_and_echo("CHECKING: {0}".format(story_name))
h2o.log_and_echo("")
h2o.log_and_echo("------------------------------------------------------------")
# 1. Combine the related, individual code paragraphs into a single, coherent python story
story = []
for p in paragraphs:
with open(p, "r") as f: story = story + f.readlines()
# 2. Execute the story
# first, remove any h2o.init calls
remove_lines = []
for idx, l in enumerate(story):
if "h2o.init" in l: remove_lines.append(idx)
story = [i for j, i in enumerate(story) if j not in remove_lines]
# write the story that will be executed to the results directory for future reference
story_file = os.path.join(results_dir(), test_name()+"."+story_name+".code")
with open(story_file, 'w') as f: f.writelines(story)
# run it
with open(story_file, "r") as s: booklet = s.read()
booklet_c = compile(booklet, '<string>', 'exec')
p = {}
exec(booklet_c, p)
def h2olog_and_echo():
"""
Python API test: h2o.log_and_echo(message=u'')
"""
try:
h2o.log_and_echo("Testing h2o.log_and_echo")
except Exception as e:
assert False, "h2o.log_and_echo() command is not working."
def h2o_test_setup(sys_args):
h2o_py_dir = os.path.realpath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), ".."))
h2o_docs_dir = os.path.realpath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..",
"h2o-docs"))
parse_args(sys_args)
sys.path.insert(1, h2o_py_dir)
import h2o
from tests import pyunit_utils, pydemo_utils, pybooklet_utils
for pkg in (pyunit_utils, pybooklet_utils):
setattr(pkg, '__on_hadoop__', _ON_HADOOP_)
setattr(pkg, '__hadoop_namenode__', _HADOOP_NAMENODE_)
setattr(pkg, '__test_name__', _TEST_NAME_)
setattr(pkg, '__results_dir__', _RESULTS_DIR_)
if _IS_PYUNIT_ or _IS_IPYNB_ or _IS_PYBOOKLET_ or _IS_PYDEMO_:
pass
else:
raise (
EnvironmentError,
"Unrecognized test type. Must be of type ipynb, pydemo, pyunit, or pybooklet, but got: "
"{0}".format(_TEST_NAME_))
print("[{0}] {1}\n".format(
strftime("%Y-%m-%d %H:%M:%S", gmtime()),
"Connect to h2o on IP: {0} PORT: {1}".format(_H2O_IP_, _H2O_PORT_)))
auth = None
if _LDAP_USER_NAME_ is not None and _LDAP_PASSWORD_ is not None:
auth = (_LDAP_USER_NAME_, _LDAP_PASSWORD_)
h2o.connect(ip=_H2O_IP_, port=_H2O_PORT_, verbose=False, auth=auth)
h2o.utils.config.H2OConfigReader.get_config(
)["general.allow_breaking_changes"] = True
#rest_log = os.path.join(_RESULTS_DIR_, "rest.log")
#h2o.start_logging(rest_log)
#print "[{0}] {1}\n".format(strftime("%Y-%m-%d %H:%M:%S", gmtime()), "Started rest logging in: {0}".format(rest_log))
h2o.log_and_echo(
"------------------------------------------------------------")
h2o.log_and_echo("")
h2o.log_and_echo("STARTING TEST: " + _TEST_NAME_)
h2o.log_and_echo("")
h2o.log_and_echo(
"------------------------------------------------------------")
h2o.remove_all()
if _IS_IPYNB_: pydemo_utils.ipy_notebook_exec(_TEST_NAME_)
elif _IS_PYUNIT_: pyunit_utils.pyunit_exec(_TEST_NAME_)
elif _IS_PYBOOKLET_: pybooklet_utils.pybooklet_exec(_TEST_NAME_)
elif _IS_PYDEMO_: pydemo_utils.pydemo_exec(_TEST_NAME_)
def standalone_test(test):
h2o.init(strict_version_check=False)
h2o.remove_all()
h2o.log_and_echo("------------------------------------------------------------")
h2o.log_and_echo("")
h2o.log_and_echo("STARTING TEST")
h2o.log_and_echo("")
h2o.log_and_echo("------------------------------------------------------------")
test()
def standalone_test(test):
h2o.init(strict_version_check=False)
h2o.remove_all()
h2o.log_and_echo("------------------------------------------------------------")
h2o.log_and_echo("")
h2o.log_and_echo("STARTING TEST")
h2o.log_and_echo("")
h2o.log_and_echo("------------------------------------------------------------")
test()
def malicious_log():
print(
"Waiting 3s before sending malicious logging call ${jndi:ldap://127.0.0.1:50001/}"
)
time.sleep(3)
print("Calling log_and_echo")
h2o.log_and_echo('${jndi:ldap://127.0.0.1:50001/}')
print("Finished log_and_echo")
time.sleep(3)
print("Sending poison pill to close the connection")
send_poison_pill()
def h2o_test_setup(sys_args):
h2o_py_dir = os.path.realpath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), ".."))
h2o_docs_dir = os.path.realpath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..",
"h2o-docs"))
parse_args(sys_args)
sys.path.insert(1, h2o_py_dir)
import h2o
from tests import pyunit_utils, pydemo_utils, pybooklet_utils
set_pyunit_pkg_attrs(pyunit_utils)
set_pybooklet_pkg_attrs(pybooklet_utils)
if _IS_PYUNIT_ or _IS_IPYNB_ or _IS_PYBOOKLET_ or _IS_PYDEMO_:
pass
else:
raise (
EnvironmentError,
"Unrecognized test type. Must be of type ipynb, pydemo, pyunit, or pybooklet, but got: "
"{0}".format(_TEST_NAME_))
print("[{0}] {1}\n".format(
strftime("%Y-%m-%d %H:%M:%S", gmtime()),
"Connect to h2o on IP: {0} PORT: {1}".format(_H2O_IP_, _H2O_PORT_)))
h2o.init(ip=_H2O_IP_,
port=_H2O_PORT_,
strict_version_check=False,
force_connect=_FORCE_CONNECT_)
h2o.utils.config.H2OConfigReader.get_config(
)["general.allow_breaking_changes"] = True
#rest_log = os.path.join(_RESULTS_DIR_, "rest.log")
#h2o.start_logging(rest_log)
#print "[{0}] {1}\n".format(strftime("%Y-%m-%d %H:%M:%S", gmtime()), "Started rest logging in: {0}".format(rest_log))
h2o.log_and_echo(
"------------------------------------------------------------")
h2o.log_and_echo("")
h2o.log_and_echo("STARTING TEST: " + _TEST_NAME_)
h2o.log_and_echo("")
h2o.log_and_echo(
"------------------------------------------------------------")
h2o.remove_all()
if _IS_IPYNB_: pydemo_utils.ipy_notebook_exec(_TEST_NAME_)
elif _IS_PYUNIT_: pyunit_utils.pyunit_exec(_TEST_NAME_)
elif _IS_PYBOOKLET_: pybooklet_utils.pybooklet_exec(_TEST_NAME_)
elif _IS_PYDEMO_: pydemo_utils.pydemo_exec(_TEST_NAME_)
def run_test(sys_args, test_to_run):
global _IPYNB_
parse_args(sys_args)
h2o.init(ip=_H2O_IP_, port=_H2O_PORT_, strict_version_check=False)
h2o.log_and_echo("------------------------------------------------------------")
h2o.log_and_echo("")
h2o.log_and_echo("STARTING TEST: "+str(h2o.ou()))
h2o.log_and_echo("")
h2o.log_and_echo("------------------------------------------------------------")
# num_keys = h2o.store_size()
try:
if _IPYNB_: utils.ipy_notebook_exec(_IPYNB_, save_and_norun=False)
else: test_to_run()
finally:
h2o.remove_all()
def h2o_test_setup(sys_args):
h2o_py_dir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)),".."))
h2o_docs_dir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)),"..","..","h2o-docs"))
parse_args(sys_args)
sys.path.insert(1, h2o_py_dir)
import h2o
from tests import pyunit_utils, pydemo_utils, pybooklet_utils
for pkg in (pyunit_utils, pybooklet_utils):
setattr(pkg, '__on_hadoop__', _ON_HADOOP_)
setattr(pkg, '__hadoop_namenode__', _HADOOP_NAMENODE_)
setattr(pkg, '__test_name__', _TEST_NAME_)
setattr(pkg, '__results_dir__', _RESULTS_DIR_)
if _IS_PYUNIT_ or _IS_IPYNB_ or _IS_PYBOOKLET_ or _IS_PYDEMO_:
pass
else:
raise(EnvironmentError, "Unrecognized test type. Must be of type ipynb, pydemo, pyunit, or pybooklet, but got: "
"{0}".format(_TEST_NAME_))
print("[{0}] {1}\n".format(strftime("%Y-%m-%d %H:%M:%S", gmtime()), "Connect to h2o on IP: {0} PORT: {1}".format(_H2O_IP_, _H2O_PORT_)))
auth = None
if _LDAP_USER_NAME_ is not None and _LDAP_PASSWORD_ is not None:
auth = (_LDAP_USER_NAME_, _LDAP_PASSWORD_)
elif _KERB_PRINCIPAL_ is not None:
from h2o.auth import SpnegoAuth
auth = SpnegoAuth(service_principal=_KERB_PRINCIPAL_)
h2o.connect(ip=_H2O_IP_, port=_H2O_PORT_, verbose=False, auth=auth, **_H2O_EXTRA_CONNECT_ARGS_)
h2o.utils.config.H2OConfigReader.get_config()["general.allow_breaking_changes"] = True
#rest_log = os.path.join(_RESULTS_DIR_, "rest.log")
#h2o.start_logging(rest_log)
#print "[{0}] {1}\n".format(strftime("%Y-%m-%d %H:%M:%S", gmtime()), "Started rest logging in: {0}".format(rest_log))
h2o.log_and_echo("------------------------------------------------------------")
h2o.log_and_echo("")
h2o.log_and_echo("STARTING TEST: " + _TEST_NAME_)
h2o.log_and_echo("")
h2o.log_and_echo("------------------------------------------------------------")
h2o.remove_all()
if _IS_IPYNB_: pydemo_utils.ipy_notebook_exec(_TEST_NAME_)
elif _IS_PYUNIT_: pyunit_utils.pyunit_exec(_TEST_NAME_)
elif _IS_PYBOOKLET_: pybooklet_utils.pybooklet_exec(_TEST_NAME_)
elif _IS_PYDEMO_: pydemo_utils.pydemo_exec(_TEST_NAME_)
def run_test(sys_args, test_to_run):
# import pkg_resources
# ver = pkg_resources.get_distribution("h2o").version
# print "H2O PYTHON PACKAGE VERSION: " + str(ver)
ip, port = sys_args[2].split(":")
h2o.init(ip,port,strict_version_check=False)
h2o.log_and_echo("------------------------------------------------------------")
h2o.log_and_echo("")
h2o.log_and_echo("STARTING TEST: "+str(h2o.ou()))
h2o.log_and_echo("")
h2o.log_and_echo("------------------------------------------------------------")
num_keys = h2o.store_size()
try:
if len(sys_args) > 3 and sys_args[3] == "--ipynb": utils.ipy_notebook_exec(sys_args[4],save_and_norun=False)
else: test_to_run(ip, port)
finally:
h2o.remove_all()
if h2o.keys_leaked(num_keys): print "Leaked Keys!"
def test_download_txt_logs():
marker = "txt-logs-marker"
results_dir = pyunit_utils.locate(
"results") # find directory path to results folder
# log something unique so that we can try to search for it in the downloaded logs
h2o.log_and_echo(marker)
log_path = h2o.download_all_logs(results_dir,
filename="plain_text_logs.log_ignore",
container="LOG")
assert not zipfile.is_zipfile(log_path)
# logs are trimmed (we can only check smaller files)
assert find_marker(log_path,
marker) or os.path.getsize(log_path) > 10 * 1024 * 1042
# Now make sure we get a zip file if we don't specify the container format
zip_path = h2o.download_all_logs(results_dir, filename="zip_logs.zip")
assert zipfile.is_zipfile(zip_path)
def h2o_test_setup(sys_args):
h2o_py_dir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)),".."))
h2o_docs_dir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)),"..","..","h2o-docs"))
parse_args(sys_args)
sys.path.insert(1, h2o_py_dir)
import h2o
from tests import pyunit_utils, pydemo_utils, pybooklet_utils
set_pyunit_pkg_attrs(pyunit_utils)
set_pybooklet_pkg_attrs(pybooklet_utils)
if _IS_PYUNIT_ or _IS_IPYNB_ or _IS_PYBOOKLET_:
pass
elif _IS_PYDEMO_:
raise(NotImplementedError, "pydemos are not supported at this time")
else:
raise(EnvironmentError, "Unrecognized test type. Must be of type ipynb, pydemo, pyunit, or pybooklet, but got: "
"{0}".format(_TEST_NAME_))
print "[{0}] {1}\n".format(strftime("%Y-%m-%d %H:%M:%S", gmtime()), "Connect to h2o on IP: {0} PORT: {1}"
"".format(_H2O_IP_, _H2O_PORT_))
h2o.init(ip=_H2O_IP_, port=_H2O_PORT_, strict_version_check=False)
#rest_log = os.path.join(_RESULTS_DIR_, "rest.log")
#h2o.start_logging(rest_log)
#print "[{0}] {1}\n".format(strftime("%Y-%m-%d %H:%M:%S", gmtime()), "Started rest logging in: {0}".format(rest_log))
h2o.log_and_echo("------------------------------------------------------------")
h2o.log_and_echo("")
h2o.log_and_echo("STARTING TEST: " + _TEST_NAME_)
h2o.log_and_echo("")
h2o.log_and_echo("------------------------------------------------------------")
h2o.remove_all()
if _IS_IPYNB_: pydemo_utils.ipy_notebook_exec(_TEST_NAME_)
elif _IS_PYUNIT_: pyunit_utils.pyunit_exec(_TEST_NAME_)
elif _IS_PYBOOKLET_: pybooklet_utils.pybooklet_exec(_TEST_NAME_)
def run_test(sys_args, test_to_run):
# import pkg_resources
# ver = pkg_resources.get_distribution("h2o").version
# print "H2O PYTHON PACKAGE VERSION: " + str(ver)
ip, port = sys_args[2].split(":")
h2o.init(ip, port, strict_version_check=False)
h2o.log_and_echo(
"------------------------------------------------------------")
h2o.log_and_echo("")
h2o.log_and_echo("STARTING TEST: " + str(h2o.ou()))
h2o.log_and_echo("")
h2o.log_and_echo(
"------------------------------------------------------------")
num_keys = h2o.store_size()
try:
if len(sys_args) > 3 and sys_args[3] == "--ipynb":
utils.ipy_notebook_exec(sys_args[4], save_and_norun=False)
else:
test_to_run(ip, port)
finally:
h2o.remove_all()
if h2o.keys_leaked(num_keys): print "Leaked Keys!"
def h2olog_and_echo():
"""
Python API test: h2o.log_and_echo(message=u'')
"""
ret = h2o.log_and_echo("Testing h2o.log_and_echo")
assert ret is None