def test_notebook_ui(): project = utils.get_test_project() project = utils.get_test_project() head_ip = project.cluster.head.ip r = requests.get('http://%s:8888/' % head_ip) assert r.status_code == 200
def test_ibis_conn(): project = utils.get_test_project() head_ip = project.cluster.head.ip compute_ip = project.cluster.instances[1].ip hdfs = ibis.hdfs_connect(host=head_ip, port=50070) impala = ibis.impala.connect(compute_ip, hdfs_client=hdfs)
def test_salt_formulas(): project = utils.get_test_project() kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'} out = project.salt('state.sls', args=['ipython.notebook'], target='head', kwargs=kwargs) utils.check_all_true(out, none_is_ok=True)
def test_namenode_ui(): ''' Note: Namenode UI uses a lot of javascript requests alone is not good enough ''' project = utils.get_test_project() nn_ip = project.cluster.head.ip r = requests.get('http://%s:50070/dfshealth.html#tab-overview' % nn_ip) assert r.status_code == 200
def test_namenode_ui(): """ Note: Namenode UI uses a lot of javascript requests alone is not good enough """ project = utils.get_test_project() nn_ip = project.cluster.head.ip r = requests.get("http://%s:50070/dfshealth.html#tab-overview" % nn_ip) assert r.status_code == 200
def test_zk_conn(): project = utils.get_test_project() zk_ip = project.cluster.head.ip zk = KazooClient(hosts='%s:2181' % zk_ip) zk.start() assert zk children = zk.get_children('/') assert 'zookeeper' in children
def test_hdfs_files(): project = utils.get_test_project() head_ip = project.cluster.head.ip hdfs = Client('http://%s:50070' % head_ip) root_dirs = hdfs.list('/') assert 'spark' in root_dirs spark_dirs = hdfs.list('/spark') assert 'spark-1.4.1-bin-hadoop2.6.tgz' in spark_dirs
def test_mesos_ui(): ''' Note 1: Mesos UI uses a lot of javascript requests alone is not good enough Note 2: Mesos UI does not bing to 0.0.0.0 so need explicit IP if using vagrant ''' project = utils.get_test_project() nn_ip = project.cluster.head.ip r = requests.get('http://%s:5050/' % nn_ip) assert r.status_code == 200
def test_salt_formulas(): project = utils.get_test_project() kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'} out = project.salt('state.sls', args=['cdh5.zookeeper.cluster'], target='head', kwargs=kwargs) utils.check_all_true(out, none_is_ok=True) kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'} out = project.salt('state.sls', args=['mesos.cluster'], kwargs=kwargs) utils.check_all_true(out, none_is_ok=True)
def test_hdfs_dirs(): project = utils.get_test_project() head_ip = project.cluster.head.ip hdfs = Client('http://%s:50070' % head_ip) users_dirs = hdfs.list('/user') assert 'hive' in users_dirs assert 'impala' in users_dirs users_dirs = hdfs.list('/user/hive') assert 'warehouse' in users_dirs
def test_marathon_ui(): ''' Note 1: Marathon UI uses a lot of javascript requests alone is not good enough Note 2: Marathon UI does not bing to 0.0.0.0 so need explicit vagrant IP ''' project = utils.get_test_project() nn_ip = project.cluster.head.ip nn_ip = '10.10.10.100' r = requests.get('http://%s:18080/' % nn_ip) assert r.status_code == 200
def test_hdfs_dirs(): project = utils.get_test_project() nn_ip = project.cluster.head.ip hdfs = Client("http://%s:50070" % nn_ip) assert hdfs root_dirs = hdfs.list("/") assert "tmp" in root_dirs assert "user" in root_dirs users_dirs = hdfs.list("/user") assert project.settings["USERNAME"] in users_dirs
def test_hdfs_dirs(): project = utils.get_test_project() nn_ip = project.cluster.head.ip hdfs = Client('http://%s:50070' % nn_ip) assert hdfs root_dirs = hdfs.list('/') assert 'tmp' in root_dirs assert 'user' in root_dirs users_dirs = hdfs.list('/user') assert project.settings['USERNAME'] in users_dirs
def test_conda_create(): env = 'test_conda_create' project = utils.get_test_project() home_dir = '/home/%s' % project.settings['USERNAME'] out = project.salt('cmd.run', args=['"rm -rf {}/anaconda/envs/{}"'.format(home_dir, env)]) out = project.salt('conda.create', args=[env]) # Test env dir kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'} test = '"test -e {}/anaconda/envs/{}"'.format(home_dir, env) out = project.salt('cmd.run_all', args=[test], kwargs=kwargs) utils.check_all_cmd_retcode0(out)
def test_salt_formulas(): project = utils.get_test_project() kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'} out = project.salt('state.sls', args=['cdh5.hdfs.cluster'], kwargs=kwargs) utils.check_all_true(out, none_is_ok=True) kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'} out = project.salt('state.sls', args=['cdh5.hive.metastore'], target='head', kwargs=kwargs) utils.check_all_true(out, none_is_ok=True) # kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'} out = project.salt('state.sls', args=['cdh5.impala.cluster'], kwargs=kwargs) utils.check_all_true(out, none_is_ok=True)
def test_conda_install(): env = 'test_conda_install' project = utils.get_test_project() home_dir = '/home/%s' % project.settings['USERNAME'] out = project.salt('cmd.run', args=['"rm -rf {}/anaconda/envs/{}"'.format(home_dir, env)]) out = project.salt('conda.create', args=[env]) kwargs = {'env': env, '--out': 'json', '--out-indent': '-1'} out = project.salt('conda.list', kwargs=kwargs) with pytest.raises(AssertionError): check_pkg(out, 'numpy') with pytest.raises(AssertionError): check_pkg(out, 'scipy') with pytest.raises(AssertionError): check_pkg(out, 'pandas') out = project.salt('conda.install', args=['numpy==1.7.1'], kwargs={'env': env}) out = project.salt('conda.list', kwargs=kwargs) check_pkg(out, 'numpy', version='1.7.1') with pytest.raises(AssertionError): check_pkg(out, 'scipy') with pytest.raises(AssertionError): check_pkg(out, 'pandas') out = project.salt('conda.install', args=['scipy'], kwargs={'env': env}, target='head') out = project.salt('conda.list', kwargs=kwargs) check_pkg(out, 'numpy') check_pkg(out, 'scipy', minion='head') with pytest.raises(AssertionError): check_pkg(out, 'scipy', minion='compute-1') with pytest.raises(AssertionError): check_pkg(out, 'pandas') out = project.salt('conda.install', args=['scipy'], kwargs={'env': env}) out = project.salt('conda.list', kwargs=kwargs) check_pkg(out, 'numpy') check_pkg(out, 'scipy', minion='head') check_pkg(out, 'scipy', minion='compute-1') with pytest.raises(AssertionError): check_pkg(out, 'pandas') out = project.salt('conda.install', args=['pandas'], kwargs=kwargs, target='*') out = project.salt('conda.list', kwargs=kwargs) check_pkg(out, 'numpy') check_pkg(out, 'scipy') check_pkg(out, 'pandas')
def test_conda_install_update_remove(): env = 'test_conda_install_update_remove' project = utils.get_test_project() home_dir = '/home/%s' % project.settings['USERNAME'] out = project.salt('cmd.run', args=['"rm -rf {}/anaconda/envs/{}"'.format(home_dir, env)]) out = project.salt('conda.create', args=[env]) kwargs = {'env': env, '--out': 'json', '--out-indent': '-1'} out = project.salt('conda.list', kwargs=kwargs) with pytest.raises(AssertionError): check_pkg(out, 'boto') with pytest.raises(AssertionError): check_pkg(out, 'scikit-learn') out = project.salt('conda.install', args=['boto==2.7.0'], kwargs={'env': env}) out = project.salt('conda.list', kwargs=kwargs) check_pkg(out, 'boto', version='2.7.0') out = project.salt('conda.update', args=['boto'], kwargs={'env': env}) out = project.salt('conda.list', kwargs=kwargs) check_pkg(out, 'boto', version_greater_than='2.7.0') out = project.salt('conda.install', args=['scikit-learn'], kwargs={'env': env}) out = project.salt('conda.list', kwargs=kwargs) check_pkg(out, 'scikit-learn') out = project.salt('conda.remove', args=['scikit-learn'], kwargs={'env': env}, target='compute-1') out = project.salt('conda.list', kwargs=kwargs) check_pkg(out, 'scikit-learn', minion='head') with pytest.raises(AssertionError): check_pkg(out, 'scikit-learn', minion='compute-1') out = project.salt('conda.remove', args=['scikit-learn'], kwargs={'env': env}) out = project.salt('conda.list', kwargs=kwargs) with pytest.raises(AssertionError): check_pkg(out, 'scikit-learn', minion='head') with pytest.raises(AssertionError): check_pkg(out, 'scikit-learn', minion='compute-1')
def test_conda_create_w_pkgs(): import re from collections import Counter env = 'test_conda_create_w_pkgs' project = utils.get_test_project() home_dir = '/home/%s' % project.settings['USERNAME'] out = project.salt('cmd.run', args=['"rm -rf {}/anaconda/envs/{}"'.format(home_dir, env)]) kwargs = {'packages': 'numpy,scipy,pandas'} out = project.salt('conda.create', args=[env], kwargs=kwargs) # Test env dir kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'} test = '"test -e {}/anaconda/envs/{}"'.format(home_dir, env) out = project.salt('cmd.run_all', args=[test], kwargs=kwargs) utils.check_all_cmd_retcode0(out) kwargs = {'env': env, '--out': 'json', '--out-indent': '-1'} out = project.salt('conda.list', kwargs=kwargs) check_pkg(out, 'numpy') check_pkg(out, 'numpy', minion='head') check_pkg(out, 'numpy', minion='compute') check_pkg(out, 'scipy') check_pkg(out, 'pandas')
def test_salt_formulas(): project = utils.get_test_project() kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'} out = project.salt('state.sls', args=['miniconda.status'], kwargs=kwargs) utils.check_all_true(out)
def test_salt_formulas(): project = utils.get_test_project() kwargs = {"test": "true", "--out": "json", "--out-indent": "-1"} out = project.salt("state.sls", args=["cdh5.hdfs.cluster"], kwargs=kwargs) utils.check_all_true(out, none_is_ok=True)