Python get_test_project Examples

Example #1

File: test_notebook.py Project: hflamboauto1/datasciencebox

def test_notebook_ui():
    project = utils.get_test_project()

    project = utils.get_test_project()
    head_ip = project.cluster.head.ip

    r = requests.get('http://%s:8888/' % head_ip)
    assert r.status_code == 200

Example #2

File: test_notebook.py Project: aryanet/datasciencebox

def test_notebook_ui():
    project = utils.get_test_project()

    project = utils.get_test_project()
    head_ip = project.cluster.head.ip

    r = requests.get('http://%s:8888/' % head_ip)
    assert r.status_code == 200

Example #3

File: test_impala.py Project: hflamboauto1/datasciencebox

def test_ibis_conn():
    project = utils.get_test_project()

    head_ip = project.cluster.head.ip
    compute_ip = project.cluster.instances[1].ip

    hdfs = ibis.hdfs_connect(host=head_ip, port=50070)
    impala = ibis.impala.connect(compute_ip, hdfs_client=hdfs)

Example #4

File: test_impala.py Project: aryanet/datasciencebox

def test_ibis_conn():
    project = utils.get_test_project()

    head_ip = project.cluster.head.ip
    compute_ip = project.cluster.instances[1].ip

    hdfs = ibis.hdfs_connect(host=head_ip, port=50070)
    impala = ibis.impala.connect(compute_ip, hdfs_client=hdfs)

Example #5

File: test_notebook.py Project: aryanet/datasciencebox

def test_salt_formulas():
    project = utils.get_test_project()

    kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'}
    out = project.salt('state.sls',
                       args=['ipython.notebook'],
                       target='head',
                       kwargs=kwargs)
    utils.check_all_true(out, none_is_ok=True)

Example #6

File: test_hdfs.py Project: aryanet/datasciencebox

def test_namenode_ui():
    '''
    Note: Namenode UI uses a lot of javascript requests alone is not good enough
    '''
    project = utils.get_test_project()
    nn_ip = project.cluster.head.ip

    r = requests.get('http://%s:50070/dfshealth.html#tab-overview' % nn_ip)
    assert r.status_code == 200

Example #7

File: test_hdfs.py Project: hflamboauto1/datasciencebox

def test_namenode_ui():
    """
    Note: Namenode UI uses a lot of javascript requests alone is not good enough
    """
    project = utils.get_test_project()
    nn_ip = project.cluster.head.ip

    r = requests.get("http://%s:50070/dfshealth.html#tab-overview" % nn_ip)
    assert r.status_code == 200

Example #8

def test_zk_conn():
    project = utils.get_test_project()

    zk_ip = project.cluster.head.ip
    zk = KazooClient(hosts='%s:2181' % zk_ip)
    zk.start()
    assert zk

    children = zk.get_children('/')
    assert 'zookeeper' in children

Example #9

File: test_mesos_spark.py Project: aryanet/datasciencebox

def test_hdfs_files():
    project = utils.get_test_project()
    head_ip = project.cluster.head.ip
    hdfs = Client('http://%s:50070' % head_ip)

    root_dirs = hdfs.list('/')
    assert 'spark' in root_dirs

    spark_dirs = hdfs.list('/spark')
    assert 'spark-1.4.1-bin-hadoop2.6.tgz' in spark_dirs

Example #10

File: test_mesos.py Project: aryanet/datasciencebox

def test_mesos_ui():
    '''
    Note 1: Mesos UI uses a lot of javascript requests alone is not good enough
    Note 2: Mesos UI does not bing to 0.0.0.0 so need explicit IP if using vagrant 
    '''
    project = utils.get_test_project()
    nn_ip = project.cluster.head.ip

    r = requests.get('http://%s:5050/' % nn_ip)
    assert r.status_code == 200

Example #11

File: test_mesos_spark.py Project: hflamboauto1/datasciencebox

def test_hdfs_files():
    project = utils.get_test_project()
    head_ip = project.cluster.head.ip
    hdfs = Client('http://%s:50070' % head_ip)

    root_dirs = hdfs.list('/')
    assert 'spark' in root_dirs

    spark_dirs = hdfs.list('/spark')
    assert 'spark-1.4.1-bin-hadoop2.6.tgz' in spark_dirs

Example #12

File: test_zookeeper.py Project: hflamboauto1/datasciencebox

def test_zk_conn():
    project = utils.get_test_project()

    zk_ip = project.cluster.head.ip
    zk = KazooClient(hosts='%s:2181' % zk_ip)
    zk.start()
    assert zk

    children = zk.get_children('/')
    assert 'zookeeper' in children

Example #13

File: test_mesos.py Project: aryanet/datasciencebox

def test_salt_formulas():
    project = utils.get_test_project()

    kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'}
    out = project.salt('state.sls', args=['cdh5.zookeeper.cluster'], target='head', kwargs=kwargs)
    utils.check_all_true(out, none_is_ok=True)

    kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'}
    out = project.salt('state.sls', args=['mesos.cluster'], kwargs=kwargs)
    utils.check_all_true(out, none_is_ok=True)

Example #14

File: test_impala.py Project: aryanet/datasciencebox

def test_hdfs_dirs():
    project = utils.get_test_project()
    head_ip = project.cluster.head.ip
    hdfs = Client('http://%s:50070' % head_ip)

    users_dirs = hdfs.list('/user')
    assert 'hive' in users_dirs
    assert 'impala' in users_dirs

    users_dirs = hdfs.list('/user/hive')
    assert 'warehouse' in users_dirs

Example #15

File: test_impala.py Project: hflamboauto1/datasciencebox

def test_hdfs_dirs():
    project = utils.get_test_project()
    head_ip = project.cluster.head.ip
    hdfs = Client('http://%s:50070' % head_ip)

    users_dirs = hdfs.list('/user')
    assert 'hive' in users_dirs
    assert 'impala' in users_dirs

    users_dirs = hdfs.list('/user/hive')
    assert 'warehouse' in users_dirs

Example #16

File: test_marathon.py Project: hflamboauto1/datasciencebox

def test_marathon_ui():
    '''
    Note 1: Marathon UI uses a lot of javascript requests alone is not good enough
    Note 2: Marathon UI does not bing to 0.0.0.0 so need explicit vagrant IP
    '''
    project = utils.get_test_project()
    nn_ip = project.cluster.head.ip
    nn_ip = '10.10.10.100'

    r = requests.get('http://%s:18080/' % nn_ip)
    assert r.status_code == 200

Example #17

File: test_marathon.py Project: aryanet/datasciencebox

def test_marathon_ui():
    '''
    Note 1: Marathon UI uses a lot of javascript requests alone is not good enough
    Note 2: Marathon UI does not bing to 0.0.0.0 so need explicit vagrant IP
    '''
    project = utils.get_test_project()
    nn_ip = project.cluster.head.ip
    nn_ip = '10.10.10.100'

    r = requests.get('http://%s:18080/' % nn_ip)
    assert r.status_code == 200

Example #18

File: test_hdfs.py Project: hflamboauto1/datasciencebox

def test_hdfs_dirs():
    project = utils.get_test_project()
    nn_ip = project.cluster.head.ip

    hdfs = Client("http://%s:50070" % nn_ip)
    assert hdfs

    root_dirs = hdfs.list("/")
    assert "tmp" in root_dirs
    assert "user" in root_dirs

    users_dirs = hdfs.list("/user")
    assert project.settings["USERNAME"] in users_dirs

Example #19

File: test_hdfs.py Project: aryanet/datasciencebox

def test_hdfs_dirs():
    project = utils.get_test_project()
    nn_ip = project.cluster.head.ip

    hdfs = Client('http://%s:50070' % nn_ip)
    assert hdfs

    root_dirs = hdfs.list('/')
    assert 'tmp' in root_dirs
    assert 'user' in root_dirs

    users_dirs = hdfs.list('/user')
    assert project.settings['USERNAME'] in users_dirs

Example #20

def test_conda_create():
    env = 'test_conda_create'
    project = utils.get_test_project()
    home_dir = '/home/%s' % project.settings['USERNAME']
    out = project.salt('cmd.run', args=['"rm -rf {}/anaconda/envs/{}"'.format(home_dir, env)])

    out = project.salt('conda.create', args=[env])

    # Test env dir
    kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'}
    test = '"test -e {}/anaconda/envs/{}"'.format(home_dir, env)
    out = project.salt('cmd.run_all', args=[test], kwargs=kwargs)
    utils.check_all_cmd_retcode0(out)

Example #21

File: test_impala.py Project: hflamboauto1/datasciencebox

def test_salt_formulas():
    project = utils.get_test_project()

    kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'}
    out = project.salt('state.sls', args=['cdh5.hdfs.cluster'], kwargs=kwargs)
    utils.check_all_true(out, none_is_ok=True)

    kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'}
    out = project.salt('state.sls', args=['cdh5.hive.metastore'], target='head', kwargs=kwargs)
    utils.check_all_true(out, none_is_ok=True)
    #
    kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'}
    out = project.salt('state.sls', args=['cdh5.impala.cluster'], kwargs=kwargs)
    utils.check_all_true(out, none_is_ok=True)

Example #22

def test_conda_install():
    env = 'test_conda_install'
    project = utils.get_test_project()
    home_dir = '/home/%s' % project.settings['USERNAME']
    out = project.salt('cmd.run', args=['"rm -rf {}/anaconda/envs/{}"'.format(home_dir, env)])

    out = project.salt('conda.create', args=[env])

    kwargs = {'env': env, '--out': 'json', '--out-indent': '-1'}
    out = project.salt('conda.list', kwargs=kwargs)
    with pytest.raises(AssertionError):
        check_pkg(out, 'numpy')
    with pytest.raises(AssertionError):
        check_pkg(out, 'scipy')
    with pytest.raises(AssertionError):
        check_pkg(out, 'pandas')

    out = project.salt('conda.install', args=['numpy==1.7.1'], kwargs={'env': env})
    out = project.salt('conda.list', kwargs=kwargs)
    check_pkg(out, 'numpy', version='1.7.1')
    with pytest.raises(AssertionError):
        check_pkg(out, 'scipy')
    with pytest.raises(AssertionError):
        check_pkg(out, 'pandas')

    out = project.salt('conda.install', args=['scipy'], kwargs={'env': env}, target='head')
    out = project.salt('conda.list', kwargs=kwargs)
    check_pkg(out, 'numpy')
    check_pkg(out, 'scipy', minion='head')
    with pytest.raises(AssertionError):
        check_pkg(out, 'scipy', minion='compute-1')
    with pytest.raises(AssertionError):
        check_pkg(out, 'pandas')

    out = project.salt('conda.install', args=['scipy'], kwargs={'env': env})
    out = project.salt('conda.list', kwargs=kwargs)
    check_pkg(out, 'numpy')
    check_pkg(out, 'scipy', minion='head')
    check_pkg(out, 'scipy', minion='compute-1')
    with pytest.raises(AssertionError):
        check_pkg(out, 'pandas')

    out = project.salt('conda.install', args=['pandas'], kwargs=kwargs, target='*')
    out = project.salt('conda.list', kwargs=kwargs)
    check_pkg(out, 'numpy')
    check_pkg(out, 'scipy')
    check_pkg(out, 'pandas')

Example #23

File: test_impala.py Project: aryanet/datasciencebox

def test_salt_formulas():
    project = utils.get_test_project()

    kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'}
    out = project.salt('state.sls', args=['cdh5.hdfs.cluster'], kwargs=kwargs)
    utils.check_all_true(out, none_is_ok=True)

    kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'}
    out = project.salt('state.sls',
                       args=['cdh5.hive.metastore'],
                       target='head',
                       kwargs=kwargs)
    utils.check_all_true(out, none_is_ok=True)
    #
    kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'}
    out = project.salt('state.sls',
                       args=['cdh5.impala.cluster'],
                       kwargs=kwargs)
    utils.check_all_true(out, none_is_ok=True)

Example #24

def test_conda_install_update_remove():
    env = 'test_conda_install_update_remove'
    project = utils.get_test_project()
    home_dir = '/home/%s' % project.settings['USERNAME']
    out = project.salt('cmd.run', args=['"rm -rf {}/anaconda/envs/{}"'.format(home_dir, env)])

    out = project.salt('conda.create', args=[env])

    kwargs = {'env': env, '--out': 'json', '--out-indent': '-1'}
    out = project.salt('conda.list', kwargs=kwargs)
    with pytest.raises(AssertionError):
        check_pkg(out, 'boto')
    with pytest.raises(AssertionError):
        check_pkg(out, 'scikit-learn')

    out = project.salt('conda.install', args=['boto==2.7.0'], kwargs={'env': env})
    out = project.salt('conda.list', kwargs=kwargs)
    check_pkg(out, 'boto', version='2.7.0')

    out = project.salt('conda.update', args=['boto'], kwargs={'env': env})
    out = project.salt('conda.list', kwargs=kwargs)
    check_pkg(out, 'boto', version_greater_than='2.7.0')

    out = project.salt('conda.install', args=['scikit-learn'], kwargs={'env': env})
    out = project.salt('conda.list', kwargs=kwargs)
    check_pkg(out, 'scikit-learn')

    out = project.salt('conda.remove',
                       args=['scikit-learn'],
                       kwargs={'env': env},
                       target='compute-1')
    out = project.salt('conda.list', kwargs=kwargs)
    check_pkg(out, 'scikit-learn', minion='head')
    with pytest.raises(AssertionError):
        check_pkg(out, 'scikit-learn', minion='compute-1')

    out = project.salt('conda.remove', args=['scikit-learn'], kwargs={'env': env})
    out = project.salt('conda.list', kwargs=kwargs)
    with pytest.raises(AssertionError):
        check_pkg(out, 'scikit-learn', minion='head')
    with pytest.raises(AssertionError):
        check_pkg(out, 'scikit-learn', minion='compute-1')

Example #25

def test_conda_create_w_pkgs():
    import re
    from collections import Counter
    env = 'test_conda_create_w_pkgs'
    project = utils.get_test_project()
    home_dir = '/home/%s' % project.settings['USERNAME']
    out = project.salt('cmd.run', args=['"rm -rf {}/anaconda/envs/{}"'.format(home_dir, env)])

    kwargs = {'packages': 'numpy,scipy,pandas'}
    out = project.salt('conda.create', args=[env], kwargs=kwargs)

    # Test env dir
    kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'}
    test = '"test -e {}/anaconda/envs/{}"'.format(home_dir, env)
    out = project.salt('cmd.run_all', args=[test], kwargs=kwargs)
    utils.check_all_cmd_retcode0(out)

    kwargs = {'env': env, '--out': 'json', '--out-indent': '-1'}
    out = project.salt('conda.list', kwargs=kwargs)
    check_pkg(out, 'numpy')
    check_pkg(out, 'numpy', minion='head')
    check_pkg(out, 'numpy', minion='compute')
    check_pkg(out, 'scipy')
    check_pkg(out, 'pandas')

Example #26

File: test_notebook.py Project: hflamboauto1/datasciencebox

def test_salt_formulas():
    project = utils.get_test_project()

    kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'}
    out = project.salt('state.sls', args=['ipython.notebook'], target='head', kwargs=kwargs)
    utils.check_all_true(out, none_is_ok=True)

Example #27

def test_salt_formulas():
    project = utils.get_test_project()

    kwargs = {'test': 'true', '--out': 'json', '--out-indent': '-1'}
    out = project.salt('state.sls', args=['miniconda.status'], kwargs=kwargs)
    utils.check_all_true(out)

Example #28

File: test_hdfs.py Project: hflamboauto1/datasciencebox

def test_salt_formulas():
    project = utils.get_test_project()

    kwargs = {"test": "true", "--out": "json", "--out-indent": "-1"}
    out = project.salt("state.sls", args=["cdh5.hdfs.cluster"], kwargs=kwargs)
    utils.check_all_true(out, none_is_ok=True)