def test_set_config_path(self):
# The good config is the default_config with the credentials path changed
good_config = copy.deepcopy(spark_notebook.config.default_config)
good_config["credentials"]["path"] = self.temp_credentials_file
with app.test_client() as c:
c.get('/?config_path=%s' % self.temp_config_file)
# Test saving the config file to a path that doesn't exist
rv = c.post(url_for('save_config_location'), data=dict(path="bad/path.yaml"),
follow_redirects=True)
assert '<strong>Error:</strong> Base directory bad does not exist.' \
in rv.data.decode('utf-8')
# Test saving the config file to a path that does exist
rv = c.post(url_for('save_config_location'), data=dict(path=self.temp_credentials_file),
follow_redirects=True)
# Make sure there were no errors
assert '<p class="error"><strong>Error:</strong>' not in rv.data.decode('utf-8')
assert '<div class="flash">Credentials saved to %s</div>' % \
self.temp_credentials_file in rv.data.decode('utf-8')
# Read the YAML from temp_config_file
if os.path.isfile(self.temp_config_file):
with open(self.temp_config_file, 'r') as stream:
test_config_yaml = yaml.load(stream)
else:
self.fail("Missing: %s " % self.temp_config_file)
self.assertEqual(test_config_yaml, good_config)
def test_root_redirect(self):
with app.test_client() as c:
# Test the root redirect for a config file that doesn't exist
rv = c.get('/?config_path=%s' % self.temp_config_file)
assert '<a href="%s">%s</a>' % (url_for('save_config_location'),
url_for('save_config_location')) in \
rv.data.decode('utf-8')
# Test the root redirect for a config file that does exist
rv = c.get('/?config_path=./tests/test_files/test_config.yaml')
assert '<a href="%s">%s</a>' % (url_for('accounts'),
url_for('accounts')) in \
rv.data.decode('utf-8')
def test_emr_list_create(self, mock_run_job_flow_expected):
with app.test_client() as c:
c.get('/?config_path=%s' % self.test_config_file)
# Verifying the correct credentials file is being used from the self.test_config_file
rv = c.get(url_for('accounts'))
assert '<!-- ./tests/test_files/test_credentials.yaml -->' in rv.data.decode(
'utf-8')
# Verifying that there are currently not clusters running
rv = c.get(url_for('cluster_list_create', account="test-4"))
assert '<p>No clusters are running.</p>' in rv.data.decode('utf-8')
#
# Test launching a spot cluster with pyspark python 3
#
expected = copy.deepcopy(self.expected)
expected["Name"] = u"cluster-1"
expected["Instances"]["InstanceGroups"] = [{
'InstanceCount':
1,
'Name':
'Master nodes',
'InstanceRole':
'MASTER',
'BidPrice':
'1.0',
'InstanceType':
u'r3.xlarge',
'Market':
'SPOT',
'Configurations':
self.pyspark_python_3
}, {
'InstanceCount':
1,
'Name':
'Core nodes',
'InstanceRole':
'CORE',
'BidPrice':
'1.0',
'InstanceType':
u'r3.xlarge',
'Market':
'SPOT',
'Configurations':
self.pyspark_python_3
}]
# Append bootstrap arg to specify Python 3
expected["BootstrapActions"][0]["ScriptBootstrapAction"][
"Args"].append("3")
mock_run_job_flow_expected.return_value = expected
rv = c.post(url_for('cluster_list_create', account="test-4"),
data=dict(
name="cluster-1",
password="******",
worker_count="1",
subnet_id="subnet-12345678",
instance_type="r3.xlarge",
use_spot="true",
spot_price="1.0",
bootstrap_path="s3://test_bucket/test_script.sh",
pyspark_python_version="3"),
follow_redirects=True)
# Make sure there were no errors
assert '<p class="error"><strong>Error:</strong>' not in rv.data.decode(
'utf-8')
assert '<div class="flash">Cluster launched: cluster-1</div>' in rv.data.decode(
'utf-8')
assert '<p>Launching. Please refresh again later.' in rv.data.decode(
'utf-8')
# Test that cluster-1 is in the cluster list
assert 'Cluster: cluster-1'
#
# Test launching an on-demand cluster with pyspark python 2
#
expected = copy.deepcopy(self.expected)
expected["Name"] = u"cluster-2"
expected["Instances"]["InstanceGroups"] = [{
'InstanceCount': 1,
'Name': 'Master nodes',
'InstanceRole': 'MASTER',
'InstanceType': u'r3.xlarge',
'Market': 'ON_DEMAND'
}, {
'InstanceCount': 1,
'Name': 'Core nodes',
'InstanceRole': 'CORE',
'InstanceType': u'r3.xlarge',
'Market': 'ON_DEMAND'
}]
# Append bootstrap arg to specify Python 2
expected["BootstrapActions"][0]["ScriptBootstrapAction"][
"Args"].append("2")
mock_run_job_flow_expected.return_value = expected
rv = c.post(url_for('cluster_list_create', account="test-4"),
data=dict(
name="cluster-2",
password="******",
worker_count="1",
subnet_id="subnet-12345678",
instance_type="r3.xlarge",
bootstrap_path="s3://test_bucket/test_script.sh",
pyspark_python_version="2"),
follow_redirects=True)
# Make sure there were no errors
assert '<p class="error"><strong>Error:</strong>' not in rv.data.decode(
'utf-8')
assert '<div class="flash">Cluster launched: cluster-2</div>' in rv.data.decode(
'utf-8')
assert '<p>Launching. Please refresh again later.' in rv.data.decode(
'utf-8')
# Test that cluster-1 & cluster-2 are in the cluster list
assert 'Cluster: cluster-1'
assert 'Cluster: cluster-2'
def test_emr_details(self, mock_bootstrap_actions, mock_describe_cluster):
with app.test_client() as c:
c.get('/?config_path=%s' % self.test_config_file)
#
# Test a Staring Cluster
#
cluster_details = {
'Cluster': {
'Id': 'J-starting-cluster',
'Name': 'starting-cluster',
'Status': {
'State': 'STARTING',
},
'MasterPublicDnsName': 'starting.cluster',
}
}
mock_describe_cluster.return_value = cluster_details
rv = c.get(
url_for('cluster_details',
account="test-4",
cluster_id="J-starting-cluster"))
# Make sure there were no errors
assert '<p class="error"><strong>Error:</strong>' not in rv.data.decode(
'utf-8')
# Test the cluster information is correct
assert 'Account: <a href="/g/test-4">test-4</a>' in rv.data.decode(
'utf-8')
assert 'Cluster: starting-cluster' in rv.data.decode('utf-8')
assert 'State: STARTING' in rv.data.decode('utf-8')
# Test that the launching message is displayed
assert '<p>Launching. Please refresh again later.' in rv.data.decode(
'utf-8')
#
# Test a Running Cluster
#
cluster_details = {
'Cluster': {
'Id': 'J-running-cluster',
'Name': 'running-cluster',
'Status': {
'State': 'RUNNING',
},
'MasterPublicDnsName': 'running.cluster',
}
}
bootstrap_actions = {
'BootstrapActions': [{
'Name': 'jupyter-provision',
'Args': ['running-password']
}]
}
mock_describe_cluster.return_value = cluster_details
mock_bootstrap_actions.return_value = bootstrap_actions
rv = c.get(
url_for('cluster_details',
account="test-4",
cluster_id="J-running-cluster"))
# Make sure there were no errors
assert '<p class="error"><strong>Error:</strong>' not in rv.data.decode(
'utf-8')
# Test the cluster information is correct
assert 'Account: <a href="/g/test-4">test-4</a>' in rv.data.decode(
'utf-8')
assert 'Cluster: running-cluster' in rv.data.decode('utf-8')
assert 'State: RUNNING' in rv.data.decode('utf-8')
# Test that the MasterPublicDnsName is returned from cluster_details
assert '<a target="_blank" href="http://running.cluster:8888">' \
'http://running.cluster:8888</a>' in rv.data.decode('utf-8')
# Test that the password is returned from bootstrap_actions
assert '<p><b>Notebook Password:</b> running-password</p>' in rv.data.decode(
'utf-8')
# Test that the SSH command correct
assert '<pre>ssh -i ./tests/test_files/identity_file.pem [email protected]' \
'</pre></p>' in rv.data.decode('utf-8')
#
# Test a Terminated Cluster
#
cluster_details = {
'Cluster': {
'Id': 'J-terminated-cluster',
'Name': 'terminated-cluster',
'Status': {
'State': 'TERMINATED',
},
'MasterPublicDnsName': 'terminated.cluster',
}
}
mock_describe_cluster.return_value = cluster_details
rv = c.get(
url_for('cluster_details',
account="test-4",
cluster_id="J-terminated-cluster"))
# Make sure there were no errors
assert '<p class="error"><strong>Error:</strong>' not in rv.data.decode(
'utf-8')
# Test the cluster information is correct
assert 'Account: <a href="/g/test-4">test-4</a>' in rv.data.decode(
'utf-8')
assert 'Cluster: terminated-cluster' in rv.data.decode('utf-8')
assert 'State: TERMINATED' in rv.data.decode('utf-8')
def test_accounts(self, mock_time, mock_get_hostname):
with app.test_client() as c:
c.get('/?config_path=%s' % self.test_config_file)
# Test that no AWS accounts are present and that test_config_file was read properly
# by verifying the temp credentials file is /tmp/temp_credentials.yaml
rv = c.get(url_for('accounts'))
assert '<p>No AWS accounts found.</p>' in rv.data.decode('utf-8')
assert '<!-- /tmp/temp_credentials.yaml -->' in rv.data.decode('utf-8')
# Test invalid AWS credentials
rv = c.post(url_for('accounts'),
data=dict(name="bad-credentials",
email_address="bad@email",
access_key_id="bad_access_key_id",
secret_access_key="bad_secret_access_key",
ssh_key="existing",
key_name="key_name",
identity_file="./tests/test_files/identity_file.pem"),
follow_redirects=True)
assert '<p class="error"><strong>Error:</strong> Invalid AWS access key id or aws ' \
'secret access key' in rv.data.decode('utf-8')
# Test valid AWS credentials but invalid ssh key_name
rv = c.post(url_for('accounts'),
data=dict(name="test-1",
email_address="test-1@email",
access_key_id="access_key_id",
secret_access_key="secret_access_key",
ssh_key="existing",
key_name="bad_key_name",
identity_file="./tests/test_files/identity_file.pem"),
follow_redirects=True)
assert '<p class="error"><strong>Error:</strong> Key bad_key_name not found on AWS' \
in rv.data.decode('utf-8')
# Test valid AWS credentials but invalid ssh_key file
rv = c.post(url_for('accounts'),
data=dict(name="test-2",
email_address="test-2@email",
access_key_id="access_key_id",
secret_access_key="secret_access_key",
ssh_key="existing",
key_name="key_name",
identity_file="/invalid/ssh_key"),
follow_redirects=True)
assert '<p class="error"><strong>Error:</strong> Key identity file /invalid/ssh_key ' \
'not found' in rv.data.decode('utf-8')
# Test valid AWS credentials and generate ssh key
mock_get_hostname.return_value = "hostname"
mock_time.return_value = "0"
rv = c.post(url_for('accounts'),
data=dict(name="test-3",
email_address="test-3@email",
access_key_id="access_key_id",
secret_access_key="secret_access_key",
ssh_key="generate"),
follow_redirects=True)
# Make sure there were no errors
assert '<p class="error"><strong>Error:</strong>' not in rv.data.decode('utf-8')
# Verify account added message is displayed and the test-3 account is in account list
assert '<div class="flash">Account test-3 added</div>' in rv.data.decode('utf-8')
assert '<li><a href="/g/test-3">test-3</a></li>' in rv.data.decode('utf-8')
# Test valid AWS credentials and valid SSH key
rv = c.post(url_for('accounts'),
data=dict(name="test-4",
email_address="test-4@email",
access_key_id="access_key_id",
secret_access_key="secret_access_key",
ssh_key="existing",
key_name="key_name",
identity_file="./tests/test_files/identity_file.pem"),
follow_redirects=True)
# Make sure there were no errors
assert '<p class="error"><strong>Error:</strong>' not in rv.data.decode('utf-8')
# Verify account added message is displayed and the test-3 & test-4 accounts are in
# account list
assert '<div class="flash">Account test-4 added</div>' in rv.data.decode('utf-8')
assert '<li><a href="/g/test-3">test-3</a></li>' in rv.data.decode('utf-8')
assert '<li><a href="/g/test-4">test-4</a></li>' in rv.data.decode('utf-8')
# Verify the saved temp_credentials_file matches the expected output
if os.path.isfile(self.temp_credentials_file):
with open(self.temp_credentials_file, 'r') as stream:
temp_credentials_yaml = yaml.load(stream)
else:
self.fail("Missing: %s " % self.temp_credentials_file)
if os.path.isfile(self.good_credentials_file):
with open(self.good_credentials_file, 'r') as stream:
good_credentials_yaml = yaml.load(stream)
else:
self.fail("Missing: %s " % self.good_credentials_file)
self.assertEqual(temp_credentials_yaml, good_credentials_yaml)