def test_cleanup_configs(self):
remote = mock.Mock()
instance = mock.Mock()
extra_conf = {'job_cleanup': {
'valid': True,
'script': 'script_text',
'cron': 'cron_text'}}
instance.node_group.node_processes = ["master"]
instance.node_group.id = id
cluster_dict = self._init_cluster_dict('2.2')
cluster = conductor.cluster_create(context.ctx(), cluster_dict)
plugin = pb.PLUGINS.get_plugin(cluster.plugin_name)
plugin._push_cleanup_job(remote, cluster, extra_conf, instance)
remote.write_file_to.assert_called_with(
'/etc/hadoop/tmp-cleanup.sh',
'script_text')
remote.execute_command.assert_called_with(
'sudo sh -c \'echo "cron_text" > /etc/cron.d/spark-cleanup\'')
remote.reset_mock()
instance.node_group.node_processes = ["worker"]
plugin._push_cleanup_job(remote, cluster, extra_conf, instance)
self.assertFalse(remote.called)
remote.reset_mock()
instance.node_group.node_processes = ["master"]
extra_conf['job_cleanup']['valid'] = False
plugin._push_cleanup_job(remote, cluster, extra_conf, instance)
remote.execute_command.assert_called_with(
'sudo rm -f /etc/crond.d/spark-cleanup')
def test_validate(self, mock_utils):
cluster_data = self._get_cluster('cluster', '1.1.0')
cluster = conductor.cluster_create(context.ctx(), cluster_data)
plugin = pb.PLUGINS.get_plugin(cluster.plugin_name)
# number of nimbus nodes != 1 should raise an exception
fake_ng = mock.Mock()
fake_ng.count = 0
mock_ng = mock.Mock(return_value=[fake_ng])
mock_utils.get_node_groups = mock_ng
self.assertRaises(ex.RequiredServiceMissingException, plugin.validate,
cluster)
mock_ng.assert_called_once_with(cluster, "nimbus")
fake_ng.count = 2
self.assertRaises(ex.InvalidComponentCountException, plugin.validate,
cluster)
mock_ng.assert_called_with(cluster, "nimbus")
self.assertEqual(2, mock_ng.call_count)
# no supervisor should raise an exception
fake_nimbus = mock.Mock()
fake_nimbus.count = 1
fake_supervisor = mock.Mock()
fake_supervisor.count = 0
mock_ng = mock.Mock(side_effect=[[fake_nimbus], [fake_supervisor]])
mock_utils.get_node_groups = mock_ng
self.assertRaises(ex.InvalidComponentCountException, plugin.validate,
cluster)
mock_ng.assert_any_call(cluster, "nimbus")
mock_ng.assert_any_call(cluster, "supervisor")
self.assertEqual(2, mock_ng.call_count)
# one nimbus and one or more supervisors should not raise an exception
fake_nimbus.count = 1
fake_supervisor.count = 2
mock_ng = mock.Mock(side_effect=[[fake_nimbus], [fake_supervisor]])
mock_utils.get_node_groups = mock_ng
plugin.validate(cluster)
mock_ng.assert_any_call(cluster, "nimbus")
mock_ng.assert_any_call(cluster, "supervisor")
self.assertEqual(2, mock_ng.call_count)
def test_plugin_edp_engine(self):
cluster_dict = {
'name': 'cluster',
'plugin_name': 'cdh',
'hadoop_version': '5.7.0',
'default_image_id': 'image'}
cluster = conductor.cluster_create(context.ctx(), cluster_dict)
plugin = pb.PLUGINS.get_plugin(cluster.plugin_name)
self.assertIsInstance(
plugin.get_edp_engine(cluster, edp.JOB_TYPE_SPARK),
edp.PluginsSparkJobEngine)
def test_edp_calls_hadoop2_create_dir(self, create_dir):
for version in ['2.7.1']:
cluster_dict = {
'name': 'cluster' + version.replace('.', '_'),
'plugin_name': 'vanilla',
'hadoop_version': version,
'default_image_id': 'image'}
cluster = conductor.cluster_create(context.ctx(), cluster_dict)
plugin = pb.PLUGINS.get_plugin(cluster.plugin_name)
create_dir.reset_mock()
plugin.get_edp_engine(cluster, edp.JOB_TYPE_PIG).create_hdfs_dir(
mock.Mock(), '/tmp')
self.assertEqual(1, create_dir.call_count)
def test_validate_additional_ng_scaling_raises(self):
data = [{
'name': 'master',
'flavor_id': '42',
'count': 1,
'node_processes': ['nimbus']
}, {
'name': 'slave',
'flavor_id': '42',
'count': 1,
'node_processes': ['supervisor']
}, {
'name': 'zookeeper',
'flavor_id': '42',
'count': 1,
'node_processes': ['zookeeper']
}, {
'name': 'master2',
'flavor_id': '42',
'count': 0,
'node_processes': ['nimbus']
}]
cluster_data_101 = self._get_cluster('cluster_1.0.1', '1.0.1')
cluster_data_110 = self._get_cluster('cluster_1.1.0', '1.1.0')
cluster_data_120 = self._get_cluster('cluster_1.2.0', '1.2')
cluster_data_101['node_groups'] = data
cluster_data_110['node_groups'] = data
cluster_data_120['node_groups'] = data
clusters = [cluster_data_101, cluster_data_110, cluster_data_120]
for cluster_data in clusters:
cluster = conductor.cluster_create(context.ctx(), cluster_data)
plugin = pb.PLUGINS.get_plugin(cluster.plugin_name)
master_id = [
node.id for node in cluster.node_groups
if node.name == 'master2'
]
self.assertRaises(ex.NodeGroupCannotBeScaled,
plugin._validate_existing_ng_scaling, cluster,
master_id)
def test_plugin_edp_engine_no_spark(self):
cluster_dict = {
'name': 'cluster',
'plugin_name': 'cdh',
'hadoop_version': '5.7.0',
'default_image_id': 'image'}
job = mock.Mock()
job.type = edp.JOB_TYPE_SPARK
cluster = conductor.cluster_create(context.ctx(), cluster_dict)
plugin = pb.PLUGINS.get_plugin(cluster.plugin_name)
edp_engine = plugin.get_edp_engine(cluster, edp.JOB_TYPE_SPARK)
with testtools.ExpectedException(
ex.InvalidComponentCountException,
value_re="Hadoop cluster should contain 1 "
"SPARK_YARN_HISTORY_SERVER component\(s\). Actual "
"SPARK_YARN_HISTORY_SERVER count is 0\nError ID: .*"):
edp_engine.validate_job_execution(cluster, job, mock.Mock())
def _test_engine(self, version, job_type, eng):
cluster_dict = self._get_cluster('demo', version)
cluster = conductor.cluster_create(context.ctx(), cluster_dict)
plugin = pb.PLUGINS.get_plugin(cluster.plugin_name)
self.assertIsInstance(plugin.get_edp_engine(cluster, job_type), eng)
