def test_create_with_interface(self):
self.mock_job_binary_list.return_value = []
self.mock_job_types_list.return_value = self.job_types.list()
form_data = {'job_name': 'test_interface',
'job_type': 'pig',
'lib_binaries': [],
'lib_ids': '[]',
'job_description': 'test create',
'hidden_arguments_field': [],
'argument_ids': '["0", "1"]',
'argument_id_0': '0',
'argument_name_0': 'argument',
'argument_description_0': '',
'argument_mapping_type_0': 'args',
'argument_location_0': '0',
'argument_value_type_0': 'number',
'argument_required_0': True,
'argument_default_value_0': '',
'argument_id_1': '1',
'argument_name_1': 'config',
'argument_description_1': 'Really great config',
'argument_mapping_type_1': 'configs',
'argument_location_1': 'edp.important.config',
'argument_value_type_1': 'string',
'argument_default_value_1': 'A value'}
url = reverse('horizon:project:data_processing.jobs:create-job')
res = self.client.post(url, form_data)
self.assertNoFormErrors(res)
self.assert_mock_multiple_calls_with_same_arguments(
self.mock_job_binary_list, 2,
mock.call(test.IsHttpRequest()))
self.mock_job_types_list.assert_called_once_with(
test.IsHttpRequest())
self.mock_job_create.assert_called_once_with(
test.IsHttpRequest(),
'test_interface', 'Pig', [], [], 'test create',
interface=[
{
"name": "argument",
"description": None,
"mapping_type": "args",
"location": "0",
"value_type": "number",
"required": True,
"default": None
},
{
"name": "config",
"description": "Really great config",
"mapping_type": "configs",
"location": "edp.important.config",
"value_type": "string",
"required": False,
"default": "A value"
}],
is_public=False,
is_protected=False)
def test_delete(self):
job = self.jobs.first()
self.mock_job_list.return_value = self.jobs.list()
form_data = {'action': 'job_templates__delete__%s' % job.id}
res = self.client.post(INDEX_URL, form_data)
self.assertNoFormErrors(res)
self.assertRedirectsNoFollow(res, INDEX_URL)
self.assertMessageCount(success=1)
self.mock_job_list.assert_called_once_with(
test.IsHttpRequest(), {})
self.mock_job_delete.assert_called_once_with(
test.IsHttpRequest(), job.id)
def test_delete(self):
ngt = self.nodegroup_templates.first()
self.mock_nodegroup_template_list.return_value = \
self.nodegroup_templates.list()
self.mock_nodegroup_template_delete.return_value = None
form_data = {'action': 'nodegroup_templates__delete__%s' % ngt.id}
res = self.client.post(INDEX_URL, form_data)
self.assertNoFormErrors(res)
self.assertRedirectsNoFollow(res, INDEX_URL)
self.assertMessageCount(success=1)
self.mock_nodegroup_template_list.assert_called_once_with(
test.IsHttpRequest(), {})
self.mock_nodegroup_template_delete.assert_called_once_with(
test.IsHttpRequest(), ngt.id)
def _setup_copy_test(self, service_checker):
service_checker.return_value = True
ngt = self.nodegroup_templates.first()
configs = self.plugins_configs.first()
self.mock_extension_supported.return_value = True
self.mock_availability_zone_list.return_value = \
self.availability_zones.list()
self.mock_volume_type_list.return_value = []
self.mock_nodegroup_template_get.return_value = ngt
self.mock_plugin_get_version_details.return_value = configs
self.mock_floating_ip_pools_list.return_value = []
self.mock_security_group_list.return_value = []
url = reverse('horizon:project:data_processing.clusters:copy',
args=[ngt.id])
res = self.client.get(url)
self.mock_availability_zone_list.assert_called_once_with(
test.IsHttpRequest())
self.mock_availability_zone_list.assert_called_once_with(
test.IsHttpRequest())
self.mock_volume_type_list.assert_called_once_with(
test.IsHttpRequest())
self.mock_nodegroup_template_get.assert_called_once_with(
test.IsHttpRequest(), ngt.id)
self.assert_mock_multiple_calls_with_same_arguments(
self.mock_plugin_get_version_details, 6,
mock.call(test.IsHttpRequest(), ngt.plugin_name,
ngt.hadoop_version))
self.mock_floating_ip_pools_list.assert_called_once_with(
test.IsHttpRequest())
self.mock_security_group_list.assert_called_once_with(
test.IsHttpRequest())
return ngt, configs, res
def test_details(self):
self.mock_plugin_get.return_value = self.plugins.list()[0]
res = self.client.get(DETAILS_URL)
self.assert_mock_multiple_calls_with_same_arguments(
self.mock_plugin_get, 2,
mock.call(test.IsHttpRequest(), test.IsA(six.text_type)))
self.assertTemplateUsed(res, 'horizon/common/_detail.html')
self.assertContains(res, 'vanilla')
self.assertContains(res, 'plugin')
def test_index(self):
self.mock_cluster_template_list.return_value = \
self.cluster_templates.list()
res = self.client.get(INDEX_URL)
self.mock_cluster_template_list.assert_called_once_with(
test.IsHttpRequest(), {})
self.assertTemplateUsed(res, 'clusters/index.html')
self.assertContains(res, 'Cluster Templates')
self.assertContains(res, 'Name')
def test_details(self):
self.mock_job_get.return_value = self.jobs.first()
res = self.client.get(DETAILS_URL)
self.mock_job_get.assert_called_with(
test.IsHttpRequest(), 'id')
self.assertTemplateUsed(res, 'horizon/common/_detail.html')
self.assertContains(res, 'pigjob')
self.assertContains(res, 'example.pig')
self.assertContains(res, 'udf.jar')
def test_index(self):
self.mock_job_list.return_value = self.jobs.list()
res = self.client.get(INDEX_URL)
self.mock_job_list.assert_called_once_with(
test.IsHttpRequest(), {})
self.assertTemplateUsed(res, 'jobs/index.html')
self.assertContains(res, 'Job Templates')
self.assertContains(res, 'Name')
self.assertContains(res, 'pigjob')
def test_update(self):
ct = self.cluster_templates.first()
ngts = self.nodegroup_templates.list()
configs = self.plugins_configs.first()
new_name = "UpdatedName"
new_ct = copy.copy(ct)
new_ct.name = new_name
self.mock_cluster_template_get.return_value = ct
self.mock_plugin_get_version_details.return_value = configs
self.mock_nodegroup_template_find.return_value = ngts
self.mock_cluster_template_update.return_value = new_ct
url = reverse('horizon:project:data_processing.clusters:ct-edit',
args=[ct.id])
def serialize(obj):
return utils.serialize(jsonutils.dump_as_bytes(obj))
res = self.client.post(
url, {
'ct_id': ct.id,
'cluster_template_name': new_name,
'plugin_name': ct.plugin_name,
'hadoop_version': ct.hadoop_version,
'description': ct.description,
'hidden_configure_field': "",
'template_id_0': ct.node_groups[0]['node_group_template_id'],
'group_name_0': ct.node_groups[0]['name'],
'count_0': 1,
'serialized_0': serialize(ct.node_groups[0]),
'template_id_1': ct.node_groups[1]['node_group_template_id'],
'group_name_1': ct.node_groups[1]['name'],
'count_1': 2,
'serialized_1': serialize(ct.node_groups[1]),
'forms_ids': "[0,1]",
})
self.assertNoFormErrors(res)
self.assertRedirectsNoFollow(res, INDEX_URL)
self.assertMessageCount(success=1)
self.mock_cluster_template_update.assert_called_once_with(
request=test.IsHttpRequest(),
ct_id=ct.id,
name=new_name,
plugin_name=ct.plugin_name,
hadoop_version=ct.hadoop_version,
description=ct.description,
cluster_configs=ct.cluster_configs,
node_groups=ct.node_groups,
anti_affinity=ct.anti_affinity,
use_autoconfig=False,
shares=ct.shares,
is_public=False,
is_protected=False,
domain_name=ct.domain_name)
def test_details(self):
flavor = self.flavors.first()
ngt = self.nodegroup_templates.first()
self.mock_flavor_get.return_value = flavor
self.mock_nodegroup_template_get.return_value = ngt
res = self.client.get(DETAILS_URL)
self.assertTemplateUsed(res, 'horizon/common/_detail.html')
self.assertContains(res, 'sample-template')
self.assert_mock_multiple_calls_with_same_arguments(
self.mock_flavor_get, 1, mock.call(test.IsHttpRequest(),
flavor.id))
def test_index(self):
self.mock_nodegroup_template_list.return_value = \
self.nodegroup_templates.list()
res = self.client.get(INDEX_URL +
"?tab=cluster_tabs__node_group_templates_tab")
self.assertTemplateUsed(res, 'clusters/index.html')
self.assertContains(res, 'Node Group Templates')
self.assertContains(res, 'Name')
self.assertContains(res, 'Plugin')
self.mock_nodegroup_template_list.assert_called_once_with(
test.IsHttpRequest(), {})
def test_create(self):
self.mock_job_binary_list.return_value = []
self.mock_job_types_list.return_value = self.job_types.list()
form_data = {'job_name': 'test',
'job_type': 'pig',
'lib_binaries': [],
'lib_ids': '[]',
'job_description': 'test create',
'hidden_arguments_field': [],
'argument_ids': '[]'}
url = reverse('horizon:project:data_processing.jobs:create-job')
res = self.client.post(url, form_data)
self.assertNoFormErrors(res)
self.assert_mock_multiple_calls_with_same_arguments(
self.mock_job_binary_list, 2,
mock.call(test.IsHttpRequest()))
self.mock_job_types_list.assert_called_once_with(
test.IsHttpRequest())
self.mock_job_create.assert_called_once_with(
test.IsHttpRequest(), 'test', 'Pig', [], [],
'test create', interface=[], is_public=False,
is_protected=False)
def test_copy(self):
ct = self.cluster_templates.first()
ngts = self.nodegroup_templates.list()
configs = self.plugins_configs.first()
self.mock_cluster_template_get.return_value = ct
self.mock_plugin_get_version_details.return_value = configs
self.mock_nodegroup_template_find.return_value = ngts
url = reverse('horizon:project:data_processing.clusters:ct-copy',
args=[ct.id])
res = self.client.get(url)
self.mock_cluster_template_get.assert_called_once_with(
test.IsHttpRequest(), ct.id)
workflow = res.context['workflow']
step = workflow.get_step("generalconfigaction")
self.assertEqual(step.action['cluster_template_name'].field.initial,
ct.name + "-copy")
def test_update(self, mock_workflow, service_checker):
service_checker.return_value = True
flavor = self.flavors.first()
ngt = self.nodegroup_templates.first()
configs = self.plugins_configs.first()
new_name = ngt.name + '-updated'
UPDATE_URL = reverse('horizon:project:data_processing.clusters:edit',
kwargs={'template_id': ngt.id})
mock_workflow.return_value = {}
self.mock_extension_supported.return_value = True
self.mock_availability_zone_list.return_value = \
self.availability_zones.list()
self.mock_volume_type_list.return_value = []
self.mock_flavor_list.return_value = [flavor]
self.mock_plugin_get_version_details.return_value = configs
self.mock_floating_ip_pools_list.return_value = []
self.mock_security_group_list.return_value = []
self.mock_nodegroup_template_get.return_value = ngt
self.mock_nodegroup_template_update.return_value = True
res = self.client.post(
UPDATE_URL, {
'ng_id': ngt.id,
'nodegroup_name': new_name,
'plugin_name': ngt.plugin_name,
ngt.plugin_name + '_version': '1.2.1',
'hadoop_version': ngt.hadoop_version,
'description': ngt.description,
'flavor': flavor.id,
'availability_zone': None,
'storage': 'ephemeral_drive',
'volumes_per_node': 0,
'volumes_size': 0,
'volume_type': None,
'volume_local_to_instance': False,
'volumes_availability_zone': None,
'floating_ip_pool': None,
'is_proxy_gateway': False,
'security_autogroup': True,
'processes': 'HDFS:namenode',
'use_autoconfig': True
})
self.assertNoFormErrors(res)
self.assertRedirectsNoFollow(res, INDEX_URL)
self.assertMessageCount(success=1)
self.mock_extension_supported.assert_called_once_with(
test.IsHttpRequest(), 'AvailabilityZones')
self.mock_availability_zone_list.assert_called_once_with(
test.IsHttpRequest())
self.mock_volume_type_list.assert_called_once_with(
test.IsHttpRequest())
self.mock_flavor_list.assert_called_once_with(test.IsHttpRequest())
self.assert_mock_multiple_calls_with_same_arguments(
self.mock_plugin_get_version_details, 5,
mock.call(test.IsHttpRequest(), ngt.plugin_name,
ngt.hadoop_version))
self.mock_floating_ip_pools_list.assert_called_once_with(
test.IsHttpRequest())
self.mock_security_group_list.assert_called_once_with(
test.IsHttpRequest())
self.mock_nodegroup_template_get.assert_called_once_with(
test.IsHttpRequest(), ngt.id)
self.mock_nodegroup_template_update.assert_called_once_with(
request=test.IsHttpRequest(),
ngt_id=ngt.id,
name=new_name,
plugin_name=ngt.plugin_name,
hadoop_version=ngt.hadoop_version,
flavor_id=flavor.id,
description=ngt.description,
volumes_per_node=0,
volumes_size=None,
volume_type=None,
volume_local_to_instance=False,
volumes_availability_zone=None,
node_processes=['namenode'],
node_configs={},
floating_ip_pool=None,
security_groups=[],
auto_security_group=True,
availability_zone=None,
use_autoconfig=True,
is_proxy_gateway=False,
shares=[],
is_protected=False,
is_public=False,
image_id=ngt.image_id,
)
def test_create(self, mock_workflow, service_checker):
service_checker.return_value = True
mock_workflow.return_value = {}
flavor = self.flavors.first()
ngt = self.nodegroup_templates.first()
configs = self.plugins_configs.first()
new_name = ngt.name + '-new'
self.mock_extension_supported.return_value = True
self.mock_availability_zone_list.return_value = \
self.availability_zones.list()
self.mock_volume_type_list.return_value = []
self.mock_flavor_list.return_value = [flavor]
self.mock_plugin_get_version_details.return_value = configs
self.mock_floating_ip_pools_list.return_value = []
self.mock_security_group_list.return_value = []
self.mock_nodegroup_template_create.return_value = True
res = self.client.post(
CREATE_URL, {
'nodegroup_name': new_name,
'plugin_name': ngt.plugin_name,
ngt.plugin_name + '_version': '1.2.1',
'hadoop_version': ngt.hadoop_version,
'description': ngt.description,
'flavor': flavor.id,
'availability_zone': None,
'storage': 'ephemeral_drive',
'volumes_per_node': 0,
'volumes_size': 0,
'volume_type': None,
'volume_local_to_instance': False,
'volumes_availability_zone': None,
'floating_ip_pool': None,
'security_autogroup': True,
'processes': 'HDFS:namenode',
'use_autoconfig': True,
'shares': [],
'is_public': False,
'is_protected': False
})
self.assertNoFormErrors(res)
self.assertRedirectsNoFollow(res, INDEX_URL)
self.assertMessageCount(success=1)
self.mock_extension_supported.assert_called_once_with(
test.IsHttpRequest(), 'AvailabilityZones')
self.mock_availability_zone_list.assert_called_once_with(
test.IsHttpRequest())
self.mock_flavor_list.assert_called_once_with(test.IsHttpRequest())
self.assert_mock_multiple_calls_with_same_arguments(
self.mock_plugin_get_version_details, 4,
mock.call(test.IsHttpRequest(), ngt.plugin_name,
ngt.hadoop_version))
self.mock_floating_ip_pools_list.assert_called_once_with(
test.IsHttpRequest())
self.mock_security_group_list.assert_called_once_with(
test.IsHttpRequest())
self.mock_nodegroup_template_create(
test.IsHttpRequest(), **{
'name': new_name,
'plugin_name': ngt.plugin_name,
'hadoop_version': ngt.hadoop_version,
'description': ngt.description,
'flavor_id': flavor.id,
'volumes_per_node': None,
'volumes_size': None,
'volume_type': None,
'volume_local_to_instance': False,
'volumes_availability_zone': None,
'node_processes': ['namenode'],
'node_configs': {},
'floating_ip_pool': None,
'security_groups': [],
'image_id': None,
'auto_security_group': True,
'availability_zone': None,
'is_proxy_gateway': False,
'use_autoconfig': True,
'shares': [],
'is_public': False,
'is_protected': False
})
def test_launch(self):
job = self.jobs.first()
job_execution = self.job_executions.first()
cluster = self.clusters.first()
input_ds = self.data_sources.first()
output_ds = self.data_sources.first()
self.mock_job_get.return_value = job
self.mock_job_get_configs.return_value = job
self.mock_cluster_list.return_value = self.clusters.list()
self.mock_data_source_list.return_value = self.data_sources.list()
self.mock_job_list.return_value = self.jobs.list()
self.mock_job_get.return_value = job
self.mock_job_execution_create.return_value = job_execution
url = reverse('horizon:project:data_processing.jobs:launch-job')
form_data = {
'job': self.jobs.first().id,
'cluster': cluster.id,
'job_input': input_ds.id,
'job_output': output_ds.id,
'config': {},
'argument_ids': '{}',
'adapt_oozie': 'on',
'adapt_swift_spark': 'on',
'hbase_common_lib': 'on',
'hbase_common_lib': 'on',
'datasource_substitute': 'on',
'java_opts': '',
'job_args_array': [[], []],
'job_configs': [{}, {}],
'job_params': [{}, {}],
'job_type': 'Pig',
'streaming_mapper': '',
'streaming_reducer': ''
}
res = self.client.post(url, form_data)
# there seem not to be an easy way to check the order
# of the calls; check only if they happened
self.assertNoFormErrors(res)
self.assert_mock_multiple_calls_with_same_arguments(
self.mock_job_get, 2,
mock.call(test.IsHttpRequest(), job.id))
self.mock_job_get_configs.assert_called_once_with(
test.IsHttpRequest(), job.type)
self.assert_mock_multiple_calls_with_same_arguments(
self.mock_cluster_list, 2,
mock.call(test.IsHttpRequest()))
self.mock_data_source_list.assert_called_with(
test.IsHttpRequest())
self.mock_job_list.assert_called_once_with(
test.IsHttpRequest())
self.mock_job_execution_create.assert_called_once_with(
test.IsHttpRequest(),
job.id,
cluster.id,
input_ds.id,
output_ds.id,
{
'params': {},
'args': [],
'configs': {
'edp.substitute_data_source_for_name': True,
'edp.substitute_data_source_for_uuid': True}
},
{},
is_public=False,
is_protected=False)