def test_user_name_fail(self, cmd_mock, tar_mock, os_mock, shutil_mock,
spur_ssh, hdfs_client_mock, post_mock, put_mock,
exec_ssh_mock, os_sys_mock, dt_mock, hive_mock,
hbase_mock):
class Resp(object):
status_code = 201
def json(self):
return {'id': 'someid'}
post_mock.return_value = Resp()
cmd_mock.return_value = (0, 'dev')
with patch("__builtin__.open", mock_open(read_data="[]")):
creator = ApplicationCreator(self.config, self.environment,
self.service)
try:
creator.create_application('abcd', self.package_metadata,
'appname',
self.property_overrides_no_user)
self.assertFail(
'Expected FailedCreation exception but was not thrown')
except FailedCreation as ex:
self.assertEqual(
ex.msg,
'User somebody does not exist. Verify that this user account exists on the machine running the deployment manager.'
)
def test_app_name_fail(self, tar_mock, os_mock, shutil_mock, spur_ssh,
hdfs_client_mock, post_mock, put_mock,
exec_ssh_mock, os_sys_mock, dt_mock, hive_mock,
hbase_mock):
class Resp(object):
status_code = 201
def json(self):
return {'id': 'someid'}
post_mock.return_value = Resp()
with patch("__builtin__.open", mock_open(read_data="[]")):
creator = ApplicationCreator(self.config, self.environment,
self.service)
try:
creator.create_application('abcd', self.package_metadata,
'with&oddchar',
self.property_overrides)
self.assertFail(
'Expected FailedCreation exception but was not thrown')
except FailedCreation as ex:
self.assertEqual(
ex.msg,
'Application name with&oddchar may only contain a-z A-Z 0-9 - _'
)
def test_fail_create_application(self, cmd_mock, tar_mock, os_mock,
shutil_mock, spur_ssh, hdfs_client_mock,
post_mock, put_mock, exec_ssh_mock,
os_sys_mock, dt_mock, hive_mock,
hbase_mock):
dt_mock.utcnow.return_value = (datetime(2013, 01, 01))
class Resp(object):
status_code = 400
headers = {'oozie-error-message': 'oozie error!'}
def json(self):
return {'id': 'someid'}
default_properties = {"oozie": {"componentA": {"spark_version": "1"}}}
override_properties = {
"user": "******",
"oozie": {
"componentA": {
"spark_version": "2"
}
}
}
post_mock.return_value = Resp()
cmd_mock.return_value = (0, 'dev')
with patch("__builtin__.open", mock_open(read_data="[]")):
creator = ApplicationCreator(self.config, self.environment,
self.service)
self.assertRaises(FailedValidation,
creator.assert_application_properties,
override_properties, default_properties)
self.assertRaises(FailedCreation, creator.create_application,
'abcd', self.package_metadata_2, 'aname',
self.property_overrides)
def test_validate_package(self):
creator = ApplicationCreator(self.config, self.environment,
self.service)
result = {}
try:
creator.validate_package(self.package_metadata['package_name'],
self.package_metadata)
except FailedValidation as ex:
result = ex.msg
expected_report = {
'oozie': {
'componentA': ['missing file workflow.xml'],
'componentB': ['missing file workflow.xml']
},
'sparkStreaming': {
'componentC': [
'missing file yarn-kill.py',
'missing file application.properties',
'missing file upstart.conf',
'missing file log4j.properties'
]
}
}
self.assertEqual(result, expected_report)
def test_create_application(self, dist_mock, copy_mock, tar_mock, os_mock,
shutil_mock, spur_ssh, hdfs_client_mock,
post_mock, put_mock, exec_ssh_mock,
os_sys_mock, dt_mock, hive_mock, hbase_mock):
dt_mock.utcnow.return_value = (datetime(2013, 01, 01))
class Resp(object):
status_code = 201
def json(self):
return {'id': 'someid'}
post_mock.return_value = Resp()
dist_mock.return_value = 'ubuntu'
with patch("__builtin__.open", mock_open(read_data="[]")):
creator = ApplicationCreator(self.config, self.environment,
self.service)
creator.create_application('abcd', self.package_metadata, 'aname',
self.property_overrides)
print post_mock.call_args_list
# pylint: disable=line-too-long
post_mock.assert_any_call(
'oozie/v1/jobs',
data=
'<?xml version="1.0" encoding="UTF-8" ?><configuration><property><name>environment_cluster_private_key</name><value>keyfile.pem</value></property><property><name>environment_hbase_thrift_server</name><value>hbasehost</value></property><property><name>environment_webhdfs_host</name><value>webhdfshost</value></property><property><name>environment_opentsdb</name><value>1.2.3.5:1234</value></property><property><name>environment_yarn_node_managers</name><value>nm1,nm2</value></property><property><name>environment_webhdfs_port</name><value>webhdfsport</value></property><property><name>environment_hbase_rest_server</name><value>hbasehost</value></property><property><name>environment_oozie_uri</name><value>oozie</value></property><property><name>environment_hbase_rest_port</name><value>123</value></property><property><name>environment_cluster_root_user</name><value>root_user</value></property><property><name>environment_hive_port</name><value>124</value></property><property><name>environment_name_node</name><value>namenode</value></property><property><name>environment_hive_server</name><value>hivehost</value></property><property><name>component_property3</name><value>3</value></property><property><name>component_property4</name><value>nine</value></property><property><name>component_application</name><value>aname</value></property><property><name>component_name</name><value>componentA</value></property><property><name>component_job_name</name><value>aname-componentA-job</value></property><property><name>component_hdfs_root</name><value>/user/aname/componentA</value></property><property><name>deployment_start</name><value>2013-01-01T00:02Z</value></property><property><name>deployment_end</name><value>2013-01-08T00:02Z</value></property><property><name>user.name</name><value>hdfs</value></property><property><name>oozie.use.system.libpath</name><value>true</value></property><property><name>oozie.libpath</name><value>/user/deployment/platform</value></property><property><name>oozie.wf.application.path</name><value>namenode/user/aname/componentA</value></property></configuration>',
headers={'Content-Type': 'application/xml'})
post_mock.assert_any_call(
'oozie/v1/jobs',
data=
'<?xml version="1.0" encoding="UTF-8" ?><configuration><property><name>environment_cluster_private_key</name><value>keyfile.pem</value></property><property><name>environment_hbase_thrift_server</name><value>hbasehost</value></property><property><name>environment_webhdfs_host</name><value>webhdfshost</value></property><property><name>environment_opentsdb</name><value>1.2.3.5:1234</value></property><property><name>environment_yarn_node_managers</name><value>nm1,nm2</value></property><property><name>environment_webhdfs_port</name><value>webhdfsport</value></property><property><name>environment_hbase_rest_server</name><value>hbasehost</value></property><property><name>environment_oozie_uri</name><value>oozie</value></property><property><name>environment_hbase_rest_port</name><value>123</value></property><property><name>environment_cluster_root_user</name><value>root_user</value></property><property><name>environment_hive_port</name><value>124</value></property><property><name>environment_name_node</name><value>namenode</value></property><property><name>environment_hive_server</name><value>hivehost</value></property><property><name>component_application</name><value>aname</value></property><property><name>component_name</name><value>componentB</value></property><property><name>component_job_name</name><value>aname-componentB-job</value></property><property><name>component_hdfs_root</name><value>/user/aname/componentB</value></property><property><name>deployment_start</name><value>2013-01-01T00:02Z</value></property><property><name>deployment_end</name><value>2013-01-08T00:02Z</value></property><property><name>user.name</name><value>hdfs</value></property><property><name>oozie.use.system.libpath</name><value>true</value></property><property><name>oozie.libpath</name><value>/user/deployment/platform</value></property><property><name>oozie.wf.application.path</name><value>namenode/user/aname/componentB</value></property></configuration>',
headers={'Content-Type': 'application/xml'})
put_mock.assert_any_call('oozie/v1/job/someid?action=suspend')
exec_ssh_mock.assert_any_call(
'localhost', 'root_user', 'keyfile.pem', [
'mkdir -p /tmp/ns/aname/componentC',
'sudo mkdir -p /opt/ns/aname/componentC'
])
exec_ssh_mock.assert_any_call('nm1', 'root_user', 'keyfile.pem',
['mkdir -p /tmp/ns/aname/componentC'])
exec_ssh_mock.assert_any_call('nm1', 'root_user', 'keyfile.pem', [
'sudo mkdir -p /opt/ns/aname/componentC',
'sudo mv /tmp/ns/aname/componentC/log4j.properties /opt/ns/aname/componentC/log4j.properties'
])
exec_ssh_mock.assert_any_call('nm2', 'root_user', 'keyfile.pem',
['mkdir -p /tmp/ns/aname/componentC'])
exec_ssh_mock.assert_any_call('nm2', 'root_user', 'keyfile.pem', [
'sudo mkdir -p /opt/ns/aname/componentC',
'sudo mv /tmp/ns/aname/componentC/log4j.properties /opt/ns/aname/componentC/log4j.properties'
])
exec_ssh_mock.assert_any_call('localhost', 'root_user', 'keyfile.pem', [
'sudo cp /tmp/ns/aname/componentC/upstart.conf.tpl /etc/init/ns-aname-componentC.conf',
'sudo cp /tmp/ns/aname/componentC/* /opt/ns/aname/componentC',
'sudo chmod a+x /opt/ns/aname/componentC/yarn-kill.py',
'cd /opt/ns/aname/componentC && sudo jar uf abc.jar application.properties',
'sudo rm -rf /tmp/ns/aname/componentC'
])
def test_stop_application(self, put_mock, exec_ssh_mock):
creator = ApplicationCreator(self.config, self.environment,
self.service)
creator.stop_application('name', self.create_data)
exec_ssh_mock.assert_any_call(
'localhost', 'root_user', 'keyfile.pem',
['sudo initctl stop ns-aname-componentC\n'])
put_mock.assert_any_call('oozie/v1/job/someid1?action=suspend')
put_mock.assert_any_call('oozie/v1/job/someid2?action=suspend')
def test_app_name_ok(self, tar_mock, os_mock, shutil_mock, spur_ssh,
hdfs_client_mock, post_mock, put_mock, exec_ssh_mock,
os_sys_mock, dt_mock, hive_mock, hbase_mock):
class Resp(object):
status_code = 201
def json(self):
return {'id': 'someid'}
post_mock.return_value = Resp()
with patch("__builtin__.open", mock_open(read_data="[]")):
creator = ApplicationCreator(self.config, self.environment,
self.service)
creator.create_application('abcd', self.package_metadata,
'test-app', self.property_overrides)
def test_destroy_application(self, rmdir_mock, isdir_mock, put_mock,
exec_ssh_mock, hdfs_client_mock):
isdir_mock.return_value = True
creator = ApplicationCreator(self.config, self.environment,
self.service)
# pylint: disable=protected-access
creator._hdfs_client = hdfs_client_mock
creator.destroy_application('name', self.create_data)
print exec_ssh_mock.call_args_list
exec_ssh_mock.assert_any_call(
'localhost', 'root_user', 'keyfile.pem', [
'sudo initctl stop ns-aname-componentC\n',
'sudo rm -rf /opt/ns/aname/componentC\n',
'sudo rm /etc/init/ns-aname-componentC.conf\n'
])
put_mock.assert_any_call('oozie/v1/job/someid1?action=kill')
put_mock.assert_any_call('oozie/v1/job/someid2?action=kill')
def test_fail_create_application(self, tar_mock, os_mock, shutil_mock,
spur_ssh, hdfs_client_mock, post_mock,
put_mock, exec_ssh_mock, os_sys_mock,
dt_mock, hive_mock, hbase_mock):
dt_mock.utcnow.return_value = (datetime(2013, 01, 01))
class Resp(object):
status_code = 400
headers = {'oozie-error-message': 'oozie error!'}
def json(self):
return {'id': 'someid'}
post_mock.return_value = Resp()
with patch("__builtin__.open", mock_open(read_data="[]")):
creator = ApplicationCreator(self.config, self.environment,
self.service)
self.assertRaises(FailedCreation, creator.create_application,
'abcd', self.package_metadata_2, 'aname',
self.property_overrides)
def test_invalid_package(self):
creator = ApplicationCreator(self.config, self.environment,
self.service)
try:
creator.validate_package('test_package-1.0.2a',
{"package_name": "test_package-1.0.2"})
self.assertFail('Expected FailedValidation exception')
except FailedValidation as ex:
self.assertEqual(
ex.msg,
'package name must match name of enclosed folder but found test_package-1.0.2a and test_package-1.0.2'
)
try:
creator.validate_package('test_package-1.0',
{"package_name": "test_package-1.0"})
self.assertFail('Expected FailedValidation exception')
except FailedValidation as ex:
self.assertEqual(
ex.msg,
'version must be a three part major.minor.patch e.g. 1.2.3 but found 1.0'
)
try:
creator.validate_package('test_package=1.0.2',
{"package_name": "test_package=1.0.2"})
self.assertFail('Expected FailedValidation exception')
except FailedValidation as ex:
self.assertEqual(
ex.msg,
'package name must be of the form name-version e.g. name-version.1.2.3 but found test_package=1.0.2'
)
creator.validate_package('test-package-1.0.2', {
"package_name": "test-package-1.0.2",
"component_types": {}
})
def test_get_runtime_details(self, get_mock):
rm_call = Mock()
rm_call.json.return_value = {
"apps": {
"app": [{
"id": "application_1455877292606_13009",
"user": "******",
"name": "aname-componentA-job",
"queue": "root.users.hdfs",
"state": "FINISHED",
"finalStatus": "SUCCEEDED",
"progress": 100.0,
"trackingUI": "History",
"trackingUrl": "",
"diagnostics": "",
"clusterId": 1479988623709,
"applicationType": "MAPREDUCE",
"applicationTags": "",
"startedTime": 1479996060665,
"finishedTime": 1479996103786,
"elapsedTime": 43121,
"amContainerLogs": "",
"amHostHttpAddress": "",
"allocatedMB": -1,
"allocatedVCores": -1,
"runningContainers": -1,
"memorySeconds": 30600,
"vcoreSeconds": 29,
"preemptedResourceMB": 0,
"preemptedResourceVCores": 0,
"numNonAMContainerPreempted": 0,
"numAMContainerPreempted": 0,
"logAggregationStatus": "DISABLED"
}, {
"id": "application_1455877292606_13010",
"user": "******",
"name": "aname-componentC-job",
"queue": "root.users.hdfs",
"state": "FINISHED",
"finalStatus": "SUCCEEDED",
"progress": 100.0,
"trackingUI": "History",
"trackingUrl": "",
"diagnostics": "",
"clusterId": 1479988623709,
"applicationType": "MAPREDUCE",
"applicationTags": "",
"startedTime": 1479996060665,
"finishedTime": 1479996103786,
"elapsedTime": 43121,
"amContainerLogs": "",
"amHostHttpAddress": "",
"allocatedMB": -1,
"allocatedVCores": -1,
"runningContainers": -1,
"memorySeconds": 30600,
"vcoreSeconds": 29,
"preemptedResourceMB": 0,
"preemptedResourceVCores": 0,
"numNonAMContainerPreempted": 0,
"numAMContainerPreempted": 0,
"logAggregationStatus": "DISABLED"
}, {
"id": "application_1455877292606_13011",
"user": "******",
"name": "aname-componentC-job",
"queue": "root.users.hdfs",
"state": "RUNNING",
"finalStatus": "SUCCEEDED",
"progress": 100.0,
"trackingUI": "History",
"trackingUrl": "",
"diagnostics": "",
"clusterId": 1479988623709,
"applicationType": "MAPREDUCE",
"applicationTags": "",
"startedTime": 1479996060667,
"finishedTime": 1479996103786,
"elapsedTime": 43121,
"amContainerLogs": "",
"amHostHttpAddress": "",
"allocatedMB": -1,
"allocatedVCores": -1,
"runningContainers": -1,
"memorySeconds": 30600,
"vcoreSeconds": 29,
"preemptedResourceMB": 0,
"preemptedResourceVCores": 0,
"numNonAMContainerPreempted": 0,
"numAMContainerPreempted": 0,
"logAggregationStatus": "DISABLED"
}, {
"id": "application_1455877292606_13012",
"user": "******",
"name": "aname-componentC-job",
"queue": "root.users.hdfs",
"state": "NOT STARTED",
"finalStatus": "SUCCEEDED",
"progress": 100.0,
"trackingUI": "History",
"trackingUrl": "",
"diagnostics": "",
"clusterId": 1479988623709,
"applicationType": "MAPREDUCE",
"applicationTags": "",
"startedTime": None,
"finishedTime": 1479996103786,
"elapsedTime": 43121,
"amContainerLogs": "",
"amHostHttpAddress": "",
"allocatedMB": -1,
"allocatedVCores": -1,
"runningContainers": -1,
"memorySeconds": 30600,
"vcoreSeconds": 29,
"preemptedResourceMB": 0,
"preemptedResourceVCores": 0,
"numNonAMContainerPreempted": 0,
"numAMContainerPreempted": 0,
"logAggregationStatus": "DISABLED"
}]
}
}
get_mock.return_value = rm_call
creator = ApplicationCreator(self.config, self.environment,
self.service)
result = creator.get_application_runtime_details(
'name', self.create_data)
self.assertEqual(
result, {
"yarn_applications": {
"oozie-componentA": {
"type": "oozie",
"yarn-id": "application_1455877292606_13009",
"component": "componentA",
"yarn-start-time": 1479996060665,
"yarn-state": "FINISHED"
},
"sparkStreaming-componentC": {
"type": "sparkStreaming",
"yarn-id": "application_1455877292606_13011",
"component": "componentC",
"yarn-start-time": 1479996060667,
"yarn-state": "RUNNING"
}
}
})