def test_create_application(self, dist_mock, copy_mock, tar_mock, os_mock, shutil_mock, spur_ssh, hdfs_client_mock, post_mock, put_mock, exec_ssh_mock, os_sys_mock, dt_mock, hive_mock, hbase_mock): dt_mock.utcnow.return_value = (datetime(2013, 01, 01)) class Resp(object): status_code = 201 def json(self): return {'id': 'someid'} post_mock.return_value = Resp() dist_mock.return_value = 'ubuntu' with patch("__builtin__.open", mock_open(read_data="[]")): creator = ApplicationCreator(self.config, self.environment, self.service) creator.create_application('abcd', self.package_metadata, 'aname', self.property_overrides) print post_mock.call_args_list # pylint: disable=line-too-long post_mock.assert_any_call('oozie/v1/jobs', data='<?xml version="1.0" encoding="UTF-8" ?><configuration><property><name>environment_application_default_user</name><value>'+self.user+'</value></property><property><name>environment_cluster_private_key</name><value>keyfile.pem</value></property><property><name>environment_hbase_thrift_server</name><value>hbasehost</value></property><property><name>environment_webhdfs_host</name><value>webhdfshost</value></property><property><name>environment_opentsdb</name><value>1.2.3.5:1234</value></property><property><name>environment_yarn_node_managers</name><value>nm1,nm2</value></property><property><name>environment_webhdfs_port</name><value>webhdfsport</value></property><property><name>environment_hbase_rest_server</name><value>hbasehost</value></property><property><name>environment_oozie_uri</name><value>oozie</value></property><property><name>environment_hbase_rest_port</name><value>123</value></property><property><name>environment_cluster_root_user</name><value>root_user</value></property><property><name>environment_hive_port</name><value>124</value></property><property><name>environment_name_node</name><value>namenode</value></property><property><name>environment_hive_server</name><value>hivehost</value></property><property><name>component_property3</name><value>3</value></property><property><name>component_property4</name><value>nine</value></property><property><name>component_application</name><value>aname</value></property><property><name>component_name</name><value>componentA</value></property><property><name>component_job_name</name><value>aname-componentA-job</value></property><property><name>component_hdfs_root</name><value>/user/aname/componentA</value></property><property><name>application_user</name><value>'+self.user+'</value></property><property><name>deployment_start</name><value>2013-01-01T00:02Z</value></property><property><name>deployment_end</name><value>2013-01-08T00:02Z</value></property><property><name>user.name</name><value>'+self.user+'</value></property><property><name>oozie.use.system.libpath</name><value>true</value></property><property><name>oozie.libpath</name><value>/user/deployment/platform</value></property><property><name>oozie.wf.application.path</name><value>namenode/user/aname/componentA</value></property></configuration>', headers={'Content-Type': 'application/xml'}) post_mock.assert_any_call('oozie/v1/jobs', data='<?xml version="1.0" encoding="UTF-8" ?><configuration><property><name>environment_application_default_user</name><value>'+self.user+'</value></property><property><name>environment_cluster_private_key</name><value>keyfile.pem</value></property><property><name>environment_hbase_thrift_server</name><value>hbasehost</value></property><property><name>environment_webhdfs_host</name><value>webhdfshost</value></property><property><name>environment_opentsdb</name><value>1.2.3.5:1234</value></property><property><name>environment_yarn_node_managers</name><value>nm1,nm2</value></property><property><name>environment_webhdfs_port</name><value>webhdfsport</value></property><property><name>environment_hbase_rest_server</name><value>hbasehost</value></property><property><name>environment_oozie_uri</name><value>oozie</value></property><property><name>environment_hbase_rest_port</name><value>123</value></property><property><name>environment_cluster_root_user</name><value>root_user</value></property><property><name>environment_hive_port</name><value>124</value></property><property><name>environment_name_node</name><value>namenode</value></property><property><name>environment_hive_server</name><value>hivehost</value></property><property><name>component_application</name><value>aname</value></property><property><name>component_name</name><value>componentB</value></property><property><name>component_job_name</name><value>aname-componentB-job</value></property><property><name>component_hdfs_root</name><value>/user/aname/componentB</value></property><property><name>application_user</name><value>'+self.user+'</value></property><property><name>deployment_start</name><value>2013-01-01T00:02Z</value></property><property><name>deployment_end</name><value>2013-01-08T00:02Z</value></property><property><name>user.name</name><value>'+self.user+'</value></property><property><name>oozie.use.system.libpath</name><value>true</value></property><property><name>oozie.libpath</name><value>/user/deployment/platform</value></property><property><name>oozie.wf.application.path</name><value>namenode/user/aname/componentB</value></property></configuration>', headers={'Content-Type': 'application/xml'}) put_mock.assert_any_call('oozie/v1/job/someid?action=suspend&user.name='+self.user) exec_ssh_mock.assert_any_call('localhost', 'root_user', 'keyfile.pem', ['mkdir -p /tmp/ns/aname/componentC', 'sudo mkdir -p /opt/ns/aname/componentC']) exec_ssh_mock.assert_any_call('nm1', 'root_user', 'keyfile.pem', ['mkdir -p /tmp/ns/aname/componentC']) exec_ssh_mock.assert_any_call('nm1', 'root_user', 'keyfile.pem', ['sudo mkdir -p /opt/ns/aname/componentC', 'sudo mv /tmp/ns/aname/componentC/log4j.properties /opt/ns/aname/componentC/log4j.properties']) exec_ssh_mock.assert_any_call('nm2', 'root_user', 'keyfile.pem', ['mkdir -p /tmp/ns/aname/componentC']) exec_ssh_mock.assert_any_call('nm2', 'root_user', 'keyfile.pem', ['sudo mkdir -p /opt/ns/aname/componentC', 'sudo mv /tmp/ns/aname/componentC/log4j.properties /opt/ns/aname/componentC/log4j.properties']) exec_ssh_mock.assert_any_call('localhost', 'root_user', 'keyfile.pem', ['sudo cp /tmp/ns/aname/componentC/upstart.conf.tpl /etc/init/ns-aname-componentC.conf', 'sudo cp /tmp/ns/aname/componentC/* /opt/ns/aname/componentC', 'sudo chmod a+x /opt/ns/aname/componentC/yarn-kill.py', 'cd /opt/ns/aname/componentC && sudo jar uf abc.jar application.properties', 'sudo rm -rf /tmp/ns/aname/componentC'])
def test_validate_package(self): creator = ApplicationCreator(self.config, self.environment, self.service) result = {} try: creator.validate_package(self.package_metadata['package_name'], self.package_metadata) except FailedValidation as ex: result = ex.msg expected_report = { 'oozie': { 'componentA': ['missing file workflow.xml'], 'componentB': ['missing file workflow.xml'] }, 'sparkStreaming': { 'componentC': [ 'missing file yarn-kill.py', 'missing file application.properties', 'missing file upstart.conf', 'missing file log4j.properties' ] } } self.assertEqual(result, expected_report)
def test_user_name_fail(self, cmd_mock, tar_mock, os_mock, shutil_mock, spur_ssh, hdfs_client_mock, post_mock, put_mock, exec_ssh_mock, os_sys_mock, dt_mock, hive_mock, hbase_mock): class Resp(object): status_code = 201 def json(self): return {'id': 'someid'} post_mock.return_value = Resp() cmd_mock.return_value = (0, 'dev') with patch("__builtin__.open", mock_open(read_data="[]")): creator = ApplicationCreator(self.config, self.environment, self.service) try: creator.create_application('abcd', self.package_metadata, 'appname', self.property_overrides_no_user) self.assertFail( 'Expected FailedCreation exception but was not thrown') except FailedCreation as ex: self.assertEqual( ex.msg, 'User somebody does not exist. Verify that this user account exists on the machine running the deployment manager.' )
def test_app_name_fail(self, tar_mock, os_mock, shutil_mock, spur_ssh, hdfs_client_mock, post_mock, put_mock, exec_ssh_mock, os_sys_mock, dt_mock, hive_mock, hbase_mock): class Resp(object): status_code = 201 def json(self): return {'id': 'someid'} post_mock.return_value = Resp() with patch("__builtin__.open", mock_open(read_data="[]")): creator = ApplicationCreator(self.config, self.environment, self.service) try: creator.create_application('abcd', self.package_metadata, 'with&oddchar', self.property_overrides) self.assertFail( 'Expected FailedCreation exception but was not thrown') except FailedCreation as ex: self.assertEqual( ex.msg, 'Application name with&oddchar may only contain a-z A-Z 0-9 - _' )
def test_start_application(self, put_mock, exec_ssh_mock): creator = ApplicationCreator(self.config, self.environment, self.service) creator.start_application('name', self.create_data) exec_ssh_mock.assert_any_call('localhost', 'root_user', 'keyfile.pem', ['sudo initctl start ns-aname-componentC\n']) put_mock.assert_any_call('oozie/v1/job/someid1?action=resume&user.name='+self.user) put_mock.assert_any_call('oozie/v1/job/someid2?action=resume&user.name='+self.user) put_mock.assert_any_call('oozie/v1/job/someid1?action=start&user.name='+self.user) put_mock.assert_any_call('oozie/v1/job/someid2?action=start&user.name='+self.user)
def test_create_application(self, dist_mock, copy_mock, tar_mock, os_mock, shutil_mock, spur_ssh, hdfs_client_mock, post_mock, put_mock, exec_ssh_mock, os_sys_mock, dt_mock, hive_mock, hbase_mock): dt_mock.utcnow.return_value = (datetime(2013, 01, 01)) class Resp(object): status_code = 201 def json(self): return {'id': 'someid'} post_mock.return_value = Resp() dist_mock.return_value = 'ubuntu' with patch("__builtin__.open", mock_open(read_data="[]")): creator = ApplicationCreator(self.config, self.environment, self.service) creator.create_application('abcd', self.package_metadata, 'aname', self.property_overrides) print post_mock.call_args_list # pylint: disable=line-too-long post_mock.assert_any_call( 'oozie/v1/jobs', data= '<?xml version="1.0" encoding="UTF-8" ?><configuration><property><name>environment_cluster_private_key</name><value>keyfile.pem</value></property><property><name>environment_hbase_thrift_server</name><value>hbasehost</value></property><property><name>environment_webhdfs_host</name><value>webhdfshost</value></property><property><name>environment_opentsdb</name><value>1.2.3.5:1234</value></property><property><name>environment_yarn_node_managers</name><value>nm1,nm2</value></property><property><name>environment_webhdfs_port</name><value>webhdfsport</value></property><property><name>environment_hbase_rest_server</name><value>hbasehost</value></property><property><name>environment_oozie_uri</name><value>oozie</value></property><property><name>environment_hbase_rest_port</name><value>123</value></property><property><name>environment_cluster_root_user</name><value>root_user</value></property><property><name>environment_hive_port</name><value>124</value></property><property><name>environment_name_node</name><value>namenode</value></property><property><name>environment_hive_server</name><value>hivehost</value></property><property><name>component_property3</name><value>3</value></property><property><name>component_property4</name><value>nine</value></property><property><name>component_application</name><value>aname</value></property><property><name>component_name</name><value>componentA</value></property><property><name>component_job_name</name><value>aname-componentA-job</value></property><property><name>component_hdfs_root</name><value>/user/aname/componentA</value></property><property><name>deployment_start</name><value>2013-01-01T00:02Z</value></property><property><name>deployment_end</name><value>2013-01-08T00:02Z</value></property><property><name>user.name</name><value>hdfs</value></property><property><name>oozie.use.system.libpath</name><value>true</value></property><property><name>oozie.libpath</name><value>/user/deployment/platform</value></property><property><name>oozie.wf.application.path</name><value>namenode/user/aname/componentA</value></property></configuration>', headers={'Content-Type': 'application/xml'}) post_mock.assert_any_call( 'oozie/v1/jobs', data= '<?xml version="1.0" encoding="UTF-8" ?><configuration><property><name>environment_cluster_private_key</name><value>keyfile.pem</value></property><property><name>environment_hbase_thrift_server</name><value>hbasehost</value></property><property><name>environment_webhdfs_host</name><value>webhdfshost</value></property><property><name>environment_opentsdb</name><value>1.2.3.5:1234</value></property><property><name>environment_yarn_node_managers</name><value>nm1,nm2</value></property><property><name>environment_webhdfs_port</name><value>webhdfsport</value></property><property><name>environment_hbase_rest_server</name><value>hbasehost</value></property><property><name>environment_oozie_uri</name><value>oozie</value></property><property><name>environment_hbase_rest_port</name><value>123</value></property><property><name>environment_cluster_root_user</name><value>root_user</value></property><property><name>environment_hive_port</name><value>124</value></property><property><name>environment_name_node</name><value>namenode</value></property><property><name>environment_hive_server</name><value>hivehost</value></property><property><name>component_application</name><value>aname</value></property><property><name>component_name</name><value>componentB</value></property><property><name>component_job_name</name><value>aname-componentB-job</value></property><property><name>component_hdfs_root</name><value>/user/aname/componentB</value></property><property><name>deployment_start</name><value>2013-01-01T00:02Z</value></property><property><name>deployment_end</name><value>2013-01-08T00:02Z</value></property><property><name>user.name</name><value>hdfs</value></property><property><name>oozie.use.system.libpath</name><value>true</value></property><property><name>oozie.libpath</name><value>/user/deployment/platform</value></property><property><name>oozie.wf.application.path</name><value>namenode/user/aname/componentB</value></property></configuration>', headers={'Content-Type': 'application/xml'}) put_mock.assert_any_call('oozie/v1/job/someid?action=suspend') exec_ssh_mock.assert_any_call( 'localhost', 'root_user', 'keyfile.pem', [ 'mkdir -p /tmp/ns/aname/componentC', 'sudo mkdir -p /opt/ns/aname/componentC' ]) exec_ssh_mock.assert_any_call('nm1', 'root_user', 'keyfile.pem', ['mkdir -p /tmp/ns/aname/componentC']) exec_ssh_mock.assert_any_call('nm1', 'root_user', 'keyfile.pem', [ 'sudo mkdir -p /opt/ns/aname/componentC', 'sudo mv /tmp/ns/aname/componentC/log4j.properties /opt/ns/aname/componentC/log4j.properties' ]) exec_ssh_mock.assert_any_call('nm2', 'root_user', 'keyfile.pem', ['mkdir -p /tmp/ns/aname/componentC']) exec_ssh_mock.assert_any_call('nm2', 'root_user', 'keyfile.pem', [ 'sudo mkdir -p /opt/ns/aname/componentC', 'sudo mv /tmp/ns/aname/componentC/log4j.properties /opt/ns/aname/componentC/log4j.properties' ]) exec_ssh_mock.assert_any_call('localhost', 'root_user', 'keyfile.pem', [ 'sudo cp /tmp/ns/aname/componentC/upstart.conf.tpl /etc/init/ns-aname-componentC.conf', 'sudo cp /tmp/ns/aname/componentC/* /opt/ns/aname/componentC', 'sudo chmod a+x /opt/ns/aname/componentC/yarn-kill.py', 'cd /opt/ns/aname/componentC && sudo jar uf abc.jar application.properties', 'sudo rm -rf /tmp/ns/aname/componentC' ])
def test_stop_application(self, put_mock, exec_ssh_mock): creator = ApplicationCreator(self.config, self.environment, self.service) creator.stop_application('name', self.create_data) exec_ssh_mock.assert_any_call( 'localhost', 'root_user', 'keyfile.pem', ['sudo initctl stop ns-aname-componentC\n']) put_mock.assert_any_call('oozie/v1/job/someid1?action=suspend') put_mock.assert_any_call('oozie/v1/job/someid2?action=suspend')
def test_destroy_application(self, rmdir_mock, isdir_mock, put_mock, exec_ssh_mock, hdfs_client_mock): isdir_mock.return_value = True creator = ApplicationCreator(self.config, self.environment, self.service) # pylint: disable=protected-access creator._hdfs_client = hdfs_client_mock creator.destroy_application('name', self.create_data) print exec_ssh_mock.call_args_list exec_ssh_mock.assert_any_call('localhost', 'root_user', 'keyfile.pem', [ 'sudo initctl stop ns-aname-componentC\n', 'sudo rm -rf /opt/ns/aname/componentC\n', 'sudo rm /etc/init/ns-aname-componentC.conf\n']) put_mock.assert_any_call('oozie/v1/job/someid1?action=kill&user.name='+self.user) put_mock.assert_any_call('oozie/v1/job/someid2?action=kill&user.name='+self.user)
def test_app_name_ok(self, copy_mock, tar_mock, os_mock, shutil_mock, spur_ssh, hdfs_client_mock, post_mock, put_mock, exec_ssh_mock, os_sys_mock, dt_mock, hive_mock, hbase_mock): class Resp(object): status_code = 201 def json(self): return {'id': 'someid'} post_mock.return_value = Resp() with patch("__builtin__.open", mock_open(read_data="[]")): creator = ApplicationCreator(self.config, self.environment, self.service) creator.create_application('abcd', self.package_metadata, 'test-app', self.property_overrides)
def test_validate_package(self): creator = ApplicationCreator(self.config, self.environment, self.service) result = {} try: creator.validate_package(self.package_metadata['package_name'], self.package_metadata) except FailedValidation as ex: result = ex.msg expected_report = { 'oozie': { 'componentA': ['missing file workflow.xml'], 'componentB': ['missing file workflow.xml']}, 'sparkStreaming': { 'componentC': ['missing file application.properties', 'missing file log4j.properties']}} self.assertEqual(result, expected_report)
def test_fail_create_application(self, cmd_mock, tar_mock, os_mock, shutil_mock, spur_ssh, hdfs_client_mock, post_mock, put_mock, exec_ssh_mock, os_sys_mock, dt_mock, hive_mock, hbase_mock): dt_mock.utcnow.return_value = (datetime(2013, 01, 01)) class Resp(object): status_code = 400 headers = {'oozie-error-message': 'oozie error!'} def json(self): return {'id': 'someid'} default_properties = {"oozie": {"componentA": {"spark_version": "1"}}} override_properties = { "user": "******", "oozie": { "componentA": { "spark_version": "2" } } } post_mock.return_value = Resp() cmd_mock.return_value = (0, 'dev') with patch("__builtin__.open", mock_open(read_data="[]")): creator = ApplicationCreator(self.config, self.environment, self.service) self.assertRaises(FailedValidation, creator.assert_application_properties, override_properties, default_properties) self.assertRaises(FailedCreation, creator.create_application, 'abcd', self.package_metadata_2, 'aname', self.property_overrides)
def test_app_name_ok(self, tar_mock, os_mock, shutil_mock, spur_ssh, hdfs_client_mock, post_mock, put_mock, exec_ssh_mock, os_sys_mock, dt_mock, hive_mock, hbase_mock): class Resp(object): status_code = 201 def json(self): return {'id': 'someid'} post_mock.return_value = Resp() with patch("__builtin__.open", mock_open(read_data="[]")): creator = ApplicationCreator(self.config, self.environment, self.service) creator.create_application('abcd', self.package_metadata, 'test-app', self.property_overrides)
def test_destroy_application(self, rmdir_mock, isdir_mock, put_mock, exec_ssh_mock, hdfs_client_mock): isdir_mock.return_value = True creator = ApplicationCreator(self.config, self.environment, self.service) # pylint: disable=protected-access creator._hdfs_client = hdfs_client_mock creator.destroy_application('name', self.create_data) print exec_ssh_mock.call_args_list exec_ssh_mock.assert_any_call( 'localhost', 'root_user', 'keyfile.pem', [ 'sudo initctl stop ns-aname-componentC\n', 'sudo rm -rf /opt/ns/aname/componentC\n', 'sudo rm /etc/init/ns-aname-componentC.conf\n' ]) put_mock.assert_any_call('oozie/v1/job/someid1?action=kill') put_mock.assert_any_call('oozie/v1/job/someid2?action=kill')
def test_app_name_fail(self, tar_mock, os_mock, shutil_mock, spur_ssh, hdfs_client_mock, post_mock, put_mock, exec_ssh_mock, os_sys_mock, dt_mock, hive_mock, hbase_mock): class Resp(object): status_code = 201 def json(self): return {'id': 'someid'} post_mock.return_value = Resp() with patch("__builtin__.open", mock_open(read_data="[]")): creator = ApplicationCreator(self.config, self.environment, self.service) try: creator.create_application('abcd', self.package_metadata, 'with&oddchar', self.property_overrides) self.assertFail('Expected FailedCreation exception but was not thrown') except FailedCreation as ex: self.assertEqual(ex.msg, 'Application name with&oddchar may only contain a-z A-Z 0-9 - _')
def test_invalid_package(self): creator = ApplicationCreator(self.config, self.environment, self.service) try: creator.validate_package('test_package-1.0.2a', {"package_name": "test_package-1.0.2"}) self.assertFail('Expected FailedValidation exception') except FailedValidation as ex: self.assertEqual( ex.msg, 'package name must match name of enclosed folder but found test_package-1.0.2a and test_package-1.0.2' ) try: creator.validate_package('test_package-1.0', {"package_name": "test_package-1.0"}) self.assertFail('Expected FailedValidation exception') except FailedValidation as ex: self.assertEqual( ex.msg, 'version must be a three part major.minor.patch e.g. 1.2.3 but found 1.0' ) try: creator.validate_package('test_package=1.0.2', {"package_name": "test_package=1.0.2"}) self.assertFail('Expected FailedValidation exception') except FailedValidation as ex: self.assertEqual( ex.msg, 'package name must be of the form name-version e.g. name-version.1.2.3 but found test_package=1.0.2' ) creator.validate_package('test-package-1.0.2', { "package_name": "test-package-1.0.2", "component_types": {} })
def test_invalid_package(self): creator = ApplicationCreator(self.config, self.environment, self.service) try: creator.validate_package('test_package-1.0.2a', {"package_name": "test_package-1.0.2"}) self.assertFail('Expected FailedValidation exception') except FailedValidation as ex: self.assertEqual(ex.msg, 'package name must match name of enclosed folder but found test_package-1.0.2a and test_package-1.0.2') try: creator.validate_package('test_package-1.0', {"package_name": "test_package-1.0"}) self.assertFail('Expected FailedValidation exception') except FailedValidation as ex: self.assertEqual(ex.msg, 'version must be a three part major.minor.patch e.g. 1.2.3 but found 1.0') try: creator.validate_package('test_package=1.0.2', {"package_name": "test_package=1.0.2"}) self.assertFail('Expected FailedValidation exception') except FailedValidation as ex: self.assertEqual(ex.msg, 'package name must be of the form name-version e.g. name-version.1.2.3 but found test_package=1.0.2') creator.validate_package('test-package-1.0.2', {"package_name": "test-package-1.0.2", "component_types": {}})
def test_fail_create_application(self, tar_mock, os_mock, shutil_mock, spur_ssh, hdfs_client_mock, post_mock, put_mock, exec_ssh_mock, os_sys_mock, dt_mock, hive_mock, hbase_mock): dt_mock.utcnow.return_value = (datetime(2013, 01, 01)) class Resp(object): status_code = 400 headers = {'oozie-error-message': 'oozie error!'} def json(self): return {'id': 'someid'} post_mock.return_value = Resp() with patch("__builtin__.open", mock_open(read_data="[]")): creator = ApplicationCreator(self.config, self.environment, self.service) self.assertRaises(FailedCreation, creator.create_application, 'abcd', self.package_metadata_2, 'aname', self.property_overrides)
def test_get_runtime_details(self, get_mock): rm_call = Mock() rm_call.json.return_value = { "apps": { "app": [{ "id": "application_1455877292606_13009", "user": self.user, "name": "aname-componentA-job", "queue": "root.users."+self.user, "state": "FINISHED", "finalStatus": "SUCCEEDED", "progress": 100.0, "trackingUI": "History", "trackingUrl": "", "diagnostics": "", "clusterId": 1479988623709, "applicationType": "MAPREDUCE", "applicationTags": "", "startedTime": 1479996060665, "finishedTime": 1479996103786, "elapsedTime": 43121, "amContainerLogs": "", "amHostHttpAddress": "", "allocatedMB": -1, "allocatedVCores": -1, "runningContainers": -1, "memorySeconds": 30600, "vcoreSeconds": 29, "preemptedResourceMB": 0, "preemptedResourceVCores": 0, "numNonAMContainerPreempted": 0, "numAMContainerPreempted": 0, "logAggregationStatus": "DISABLED" }, { "id": "application_1455877292606_13010", "user": self.user, "name": "aname-componentC-job", "queue": "root.users."+self.user, "state": "FINISHED", "finalStatus": "SUCCEEDED", "progress": 100.0, "trackingUI": "History", "trackingUrl": "", "diagnostics": "", "clusterId": 1479988623709, "applicationType": "MAPREDUCE", "applicationTags": "", "startedTime": 1479996060665, "finishedTime": 1479996103786, "elapsedTime": 43121, "amContainerLogs": "", "amHostHttpAddress": "", "allocatedMB": -1, "allocatedVCores": -1, "runningContainers": -1, "memorySeconds": 30600, "vcoreSeconds": 29, "preemptedResourceMB": 0, "preemptedResourceVCores": 0, "numNonAMContainerPreempted": 0, "numAMContainerPreempted": 0, "logAggregationStatus": "DISABLED" }, { "id": "application_1455877292606_13011", "user": self.user, "name": "aname-componentC-job", "queue": "root.users"+self.user, "state": "RUNNING", "finalStatus": "SUCCEEDED", "progress": 100.0, "trackingUI": "History", "trackingUrl": "", "diagnostics": "", "clusterId": 1479988623709, "applicationType": "MAPREDUCE", "applicationTags": "", "startedTime": 1479996060667, "finishedTime": 1479996103786, "elapsedTime": 43121, "amContainerLogs": "", "amHostHttpAddress": "", "allocatedMB": -1, "allocatedVCores": -1, "runningContainers": -1, "memorySeconds": 30600, "vcoreSeconds": 29, "preemptedResourceMB": 0, "preemptedResourceVCores": 0, "numNonAMContainerPreempted": 0, "numAMContainerPreempted": 0, "logAggregationStatus": "DISABLED" }, { "id": "application_1455877292606_13012", "user": self.user, "name": "aname-componentC-job", "queue": "root.users."+self.user, "state": "NOT STARTED", "finalStatus": "SUCCEEDED", "progress": 100.0, "trackingUI": "History", "trackingUrl": "", "diagnostics": "", "clusterId": 1479988623709, "applicationType": "MAPREDUCE", "applicationTags": "", "startedTime": None, "finishedTime": 1479996103786, "elapsedTime": 43121, "amContainerLogs": "", "amHostHttpAddress": "", "allocatedMB": -1, "allocatedVCores": -1, "runningContainers": -1, "memorySeconds": 30600, "vcoreSeconds": 29, "preemptedResourceMB": 0, "preemptedResourceVCores": 0, "numNonAMContainerPreempted": 0, "numAMContainerPreempted": 0, "logAggregationStatus": "DISABLED" }] } } get_mock.return_value = rm_call creator = ApplicationCreator(self.config, self.environment, self.service) result = creator.get_application_runtime_details('name', self.create_data) self.assertEqual(result, {"yarn_applications": { "oozie-componentA": { "type": "oozie", "yarn-id": "application_1455877292606_13009", "component": "componentA", "yarn-start-time": 1479996060665, "yarn-state": "FINISHED" }, "sparkStreaming-componentC": { "type": "sparkStreaming", "yarn-id": "application_1455877292606_13011", "component": "componentC", "yarn-start-time": 1479996060667, "yarn-state": "RUNNING" } }})
def test_get_runtime_details(self, get_mock): rm_call = Mock() rm_call.json.return_value = { "apps": { "app": [{ "id": "application_1455877292606_13009", "user": "******", "name": "aname-componentA-job", "queue": "root.users.hdfs", "state": "FINISHED", "finalStatus": "SUCCEEDED", "progress": 100.0, "trackingUI": "History", "trackingUrl": "", "diagnostics": "", "clusterId": 1479988623709, "applicationType": "MAPREDUCE", "applicationTags": "", "startedTime": 1479996060665, "finishedTime": 1479996103786, "elapsedTime": 43121, "amContainerLogs": "", "amHostHttpAddress": "", "allocatedMB": -1, "allocatedVCores": -1, "runningContainers": -1, "memorySeconds": 30600, "vcoreSeconds": 29, "preemptedResourceMB": 0, "preemptedResourceVCores": 0, "numNonAMContainerPreempted": 0, "numAMContainerPreempted": 0, "logAggregationStatus": "DISABLED" }, { "id": "application_1455877292606_13010", "user": "******", "name": "aname-componentC-job", "queue": "root.users.hdfs", "state": "FINISHED", "finalStatus": "SUCCEEDED", "progress": 100.0, "trackingUI": "History", "trackingUrl": "", "diagnostics": "", "clusterId": 1479988623709, "applicationType": "MAPREDUCE", "applicationTags": "", "startedTime": 1479996060665, "finishedTime": 1479996103786, "elapsedTime": 43121, "amContainerLogs": "", "amHostHttpAddress": "", "allocatedMB": -1, "allocatedVCores": -1, "runningContainers": -1, "memorySeconds": 30600, "vcoreSeconds": 29, "preemptedResourceMB": 0, "preemptedResourceVCores": 0, "numNonAMContainerPreempted": 0, "numAMContainerPreempted": 0, "logAggregationStatus": "DISABLED" }, { "id": "application_1455877292606_13011", "user": "******", "name": "aname-componentC-job", "queue": "root.users.hdfs", "state": "RUNNING", "finalStatus": "SUCCEEDED", "progress": 100.0, "trackingUI": "History", "trackingUrl": "", "diagnostics": "", "clusterId": 1479988623709, "applicationType": "MAPREDUCE", "applicationTags": "", "startedTime": 1479996060667, "finishedTime": 1479996103786, "elapsedTime": 43121, "amContainerLogs": "", "amHostHttpAddress": "", "allocatedMB": -1, "allocatedVCores": -1, "runningContainers": -1, "memorySeconds": 30600, "vcoreSeconds": 29, "preemptedResourceMB": 0, "preemptedResourceVCores": 0, "numNonAMContainerPreempted": 0, "numAMContainerPreempted": 0, "logAggregationStatus": "DISABLED" }, { "id": "application_1455877292606_13012", "user": "******", "name": "aname-componentC-job", "queue": "root.users.hdfs", "state": "NOT STARTED", "finalStatus": "SUCCEEDED", "progress": 100.0, "trackingUI": "History", "trackingUrl": "", "diagnostics": "", "clusterId": 1479988623709, "applicationType": "MAPREDUCE", "applicationTags": "", "startedTime": None, "finishedTime": 1479996103786, "elapsedTime": 43121, "amContainerLogs": "", "amHostHttpAddress": "", "allocatedMB": -1, "allocatedVCores": -1, "runningContainers": -1, "memorySeconds": 30600, "vcoreSeconds": 29, "preemptedResourceMB": 0, "preemptedResourceVCores": 0, "numNonAMContainerPreempted": 0, "numAMContainerPreempted": 0, "logAggregationStatus": "DISABLED" }] } } get_mock.return_value = rm_call creator = ApplicationCreator(self.config, self.environment, self.service) result = creator.get_application_runtime_details( 'name', self.create_data) self.assertEqual( result, { "yarn_applications": { "oozie-componentA": { "type": "oozie", "yarn-id": "application_1455877292606_13009", "component": "componentA", "yarn-start-time": 1479996060665, "yarn-state": "FINISHED" }, "sparkStreaming-componentC": { "type": "sparkStreaming", "yarn-id": "application_1455877292606_13011", "component": "componentC", "yarn-start-time": 1479996060667, "yarn-state": "RUNNING" } } })