def test_finish(self): """Tests container finish procedure and freeing of the resources. """ # Access protected module _kill_apps_by_root # pylint: disable=W0212 manifest = { 'app': 'proid.myapp', 'cell': 'test', 'cpu': '100%', 'disk': '100G', 'environment': 'dev', 'host_ip': '172.31.81.67', 'memory': '100M', 'name': 'proid.myapp#001', 'proid': 'foo', 'shared_network': False, 'task': '001', 'uniqueid': '0000000ID1234', 'archive': ['/var/tmp/treadmill'], 'endpoints': [{ 'port': 8000, 'name': 'http', 'real_port': 5000, 'proto': 'tcp', }, { 'port': 54321, 'type': 'infra', 'name': 'ssh', 'real_port': 54321, 'proto': 'tcp', }], 'ephemeral_ports': { 'tcp': [45024], 'udp': [62422], }, 'services': [{ 'name': 'web_server', 'command': '/bin/false', 'restart': { 'limit': 3, 'interval': 60, }, }], } treadmill.appmgr.manifest.read.return_value = manifest app_unique_name = 'proid.myapp-001-0000000ID1234' mock_cgroup_client = self.app_env.svc_cgroup.make_client.return_value mock_ld_client = self.app_env.svc_localdisk.make_client.return_value mock_nwrk_client = self.app_env.svc_network.make_client.return_value localdisk = { 'block_dev': '/dev/foo', } mock_ld_client.get.return_value = localdisk network = { 'vip': '192.168.0.2', 'gateway': '192.168.254.254', 'veth': 'testveth.0', } mock_nwrk_client.get.return_value = network app_dir = os.path.join(self.app_env.apps_dir, app_unique_name) # Create content in app root directory, verify that it is archived. fs.mkdir_safe(os.path.join(app_dir, 'root', 'xxx')) fs.mkdir_safe(os.path.join(app_dir, 'services')) # Simulate daemontools finish script, marking the app is done. with open(os.path.join(app_dir, 'exitinfo'), 'w') as f: f.write(yaml.dump({'service': 'web_server', 'rc': 0, 'sig': 0})) mock_zkclient = kazoo.client.KazooClient() app_finish.finish(self.app_env, mock_zkclient, app_dir) self.app_env.watchdogs.create.assert_called_with( 'treadmill.appmgr.finish-' + app_unique_name, '5m', mock.ANY) treadmill.subproc.check_call.assert_has_calls([ mock.call([ 's6-svc', '-d', app_dir, ]), mock.call([ 's6-svwait', '-d', app_dir, ]), ]) # All resource service clients are properly created self.app_env.svc_cgroup.make_client.assert_called_with( os.path.join(app_dir, 'cgroups')) self.app_env.svc_localdisk.make_client.assert_called_with( os.path.join(app_dir, 'localdisk')) self.app_env.svc_network.make_client.assert_called_with( os.path.join(app_dir, 'network')) treadmill.appmgr.finish._kill_apps_by_root.assert_called_with( os.path.join(app_dir, 'root')) # Verify that we tested the archiving for the app root volume treadmill.fs.archive_filesystem.assert_called_with( '/dev/foo', os.path.join(app_dir, 'root'), os.path.join(app_dir, '001_xxx.xx.com_20150122_141436537918.tar'), mock.ANY) # Verify that the file is uploaded by Uploader app = utils.to_obj(manifest) treadmill.appmgr.finish._send_container_archive.assert_called_with( mock_zkclient, app, os.path.join(app_dir, '001_xxx.xx.com_20150122_141436537918.tar.gz'), ) # Verify that the app folder was deleted self.assertFalse(os.path.exists(app_dir)) # Cleanup the block device mock_ld_client.delete.assert_called_with(app_unique_name) # Cleanup the cgroup resource mock_cgroup_client.delete.assert_called_with(app_unique_name) # Cleanup network resources mock_nwrk_client.get.assert_called_with(app_unique_name) self.app_env.rules.unlink_rule.assert_has_calls([ mock.call(rule=firewall.DNATRule('tcp', '172.31.81.67', 5000, '192.168.0.2', 8000), owner=app_unique_name), mock.call(rule=firewall.DNATRule('tcp', '172.31.81.67', 54321, '192.168.0.2', 54321), owner=app_unique_name), mock.call(rule=firewall.DNATRule('tcp', '172.31.81.67', 45024, '192.168.0.2', 45024), owner=app_unique_name), mock.call(rule=firewall.DNATRule('udp', '172.31.81.67', 62422, '192.168.0.2', 62422), owner=app_unique_name), ]) treadmill.iptables.rm_ip_set.assert_has_calls([ mock.call(treadmill.iptables.SET_INFRA_SVC, '192.168.0.2,tcp:54321'), mock.call(treadmill.iptables.SET_INFRA_SVC, '192.168.0.2,tcp:45024'), mock.call(treadmill.iptables.SET_INFRA_SVC, '192.168.0.2,udp:62422'), ]) mock_nwrk_client.delete.assert_called_with(app_unique_name) treadmill.appevents.post.assert_called_with( mock.ANY, events.FinishedTraceEvent(instanceid='proid.myapp#001', rc=0, signal=0, payload={ 'service': 'web_server', 'sig': 0, 'rc': 0 })) treadmill.rrdutils.flush_noexc.assert_called_with( os.path.join(self.root, 'metrics', 'apps', app_unique_name + '.rrd')) shutil.copy.assert_called_with( os.path.join(self.root, 'metrics', 'apps', app_unique_name + '.rrd'), os.path.join(app_dir, 'metrics.rrd'))
def test_finish_no_resources(self): """Test app finish on directory when all resources are already freed. """ # Access protected module _kill_apps_by_root # pylint: disable=W0212 manifest = { 'app': 'proid.myapp', 'cell': 'test', 'cpu': '100%', 'disk': '100G', 'environment': 'dev', 'host_ip': '172.31.81.67', 'memory': '100M', 'name': 'proid.myapp#001', 'proid': 'foo', 'shared_network': False, 'task': '001', 'uniqueid': '0000000ID1234', 'archive': ['/var/tmp/treadmill'], 'endpoints': [{ 'port': 8000, 'name': 'http', 'real_port': 5000 }, { 'port': 54321, 'type': 'infra', 'name': 'ssh', 'real_port': 54321 }], 'ephemeral_ports': { 'tcp': [45024], 'udp': [62422], }, 'services': [{ 'command': '/bin/false', 'restart_count': 3, 'name': 'web_server' }], } treadmill.appmgr.manifest.read.return_value = manifest app_unique_name = 'proid.myapp-001-0000000ID1234' mock_cgroup_client = self.app_env.svc_cgroup.make_client.return_value mock_ld_client = self.app_env.svc_localdisk.make_client.return_value mock_nwrk_client = self.app_env.svc_network.make_client.return_value # All resource managers return None mock_cgroup_client.get.return_value = None mock_ld_client.get.return_value = None mock_nwrk_client.get.return_value = None app_dir = os.path.join(self.app_env.apps_dir, app_unique_name) # Create content in app root directory, verify that it is archived. fs.mkdir_safe(os.path.join(app_dir, 'root', 'xxx')) fs.mkdir_safe(os.path.join(app_dir, 'services')) # Simulate daemontools finish script, marking the app is done. with open(os.path.join(app_dir, 'exitinfo'), 'w') as f: f.write(yaml.dump({'service': 'web_server', 'rc': 0, 'sig': 0})) mock_zkclient = kazoo.client.KazooClient() app_finish.finish(self.app_env, mock_zkclient, app_dir) self.app_env.watchdogs.create.assert_called_with( 'treadmill.appmgr.finish-' + app_unique_name, '5m', mock.ANY) treadmill.subproc.check_call.assert_has_calls([ mock.call([ 's6-svc', '-d', app_dir, ], ), mock.call([ 's6-svwait', '-d', app_dir, ], ), ]) self.app_env.svc_cgroup.make_client.assert_called_with( os.path.join(app_dir, 'cgroups')) self.app_env.svc_localdisk.make_client.assert_called_with( os.path.join(app_dir, 'localdisk')) self.app_env.svc_network.make_client.assert_called_with( os.path.join(app_dir, 'network')) treadmill.appmgr.finish._kill_apps_by_root.assert_called_with( os.path.join(app_dir, 'root')) # Verify that the app folder was deleted self.assertFalse(os.path.exists(app_dir)) # Cleanup the network resources mock_nwrk_client.get.assert_called_with(app_unique_name) # Cleanup the block device mock_ld_client.delete.assert_called_with(app_unique_name) # Cleanup the cgroup resource mock_cgroup_client.delete.assert_called_with(app_unique_name) treadmill.appevents.post.assert_called_with( mock.ANY, events.FinishedTraceEvent(instanceid='proid.myapp#001', rc=0, signal=0, payload={ 'service': 'web_server', 'sig': 0, 'rc': 0 })) treadmill.rrdutils.flush_noexc.assert_called_with( os.path.join(self.root, 'metrics', 'apps', app_unique_name + '.rrd')) shutil.copy.assert_called_with( os.path.join(self.root, 'metrics', 'apps', app_unique_name + '.rrd'), os.path.join(app_dir, 'metrics.rrd'))
def test_finish_no_manifest(self): """Test app finish on directory with no app.yml. """ app_finish.finish(self.app_env, None, self.root)
def test_finish_aborted(self): """Tests container finish procedure when node is aborted. """ manifest = { 'app': 'proid.myapp', 'cell': 'test', 'cpu': '100%', 'disk': '100G', 'environment': 'dev', 'host_ip': '172.31.81.67', 'memory': '100M', 'name': 'proid.myapp#001', 'proid': 'foo', 'shared_network': False, 'task': '001', 'uniqueid': '0000000ID1234', 'archive': ['/var/tmp/treadmill'], 'endpoints': [{ 'port': 8000, 'name': 'http', 'real_port': 5000, 'proto': 'tcp', }], 'services': [{ 'name': 'web_server', 'command': '/bin/false', 'restart': { 'limit': 3, 'interval': 60, }, }], 'ephemeral_ports': { 'tcp': [], 'udp': [], } } treadmill.appmgr.manifest.read.return_value = manifest app_unique_name = 'proid.myapp-001-0000000ID1234' mock_ld_client = self.app_env.svc_localdisk.make_client.return_value localdisk = { 'block_dev': '/dev/foo', } mock_ld_client.get.return_value = localdisk mock_nwrk_client = self.app_env.svc_network.make_client.return_value network = { 'vip': '192.168.0.2', 'gateway': '192.168.254.254', 'veth': 'testveth.0', } mock_nwrk_client.get.return_value = network app_dir = os.path.join(self.root, 'apps', app_unique_name) # Create content in app root directory, verify that it is archived. fs.mkdir_safe(os.path.join(app_dir, 'root', 'xxx')) fs.mkdir_safe(os.path.join(app_dir, 'services')) # Simulate daemontools finish script, marking the app is done. with open(os.path.join(app_dir, 'aborted'), 'w') as aborted: aborted.write('something went wrong') mock_zkclient = kazoo.client.KazooClient() app_finish.finish(self.app_env, mock_zkclient, app_dir) treadmill.appevents.post( mock.ANY, events.AbortedTraceEvent(instanceid='proid.myapp#001', why=None, payload={ 'why': 'something went wrong', 'node': 'hostname', })) treadmill.rrdutils.flush_noexc.assert_called_with( os.path.join(self.root, 'metrics', 'apps', app_unique_name + '.rrd')) shutil.copy.assert_called_with( os.path.join(self.root, 'metrics', 'apps', app_unique_name + '.rrd'), os.path.join(app_dir, 'metrics.rrd'))
def test_finish_no_manifest(self): """Test app finish on directory with no app.yml. """ app_env = appmgr.AppEnvironment(root=self.root) app_finish.finish(app_env, None, self.root)
def test_finish_error(self): """Tests container finish procedure when app is improperly finished.""" manifest = { 'app': 'proid.myapp', 'cell': 'test', 'cpu': '100%', 'disk': '100G', 'environment': 'dev', 'host_ip': '172.31.81.67', 'memory': '100M', 'name': 'proid.myapp#001', 'proid': 'foo', 'shared_network': False, 'task': '001', 'uniqueid': '0000000001234', 'archive': ['/var/tmp/treadmill'], 'endpoints': [{ 'port': 8000, 'name': 'http', 'real_port': 5000 }], 'services': [{ 'command': '/bin/false', 'restart_count': 3, 'name': 'web_server' }], } treadmill.appmgr.manifest.read.return_value = manifest app_unique_name = 'proid.myapp-001-0000000001234' mock_ld_client = self.app_env.svc_localdisk.make_client.return_value localdisk = { 'block_dev': '/dev/foo', } mock_ld_client.get.return_value = localdisk mock_nwrk_client = self.app_env.svc_network.make_client.return_value network = { 'vip': '192.168.0.2', 'gateway': '192.168.254.254', 'veth': 'testveth.0', } mock_nwrk_client.get.return_value = network app_dir = os.path.join(self.app_env.apps_dir, app_unique_name) # Create content in app root directory, verify that it is archived. fs.mkdir_safe(os.path.join(app_dir, 'root', 'xxx')) fs.mkdir_safe(os.path.join(app_dir, 'services')) # Simulate daemontools finish script, marking the app is done. with open(os.path.join(app_dir, 'exitinfo'), 'w') as f: f.write(yaml.dump({'service': 'web_server', 'rc': 1, 'sig': 3})) kazoo.client.KazooClient.exists.return_value = True kazoo.client.KazooClient.get_children.return_value = [] app_finish.finish(self.app_env, kazoo.client.KazooClient(), app_dir) treadmill.appevents.post.assert_called_with( mock.ANY, 'proid.myapp#001', 'finished', '1.3', { 'sig': 3, 'service': 'web_server', 'rc': 1 }) treadmill.rrdutils.flush_noexc.assert_called_with( os.path.join(self.root, 'metrics', 'apps', app_unique_name + '.rrd')) shutil.copy.assert_called_with( os.path.join(self.app_env.metrics_dir, 'apps', app_unique_name + '.rrd'), os.path.join(app_dir, 'metrics.rrd'))