def load_runtime_manifest(tm_env, event, runtime): """load manifest data and modify based on runtime :param tm_env: Full path to the application node event in the zookeeper cache. :type event: ``treadmill.appenv.AppEnvironment`` :param event: Full path to the application node event in the zookeeper cache. :type event: ``str`` :param runtime: The name of the runtime to use. :type runtime: ``str`` :return: Application manifest object :rtype: ``dict`` """ manifest_data = app_manifest.load(event) # apply runtime manipulation on manifest data runtime_cls = app_runtime.get_runtime_cls(runtime) runtime_cls.manifest(tm_env, manifest_data) return manifest_data
def test_load_with_env(self): """Tests loading app manifest with resource allocation.""" manifest = { 'services': [{ 'name': 'web_server', 'command': '/bin/sleep 5', 'restart': { 'limit': 3, 'interval': 60, } }], 'proid': 'foo', 'environment': 'dev', 'disk': '100G', 'cpu': '100%', 'memory': '100M', 'environ': [{ 'name': 'xxx', 'value': 'yyy' }], } treadmill.appcfg.manifest.read.return_value = manifest event_filename0 = os.path.join(self.root, 'proid.myapp#0') app0 = app_manifest.load(self.tm_env, event_filename0) self.assertEqual(app0['environ'], [{'name': 'xxx', 'value': 'yyy'}])
def test_load_tar_image(self): """Tests loading app manifest with a docker image defined.""" manifest = { 'services': [{ 'name': 'web_server', 'command': '/bin/sleep 5', 'restart': { 'limit': 3, 'interval': 60, } }], 'proid': 'foo', 'environment': 'dev', 'disk': '100G', 'cpu': '100%', 'memory': '100M', 'image': 'http://test' } treadmill.appcfg.manifest.read.return_value = manifest event_filename0 = os.path.join(self.root, 'proid.myapp#0') app0 = app_manifest.load(self.tm_env, event_filename0) self.assertEqual(app0['image'], 'http://test') self.assertEqual(app0['type'], 'tar') manifest = { 'proid': 'foo', 'disk': '100G', 'cpu': '100%', 'memory': '100M', 'image': 'http://test' } treadmill.appcfg.manifest.read.return_value = manifest with self.assertRaises(exc.InvalidInputError): app_manifest.load(self.tm_env, event_filename0)
def test_load_docker_image(self): """Tests loading app manifest with a docker image defined.""" manifest = { 'proid': 'foo', 'environment': 'dev', 'disk': '100G', 'cpu': '100%', 'memory': '100M', 'image': 'docker://test' } treadmill.appcfg.manifest.read.return_value = manifest event_filename0 = os.path.join(self.root, 'proid.myapp#0') app0 = app_manifest.load(self.tm_env, event_filename0) self.assertEqual(app0['image'], 'docker://test') self.assertEqual(app0['type'], 'docker')
def load_runtime_manifest(tm_env, event, runtime): """load manifest data and modify based on runtime :param tm_env: Full path to the application node event in the zookeeper cache. :type event: ``treadmill.appenv.AppEnvironment`` :param event: Full path to the application node event in the zookeeper cache. :type event: ``str`` :param runtime: The name of the runtime to use. :type runtime: ``str`` :return: Application manifest object :rtype: ``dict`` """ manifest_data = app_manifest.load(event) # apply runtime manipulation on manifest data runtime_cls = app_runtime.get_runtime_cls(runtime) runtime_cls.manifest(tm_env, manifest_data) app_manifest.add_manifest_features(manifest_data, runtime_cls.name, tm_env) # TODO: This is HORRIBLE hack. # The code inside "cls".manifest used to create a copy of service # environment (manifest environ + svc environ). # # As such, any later changes to environment that were added by # loaded features did not have any effect. # # Service startup need to honor proper environment hierarchy. Load # "treadmill" envdif, then envdir of the container, then per svc. # # Need to move on, so making this change for now. for svc in manifest_data['services']: assert isinstance(manifest_data['environ'], list) assert isinstance(svc['environ'], list) svc['environ'] = manifest_data['environ'] + svc['environ'] return manifest_data
def test_load(self): """Tests loading app manifest with resource allocation.""" manifest = { 'services': [ { 'name': 'web_server', 'command': '/bin/sleep 5', 'restart': { 'limit': 3, 'interval': 60, }, }, ], 'proid': 'foo', 'environment': 'dev', 'disk': '100G', 'cpu': '100%', 'memory': '100M' } treadmill.appcfg.manifest.read.return_value = manifest event_filename0 = os.path.join(self.root, 'proid.myapp#0') app0 = app_manifest.load(self.tm_env, event_filename0) treadmill.appcfg.manifest.read.assert_called_with( event_filename0, 'yaml') treadmill.appcfg.gen_uniqueid.assert_called_with(event_filename0) self.assertEqual(app0['name'], 'proid.myapp#0') self.assertEqual(app0['type'], 'native') self.assertEqual(app0['app'], 'proid.myapp') self.assertEqual(app0['proid'], 'foo') self.assertEqual(app0['environment'], 'dev') self.assertEqual(app0['cpu'], 100) self.assertEqual(app0['memory'], '100M') self.assertEqual(app0['disk'], '100G') self.assertEqual(app0['services'], [ { 'name': 'web_server', 'command': '/bin/sleep 5', 'restart': { 'limit': 3, 'interval': 60, }, 'root': False, }, ]) self.assertEqual( app0['system_services'], [ { 'command': ('/path/to/mkdir -p /var/empty/sshd && ' '/path/to/chmod 0755 /var/empty/sshd && ' 'exec /path/to/sshd -D -f /etc/ssh/sshd_config' ' -p $TREADMILL_ENDPOINT_SSH'), 'name': 'sshd', 'proid': None, 'restart': { 'interval': 60, 'limit': 5, }, }, ], ) self.assertEqual(app0['endpoints'], [ { 'name': 'ssh', 'port': 0, 'proto': 'tcp', 'type': 'infra', }, ]) self.assertEqual(app0['uniqueid'], '42') self.assertEqual(app0['cell'], 'testcell') self.assertEqual(app0['zookeeper'], 'zookeeper://foo@foo:123')
def test_load_normalize(self): """Test the normalization of manifests. """ manifest = { 'services': [ { 'name': 'test1', 'command': '/bin/sleep 5', 'restart': { 'limit': 3, 'interval': 60, } }, { 'name': 'test2', 'command': '/bin/sleep 5', 'restart': { 'limit': 3, 'interval': 60, } }, ], 'endpoints': [ { 'name': 'test1', 'proto': 'udp', 'port': 12345, }, { 'name': 'test2', 'port': '0', }, { 'name': 'test3', 'port': 32, 'type': 'infra', }, ], 'ephemeral_ports': { 'tcp': '2', }, 'proid': 'foo', 'environment': 'dev', 'disk': '100G', 'cpu': '100%', 'memory': '100M' } treadmill.appcfg.manifest.read.return_value = manifest event_filename0 = os.path.join(self.root, 'proid.myapp#0') app0 = app_manifest.load(self.tm_env, event_filename0) self.assertEqual(app0['type'], 'native') self.assertEqual(app0['services'], [ { 'name': 'test1', 'command': '/bin/sleep 5', 'restart': { 'limit': 3, 'interval': 60, }, 'root': False, }, { 'name': 'test2', 'command': '/bin/sleep 5', 'restart': { 'limit': 3, 'interval': 60, }, 'root': False, }, ]) self.assertEqual(app0['system_services'], [ { 'command': ('/path/to/mkdir -p /var/empty/sshd && ' '/path/to/chmod 0755 /var/empty/sshd && ' 'exec /path/to/sshd' ' -D -f /etc/ssh/sshd_config' ' -p $TREADMILL_ENDPOINT_SSH'), 'name': 'sshd', 'proid': None, 'restart': { 'limit': 5, 'interval': 60, }, }, ]) self.assertEqual(app0['shared_ip'], False) self.assertEqual(app0['shared_network'], False) self.assertEqual(app0['endpoints'], [ { 'name': 'test1', 'type': None, 'port': 12345, 'proto': 'udp', }, { 'name': 'test2', 'type': None, 'port': 0, 'proto': 'tcp', }, { 'name': 'test3', 'type': 'infra', 'port': 32, 'proto': 'tcp', }, { 'name': 'ssh', 'type': 'infra', 'port': 0, 'proto': 'tcp', }, ]) self.assertEqual(app0['passthrough'], []) self.assertEqual(app0['vring']['cells'], []) self.assertEqual(app0['identity'], None) self.assertEqual(app0['identity_group'], None) self.assertEqual(app0['environ'], []) self.assertEqual(app0['ephemeral_ports'], {'tcp': 2, 'udp': 0})
def configure(tm_env, event): """Creates directory necessary for starting the application. This operation is idem-potent (it can be repeated). The directory layout is:: - (treadmill root) - apps - (app unique name) - app.yml run finish log/run The 'run' script is responsible for creating container environment and starting svscan inside the container. The 'finish' script is invoked when container terminates and will deallocate any resources (NAT rules, etc) that were allocated for the container. """ # R0915: Need to refactor long function into smaller pieces. # # pylint: disable=R0915 # Load the app from the event try: manifest_data = app_manifest.load(tm_env, event) except IOError: # File is gone. Nothing to do. _LOGGER.exception("No event to load: %r", event) return # Freeze the app data into a namedtuple object app = utils.to_obj(manifest_data) # Check the identity we are going to run as _check_identity(app.proid) # Generate a unique name for the app uniq_name = appcfg.app_unique_name(app) # Create the app's running directory container_dir = os.path.join(tm_env.apps_dir, uniq_name) if not fs.mkdir_safe(container_dir): _LOGGER.info('Resuming container %r', uniq_name) # Copy the event as 'manifest.yml' in the container dir shutil.copyfile(event, os.path.join(container_dir, 'manifest.yml')) # Store the app int the container_dir app_yml = os.path.join(container_dir, _APP_YML) with open(app_yml, 'w') as f: yaml.dump(manifest_data, stream=f) # Mark the container as defaulting to down state utils.touch(os.path.join(container_dir, 'down')) # Generate the supervisor's run script app_run_cmd = ' '.join( [treadmill.TREADMILL_BIN, 'sproc', 'run', container_dir]) utils.create_script(os.path.join(container_dir, 'run'), 'supervisor.run_no_log', cmd=app_run_cmd) fs.mkdir_safe(os.path.join(container_dir, 'log')) utils.create_script(os.path.join(container_dir, 'log', 'run'), 'logger.run') # Unique name for the link, based on creation time. cleanup_link = os.path.join(tm_env.cleanup_dir, uniq_name) if os.name == 'nt': finish_cmd = '%%COMSPEC%% /C mklink /D %s %s' % \ (cleanup_link, container_dir) else: finish_cmd = '/bin/ln -snvf %s %s' % (container_dir, cleanup_link) utils.create_script(os.path.join(container_dir, 'finish'), 'supervisor.finish', service=app.name, proid=None, cmds=[finish_cmd]) appevents.post( tm_env.app_events_dir, events.ConfiguredTraceEvent(instanceid=app.name, uniqueid=app.uniqueid)) return container_dir
def test_load(self): """Tests loading app manifest with resource allocation.""" manifest = { 'services': [ { 'name': 'web_server', 'command': '/bin/sleep 5', 'restart': { 'limit': 3, 'interval': 60, }, }, ], 'proid': 'foo', 'environment': 'dev', 'disk': '100G', 'cpu': '100%', 'memory': '100M' } treadmill.appcfg.manifest.read.return_value = manifest event_filename0 = os.path.join(self.root, 'proid.myapp#0') app0 = app_manifest.load(self.tm_env, event_filename0, 'linux') treadmill.appcfg.manifest.read.assert_called_with(event_filename0, 'yaml') treadmill.appcfg.gen_uniqueid.assert_called_with(event_filename0) self.assertEqual(app0['name'], 'proid.myapp#0') self.assertEqual(app0['type'], 'native') self.assertEqual(app0['app'], 'proid.myapp') self.assertEqual(app0['proid'], 'foo') self.assertEqual(app0['environment'], 'dev') self.assertEqual(app0['cpu'], 100) self.assertEqual(app0['memory'], '100M') self.assertEqual(app0['disk'], '100G') self.assertEqual(app0['uniqueid'], '42') self.assertEqual(app0['cell'], 'testcell') self.assertEqual(app0['zookeeper'], 'zookeeper://foo@foo:123') self.assertEqual( app0['services'][0], { 'proid': 'foo', 'root': False, 'name': 'web_server', 'command': '/bin/sleep 5', 'restart': { 'limit': 3, 'interval': 60, }, 'environ': [], 'config': None, 'downed': False, 'trace': True, }, ) self.assertTrue( any(x['name'] == 'sshd' for x in app0['services']) ) self.assertTrue( any(x['name'] == 'register' for x in app0['system_services']) ) self.assertTrue( any(x['name'] == 'hostaliases' for x in app0['system_services']) ) self.assertTrue( any(x['name'] == 'monitor' for x in app0['system_services']) ) self.assertTrue( any(x['name'] == 'start_container' for x in app0['system_services']) ) self.assertEqual( app0['endpoints'], [ { 'name': 'ssh', 'port': 0, 'proto': 'tcp', 'type': 'infra', }, ] )
def configure(tm_env, event, runtime): """Creates directory necessary for starting the application. This operation is idem-potent (it can be repeated). The directory layout is:: - (treadmill root)/ - apps/ - (app unique name)/ - data/ - app_start - app.json - manifest.yml - policy.json env/ - TREADMILL_* run finish log/ - run The 'run' script is responsible for creating container environment and starting the container. The 'finish' script is invoked when container terminates and will deallocate any resources (NAT rules, etc) that were allocated for the container. """ # Load the app from the event try: manifest_data = app_manifest.load(tm_env, event, runtime) except IOError: # File is gone. Nothing to do. _LOGGER.exception('No event to load: %r', event) return # Freeze the app data into a namedtuple object app = utils.to_obj(manifest_data) # Generate a unique name for the app uniq_name = appcfg.app_unique_name(app) # Write the actual container start script if os.name == 'nt': run_script = ' '.join( [sys.executable, '-m', 'treadmill.ms', 'sproc', 'run', '.']) else: run_script = ' '.join( ['exec', dist.TREADMILL_BIN, 'sproc', 'run', '../']) # Create the service for that container container_svc = supervisor.create_service(tm_env.apps_dir, name=uniq_name, app_run_script=run_script, userid='root', downed=False, monitor_policy={ 'limit': 0, 'interval': 60 }, environ={}, environment=app.environment) data_dir = container_svc.data_dir # Copy the original event as 'manifest.yml' in the container dir shutil.copyfile(event, os.path.join(data_dir, 'manifest.yml')) # Store the app.json in the container directory fs.write_safe(os.path.join(data_dir, appcfg.APP_JSON), lambda f: f.writelines(utils.json_genencode(manifest_data)), mode='w', permission=0o644) appevents.post( tm_env.app_events_dir, events.ConfiguredTraceEvent(instanceid=app.name, uniqueid=app.uniqueid)) return container_svc.directory