def _cleanup(tm_env, container_dir, app): """Cleanup a container that actually ran. """ # Generate a unique name for the app unique_name = appcfg.app_unique_name(app) # Create service clients cgroup_client = tm_env.svc_cgroup.make_client( os.path.join(container_dir, 'resources', 'cgroups')) localdisk_client = tm_env.svc_localdisk.make_client( os.path.join(container_dir, 'resources', 'localdisk')) network_client = tm_env.svc_network.make_client( os.path.join(container_dir, 'resources', 'network')) presence_client = tm_env.svc_presence.make_client( os.path.join(container_dir, 'resources', 'presence')) # Unregister presence presence_client.delete(app.name) # Destroy the volume try: localdisk_client.delete(unique_name) except (IOError, OSError) as err: if err.errno == errno.ENOENT: pass else: raise if not app.shared_network: _cleanup_network(tm_env, container_dir, app, network_client) # Add metrics to archive rrd_file = os.path.join( tm_env.metrics_dir, 'apps', '{name}-{instanceid}-{uniqueid}.rrd'.format( name=app.app, instanceid=app.task, uniqueid=app.uniqueid, )) rrdutils.flush_noexc(rrd_file) _copy_metrics(rrd_file, container_dir) # Cleanup our cgroup resources try: cgroup_client.delete(unique_name) except (IOError, OSError) as err: if err.errno == errno.ENOENT: pass else: raise try: runtime.archive_logs(tm_env, appcfg.app_unique_name(app), container_dir) except Exception: # pylint: disable=W0703 _LOGGER.exception('Unexpected exception storing local logs.')
def test__create_root_dir(self): """Test creation on the container root directory.""" # Access protected module _create_root_dir # pylint: disable=W0212 app = utils.to_obj({ 'type': 'native', 'proid': 'myproid', 'name': 'myproid.test#0', 'uniqueid': 'ID1234', 'environment': 'dev', 'disk': '100G', }) app_unique_name = appcfg.app_unique_name(app) container_dir = os.path.join(self.root, 'apps', app_unique_name) mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value localdisk = { 'block_dev': '/dev/foo', } mock_ld_client.wait.return_value = localdisk treadmill.runtime.linux._run._create_root_dir(self.tm_env, container_dir, '/some/root_dir', app) treadmill.fs.create_filesystem.assert_called_with('/dev/foo') unshare.unshare.assert_called_with(unshare.CLONE_NEWNS) treadmill.fs.mount_filesystem('/dev/foo', '/some/root_dir')
def test__create_root_dir(self): """Test creation on the container root directory.""" # Access protected module _create_root_dir # pylint: disable=W0212 app = utils.to_obj( { 'proid': 'myproid', 'name': 'myproid.test#0', 'uniqueid': 'ID1234', 'environment': 'dev', 'disk': '100G', } ) app_unique_name = appcfg.app_unique_name(app) container_dir = os.path.join(self.root, 'apps', app_unique_name) mock_ld_client = self.tm_env.svc_localdisk.make_client.return_value localdisk = { 'block_dev': '/dev/foo', } mock_ld_client.wait.return_value = localdisk treadmill.runtime.linux._run._create_root_dir(self.tm_env, container_dir, '/some/root_dir', app) treadmill.fs.chroot_init.assert_called_with('/some/root_dir') treadmill.fs.create_filesystem.assert_called_with('/dev/foo') treadmill.fs.mount_filesystem('/dev/foo', '/some/root_dir') treadmill.fs.make_rootfs.assert_called_with('/some/root_dir', 'myproid') treadmill.fs.configure_plugins.assert_called_with( self.root, '/some/root_dir', app ) shutil.rmtree.assert_called_with( '/some/root_dir/.etc' ) shutil.copytree.assert_called_with( os.path.join(self.tm_env.root, 'etc'), '/some/root_dir/.etc' ) shutil.copyfile.assert_has_calls([ mock.call('/etc/hosts', '/some/root_dir/.etc/hosts'), mock.call('/etc/hosts', '/some/root_dir/.etc/hosts.original'), ]) treadmill.subproc.check_call.assert_has_calls([ mock.call( [ 'mount', '-n', '--bind', os.path.join(self.tm_env.root, 'etc/resolv.conf'), '/etc/resolv.conf' ] ) ])
def apply_cgroup_limits(tm_env, container_dir, manifest): """Configures cgroups and limits. :param tm_env: Treadmill application environment :type tm_env: `appenv.AppEnvironment` :param container_dir: Full path to the container :type container_dir: ``str`` :param manifest: App manifest. :type manifest: ``dict`` """ app = utils.to_obj(manifest) # Generate a unique name for the app unique_name = appcfg.app_unique_name(app) # Setup the service clients cgroup_client = tm_env.svc_cgroup.make_client( os.path.join(container_dir, 'cgroups')) localdisk_client = tm_env.svc_localdisk.make_client( os.path.join(container_dir, 'localdisk')) network_client = tm_env.svc_network.make_client( os.path.join(container_dir, 'network')) # Cgroup cgroup_req = { 'memory': app.memory, 'cpu': app.cpu, } # Local Disk localdisk_req = { 'size': app.disk, } # Network network_req = { 'environment': app.environment, } cgroup_client.put(unique_name, cgroup_req) localdisk_client.put(unique_name, localdisk_req) if not app.shared_network: network_client.put(unique_name, network_req) app_cgroups = cgroup_client.wait(unique_name) _LOGGER.info('Joining cgroups: %r', app_cgroups) for subsystem, cgrp in app_cgroups.items(): cgroups.join(subsystem, cgrp)
def _create_root_dir(tm_env, container_dir, root_dir, app): """Prepares chrooted environment and creates all mountpoints. :param tm_env: Treadmill application environment :type tm_env: `appenv.AppEnvironment` """ # Generate a unique name for the app unique_name = appcfg.app_unique_name(app) # First wait for the block device to be ready localdisk_client = tm_env.svc_localdisk.make_client( os.path.join(container_dir, 'localdisk')) localdisk = localdisk_client.wait(unique_name) already_initialized = fs.test_filesystem(localdisk['block_dev']) if not already_initialized: # Format the block device fs.create_filesystem(localdisk['block_dev']) _LOGGER.info('Creating container root directory: %s', root_dir) fs.chroot_init(root_dir) fs.mount_filesystem(localdisk['block_dev'], root_dir) fs.make_rootfs(root_dir, app.proid) fs.configure_plugins(tm_env.root, root_dir, app) # Ensure .etc directory is clean in case of volume restore. try: shutil.rmtree(os.path.join(root_dir, '.etc')) except OSError as err: if err.errno != errno.ENOENT: raise shutil.copytree(os.path.join(tm_env.root, 'etc'), os.path.join(root_dir, '.etc')) shutil.copyfile('/etc/hosts', os.path.join(root_dir, '.etc/hosts')) shutil.copyfile('/etc/hosts', os.path.join(root_dir, '.etc/hosts.original')) hosts_aliases = os.path.join(root_dir, '.etc', 'hosts-aliases') fs.mkdir_safe(hosts_aliases) pwnam = pwd.getpwnam(app.proid) os.chown(hosts_aliases, pwnam.pw_uid, pwnam.pw_gid) # Always use our own resolv.conf. Safe to rbind, as we are running in # private mount subsystem by now. resolv_conf_path = os.path.join(tm_env.root, 'etc/resolv.conf') if os.path.exists(resolv_conf_path): subproc.check_call( ['mount', '-n', '--bind', resolv_conf_path, '/etc/resolv.conf'])
def kill(self): app = runtime.load_app(self._service.data_dir, runtime.STATE_JSON) if not app: return name = appcfg.app_unique_name(app) try: client = self._get_client() container = client.containers.get(name) container.kill() except docker.errors.NotFound: pass
def _create_container(tm_env, conf, client, app): """Create docker container from given app. """ ports = {} for endpoint in app.endpoints: port_key = '{0}/{1}'.format(endpoint.port, endpoint.proto) ports[port_key] = endpoint.real_port _log_port_mapping_config(ports) # app.image contains a uri which starts with docker:// image_name = app.image[9:] client.images.pull(image_name) name = appcfg.app_unique_name(app) container_args = { 'image': image_name, 'name': name, 'environment': _create_environ(app), 'entrypoint': app.command, 'command': app.args, 'detach': True, 'tty': True, 'ports': ports, 'network': conf.get('network', 'nat'), # 1024 is max number of shares for docker 'cpu_shares': int( (app.cpu / (multiprocessing.cpu_count() * 100.0)) * 1024), 'mem_limit': app.memory, 'storage_opt': { 'size': app.disk }, 'ulimits': [ {'core': 'unlimited'}, {'nofile': 32768}, ], } if os.name == 'nt': _get_gmsa(tm_env, client, app, container_args) try: # The container might exist already # TODO: start existing container with different ports container = client.containers.get(name) container.remove(force=True) except docker.errors.NotFound: pass return client.containers.create(**container_args)
def _finish(self): app = runtime.load_app(self._service.data_dir, runtime.STATE_JSON) if app: client = self._get_client() container = state = None name = appcfg.app_unique_name(app) try: container = client.containers.get(name) state = container.attrs.get('State') except docker.errors.NotFound: pass if container is not None: try: container.remove(force=True) except docker.errors.APIError: _LOGGER.error('Failed to remove %s', container.id) aborted = _check_aborted(self._service.data_dir) if aborted is not None: app_abort.report_aborted(self._tm_env, app.name, why=aborted.get('why'), payload=aborted.get('payload')) elif state is not None: if state.get('OOMKilled', False): event = events.KilledTraceEvent( instanceid=app.name, is_oom=True, ) else: event = events.FinishedTraceEvent(instanceid=app.name, rc=state.get( 'ExitCode', 256), signal=0, payload=state) trace.post(self._tm_env.app_events_dir, event) if os.name == 'nt': credential_spec.cleanup(name, client) try: runtime.archive_logs(self._tm_env, name, self._service.data_dir) except Exception: # pylint: disable=W0703 _LOGGER.exception('Unexpected exception storing local logs.')
def share_cgroup_info(app, root_dir): """Shares subset of cgroup tree with the container.""" # Bind /cgroup/memory inside chrooted environment to /cgroup/.../memory # of the container. unique_name = appcfg.app_unique_name(app) cgrp = os.path.join('treadmill', 'apps', unique_name) # FIXME: This should be removed and proper cgroups should be # exposed (readonly). This is so that tools that # (correctly) read /proc/self/cgroups can access cgroup # data. shared_subsystems = ['memory'] for subsystem in shared_subsystems: fs.mkdir_safe(os.path.join(root_dir, 'cgroup', subsystem)) fs.mount_bind(root_dir, os.path.join('/cgroup', subsystem), cgroups.makepath(subsystem, cgrp))
def _create_root_dir(tm_env, container_dir, root_dir, app): """Prepares chrooted environment.""" # Generate a unique name for the app unique_name = appcfg.app_unique_name(app) # First wait for the block device to be ready localdisk_client = tm_env.svc_localdisk.make_client( os.path.join(container_dir, 'localdisk')) localdisk = localdisk_client.wait(unique_name) already_initialized = fs.test_filesystem(localdisk['block_dev']) if not already_initialized: # Format the block device fs.create_filesystem(localdisk['block_dev']) _LOGGER.info('Creating container root directory: %s', root_dir) # Creates directory that will serve as new root. fs.mkdir_safe(fs.norm_safe(root_dir)) # Unshare the mount namespace unshare.unshare(unshare.CLONE_NEWNS) # Mount the container root volume fs.mount_filesystem(localdisk['block_dev'], root_dir)
def test__create_root_dir(self): """Test creation on the container root directory.""" # Access protected module _create_root_dir # pylint: disable=W0212 app = utils.to_obj({ 'type': 'native', 'proid': 'myproid', 'name': 'myproid.test#0', 'uniqueid': 'ID1234', 'environment': 'dev', 'disk': '100G', }) app_unique_name = appcfg.app_unique_name(app) container_dir = '/some/dir' localdisk = { 'block_dev': '/dev/foo', } treadmill.runtime.linux._run._create_root_dir(container_dir, localdisk) treadmill.fs.linux.blk_fs_create.assert_called_with('/dev/foo') unshare.unshare.assert_called_with(unshare.CLONE_NEWNS) treadmill.fs.linux.mount_filesystem.assert_called_with( '/dev/foo', os.path.join(container_dir, 'root'), fs_type='ext4')
def configure(tm_env, event, runtime): """Creates directory necessary for starting the application. This operation is idem-potent (it can be repeated). The directory layout is:: - (treadmill root)/ - apps/ - (app unique name)/ - data/ - app_start - app.json - manifest.yml - policy.json env/ - TREADMILL_* run finish log/ - run The 'run' script is responsible for creating container environment and starting the container. The 'finish' script is invoked when container terminates and will deallocate any resources (NAT rules, etc) that were allocated for the container. """ # Load the app from the event try: manifest_data = app_manifest.load(tm_env, event, runtime) except IOError: # File is gone. Nothing to do. _LOGGER.exception('No event to load: %r', event) return # Freeze the app data into a namedtuple object app = utils.to_obj(manifest_data) # Generate a unique name for the app uniq_name = appcfg.app_unique_name(app) # Write the actual container start script if os.name == 'nt': run_script = ' '.join( [sys.executable, '-m', 'treadmill.ms', 'sproc', 'run', '.']) else: run_script = ' '.join( ['exec', dist.TREADMILL_BIN, 'sproc', 'run', '../']) # Create the service for that container container_svc = supervisor.create_service(tm_env.apps_dir, name=uniq_name, app_run_script=run_script, userid='root', downed=False, monitor_policy={ 'limit': 0, 'interval': 60 }, environ={}, environment=app.environment) data_dir = container_svc.data_dir # Copy the original event as 'manifest.yml' in the container dir shutil.copyfile(event, os.path.join(data_dir, 'manifest.yml')) # Store the app.json in the container directory fs.write_safe(os.path.join(data_dir, appcfg.APP_JSON), lambda f: f.writelines(utils.json_genencode(manifest_data)), mode='w', permission=0o644) appevents.post( tm_env.app_events_dir, events.ConfiguredTraceEvent(instanceid=app.name, uniqueid=app.uniqueid)) return container_svc.directory
def _cleanup(tm_env, zkclient, container_dir, app): """Cleanup a container that actually ran. """ # Too many branches. # # pylint: disable=R0912 rootdir = os.path.join(container_dir, 'root') # Generate a unique name for the app unique_name = appcfg.app_unique_name(app) # Create service clients cgroup_client = tm_env.svc_cgroup.make_client( os.path.join(container_dir, 'cgroups')) localdisk_client = tm_env.svc_localdisk.make_client( os.path.join(container_dir, 'localdisk')) network_client = tm_env.svc_network.make_client( os.path.join(container_dir, 'network')) # Make sure all processes are killed # FIXME(boysson): Should we use `kill_apps_in_cgroup` instead? _kill_apps_by_root(rootdir) # Setup the archive filename that will hold this container's data filetime = utils.datetime_utcnow().strftime('%Y%m%d_%H%M%S%f') archive_filename = os.path.join( container_dir, '{instance_name}_{hostname}_{timestamp}.tar'.format( instance_name=appcfg.appname_task_id(app.name), hostname=sysinfo.hostname(), timestamp=filetime)) # Tar up container root filesystem if archive list is in manifest try: localdisk = localdisk_client.get(unique_name) fs.archive_filesystem(localdisk['block_dev'], rootdir, archive_filename, app.archive) except services.ResourceServiceError: _LOGGER.warning('localdisk never allocated') except subprocess.CalledProcessError: _LOGGER.exception('Unable to archive root device of %r', unique_name) except: # pylint: disable=W0702 _LOGGER.exception('Unknown exception while archiving %r', unique_name) # Destroy the volume try: localdisk = localdisk_client.delete(unique_name) except (IOError, OSError) as err: if err.errno == errno.ENOENT: pass else: raise if not app.shared_network: _cleanup_network(tm_env, app, network_client) # Add metrics to archive rrd_file = os.path.join( tm_env.metrics_dir, 'apps', '{name}-{instanceid}-{uniqueid}.rrd'.format( name=app.app, instanceid=app.task, uniqueid=app.uniqueid, )) rrdutils.flush_noexc(rrd_file) _copy_metrics(rrd_file, container_dir) # Cleanup our cgroup resources try: cgroup_client.delete(unique_name) except (IOError, OSError) as err: if err.errno == errno.ENOENT: pass else: raise try: _archive_logs(tm_env, container_dir) except Exception: # pylint: disable=W0703 _LOGGER.exception('Unexpected exception storing local logs.') # Append or create the tarball with folders outside of container # Compress and send the tarball to HCP try: archive_filename = fs.tar(sources=container_dir, target=archive_filename, compression='gzip').name _send_container_archive(zkclient, app, archive_filename) except: # pylint: disable=W0702 _LOGGER.exception("Failed to update archive")
def create_supervision_tree(tm_env, container_dir, root_dir, app, cgroups_path): """Creates s6 supervision tree.""" uniq_name = appcfg.app_unique_name(app) ctl_uds = os.path.join(os.sep, 'run', 'tm_ctl') tombstone_ctl_uds = os.path.join(ctl_uds, 'tombstone') sys_dir = os.path.join(container_dir, 'sys') try: old_system_services = [ svc_name for svc_name in os.listdir(sys_dir) if (not svc_name.startswith('.') and os.path.isdir(os.path.join(sys_dir, svc_name))) ] except FileNotFoundError: old_system_services = [] new_system_services = [svc_def.name for svc_def in app.system_services] for svc_name in set(old_system_services) - set(new_system_services): _LOGGER.info('Removing old system service: %s', svc_name) fs.rmtree_safe(os.path.join(sys_dir, svc_name)) sys_scandir = supervisor.create_scan_dir( sys_dir, finish_timeout=6000, wait_cgroups=cgroups_path, ) for svc_def in app.system_services: if svc_def.restart is not None: monitor_policy = { 'limit': svc_def.restart.limit, 'interval': svc_def.restart.interval, 'tombstone': { 'uds': False, 'path': tm_env.services_tombstone_dir, 'id': '{},{}'.format(uniq_name, svc_def.name) } } else: monitor_policy = None supervisor.create_service( sys_scandir, name=svc_def.name, app_run_script=svc_def.command, userid='root', environ_dir=os.path.join(container_dir, _CONTAINER_ENV_DIR), environ={envvar.name: envvar.value for envvar in svc_def.environ}, environment=app.environment, downed=svc_def.downed, trace=None, monitor_policy=monitor_policy) sys_scandir.write() services_dir = os.path.join(container_dir, 'services') services_scandir = supervisor.create_scan_dir(services_dir, finish_timeout=5000) for svc_def in app.services: if svc_def.restart is not None: monitor_policy = { 'limit': svc_def.restart.limit, 'interval': svc_def.restart.interval, 'tombstone': { 'uds': True, 'path': tombstone_ctl_uds, 'id': '{},{}'.format(uniq_name, svc_def.name) } } else: monitor_policy = None if svc_def.trace is not None: trace = { 'instanceid': app.name, 'uniqueid': app.uniqueid, 'service': svc_def.name, 'path': os.path.join(ctl_uds, 'appevents') } else: trace = None logger_template = getattr(svc_def, 'logger', 's6.app-logger.run') _LOGGER.info('Using logger: %s', logger_template) supervisor.create_service( services_scandir, name=svc_def.name, app_run_script=svc_def.command, userid=svc_def.proid, environ_dir='/' + _CONTAINER_ENV_DIR, environ={envvar.name: envvar.value for envvar in svc_def.environ}, environment=app.environment, downed=svc_def.downed, trace=trace if svc_def.trace else None, log_run_script=logger_template, monitor_policy=monitor_policy) services_scandir.write() # Bind the service directory in the container volume fs.mkdir_safe(os.path.join(root_dir, 'services')) fs_linux.mount_bind(root_dir, os.path.join(os.sep, 'services'), source=os.path.join(container_dir, 'services'), recursive=False, read_only=False) # Bind the ctrl directory in the container volume which has all the # unix domain sockets to communicate outside the container to treadmill fs.mkdir_safe(os.path.join(root_dir, 'run', 'tm_ctl')) fs_linux.mount_bind(root_dir, os.path.join(os.sep, 'run', 'tm_ctl'), source=tm_env.ctl_dir, recursive=False, read_only=False)
def _unshare_network(tm_env, container_dir, app): """Configures private app network. :param ``appenv.AppEnvironment`` tm_env: Treadmill application environment """ unique_name = appcfg.app_unique_name(app) # Configure DNAT rules while on host network. for endpoint in app.endpoints: _LOGGER.info('Creating DNAT rule: %s:%s -> %s:%s', app.network.external_ip, endpoint.real_port, app.network.vip, endpoint.port) dnatrule = firewall.DNATRule(proto=endpoint.proto, dst_ip=app.network.external_ip, dst_port=endpoint.real_port, new_ip=app.network.vip, new_port=endpoint.port) snatrule = firewall.SNATRule(proto=endpoint.proto, src_ip=app.network.vip, src_port=endpoint.port, new_ip=app.network.external_ip, new_port=endpoint.real_port) tm_env.rules.create_rule(chain=iptables.PREROUTING_DNAT, rule=dnatrule, owner=unique_name) tm_env.rules.create_rule(chain=iptables.POSTROUTING_SNAT, rule=snatrule, owner=unique_name) # See if this container requires vring service if app.vring: _LOGGER.debug('adding %r to VRing set', app.network.vip) iptables.add_ip_set( iptables.SET_VRING_CONTAINERS, app.network.vip ) # See if this was an "infra" endpoint and if so add it to the whitelist # set. if getattr(endpoint, 'type', None) == 'infra': _LOGGER.debug('adding %s:%s to infra services set', app.network.vip, endpoint.port) iptables.add_ip_set( iptables.SET_INFRA_SVC, '{ip},{proto}:{port}'.format( ip=app.network.vip, proto=endpoint.proto, port=endpoint.port, ) ) for port in app.ephemeral_ports.tcp: _LOGGER.info('Creating ephemeral DNAT rule: %s:%s -> %s:%s', app.network.external_ip, port, app.network.vip, port) dnatrule = firewall.DNATRule(proto='tcp', dst_ip=app.network.external_ip, dst_port=port, new_ip=app.network.vip, new_port=port) tm_env.rules.create_rule(chain=iptables.PREROUTING_DNAT, rule=dnatrule, owner=unique_name) # Treat ephemeral ports as infra, consistent with current prodperim # behavior. iptables.add_ip_set(iptables.SET_INFRA_SVC, '{ip},tcp:{port}'.format(ip=app.network.vip, port=port)) for port in app.ephemeral_ports.udp: _LOGGER.info('Creating ephemeral DNAT rule: %s:%s -> %s:%s', app.network.external_ip, port, app.network.vip, port) dnatrule = firewall.DNATRule(proto='udp', dst_ip=app.network.external_ip, dst_port=port, new_ip=app.network.vip, new_port=port) tm_env.rules.create_rule(chain=iptables.PREROUTING_DNAT, rule=dnatrule, owner=unique_name) # Treat ephemeral ports as infra, consistent with current prodperim # behavior. iptables.add_ip_set(iptables.SET_INFRA_SVC, '{ip},udp:{port}'.format(ip=app.network.vip, port=port)) # configure passthrough while on main network. if getattr(app, 'passthrough', None): _LOGGER.info('adding passthrough for: %r', app.passthrough) # Resolve all the hosts (+dedup) new_ips = { socket.gethostbyname(host) for host in app.passthrough } # Create a passthrough rule from each of the source IP to the # container IP and record these source IP in a set. for ipaddr in new_ips: passthroughrule = firewall.PassThroughRule( src_ip=ipaddr, dst_ip=app.network.vip, ) tm_env.rules.create_rule(chain=iptables.PREROUTING_PASSTHROUGH, rule=passthroughrule, owner=unique_name) # configure exception filter rules try: firewall_plugin = plugin_manager.load( 'treadmill.firewall.plugins', 'firewall' ) firewall_plugin.apply_exception_rules(tm_env, container_dir, app) except Exception: # pylint: disable=W0703 _LOGGER.exception( 'Error in firewall plugin, skip applying firewall exception rules.' ) service_ip = None if app.shared_ip: service_ip = app.network.external_ip # Unshare network and create virtual device newnet.create_newnet(app.network.veth, app.network.vip, app.network.gateway, service_ip)
def _cleanup_network(tm_env, container_dir, app, network_client): """Cleanup the network part of a container. """ # Generate a unique name for the app unique_name = appcfg.app_unique_name(app) try: app_network = network_client.get(unique_name) except services.ResourceServiceError: _LOGGER.warning('network never allocated') return if app_network is None: _LOGGER.info('Network resource already freed') return # Unconfigure passthrough if hasattr(app, 'passthrough'): _LOGGER.info('Deleting passthrough for: %r', app.passthrough) # Resolve all the hosts # FIXME: There is no guarantie the hosts will resolve to # the same IPs as they did during creation. ips = set([socket.gethostbyname(host) for host in app.passthrough]) for ip in ips: tm_env.rules.unlink_rule( chain=iptables.PREROUTING_PASSTHROUGH, rule=firewall.PassThroughRule(src_ip=ip, dst_ip=app_network['vip']), owner=unique_name, ) if app.vring: # Mark the container's IP as VRing enabled _LOGGER.debug('removing %r from VRing set', app_network['vip']) iptables.rm_ip_set(iptables.SET_VRING_CONTAINERS, app_network['vip']) for endpoint in app.endpoints: tm_env.rules.unlink_rule( chain=iptables.PREROUTING_DNAT, rule=firewall.DNATRule(proto=endpoint.proto, dst_ip=app_network['external_ip'], dst_port=endpoint.real_port, new_ip=app_network['vip'], new_port=endpoint.port), owner=unique_name, ) tm_env.rules.unlink_rule( chain=iptables.POSTROUTING_SNAT, rule=firewall.SNATRule(proto=endpoint.proto, src_ip=app_network['vip'], src_port=endpoint.port, new_ip=app_network['external_ip'], new_port=endpoint.real_port), owner=unique_name, ) # See if this was an "infra" endpoint and if so remove it # from the whitelist set. if getattr(endpoint, 'type', None) == 'infra': _LOGGER.debug('removing %s:%s from infra services set', app_network['vip'], endpoint.port) iptables.rm_ip_set( iptables.SET_INFRA_SVC, '{ip},{proto}:{port}'.format( ip=app_network['vip'], proto=endpoint.proto, port=endpoint.port, )) _cleanup_ephemeral_ports(tm_env, unique_name, app_network['external_ip'], app_network['vip'], app.ephemeral_ports.tcp, 'tcp') _cleanup_ephemeral_ports(tm_env, unique_name, app_network['external_ip'], app_network['vip'], app.ephemeral_ports.udp, 'udp') _cleanup_exception_rules(tm_env, container_dir, app) # Terminate any entries in the conntrack table iptables.flush_cnt_conntrack_table(app_network['vip']) # Cleanup network resources network_client.delete(unique_name)
def test__unshare_network_simple(self): """Tests unshare network sequence. """ # Disable W0212: Access to a protected member # pylint: disable=W0212 app = utils.to_obj({ 'type': 'native', 'name': 'proid.test#0', 'uniqueid': 'ID1234', 'environment': 'dev', 'network': { 'veth': 'id1234.0', 'vip': '192.168.1.1', 'gateway': '192.168.254.254', 'external_ip': '172.31.81.67', }, 'shared_ip': True, 'ephemeral_ports': { 'tcp': [], 'udp': [], }, 'endpoints': [{ 'real_port': '5007', 'proto': 'tcp', 'port': '22', 'type': 'infra' }, { 'real_port': '5013', 'proto': 'udp', 'port': '12345' }], 'vring': { 'some': 'data' } }) app_unique_name = appcfg.app_unique_name(app) treadmill.runtime.linux._run._unshare_network(self.tm_env, 'test_container_dir', app) treadmill.iptables.add_ip_set.assert_has_calls([ mock.call(treadmill.iptables.SET_VRING_CONTAINERS, '192.168.1.1'), mock.call(treadmill.iptables.SET_INFRA_SVC, '192.168.1.1,tcp:22'), ], any_order=True) self.tm_env.rules.create_rule.assert_has_calls([ mock.call(chain=iptables.PREROUTING_DNAT, rule=firewall.DNATRule(proto='tcp', dst_ip='172.31.81.67', dst_port='5007', new_ip='192.168.1.1', new_port='22'), owner=app_unique_name), mock.call(chain=iptables.POSTROUTING_SNAT, rule=firewall.SNATRule(proto='tcp', src_ip='192.168.1.1', src_port='22', new_ip='172.31.81.67', new_port='5007'), owner=app_unique_name), mock.call(chain=iptables.PREROUTING_DNAT, rule=firewall.DNATRule(proto='udp', dst_ip='172.31.81.67', dst_port='5013', new_ip='192.168.1.1', new_port='12345'), owner=app_unique_name), mock.call(chain=iptables.POSTROUTING_SNAT, rule=firewall.SNATRule(proto='udp', src_ip='192.168.1.1', src_port='12345', new_ip='172.31.81.67', new_port='5013'), owner=app_unique_name) ], any_order=True) self.assertEqual(self.tm_env.rules.create_rule.call_count, 4) treadmill.newnet.create_newnet.assert_called_with( 'id1234.0', '192.168.1.1', '192.168.254.254', '172.31.81.67', )
def test__unshare_network_complex(self): """Test unshare network advanced sequence (ephemeral/passthrough).""" # Disable W0212: Access to a protected member # pylint: disable=W0212 app = utils.to_obj({ 'type': 'native', 'name': 'myproid.test#0', 'environment': 'dev', 'uniqueid': 'ID1234', 'network': { 'veth': 'id1234.0', 'vip': '192.168.0.2', 'gateway': '192.168.254.254', 'external_ip': '172.31.81.67', }, 'shared_ip': False, 'endpoints': [{ 'name': 'ssh', 'port': 54321, 'real_port': 54321, 'type': 'infra', 'proto': 'tcp', }, { 'name': 'test2', 'port': 54322, 'real_port': 54322, 'proto': 'udp', }], 'ephemeral_ports': { 'tcp': [10000, 10001, 10002], 'udp': [], }, 'passthrough': [ 'xxx', 'yyy', 'zzz', ], 'vring': { 'some': 'data' } }) app_unique_name = appcfg.app_unique_name(app) hosts_to_ip = { 'xxx': '4.4.4.4', 'yyy': '5.5.5.5', 'zzz': '5.5.5.5', } socket.gethostbyname.side_effect = lambda h: hosts_to_ip[h] self.tm_env.rules.get_rules.return_value = set() treadmill.runtime.linux._run._unshare_network(self.tm_env, 'test_container_dir', app) self.tm_env.rules.create_rule.assert_has_calls([ mock.call(chain=iptables.PREROUTING_DNAT, rule=firewall.DNATRule(proto='tcp', dst_ip='172.31.81.67', dst_port=54321, new_ip='192.168.0.2', new_port=54321), owner=app_unique_name), mock.call(chain=iptables.POSTROUTING_SNAT, rule=firewall.SNATRule(proto='tcp', src_ip='192.168.0.2', src_port=54321, new_ip='172.31.81.67', new_port=54321), owner=app_unique_name), mock.call(chain=iptables.PREROUTING_DNAT, rule=firewall.DNATRule(proto='udp', dst_ip='172.31.81.67', dst_port=54322, new_ip='192.168.0.2', new_port=54322), owner=app_unique_name), mock.call(chain=iptables.POSTROUTING_SNAT, rule=firewall.SNATRule(proto='udp', src_ip='192.168.0.2', src_port=54322, new_ip='172.31.81.67', new_port=54322), owner=app_unique_name), mock.call(chain=iptables.PREROUTING_DNAT, rule=firewall.DNATRule(proto='tcp', dst_ip='172.31.81.67', dst_port=10000, new_ip='192.168.0.2', new_port=10000), owner=app_unique_name), mock.call(chain=iptables.PREROUTING_DNAT, rule=firewall.DNATRule(proto='tcp', dst_ip='172.31.81.67', dst_port=10001, new_ip='192.168.0.2', new_port=10001), owner=app_unique_name), mock.call(chain=iptables.PREROUTING_DNAT, rule=firewall.DNATRule(proto='tcp', dst_ip='172.31.81.67', dst_port=10002, new_ip='192.168.0.2', new_port=10002), owner=app_unique_name), mock.call(chain=iptables.PREROUTING_PASSTHROUGH, rule=firewall.PassThroughRule('4.4.4.4', '192.168.0.2'), owner=app_unique_name), mock.call(chain=iptables.PREROUTING_PASSTHROUGH, rule=firewall.PassThroughRule('5.5.5.5', '192.168.0.2'), owner=app_unique_name), ], any_order=True) self.assertEqual(self.tm_env.rules.create_rule.call_count, 9) # Check that infra services + ephemeral ports are in the same set. treadmill.iptables.add_ip_set.assert_has_calls([ mock.call(treadmill.iptables.SET_VRING_CONTAINERS, '192.168.0.2'), mock.call(treadmill.iptables.SET_INFRA_SVC, '192.168.0.2,tcp:54321'), mock.call(treadmill.iptables.SET_INFRA_SVC, '192.168.0.2,tcp:10000'), mock.call(treadmill.iptables.SET_INFRA_SVC, '192.168.0.2,tcp:10001'), mock.call(treadmill.iptables.SET_INFRA_SVC, '192.168.0.2,tcp:10002'), ], any_order=True) treadmill.newnet.create_newnet.assert_called_with( 'id1234.0', '192.168.0.2', '192.168.254.254', None, )
def get_cgroup_path(app): """Gets the path of the cgroup.""" unique_name = appcfg.app_unique_name(app) cgrp = os.path.join('treadmill', 'apps', unique_name, 'services') return cgrp
def configure(tm_env, event): """Creates directory necessary for starting the application. This operation is idem-potent (it can be repeated). The directory layout is:: - (treadmill root) - apps - (app unique name) - app.yml run finish log/run The 'run' script is responsible for creating container environment and starting svscan inside the container. The 'finish' script is invoked when container terminates and will deallocate any resources (NAT rules, etc) that were allocated for the container. """ # R0915: Need to refactor long function into smaller pieces. # # pylint: disable=R0915 # Load the app from the event try: manifest_data = app_manifest.load(tm_env, event) except IOError: # File is gone. Nothing to do. _LOGGER.exception("No event to load: %r", event) return # Freeze the app data into a namedtuple object app = utils.to_obj(manifest_data) # Check the identity we are going to run as _check_identity(app.proid) # Generate a unique name for the app uniq_name = appcfg.app_unique_name(app) # Create the app's running directory container_dir = os.path.join(tm_env.apps_dir, uniq_name) if not fs.mkdir_safe(container_dir): _LOGGER.info('Resuming container %r', uniq_name) # Copy the event as 'manifest.yml' in the container dir shutil.copyfile(event, os.path.join(container_dir, 'manifest.yml')) # Store the app int the container_dir app_yml = os.path.join(container_dir, _APP_YML) with open(app_yml, 'w') as f: yaml.dump(manifest_data, stream=f) # Mark the container as defaulting to down state utils.touch(os.path.join(container_dir, 'down')) # Generate the supervisor's run script app_run_cmd = ' '.join( [treadmill.TREADMILL_BIN, 'sproc', 'run', container_dir]) utils.create_script(os.path.join(container_dir, 'run'), 'supervisor.run_no_log', cmd=app_run_cmd) fs.mkdir_safe(os.path.join(container_dir, 'log')) utils.create_script(os.path.join(container_dir, 'log', 'run'), 'logger.run') # Unique name for the link, based on creation time. cleanup_link = os.path.join(tm_env.cleanup_dir, uniq_name) if os.name == 'nt': finish_cmd = '%%COMSPEC%% /C mklink /D %s %s' % \ (cleanup_link, container_dir) else: finish_cmd = '/bin/ln -snvf %s %s' % (container_dir, cleanup_link) utils.create_script(os.path.join(container_dir, 'finish'), 'supervisor.finish', service=app.name, proid=None, cmds=[finish_cmd]) appevents.post( tm_env.app_events_dir, events.ConfiguredTraceEvent(instanceid=app.name, uniqueid=app.uniqueid)) return container_dir
def configure(tm_env, event, runtime): """Creates directory necessary for starting the application. This operation is idem-potent (it can be repeated). The directory layout is:: - (treadmill root)/ - apps/ - (app unique name)/ - data/ - app_start - app.json - manifest.yml env/ - TREADMILL_* run finish log/ - run The 'run' script is responsible for creating container environment and starting the container. The 'finish' script is invoked when container terminates and will deallocate any resources (NAT rules, etc) that were allocated for the container. """ # Load the app from the event try: manifest_data = load_runtime_manifest(tm_env, event, runtime) except IOError: # File is gone. Nothing to do. _LOGGER.exception('No event to load: %r', event) return None # Freeze the app data into a namedtuple object app = utils.to_obj(manifest_data) # Generate a unique name for the app uniq_name = appcfg.app_unique_name(app) # Write the actual container start script if os.name == 'nt': run_script = '{treadmill}/scripts/treadmill sproc run .'.format( treadmill=subproc.resolve('treadmill'), ) else: run_script = 'exec {treadmill}/bin/treadmill sproc run ../'.format( treadmill=subproc.resolve('treadmill'), ) # Create the service for that container container_svc = supervisor.create_service( tm_env.apps_dir, name=uniq_name, app_run_script=run_script, userid='root', downed=False, monitor_policy={ 'limit': 0, 'interval': 60, 'tombstone': { 'uds': False, 'path': tm_env.running_tombstone_dir, 'id': app.name } }, environ={}, environment=app.environment ) data_dir = container_svc.data_dir # Copy the original event as 'manifest.yml' in the container dir try: shutil.copyfile( event, os.path.join(data_dir, 'manifest.yml') ) except IOError as err: # File is gone, cleanup. if err.errno == errno.ENOENT: shutil.rmtree(container_svc.directory) _LOGGER.exception('Event gone: %r', event) return None else: raise # Store the app.json in the container directory fs.write_safe( os.path.join(data_dir, appcfg.APP_JSON), lambda f: f.writelines( utils.json_genencode(manifest_data) ), mode='w', permission=0o644 ) appevents.post( tm_env.app_events_dir, events.ConfiguredTraceEvent( instanceid=app.name, uniqueid=app.uniqueid ) ) return container_svc.directory