def _make_logs_container(): """Creates a log-processor container.""" environment = { 'LOGS_PATH': _LOGS_PATH, 'PREFIX': _escape('{app}_{module}' '_{version}_{instance}'.format(app=app, module=module, version=version, instance=instance)) } volumes = [ _describe_volume( _LOGS_PATH, _make_external_logs_path(app, module, version, instance)), _describe_volume(_DB_PATH), _describe_volume(_TD_AGENT_PATH) ] return containers.Container( self._docker_client, containers.ContainerOptions(image_opts=containers.ImageOptions( tag=_LOG_PROCESSOR_IMAGE), environment=environment, volumes=dict(volumes), name=container_name))
def start(self): runtime_config = self._runtime_config_getter() # api_host set to 'localhost' won't be accessible from a docker container # because container will have it's own 'localhost'. # TODO: this works only when /etc/hosts is configured properly. api_host = socket.gethostbyname(socket.gethostname()) if ( runtime_config.api_host == '0.0.0.0') else runtime_config.api_host # Must be HTTP_PORT from apphosting/ext/vmruntime/vmservice.py # TODO: update apphosting/ext/vmruntime/vmservice.py to use # env var set here. PORT = 8080 self._container = containers.Container( self._docker_client, containers.ContainerOptions( image_opts=containers.ImageOptions( dockerfile_dir=self._module_configuration.application_root, tag='vm.%(RUNTIME)s.%(APP_ID)s.%(MODULE)s.%(VERSION)s' % { 'APP_ID': self._module_configuration.application, 'MODULE': self._module_configuration.module_name, 'RUNTIME': self._module_configuration.effective_runtime, 'VERSION': self._module_configuration.major_version }, nocache=False), port=PORT, environment={ 'API_HOST': api_host, 'API_PORT': runtime_config.api_port, 'GAE_LONG_APP_ID': self._module_configuration.application_external_name, 'GAE_PARTITION': self._module_configuration.partition, 'GAE_MODULE_NAME': self._module_configuration.module_name, 'GAE_MODULE_VERSION': self._module_configuration.major_version, 'GAE_MINOR_VERSION': self._module_configuration.minor_version, 'GAE_MODULE_INSTANCE': runtime_config.instance_id }, volumes={ '/var/log/app_engine/app': { 'bind': '/var/log/app_engine/app' } }, volumes_from=None)) self._container.Start() self._proxy = http_proxy.HttpProxy( host=self._container.host, port=self._container.port, instance_died_unexpectedly=self._instance_died_unexpectedly, instance_logs_getter=self._get_instance_logs, error_handler_file=application_configuration.get_app_error_file( self._module_configuration))
def create_log_container(self, port): """Return log_server container instance for this module.""" container_name = self._LOG_SERVER_CONTAINER_NAME_FORMAT.format( image_name=self._LOG_SERVER_IMAGE_NAME, module_name=self._module_configuration.module_name) return containers.Container( self._docker_client, containers.ContainerOptions( image_opts=containers.ImageOptions( tag=self._LOG_SERVER_IMAGE_NAME), port=port, name=container_name, ))
def __init__(self, docker_client, log_server_port): super(_LogManager, self).__init__(docker_client, log_server_port) self._docker_client = docker_client volumes = [_describe_volume(_DB_PATH)] self._server = containers.Container( self._docker_client, containers.ContainerOptions( image_opts=containers.ImageOptions(tag=_LOG_SERVER_IMAGE), port=log_server_port, volumes=dict(volumes))) self._lock = threading.RLock() self._containers = {}
def _build_logs_container(self, external_logs_path, internal_logs_path, instance_id): """Builds logs container for current instance.""" environment = { 'LOGS_DIR': internal_logs_path, 'SERVER_NAME': self._LOG_GETTER_SERVER_LINK_NAME, 'MODULE_NAME': self._module_configuration.module_name, 'INSTANCE_ID': instance_id, } return containers.Container( self._docker_client, containers.ContainerOptions( image_opts=containers.ImageOptions( tag=self._LOG_GETTER_IMAGE_NAME), environment=environment, volumes={external_logs_path: { 'bind': internal_logs_path }}, links={ self._log_server_container.name: self._LOG_GETTER_SERVER_LINK_NAME }, ))
def start(self, dockerfile_dir=None): runtime_config = self._runtime_config_getter() if not self._module_configuration.major_version: logging.error('Version needs to be specified in your application ' 'configuration file.') raise VersionError() if not dockerfile_dir: dockerfile_dir = self._module_configuration.application_root # api_host set to 'localhost' won't be accessible from a docker container # because container will have it's own 'localhost'. # 10.0.2.2 is a special network setup by virtualbox to connect from the # guest to the host. api_host = runtime_config.api_host if runtime_config.api_host in ('0.0.0.0', 'localhost'): api_host = '10.0.2.2' image_name = _DOCKER_IMAGE_NAME_FORMAT.format( # Escape domain if it is present. display=self._escape_domain( self._module_configuration.application_external_name), module=self._module_configuration.module_name, version=self._module_configuration.major_version) port_bindings = self._port_bindings if self._port_bindings else {} port_bindings.setdefault(self._default_port, None) debug_port = None environment = { 'API_HOST': api_host, 'API_PORT': runtime_config.api_port, 'GAE_LONG_APP_ID': self._module_configuration.application_external_name, 'GAE_PARTITION': self._module_configuration.partition, 'GAE_MODULE_NAME': self._module_configuration.module_name, 'GAE_MODULE_VERSION': self._module_configuration.major_version, 'GAE_MINOR_VERSION': self._module_configuration.minor_version, 'GAE_MODULE_INSTANCE': runtime_config.instance_id, 'GAE_SERVER_PORT': runtime_config.server_port, 'MODULE_YAML_PATH': os.path.basename(self._module_configuration.config_path) } if self._additional_environment: environment.update(self._additional_environment) # Handle user defined environment variables if self._module_configuration.env_variables: ev = (environment.viewkeys() & self._module_configuration.env_variables.viewkeys()) if ev: raise InvalidEnvVariableError( 'Environment variables [%s] are reserved for App Engine use' % ', '.join(ev)) environment.update(self._module_configuration.env_variables) # Publish debug port if running in Debug mode. if self._module_configuration.env_variables.get('DBG_ENABLE'): debug_port = int( self._module_configuration.env_variables.get( 'DBG_PORT', self.DEFAULT_DEBUG_PORT)) environment['DBG_PORT'] = debug_port port_bindings[debug_port] = _GetPortToPublish(debug_port) # Publish forwarded ports # NOTE: fowarded ports are mapped as host_port => container_port, # port_bindings are mapped the other way around. for h, c in self._module_configuration.forwarded_ports.iteritems(): if c in port_bindings: raise InvalidForwardedPortError( 'Port {port} is already used by debugger or runtime specific ' 'VM Service. Please use a different forwarded_port.'. format(port=c)) port_bindings[c] = h external_logs_path = os.path.join( '/var/log/app_engine', self._escape_domain( self._module_configuration.application_external_name), self._module_configuration.module_name, self._module_configuration.major_version, runtime_config.instance_id) internal_logs_path = '/var/log/app_engine' container_name = _DOCKER_CONTAINER_NAME_FORMAT.format( image_name=image_name, instance_id=runtime_config.instance_id) self._container = containers.Container( self._docker_client, containers.ContainerOptions( image_opts=containers.ImageOptions( dockerfile_dir=dockerfile_dir, tag=image_name, nocache=False), port=self._default_port, port_bindings=port_bindings, environment=environment, volumes={external_logs_path: { 'bind': internal_logs_path }}, name=container_name)) if self._log_server_container: self._logs_container = self._build_logs_container( external_logs_path, internal_logs_path, runtime_config.instance_id) self._container.Start() if self._log_server_container: try: self._logs_container.Start() except containers.ImageError: logging.warning('Image \'log_server\' is not found.' 'Showing logs in admin console is disabled.') # Print the debug information before connecting to the container # as debugging might break the runtime during initialization, and # connecting the debugger is required to start processing requests. if debug_port: logging.info( 'To debug module {module} attach to {host}:{port}'.format( module=self._module_configuration.module_name, host=self.ContainerHost(), port=self.PortBinding(debug_port))) self._proxy = http_proxy.HttpProxy( host=self._container.host, port=self._container.port, instance_died_unexpectedly=self._instance_died_unexpectedly, instance_logs_getter=self.get_instance_logs, error_handler_file=application_configuration.get_app_error_file( self._module_configuration)) # If forwarded ports are used we do not really have to serve on 8080. # We'll log /_ah/start request fail, but with health-checks disabled # we should ignore that and continue working (accepting requests on # our forwarded ports outside of dev server control). health_check = self._module_configuration.vm_health_check health_check_enabled = health_check and health_check.enable_health_check if health_check_enabled or not self._module_configuration.forwarded_ports: self._proxy.wait_for_connection()
def start(self, dockerfile_dir=None): runtime_config = self._runtime_config_getter() if not self._module_configuration.major_version: logging.error('Version needs to be specified in your application ' 'configuration file.') raise VersionError() if not dockerfile_dir: dockerfile_dir = self._module_configuration.application_root # api_host set to 'localhost' won't be accessible from a docker container # because container will have it's own 'localhost'. # 10.0.2.2 is a special network setup by virtualbox to connect from the # guest to the host. api_host = runtime_config.api_host if runtime_config.api_host in ('0.0.0.0', 'localhost'): api_host = '10.0.2.2' image_name = _DOCKER_IMAGE_NAME_FORMAT.format( # Escape domain if it is present. display=self._escape_domain( self._module_configuration.application_external_name), module=self._module_configuration.module_name, version=self._module_configuration.major_version) port_bindings = self._port_bindings if self._port_bindings else {} port_bindings.setdefault(self._default_port, None) debug_port = None environment = { 'API_HOST': api_host, 'API_PORT': runtime_config.api_port, 'GAE_LONG_APP_ID': self._module_configuration.application_external_name, 'GAE_PARTITION': self._module_configuration.partition, 'GAE_MODULE_NAME': self._module_configuration.module_name, 'GAE_MODULE_VERSION': self._module_configuration.major_version, 'GAE_MINOR_VERSION': self._module_configuration.minor_version, 'GAE_MODULE_INSTANCE': runtime_config.instance_id, 'GAE_SERVER_PORT': runtime_config.server_port, 'MODULE_YAML_PATH': os.path.basename(self._module_configuration.config_path) } if self._additional_environment: environment.update(self._additional_environment) # Handle user defined environment variables if self._module_configuration.env_variables: ev = (environment.viewkeys() & self._module_configuration.env_variables.viewkeys()) if ev: raise InvalidEnvVariableError( 'Environment variables [%s] are reserved for App Engine use' % ', '.join(ev)) environment.update(self._module_configuration.env_variables) # Publish debug port if running in Debug mode. if self._module_configuration.env_variables.get('DBG_ENABLE'): debug_port = int( self._module_configuration.env_variables.get( 'DBG_PORT', self.DEFAULT_DEBUG_PORT)) environment['DBG_PORT'] = debug_port port_bindings[debug_port] = _GetPortToPublish(debug_port) external_logs_path = os.path.join( '/var/log/app_engine', self._escape_domain( self._module_configuration.application_external_name), self._module_configuration.module_name, self._module_configuration.major_version, runtime_config.instance_id) container_name = _DOCKER_CONTAINER_NAME_FORMAT.format( image_name=image_name, minor_version=self._module_configuration.minor_version) self._container = containers.Container( self._docker_client, containers.ContainerOptions( image_opts=containers.ImageOptions( dockerfile_dir=dockerfile_dir, tag=image_name, nocache=False), port=self._default_port, port_bindings=port_bindings, environment=environment, volumes={external_logs_path: { 'bind': '/var/log/app_engine' }}, name=container_name)) self._container.Start() # Print the debug information before connecting to the container # as debugging might break the runtime during initialization, and # connecting the debugger is required to start processing requests. if debug_port: logging.info( 'To debug module {module} attach to {host}:{port}'.format( module=self._module_configuration.module_name, host=self.ContainerHost(), port=self.PortBinding(debug_port))) self._proxy = http_proxy.HttpProxy( host=self._container.host, port=self._container.port, instance_died_unexpectedly=self._instance_died_unexpectedly, instance_logs_getter=self._get_instance_logs, error_handler_file=application_configuration.get_app_error_file( self._module_configuration)) self._proxy.wait_for_connection()