def _make_logs_container():
"""Creates a log-processor container."""
environment = {
'LOGS_PATH':
_LOGS_PATH,
'PREFIX':
_escape('{app}_{module}'
'_{version}_{instance}'.format(app=app,
module=module,
version=version,
instance=instance))
}
volumes = [
_describe_volume(
_LOGS_PATH,
_make_external_logs_path(app, module, version, instance)),
_describe_volume(_DB_PATH),
_describe_volume(_TD_AGENT_PATH)
]
return containers.Container(
self._docker_client,
containers.ContainerOptions(image_opts=containers.ImageOptions(
tag=_LOG_PROCESSOR_IMAGE),
environment=environment,
volumes=dict(volumes),
name=container_name))
def start(self):
runtime_config = self._runtime_config_getter()
# api_host set to 'localhost' won't be accessible from a docker container
# because container will have it's own 'localhost'.
# TODO: this works only when /etc/hosts is configured properly.
api_host = socket.gethostbyname(socket.gethostname()) if (
runtime_config.api_host == '0.0.0.0') else runtime_config.api_host
# Must be HTTP_PORT from apphosting/ext/vmruntime/vmservice.py
# TODO: update apphosting/ext/vmruntime/vmservice.py to use
# env var set here.
PORT = 8080
self._container = containers.Container(
self._docker_client,
containers.ContainerOptions(
image_opts=containers.ImageOptions(
dockerfile_dir=self._module_configuration.application_root,
tag='vm.%(RUNTIME)s.%(APP_ID)s.%(MODULE)s.%(VERSION)s' % {
'APP_ID': self._module_configuration.application,
'MODULE': self._module_configuration.module_name,
'RUNTIME':
self._module_configuration.effective_runtime,
'VERSION': self._module_configuration.major_version
},
nocache=False),
port=PORT,
environment={
'API_HOST': api_host,
'API_PORT': runtime_config.api_port,
'GAE_LONG_APP_ID':
self._module_configuration.application_external_name,
'GAE_PARTITION': self._module_configuration.partition,
'GAE_MODULE_NAME': self._module_configuration.module_name,
'GAE_MODULE_VERSION':
self._module_configuration.major_version,
'GAE_MINOR_VERSION':
self._module_configuration.minor_version,
'GAE_MODULE_INSTANCE': runtime_config.instance_id
},
volumes={
'/var/log/app_engine/app': {
'bind': '/var/log/app_engine/app'
}
},
volumes_from=None))
self._container.Start()
self._proxy = http_proxy.HttpProxy(
host=self._container.host,
port=self._container.port,
instance_died_unexpectedly=self._instance_died_unexpectedly,
instance_logs_getter=self._get_instance_logs,
error_handler_file=application_configuration.get_app_error_file(
self._module_configuration))
def create_log_container(self, port):
"""Return log_server container instance for this module."""
container_name = self._LOG_SERVER_CONTAINER_NAME_FORMAT.format(
image_name=self._LOG_SERVER_IMAGE_NAME,
module_name=self._module_configuration.module_name)
return containers.Container(
self._docker_client,
containers.ContainerOptions(
image_opts=containers.ImageOptions(
tag=self._LOG_SERVER_IMAGE_NAME),
port=port,
name=container_name,
))
def __init__(self, docker_client, log_server_port):
super(_LogManager, self).__init__(docker_client, log_server_port)
self._docker_client = docker_client
volumes = [_describe_volume(_DB_PATH)]
self._server = containers.Container(
self._docker_client,
containers.ContainerOptions(
image_opts=containers.ImageOptions(tag=_LOG_SERVER_IMAGE),
port=log_server_port,
volumes=dict(volumes)))
self._lock = threading.RLock()
self._containers = {}
def _build_logs_container(self, external_logs_path, internal_logs_path,
instance_id):
"""Builds logs container for current instance."""
environment = {
'LOGS_DIR': internal_logs_path,
'SERVER_NAME': self._LOG_GETTER_SERVER_LINK_NAME,
'MODULE_NAME': self._module_configuration.module_name,
'INSTANCE_ID': instance_id,
}
return containers.Container(
self._docker_client,
containers.ContainerOptions(
image_opts=containers.ImageOptions(
tag=self._LOG_GETTER_IMAGE_NAME),
environment=environment,
volumes={external_logs_path: {
'bind': internal_logs_path
}},
links={
self._log_server_container.name:
self._LOG_GETTER_SERVER_LINK_NAME
},
))
def start(self, dockerfile_dir=None):
runtime_config = self._runtime_config_getter()
if not self._module_configuration.major_version:
logging.error('Version needs to be specified in your application '
'configuration file.')
raise VersionError()
if not dockerfile_dir:
dockerfile_dir = self._module_configuration.application_root
# api_host set to 'localhost' won't be accessible from a docker container
# because container will have it's own 'localhost'.
# 10.0.2.2 is a special network setup by virtualbox to connect from the
# guest to the host.
api_host = runtime_config.api_host
if runtime_config.api_host in ('0.0.0.0', 'localhost'):
api_host = '10.0.2.2'
image_name = _DOCKER_IMAGE_NAME_FORMAT.format(
# Escape domain if it is present.
display=self._escape_domain(
self._module_configuration.application_external_name),
module=self._module_configuration.module_name,
version=self._module_configuration.major_version)
port_bindings = self._port_bindings if self._port_bindings else {}
port_bindings.setdefault(self._default_port, None)
debug_port = None
environment = {
'API_HOST':
api_host,
'API_PORT':
runtime_config.api_port,
'GAE_LONG_APP_ID':
self._module_configuration.application_external_name,
'GAE_PARTITION':
self._module_configuration.partition,
'GAE_MODULE_NAME':
self._module_configuration.module_name,
'GAE_MODULE_VERSION':
self._module_configuration.major_version,
'GAE_MINOR_VERSION':
self._module_configuration.minor_version,
'GAE_MODULE_INSTANCE':
runtime_config.instance_id,
'GAE_SERVER_PORT':
runtime_config.server_port,
'MODULE_YAML_PATH':
os.path.basename(self._module_configuration.config_path)
}
if self._additional_environment:
environment.update(self._additional_environment)
# Handle user defined environment variables
if self._module_configuration.env_variables:
ev = (environment.viewkeys()
& self._module_configuration.env_variables.viewkeys())
if ev:
raise InvalidEnvVariableError(
'Environment variables [%s] are reserved for App Engine use'
% ', '.join(ev))
environment.update(self._module_configuration.env_variables)
# Publish debug port if running in Debug mode.
if self._module_configuration.env_variables.get('DBG_ENABLE'):
debug_port = int(
self._module_configuration.env_variables.get(
'DBG_PORT', self.DEFAULT_DEBUG_PORT))
environment['DBG_PORT'] = debug_port
port_bindings[debug_port] = _GetPortToPublish(debug_port)
# Publish forwarded ports
# NOTE: fowarded ports are mapped as host_port => container_port,
# port_bindings are mapped the other way around.
for h, c in self._module_configuration.forwarded_ports.iteritems():
if c in port_bindings:
raise InvalidForwardedPortError(
'Port {port} is already used by debugger or runtime specific '
'VM Service. Please use a different forwarded_port.'.
format(port=c))
port_bindings[c] = h
external_logs_path = os.path.join(
'/var/log/app_engine',
self._escape_domain(
self._module_configuration.application_external_name),
self._module_configuration.module_name,
self._module_configuration.major_version,
runtime_config.instance_id)
internal_logs_path = '/var/log/app_engine'
container_name = _DOCKER_CONTAINER_NAME_FORMAT.format(
image_name=image_name, instance_id=runtime_config.instance_id)
self._container = containers.Container(
self._docker_client,
containers.ContainerOptions(
image_opts=containers.ImageOptions(
dockerfile_dir=dockerfile_dir,
tag=image_name,
nocache=False),
port=self._default_port,
port_bindings=port_bindings,
environment=environment,
volumes={external_logs_path: {
'bind': internal_logs_path
}},
name=container_name))
if self._log_server_container:
self._logs_container = self._build_logs_container(
external_logs_path, internal_logs_path,
runtime_config.instance_id)
self._container.Start()
if self._log_server_container:
try:
self._logs_container.Start()
except containers.ImageError:
logging.warning('Image \'log_server\' is not found.'
'Showing logs in admin console is disabled.')
# Print the debug information before connecting to the container
# as debugging might break the runtime during initialization, and
# connecting the debugger is required to start processing requests.
if debug_port:
logging.info(
'To debug module {module} attach to {host}:{port}'.format(
module=self._module_configuration.module_name,
host=self.ContainerHost(),
port=self.PortBinding(debug_port)))
self._proxy = http_proxy.HttpProxy(
host=self._container.host,
port=self._container.port,
instance_died_unexpectedly=self._instance_died_unexpectedly,
instance_logs_getter=self.get_instance_logs,
error_handler_file=application_configuration.get_app_error_file(
self._module_configuration))
# If forwarded ports are used we do not really have to serve on 8080.
# We'll log /_ah/start request fail, but with health-checks disabled
# we should ignore that and continue working (accepting requests on
# our forwarded ports outside of dev server control).
health_check = self._module_configuration.vm_health_check
health_check_enabled = health_check and health_check.enable_health_check
if health_check_enabled or not self._module_configuration.forwarded_ports:
self._proxy.wait_for_connection()
def start(self, dockerfile_dir=None):
runtime_config = self._runtime_config_getter()
if not self._module_configuration.major_version:
logging.error('Version needs to be specified in your application '
'configuration file.')
raise VersionError()
if not dockerfile_dir:
dockerfile_dir = self._module_configuration.application_root
# api_host set to 'localhost' won't be accessible from a docker container
# because container will have it's own 'localhost'.
# 10.0.2.2 is a special network setup by virtualbox to connect from the
# guest to the host.
api_host = runtime_config.api_host
if runtime_config.api_host in ('0.0.0.0', 'localhost'):
api_host = '10.0.2.2'
image_name = _DOCKER_IMAGE_NAME_FORMAT.format(
# Escape domain if it is present.
display=self._escape_domain(
self._module_configuration.application_external_name),
module=self._module_configuration.module_name,
version=self._module_configuration.major_version)
port_bindings = self._port_bindings if self._port_bindings else {}
port_bindings.setdefault(self._default_port, None)
debug_port = None
environment = {
'API_HOST':
api_host,
'API_PORT':
runtime_config.api_port,
'GAE_LONG_APP_ID':
self._module_configuration.application_external_name,
'GAE_PARTITION':
self._module_configuration.partition,
'GAE_MODULE_NAME':
self._module_configuration.module_name,
'GAE_MODULE_VERSION':
self._module_configuration.major_version,
'GAE_MINOR_VERSION':
self._module_configuration.minor_version,
'GAE_MODULE_INSTANCE':
runtime_config.instance_id,
'GAE_SERVER_PORT':
runtime_config.server_port,
'MODULE_YAML_PATH':
os.path.basename(self._module_configuration.config_path)
}
if self._additional_environment:
environment.update(self._additional_environment)
# Handle user defined environment variables
if self._module_configuration.env_variables:
ev = (environment.viewkeys()
& self._module_configuration.env_variables.viewkeys())
if ev:
raise InvalidEnvVariableError(
'Environment variables [%s] are reserved for App Engine use'
% ', '.join(ev))
environment.update(self._module_configuration.env_variables)
# Publish debug port if running in Debug mode.
if self._module_configuration.env_variables.get('DBG_ENABLE'):
debug_port = int(
self._module_configuration.env_variables.get(
'DBG_PORT', self.DEFAULT_DEBUG_PORT))
environment['DBG_PORT'] = debug_port
port_bindings[debug_port] = _GetPortToPublish(debug_port)
external_logs_path = os.path.join(
'/var/log/app_engine',
self._escape_domain(
self._module_configuration.application_external_name),
self._module_configuration.module_name,
self._module_configuration.major_version,
runtime_config.instance_id)
container_name = _DOCKER_CONTAINER_NAME_FORMAT.format(
image_name=image_name,
minor_version=self._module_configuration.minor_version)
self._container = containers.Container(
self._docker_client,
containers.ContainerOptions(
image_opts=containers.ImageOptions(
dockerfile_dir=dockerfile_dir,
tag=image_name,
nocache=False),
port=self._default_port,
port_bindings=port_bindings,
environment=environment,
volumes={external_logs_path: {
'bind': '/var/log/app_engine'
}},
name=container_name))
self._container.Start()
# Print the debug information before connecting to the container
# as debugging might break the runtime during initialization, and
# connecting the debugger is required to start processing requests.
if debug_port:
logging.info(
'To debug module {module} attach to {host}:{port}'.format(
module=self._module_configuration.module_name,
host=self.ContainerHost(),
port=self.PortBinding(debug_port)))
self._proxy = http_proxy.HttpProxy(
host=self._container.host,
port=self._container.port,
instance_died_unexpectedly=self._instance_died_unexpectedly,
instance_logs_getter=self._get_instance_logs,
error_handler_file=application_configuration.get_app_error_file(
self._module_configuration))
self._proxy.wait_for_connection()
