def _wait_child(self):
try:
# Don't block if no child processes have exited
pid, status = os.waitpid(0, os.WNOHANG)
if not pid:
return None
except OSError as exc:
if exc.errno not in (errno.EINTR, errno.ECHILD):
raise
return None
if os.WIFSIGNALED(status):
sig = os.WTERMSIG(status)
LOG.info(_LI('Child %(pid)d killed by signal %(sig)d'),
dict(pid=pid, sig=sig))
else:
code = os.WEXITSTATUS(status)
LOG.info(_LI('Child %(pid)s exited with status %(code)d'),
dict(pid=pid, code=code))
if pid not in self.children:
LOG.warning(_LW('pid %d not in child list'), pid)
return None
wrap = self.children.pop(pid)
wrap.children.remove(pid)
return wrap
def _start_child(self, wrap):
if len(wrap.forktimes) > wrap.workers:
# Limit ourselves to one process a second (over the period of
# number of workers * 1 second). This will allow workers to
# start up quickly but ensure we don't fork off children that
# die instantly too quickly.
if time.time() - wrap.forktimes[0] < wrap.workers:
LOG.info(_LI('Forking too fast, sleeping'))
time.sleep(1)
wrap.forktimes.pop(0)
wrap.forktimes.append(time.time())
pid = os.fork()
if pid == 0:
launcher = self._child_process(wrap.service)
while True:
self._child_process_handle_signal()
status, signo = self._child_wait_for_exit_or_signal(launcher)
if not _is_sighup_and_daemon(signo):
break
launcher.restart()
os._exit(status)
LOG.info(_LI('Started child %d'), pid)
wrap.children.add(pid)
self.children[pid] = wrap
return pid
def _start_child(self, wrap):
if len(wrap.forktimes) > wrap.workers:
# Limit ourselves to one process a second (over the period of
# number of workers * 1 second). This will allow workers to
# start up quickly but ensure we don't fork off children that
# die instantly too quickly.
if time.time() - wrap.forktimes[0] < wrap.workers:
LOG.info(_LI('Forking too fast, sleeping'))
time.sleep(1)
wrap.forktimes.pop(0)
wrap.forktimes.append(time.time())
pid = os.fork()
if pid == 0:
launcher = self._child_process(wrap.service)
while True:
self._child_process_handle_signal()
status, signo = self._child_wait_for_exit_or_signal(launcher)
if not _is_sighup_and_daemon(signo):
break
launcher.restart()
os._exit(status)
LOG.info(_LI('Started child %d'), pid)
wrap.children.add(pid)
self.children[pid] = wrap
return pid
def _wait_child(self):
try:
# Don't block if no child processes have exited
pid, status = os.waitpid(0, os.WNOHANG)
if not pid:
return None
except OSError as exc:
if exc.errno not in (errno.EINTR, errno.ECHILD):
raise
return None
if os.WIFSIGNALED(status):
sig = os.WTERMSIG(status)
LOG.info(_LI('Child %(pid)d killed by signal %(sig)d'),
dict(pid=pid, sig=sig))
else:
code = os.WEXITSTATUS(status)
LOG.info(_LI('Child %(pid)s exited with status %(code)d'),
dict(pid=pid, code=code))
if pid not in self.children:
LOG.warning(_LW('pid %d not in child list'), pid)
return None
wrap = self.children.pop(pid)
wrap.children.remove(pid)
return wrap
def wait(self):
"""Loop waiting on children to die and respawning as necessary."""
systemd.notify_once()
LOG.debug('Full set of CONF:')
CONF.log_opt_values(LOG, std_logging.DEBUG)
try:
while True:
self.handle_signal()
self._respawn_children()
# No signal means that stop was called. Don't clean up here.
if not self.sigcaught:
return
signame = _signo_to_signame(self.sigcaught)
LOG.info(_LI('Caught %s, stopping children'), signame)
if not _is_sighup_and_daemon(self.sigcaught):
break
for pid in self.children:
os.kill(pid, signal.SIGHUP)
self.running = True
self.sigcaught = None
except eventlet.greenlet.GreenletExit:
LOG.info(_LI("Wait called after thread killed. Cleaning up."))
self.stop()
def wait(self):
"""Loop waiting on children to die and respawning as necessary."""
systemd.notify_once()
LOG.debug('Full set of CONF:')
CONF.log_opt_values(LOG, std_logging.DEBUG)
try:
while True:
self.handle_signal()
self._respawn_children()
# No signal means that stop was called. Don't clean up here.
if not self.sigcaught:
return
signame = _signo_to_signame(self.sigcaught)
LOG.info(_LI('Caught %s, stopping children'), signame)
if not _is_sighup_and_daemon(self.sigcaught):
break
for pid in self.children:
os.kill(pid, signal.SIGHUP)
self.running = True
self.sigcaught = None
except eventlet.greenlet.GreenletExit:
LOG.info(_LI("Wait called after thread killed. Cleaning up."))
self.stop()
def update(self, req, id, body, tenant_id):
"""Updates the instance to attach/detach configuration."""
LOG.info(_LI("Updating database instance '%(instance_id)s' for tenant "
"'%(tenant_id)s'"),
{'instance_id': id, 'tenant_id': tenant_id})
LOG.debug("req: %s", req)
LOG.debug("body: %s", body)
context = req.environ[wsgi.CONTEXT_KEY]
instance = models.Instance.load(context, id)
# If configuration is set, then we will update the instance to use the
# new configuration. If configuration is empty, we want to disassociate
# the instance from the configuration group and remove the active
# overrides file.
update_args = {}
configuration_id = self._configuration_parse(context, body)
if configuration_id:
instance.assign_configuration(configuration_id)
else:
instance.unassign_configuration()
update_args['configuration_id'] = configuration_id
instance.update_db(**update_args)
return wsgi.Result(None, 202)
def _wait_for_exit_or_signal(self, ready_callback=None):
status = None
signo = 0
LOG.debug('Full set of CONF:')
CONF.log_opt_values(LOG, std_logging.DEBUG)
try:
if ready_callback:
ready_callback()
super(ServiceLauncher, self).wait()
except SignalExit as exc:
signame = _signo_to_signame(exc.signo)
LOG.info(_LI('Caught %s, exiting'), signame)
status = exc.code
signo = exc.signo
except SystemExit as exc:
status = exc.code
finally:
self.stop()
if rpc:
try:
rpc.cleanup()
except Exception:
# We're shutting down, so it doesn't matter at this point.
LOG.exception(_LE('Exception during rpc cleanup.'))
return status, signo
def action(self, req, body, tenant_id, id):
"""
Handles requests that modify existing instances in some manner. Actions
could include 'resize', 'restart', 'reset_password'
:param req: http request object
:param body: deserialized body of the request as a dict
:param tenant_id: the tenant id for whom owns the instance
:param id: ???
"""
LOG.debug("instance action req : '%s'\n\n", req)
if not body:
raise exception.BadRequest(_("Invalid request body."))
context = req.environ[wsgi.CONTEXT_KEY]
instance = models.Instance.load(context, id)
_actions = {
'restart': self._action_restart,
'resize': self._action_resize,
'reset_password': self._action_reset_password,
}
selected_action = None
action_name = None
for key in body:
if key in _actions:
selected_action = _actions[key]
action_name = key
LOG.info(_LI("Performing %(action_name)s action against "
"instance %(instance_id)s for tenant '%(tenant_id)s'"),
{'action_name': action_name, 'instance_id': id,
'tenant_id': tenant_id})
return selected_action(instance, body)
def action(self, req, body, tenant_id, id):
"""
Handles requests that modify existing instances in some manner. Actions
could include 'resize', 'restart', 'reset_password'
:param req: http request object
:param body: deserialized body of the request as a dict
:param tenant_id: the tenant id for whom owns the instance
:param id: ???
"""
LOG.debug("instance action req : '%s'\n\n", req)
if not body:
raise exception.BadRequest(_("Invalid request body."))
context = req.environ[wsgi.CONTEXT_KEY]
instance = models.Instance.load(context, id)
_actions = {
'restart': self._action_restart,
'resize': self._action_resize,
'reset_password': self._action_reset_password,
}
selected_action = None
action_name = None
for key in body:
if key in _actions:
selected_action = _actions[key]
action_name = key
LOG.info(
_LI("Performing %(action_name)s action against "
"instance %(instance_id)s for tenant '%(tenant_id)s'"), {
'action_name': action_name,
'instance_id': id,
'tenant_id': tenant_id
})
return selected_action(instance, body)
def update(self, req, id, body, tenant_id):
"""Updates the instance to attach/detach configuration."""
LOG.info(
_LI("Updating database instance '%(instance_id)s' for tenant "
"'%(tenant_id)s'"), {
'instance_id': id,
'tenant_id': tenant_id
})
LOG.debug("req: %s", req)
LOG.debug("body: %s", body)
context = req.environ[wsgi.CONTEXT_KEY]
instance = models.Instance.load(context, id)
# If configuration is set, then we will update the instance to use the
# new configuration. If configuration is empty, we want to disassociate
# the instance from the configuration group and remove the active
# overrides file.
update_args = {}
configuration_id = self._configuration_parse(context, body)
if configuration_id:
instance.assign_configuration(configuration_id)
else:
instance.unassign_configuration()
update_args['configuration_id'] = configuration_id
instance.update_db(**update_args)
return wsgi.Result(None, 202)
def _wait_for_exit_or_signal(self, ready_callback=None):
status = None
signo = 0
LOG.debug('Full set of CONF:')
CONF.log_opt_values(LOG, std_logging.DEBUG)
try:
if ready_callback:
ready_callback()
super(ServiceLauncher, self).wait()
except SignalExit as exc:
signame = _signo_to_signame(exc.signo)
LOG.info(_LI('Caught %s, exiting'), signame)
status = exc.code
signo = exc.signo
except SystemExit as exc:
status = exc.code
finally:
self.stop()
if rpc:
try:
rpc.cleanup()
except Exception:
# We're shutting down, so it doesn't matter at this point.
LOG.exception(_LE('Exception during rpc cleanup.'))
return status, signo
def _pipe_watcher(self):
# This will block until the write end is closed when the parent
# dies unexpectedly
self.readpipe.read()
LOG.info(_LI('Parent process has died unexpectedly, exiting'))
sys.exit(1)
def _pipe_watcher(self):
# This will block until the write end is closed when the parent
# dies unexpectedly
self.readpipe.read()
LOG.info(_LI('Parent process has died unexpectedly, exiting'))
sys.exit(1)
def backups(self, req, tenant_id, id):
"""Return all backups for the specified instance."""
LOG.info(_LI("Listing backups for instance '%s'"), id)
LOG.debug("req : '%s'\n\n", req)
context = req.environ[wsgi.CONTEXT_KEY]
backups, marker = backup_model.list_for_instance(context, id)
view = backup_views.BackupViews(backups)
paged = pagination.SimplePaginatedDataView(req.url, 'backups', view,
marker)
return wsgi.Result(paged.data(), 200)
def backups(self, req, tenant_id, id):
"""Return all backups for the specified instance."""
LOG.info(_LI("Listing backups for instance '%s'"),
id)
LOG.debug("req : '%s'\n\n", req)
context = req.environ[wsgi.CONTEXT_KEY]
backups, marker = backup_model.list_for_instance(context, id)
view = backup_views.BackupViews(backups)
paged = pagination.SimplePaginatedDataView(req.url, 'backups', view,
marker)
return wsgi.Result(paged.data(), 200)
def delete(self, req, tenant_id, id):
"""Delete a single instance."""
LOG.info(_LI("Deleting database instance '%(instance_id)s' for tenant "
"'%(tenant_id)s'"),
{'instance_id': id, 'tenant_id': tenant_id})
LOG.debug("req : '%s'\n\n", req)
# TODO(hub-cap): turn this into middleware
context = req.environ[wsgi.CONTEXT_KEY]
instance = models.load_any_instance(context, id)
instance.delete()
# TODO(cp16net): need to set the return code correctly
return wsgi.Result(None, 202)
def create(self, req, body, tenant_id):
# TODO(hub-cap): turn this into middleware
LOG.info(_LI("Creating a database instance for tenant '%s'"),
tenant_id)
LOG.debug(logging.mask_password("req : '%s'\n\n", req))
LOG.debug(logging.mask_password("body : '%s'\n\n", body))
context = req.environ[wsgi.CONTEXT_KEY]
datastore_args = body['instance'].get('datastore', {})
datastore, datastore_version = (
datastore_models.get_datastore_version(**datastore_args))
image_id = datastore_version.image_id
name = body['instance']['name']
flavor_ref = body['instance']['flavorRef']
flavor_id = utils.get_id_from_href(flavor_ref)
configuration = self._configuration_parse(context, body)
databases = populate_validated_databases(
body['instance'].get('databases', []))
database_names = [database.get('_name', '') for database in databases]
users = None
try:
users = populate_users(body['instance'].get('users', []),
database_names)
except ValueError as ve:
raise exception.BadRequest(msg=ve)
if 'volume' in body['instance']:
volume_size = int(body['instance']['volume']['size'])
else:
volume_size = None
if 'restorePoint' in body['instance']:
backupRef = body['instance']['restorePoint']['backupRef']
backup_id = utils.get_id_from_href(backupRef)
else:
backup_id = None
availability_zone = body['instance'].get('availability_zone')
nics = body['instance'].get('nics')
slave_of_id = body['instance'].get('replica_of',
# also check for older name
body['instance'].get('slave_of'))
instance = models.Instance.create(context, name, flavor_id,
image_id, databases, users,
datastore, datastore_version,
volume_size, backup_id,
availability_zone, nics,
configuration, slave_of_id)
view = views.InstanceDetailView(instance, req=req)
return wsgi.Result(view.data(), 200)
def create(self, req, body, tenant_id):
# TODO(hub-cap): turn this into middleware
LOG.info(_LI("Creating a database instance for tenant '%s'"),
tenant_id)
LOG.debug(logging.mask_password("req : '%s'\n\n", req))
LOG.debug(logging.mask_password("body : '%s'\n\n", body))
context = req.environ[wsgi.CONTEXT_KEY]
datastore_args = body['instance'].get('datastore', {})
datastore, datastore_version = (datastore_models.get_datastore_version(
**datastore_args))
image_id = datastore_version.image_id
name = body['instance']['name']
flavor_ref = body['instance']['flavorRef']
flavor_id = utils.get_id_from_href(flavor_ref)
configuration = self._configuration_parse(context, body)
databases = populate_validated_databases(body['instance'].get(
'databases', []))
database_names = [database.get('_name', '') for database in databases]
users = None
try:
users = populate_users(body['instance'].get('users', []),
database_names)
except ValueError as ve:
raise exception.BadRequest(msg=ve)
if 'volume' in body['instance']:
volume_size = int(body['instance']['volume']['size'])
else:
volume_size = None
if 'restorePoint' in body['instance']:
backupRef = body['instance']['restorePoint']['backupRef']
backup_id = utils.get_id_from_href(backupRef)
else:
backup_id = None
availability_zone = body['instance'].get('availability_zone')
nics = body['instance'].get('nics')
slave_of_id = body['instance'].get(
'replica_of',
# also check for older name
body['instance'].get('slave_of'))
instance = models.Instance.create(context, name, flavor_id, image_id,
databases, users, datastore,
datastore_version, volume_size,
backup_id, availability_zone, nics,
configuration, slave_of_id)
view = views.InstanceDetailView(instance, req=req)
return wsgi.Result(view.data(), 200)
def index(self, req, tenant_id):
"""Return all instances."""
LOG.info(_LI("Listing database instances for tenant '%s'"), tenant_id)
LOG.debug("req : '%s'\n\n", req)
context = req.environ[wsgi.CONTEXT_KEY]
clustered_q = req.GET.get('include_clustered', '').lower()
include_clustered = clustered_q == 'true'
servers, marker = models.Instances.load(context, include_clustered)
view = views.InstancesView(servers, req=req)
paged = pagination.SimplePaginatedDataView(req.url, 'instances', view,
marker)
return wsgi.Result(paged.data(), 200)
def show(self, req, tenant_id, id):
"""Return a single instance."""
LOG.info(_LI("Showing database instance '%(instance_id)s' for tenant "
"'%(tenant_id)s'"),
{'instance_id': id, 'tenant_id': tenant_id})
LOG.debug("req : '%s'\n\n", req)
context = req.environ[wsgi.CONTEXT_KEY]
server = models.load_instance_with_guest(models.DetailInstance,
context, id)
return wsgi.Result(views.InstanceDetailView(server,
req=req).data(), 200)
def index(self, req, tenant_id):
"""Return all instances."""
LOG.info(_LI("Listing database instances for tenant '%s'"), tenant_id)
LOG.debug("req : '%s'\n\n", req)
context = req.environ[wsgi.CONTEXT_KEY]
clustered_q = req.GET.get('include_clustered', '').lower()
include_clustered = clustered_q == 'true'
servers, marker = models.Instances.load(context, include_clustered)
view = views.InstancesView(servers, req=req)
paged = pagination.SimplePaginatedDataView(req.url, 'instances', view,
marker)
return wsgi.Result(paged.data(), 200)
def configuration(self, req, tenant_id, id):
"""
Returns the default configuration template applied to the instance.
"""
LOG.info(_LI("Getting default configuration for instance %s"), id)
context = req.environ[wsgi.CONTEXT_KEY]
instance = models.Instance.load(context, id)
LOG.debug("Server: %s", instance)
config = instance.get_default_configuration_template()
LOG.debug("Default config for instance %(instance_id)s is %(config)s",
{'instance_id': id, 'config': config})
return wsgi.Result(views.DefaultConfigurationView(
config).data(), 200)
def show(self, req, tenant_id, id):
"""Return a single instance."""
LOG.info(
_LI("Showing database instance '%(instance_id)s' for tenant "
"'%(tenant_id)s'"), {
'instance_id': id,
'tenant_id': tenant_id
})
LOG.debug("req : '%s'\n\n", req)
context = req.environ[wsgi.CONTEXT_KEY]
server = models.load_instance_with_guest(models.DetailInstance,
context, id)
return wsgi.Result(
views.InstanceDetailView(server, req=req).data(), 200)
def delete(self, req, tenant_id, id):
"""Delete a single instance."""
LOG.info(
_LI("Deleting database instance '%(instance_id)s' for tenant "
"'%(tenant_id)s'"), {
'instance_id': id,
'tenant_id': tenant_id
})
LOG.debug("req : '%s'\n\n", req)
# TODO(hub-cap): turn this into middleware
context = req.environ[wsgi.CONTEXT_KEY]
instance = models.load_any_instance(context, id)
instance.delete()
# TODO(cp16net): need to set the return code correctly
return wsgi.Result(None, 202)
def stop(self):
"""Terminate child processes and wait on each."""
self.running = False
for pid in self.children:
try:
os.kill(pid, signal.SIGTERM)
except OSError as exc:
if exc.errno != errno.ESRCH:
raise
# Wait for children to die
if self.children:
LOG.info(_LI('Waiting on %d children to exit'), len(self.children))
while self.children:
self._wait_child()
def stop(self):
"""Terminate child processes and wait on each."""
self.running = False
for pid in self.children:
try:
os.kill(pid, signal.SIGTERM)
except OSError as exc:
if exc.errno != errno.ESRCH:
raise
# Wait for children to die
if self.children:
LOG.info(_LI('Waiting on %d children to exit'), len(self.children))
while self.children:
self._wait_child()
def configuration(self, req, tenant_id, id):
"""
Returns the default configuration template applied to the instance.
"""
LOG.info(_LI("Getting default configuration for instance %s"), id)
context = req.environ[wsgi.CONTEXT_KEY]
instance = models.Instance.load(context, id)
LOG.debug("Server: %s", instance)
config = instance.get_default_configuration_template()
LOG.debug("Default config for instance %(instance_id)s is %(config)s",
{
'instance_id': id,
'config': config
})
return wsgi.Result(views.DefaultConfigurationView(config).data(), 200)
def _wait_for_exit_or_signal(self, ready_callback=None):
status = None
signo = 0
LOG.debug('Full set of CONF:')
CONF.log_opt_values(LOG, std_logging.DEBUG)
try:
if ready_callback:
ready_callback()
super(ServiceLauncher, self).wait()
except SignalExit as exc:
signame = _signo_to_signame(exc.signo)
LOG.info(_LI('Caught %s, exiting'), signame)
status = exc.code
signo = exc.signo
except SystemExit as exc:
status = exc.code
finally:
self.stop()
return status, signo
def _child_wait_for_exit_or_signal(self, launcher):
status = 0
signo = 0
# NOTE(johannes): All exceptions are caught to ensure this
# doesn't fallback into the loop spawning children. It would
# be bad for a child to spawn more children.
try:
launcher.wait()
except SignalExit as exc:
signame = _signo_to_signame(exc.signo)
LOG.info(_LI('Child caught %s, exiting'), signame)
status = exc.code
signo = exc.signo
except SystemExit as exc:
status = exc.code
except BaseException:
LOG.exception(_LE('Unhandled exception'))
status = 2
finally:
launcher.stop()
return status, signo
def _child_wait_for_exit_or_signal(self, launcher):
status = 0
signo = 0
# NOTE(johannes): All exceptions are caught to ensure this
# doesn't fallback into the loop spawning children. It would
# be bad for a child to spawn more children.
try:
launcher.wait()
except SignalExit as exc:
signame = _signo_to_signame(exc.signo)
LOG.info(_LI('Child caught %s, exiting'), signame)
status = exc.code
signo = exc.signo
except SystemExit as exc:
status = exc.code
except BaseException:
LOG.exception(_LE('Unhandled exception'))
status = 2
finally:
launcher.stop()
return status, signo
def edit(self, req, id, body, tenant_id):
"""
Updates the instance to set or unset one or more attributes.
"""
LOG.info(_LI("Editing instance for tenant id %s."), tenant_id)
LOG.debug(logging.mask_password("req: %s"), req)
LOG.debug(logging.mask_password("body: %s"), body)
context = req.environ[wsgi.CONTEXT_KEY]
instance = models.Instance.load(context, id)
if 'slave_of' in body['instance']:
LOG.debug("Detaching replica from source.")
instance.detach_replica()
# If configuration is set, then we will update the instance to
# use the new configuration. If configuration is empty, we
# want to disassociate the instance from the configuration
# group and remove the active overrides file.
# If instance name is set, then we will update the instance name.
edit_args = {}
if 'configuration' in body['instance']:
configuration_id = self._configuration_parse(context, body)
if configuration_id:
instance.assign_configuration(configuration_id)
else:
instance.unassign_configuration()
edit_args['configuration_id'] = configuration_id
if 'name' in body['instance']:
edit_args['name'] = body['instance']['name']
if edit_args:
instance.update_db(**edit_args)
return wsgi.Result(None, 202)
def edit(self, req, id, body, tenant_id):
"""
Updates the instance to set or unset one or more attributes.
"""
LOG.info(_LI("Editing instance for tenant id %s."), tenant_id)
LOG.debug(logging.mask_password("req: %s"), req)
LOG.debug(logging.mask_password("body: %s"), body)
context = req.environ[wsgi.CONTEXT_KEY]
instance = models.Instance.load(context, id)
if 'slave_of' in body['instance']:
LOG.debug("Detaching replica from source.")
instance.detach_replica()
# If configuration is set, then we will update the instance to
# use the new configuration. If configuration is empty, we
# want to disassociate the instance from the configuration
# group and remove the active overrides file.
# If instance name is set, then we will update the instance name.
edit_args = {}
if 'configuration' in body['instance']:
configuration_id = self._configuration_parse(context, body)
if configuration_id:
instance.assign_configuration(configuration_id)
else:
instance.unassign_configuration()
edit_args['configuration_id'] = configuration_id
if 'name' in body['instance']:
edit_args['name'] = body['instance']['name']
if edit_args:
instance.update_db(**edit_args)
return wsgi.Result(None, 202)
def launch_service(self, service, workers=1):
wrap = ServiceWrapper(service, workers)
LOG.info(_LI('Starting %d workers'), wrap.workers)
while self.running and len(wrap.children) < wrap.workers:
self._start_child(wrap)
def launch_service(self, service, workers=1):
wrap = ServiceWrapper(service, workers)
LOG.info(_LI('Starting %d workers'), wrap.workers)
while self.running and len(wrap.children) < wrap.workers:
self._start_child(wrap)
