def _add_periodic_task(cls, task):
"""Add a periodic task to the list of periodic tasks.
The task should already be decorated by @periodic_task.
:return: whether task was actually enabled
"""
name = task._periodic_name
if task._periodic_spacing < 0:
LOG.info(_LI('Skipping periodic task %(task)s because '
'its interval is negative'),
{'task': name})
return False
if not task._periodic_enabled:
LOG.info(_LI('Skipping periodic task %(task)s because '
'it is disabled'),
{'task': name})
return False
# A periodic spacing of zero indicates that this task should
# be run on the default interval to avoid running too
# frequently.
if task._periodic_spacing == 0:
task._periodic_spacing = DEFAULT_INTERVAL
cls._periodic_tasks.append((name, task))
cls._periodic_spacing[name] = task._periodic_spacing
return True
def sync_pending_row(self, exit_after_run=False):
# Block until all pending rows are processed
session = neutron_db_api.get_session()
while not self.event.is_set():
self.event.wait()
# Clear the event and go back to waiting after
# the sync block exits
self.event.clear()
while True:
LOG.debug("Thread walking database")
row = db.get_oldest_pending_db_row_with_lock(session)
if not row:
LOG.debug("No rows to sync")
break
# Validate the operation
validate_func = db.VALIDATION_MAP[row.object_type]
valid = validate_func(session, row.object_uuid, row.operation, row.data)
if not valid:
LOG.info(
_LI("%(operation)s %(type)s %(uuid)s is not a " "valid operation yet, skipping for now"),
{"operation": row.operation, "type": row.object_type, "uuid": row.object_uuid},
)
continue
LOG.info(
_LI("Syncing %(operation)s %(type)s %(uuid)s"),
{"operation": row.operation, "type": row.object_type, "uuid": row.object_uuid},
)
# Add code to sync this to ODL
method, urlpath, to_send = self._json_data(row)
try:
self.client.sendjson(method, urlpath, to_send)
db.update_processing_db_row_passed(session, row)
except exceptions.ConnectionError as e:
# Don't raise the retry count, just log an error
LOG.error(_LE("Cannot connect to the Opendaylight " "Controller"))
# Set row back to pending
db.update_db_row_pending(session, row)
# Break our of the loop and retry with the next
# timer interval
break
except Exception as e:
LOG.error(
_LE("Error syncing %(type)s %(operation)s," " id %(uuid)s Error: %(error)s"),
{
"type": row.object_type,
"uuid": row.object_uuid,
"operation": row.operation,
"error": e.message,
},
)
db.update_pending_db_row_retry(session, row, self._row_retry_count)
LOG.debug("Clearing sync thread event")
if exit_after_run:
# Permanently waiting thread model breaks unit tests
# Adding this arg to exit here only for unit tests
break
def _start_child(self, wrap):
if len(wrap.forktimes) > wrap.workers:
# Limit ourselves to one process a second (over the period of
# number of workers * 1 second). This will allow workers to
# start up quickly but ensure we don't fork off children that
# die instantly too quickly.
if time.time() - wrap.forktimes[0] < wrap.workers:
LOG.info(_LI('Forking too fast, sleeping'))
time.sleep(1)
wrap.forktimes.pop(0)
wrap.forktimes.append(time.time())
pid = os.fork()
if pid == 0:
launcher = self._child_process(wrap.service)
while True:
self._child_process_handle_signal()
status, signo = self._child_wait_for_exit_or_signal(launcher)
if not _is_sighup_and_daemon(signo):
break
launcher.restart()
os._exit(status)
LOG.info(_LI('Started child %d'), pid)
wrap.children.add(pid)
self.children[pid] = wrap
return pid
def wait(self):
"""Loop waiting on children to die and respawning as necessary."""
systemd.notify_once()
LOG.debug('Full set of CONF:')
CONF.log_opt_values(LOG, logging.DEBUG)
try:
while True:
self.handle_signal()
self._respawn_children()
# No signal means that stop was called. Don't clean up here.
if not self.sigcaught:
return
signame = _signo_to_signame(self.sigcaught)
LOG.info(_LI('Caught %s, stopping children'), signame)
if not _is_sighup_and_daemon(self.sigcaught):
break
for pid in self.children:
os.kill(pid, signal.SIGHUP)
self.running = True
self.sigcaught = None
except eventlet.greenlet.GreenletExit:
LOG.info(_LI("Wait called after thread killed. Cleaning up."))
self.stop()
def wait(self):
"""Loop waiting on children to die and respawning as necessary."""
systemd.notify_once()
LOG.debug('Full set of CONF:')
CONF.log_opt_values(LOG, std_logging.DEBUG)
try:
while True:
self.handle_signal()
self._respawn_children()
# No signal means that stop was called. Don't clean up here.
if not self.sigcaught:
return
signame = _signo_to_signame(self.sigcaught)
LOG.info(_LI('Caught %s, stopping children'), signame)
if not _is_sighup_and_daemon(self.sigcaught):
break
for pid in self.children:
os.kill(pid, signal.SIGHUP)
self.running = True
self.sigcaught = None
except eventlet.greenlet.GreenletExit:
LOG.info(_LI("Wait called after thread killed. Cleaning up."))
self.stop()
def _start_child(self, wrap):
if len(wrap.forktimes) > wrap.workers:
# Limit ourselves to one process a second (over the period of
# number of workers * 1 second). This will allow workers to
# start up quickly but ensure we don't fork off children that
# die instantly too quickly.
if time.time() - wrap.forktimes[0] < wrap.workers:
LOG.info(_LI('Forking too fast, sleeping'))
time.sleep(1)
wrap.forktimes.pop(0)
wrap.forktimes.append(time.time())
pid = os.fork()
if pid == 0:
launcher = self._child_process(wrap.service)
while True:
self._child_process_handle_signal()
status, signo = self._child_wait_for_exit_or_signal(launcher)
if not _is_sighup_and_daemon(signo):
break
launcher.restart()
os._exit(status)
LOG.info(_LI('Started child %d'), pid)
wrap.children.add(pid)
self.children[pid] = wrap
return pid
def _add_periodic_task(cls, task):
"""Add a periodic task to the list of periodic tasks.
The task should already be decorated by @periodic_task.
:return: whether task was actually enabled
"""
name = task._periodic_name
if task._periodic_spacing < 0:
LOG.info(
_LI('Skipping periodic task %(task)s because '
'its interval is negative'), {'task': name})
return False
if not task._periodic_enabled:
LOG.info(
_LI('Skipping periodic task %(task)s because '
'it is disabled'), {'task': name})
return False
# A periodic spacing of zero indicates that this task should
# be run on the default interval to avoid running too
# frequently.
if task._periodic_spacing == 0:
task._periodic_spacing = DEFAULT_INTERVAL
cls._periodic_tasks.append((name, task))
cls._periodic_spacing[name] = task._periodic_spacing
return True
def _wait_child(self):
try:
# Don't block if no child processes have exited
pid, status = os.waitpid(0, os.WNOHANG)
if not pid:
return None
except OSError as exc:
if exc.errno not in (errno.EINTR, errno.ECHILD):
raise
return None
if os.WIFSIGNALED(status):
sig = os.WTERMSIG(status)
LOG.info(_LI('Child %(pid)d killed by signal %(sig)d'),
dict(pid=pid, sig=sig))
else:
code = os.WEXITSTATUS(status)
LOG.info(_LI('Child %(pid)s exited with status %(code)d'),
dict(pid=pid, code=code))
if pid not in self.children:
LOG.warning(_LW('pid %d not in child list'), pid)
return None
wrap = self.children.pop(pid)
wrap.children.remove(pid)
return wrap
def _wait_child(self):
try:
# Block while any of child processes have exited
pid, status = os.waitpid(0, 0)
if not pid:
return None
except OSError as exc:
if exc.errno not in (errno.EINTR, errno.ECHILD):
raise
return None
if os.WIFSIGNALED(status):
sig = os.WTERMSIG(status)
LOG.info(_LI('Child %(pid)d killed by signal %(sig)d'),
dict(pid=pid, sig=sig))
else:
code = os.WEXITSTATUS(status)
LOG.info(_LI('Child %(pid)s exited with status %(code)d'),
dict(pid=pid, code=code))
if pid not in self.children:
LOG.warning(_LW('pid %d not in child list'), pid)
return None
wrap = self.children.pop(pid)
wrap.children.remove(pid)
return wrap
def initialize_if_enabled():
backdoor_locals = {
"exit": _dont_use_this, # So we don't exit the entire process
"quit": _dont_use_this, # So we don't exit the entire process
"fo": _find_objects,
"pgt": _print_greenthreads,
"pnt": _print_nativethreads,
}
if CONF.backdoor_port is None:
return None
start_port, end_port = _parse_port_range(str(CONF.backdoor_port))
# NOTE(johannes): The standard sys.displayhook will print the value of
# the last expression and set it to __builtin__._, which overwrites
# the __builtin__._ that gettext sets. Let's switch to using pprint
# since it won't interact poorly with gettext, and it's easier to
# read the output too.
def displayhook(val):
if val is not None:
pprint.pprint(val)
sys.displayhook = displayhook
sock = _listen("localhost", start_port, end_port, eventlet.listen)
# In the case of backdoor port being zero, a port number is assigned by
# listen(). In any case, pull the port number out here.
port = sock.getsockname()[1]
LOG.info(_LI("Eventlet backdoor listening on %(port)s for process %(pid)d") % {"port": port, "pid": os.getpid()})
eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock, locals=backdoor_locals)
return port
def get_physical_network(self):
if self.network_vlan_ranges == None:
return
for physical_network in self.network_vlan_ranges:
LOG.info(_LI("physical network is %(physical_network)s"),
{'physical_network': physical_network})
return physical_network
def load_rules(self, force_reload=False):
"""Loads policy_path's rules.
Policy file is cached and will be reloaded if modified.
:param force_reload: Whether to reload rules from config file.
"""
if force_reload:
self.use_conf = force_reload
if self.use_conf:
if not self.policy_path:
self.policy_path = self._get_policy_path(self.policy_file)
self._load_policy_file(self.policy_path, force_reload,
overwrite=self.overwrite)
for path in CONF.policy_dirs:
try:
path = self._get_policy_path(path)
except cfg.ConfigFilesNotFoundError:
LOG.info(_LI("Can not find policy directory: %s"), path)
continue
self._walk_through_policy_directory(path,
self._load_policy_file,
force_reload, False)
def load_rules(self, force_reload=False):
"""Loads policy_path's rules.
Policy file is cached and will be reloaded if modified.
:param force_reload: Whether to reload rules from config file.
"""
if force_reload:
self.use_conf = force_reload
if self.use_conf:
if not self.policy_path:
self.policy_path = self._get_policy_path(self.policy_file)
self._load_policy_file(self.policy_path,
force_reload,
overwrite=self.overwrite)
for path in CONF.policy_dirs:
try:
path = self._get_policy_path(path)
except cfg.ConfigFilesNotFoundError:
LOG.info(_LI("Can not find policy directory: %s"), path)
continue
self._walk_through_policy_directory(path,
self._load_policy_file,
force_reload, False)
def _pipe_watcher(self):
# This will block until the write end is closed when the parent
# dies unexpectedly
self.readpipe.read()
LOG.info(_LI('Parent process has died unexpectedly, exiting'))
sys.exit(1)
def _pipe_watcher(self):
# This will block until the write end is closed when the parent
# dies unexpectedly
self.readpipe.read()
LOG.info(_LI('Parent process has died unexpectedly, exiting'))
sys.exit(1)
def __init__(cls, names, bases, dict_):
"""Metaclass that allows us to collect decorated periodic tasks."""
super(_PeriodicTasksMeta, cls).__init__(names, bases, dict_)
# NOTE(sirp): if the attribute is not present then we must be the base
# class, so, go ahead an initialize it. If the attribute is present,
# then we're a subclass so make a copy of it so we don't step on our
# parent's toes.
try:
cls._periodic_tasks = cls._periodic_tasks[:]
except AttributeError:
cls._periodic_tasks = []
try:
cls._periodic_spacing = cls._periodic_spacing.copy()
except AttributeError:
cls._periodic_spacing = {}
for value in cls.__dict__.values():
if getattr(value, '_periodic_task', False):
task = value
name = task.__name__
if task._periodic_spacing < 0:
LOG.info(
_LI('Skipping periodic task %(task)s because '
'its interval is negative'), {'task': name})
continue
if not task._periodic_enabled:
LOG.info(
_LI('Skipping periodic task %(task)s because '
'it is disabled'), {'task': name})
continue
# A periodic spacing of zero indicates that this task should
# be run on the default interval to avoid running too
# frequently.
if task._periodic_spacing == 0:
task._periodic_spacing = DEFAULT_INTERVAL
cls._periodic_tasks.append((name, task))
cls._periodic_spacing[name] = task._periodic_spacing
def __init__(cls, names, bases, dict_):
"""Metaclass that allows us to collect decorated periodic tasks."""
super(_PeriodicTasksMeta, cls).__init__(names, bases, dict_)
# NOTE(sirp): if the attribute is not present then we must be the base
# class, so, go ahead an initialize it. If the attribute is present,
# then we're a subclass so make a copy of it so we don't step on our
# parent's toes.
try:
cls._periodic_tasks = cls._periodic_tasks[:]
except AttributeError:
cls._periodic_tasks = []
try:
cls._periodic_spacing = cls._periodic_spacing.copy()
except AttributeError:
cls._periodic_spacing = {}
for value in cls.__dict__.values():
if getattr(value, '_periodic_task', False):
task = value
name = task.__name__
if task._periodic_spacing < 0:
LOG.info(_LI('Skipping periodic task %(task)s because '
'its interval is negative'),
{'task': name})
continue
if not task._periodic_enabled:
LOG.info(_LI('Skipping periodic task %(task)s because '
'it is disabled'),
{'task': name})
continue
# A periodic spacing of zero indicates that this task should
# be run on the default interval to avoid running too
# frequently.
if task._periodic_spacing == 0:
task._periodic_spacing = DEFAULT_INTERVAL
cls._periodic_tasks.append((name, task))
cls._periodic_spacing[name] = task._periodic_spacing
def stop(self):
"""Terminate child processes and wait on each."""
self.running = False
for pid in self.children:
try:
os.kill(pid, signal.SIGTERM)
except OSError as exc:
if exc.errno != errno.ESRCH:
raise
# Wait for children to die
if self.children:
LOG.info(_LI('Waiting on %d children to exit'), len(self.children))
while self.children:
self._wait_child()
def stop(self):
"""Terminate child processes and wait on each."""
self.running = False
for pid in self.children:
try:
os.kill(pid, signal.SIGTERM)
except OSError as exc:
if exc.errno != errno.ESRCH:
raise
# Wait for children to die
if self.children:
LOG.info(_LI('Waiting on %d children to exit'), len(self.children))
while self.children:
self._wait_child()
def sync_resources(self, plugin, dbcontext, collection_name):
"""Sync objects from Neutron over to OpenDaylight.
This will handle syncing networks, subnets, and ports from Neutron to
OpenDaylight. It also filters out the requisite items which are not
valid for create API operations.
"""
filter_cls = self.FILTER_MAP[collection_name]
to_be_synced = []
obj_getter = getattr(plugin, 'get_%s' % collection_name)
if collection_name == odl_const.ODL_SGS:
resources = obj_getter(dbcontext, default_sg=True)
else:
resources = obj_getter(dbcontext)
for resource in resources:
try:
# Convert underscores to dashes in the URL for ODL
collection_name_url = collection_name.replace('_', '-')
urlpath = collection_name_url + '/' + resource['id']
self.client.sendjson('get', urlpath, None)
except requests.exceptions.HTTPError as e:
with excutils.save_and_reraise_exception() as ctx:
if e.response.status_code == requests.codes.not_found:
filter_cls.filter_create_attributes_with_plugin(
resource, plugin, dbcontext)
to_be_synced.append(resource)
ctx.reraise = False
else:
# TODO(yamahata): compare result with resource.
# If they don't match, update it below
pass
key = collection_name[:-1] if len(to_be_synced) == 1 else (
collection_name)
# Convert underscores to dashes in the URL for ODL
collection_name_url = collection_name.replace('_', '-')
new_objs_obj = self.client.sendjson('post', collection_name_url, {key: to_be_synced})
# add by hqf
if collection_name == odl_const.ODL_NETWORKS:
new_objs = new_objs_obj.get(key)
if key == collection_name:
for new_obj in new_objs:
LOG.info(_LI("new_obj %(new_obj)s"),
{'new_obj': new_obj})
self.update_network_segments(new_obj)
else:
self.update_network_segments(new_objs)
def _fetch_and_parse_network_topology(self, addresses):
# The cache calls this method to fecth new elements when at least one
# of the addresses is not in the cache or it has expired.
# pylint: disable=unused-argument
LOG.info(_LI('Fetch network topology from ODL.'))
response = self._client.get()
response.raise_for_status()
network_topology = response.json()
if LOG.isEnabledFor(logging.DEBUG):
topology_str = jsonutils.dumps(network_topology,
sort_keys=True,
indent=4,
separators=(',', ': '))
LOG.debug("Got network topology:\n%s", topology_str)
at_least_one_element_for_asked_addresses = False
for parser in self._parsers:
try:
for element in parser.parse_network_topology(network_topology):
if not isinstance(element, NetworkTopologyElement):
raise TypeError(
"Yield element doesn't implement interface "
"'NetworkTopologyElement': {!r}".format(element))
# the same element can be known by more host addresses
for host_address in element.host_addresses:
if host_address in addresses:
at_least_one_element_for_asked_addresses = True
yield host_address, element
except Exception:
LOG.exception(
_LE("Parser %(parser)r failed to parse network topology."),
{'parser': parser})
if not at_least_one_element_for_asked_addresses:
# this will mark entries for given addresses as failed to allow
# calling this method again as soon it is requested and avoid
# waiting for cache expiration
raise ValueError(
'No such topology element for given host addresses: {}'.format(
', '.join(addresses)))
def _fetch_and_parse_network_topology(self, addresses):
# The cache calls this method to fecth new elements when at least one
# of the addresses is not in the cache or it has expired.
# pylint: disable=unused-argument
LOG.info(_LI('Fetch network topology from ODL.'))
response = self._client.get()
response.raise_for_status()
network_topology = response.json()
if LOG.isEnabledFor(logging.DEBUG):
topology_str = jsonutils.dumps(
network_topology, sort_keys=True, indent=4,
separators=(',', ': '))
LOG.debug("Got network topology:\n%s", topology_str)
at_least_one_element_for_asked_addresses = False
for parser in self._parsers:
try:
for element in parser.parse_network_topology(network_topology):
if not isinstance(element, NetworkTopologyElement):
raise TypeError(
"Yield element doesn't implement interface "
"'NetworkTopologyElement': {!r}".format(element))
# the same element can be known by more host addresses
for host_address in element.host_addresses:
if host_address in addresses:
at_least_one_element_for_asked_addresses = True
yield host_address, element
except Exception:
LOG.exception(
_LE("Parser %(parser)r failed to parse network topology."),
{'parser': parser})
if not at_least_one_element_for_asked_addresses:
# this will mark entries for given addresses as failed to allow
# calling this method again as soon it is requested and avoid
# waiting for cache expiration
raise ValueError(
'No such topology element for given host addresses: {}'.format(
', '.join(addresses)))
def initialize_if_enabled():
backdoor_locals = {
'exit': _dont_use_this, # So we don't exit the entire process
'quit': _dont_use_this, # So we don't exit the entire process
'fo': _find_objects,
'pgt': _print_greenthreads,
'pnt': _print_nativethreads,
}
if CONF.backdoor_port is None:
return None
start_port, end_port = _parse_port_range(str(CONF.backdoor_port))
# NOTE(johannes): The standard sys.displayhook will print the value of
# the last expression and set it to __builtin__._, which overwrites
# the __builtin__._ that gettext sets. Let's switch to using pprint
# since it won't interact poorly with gettext, and it's easier to
# read the output too.
def displayhook(val):
if val is not None:
pprint.pprint(val)
sys.displayhook = displayhook
sock = _listen('localhost', start_port, end_port, eventlet.listen)
# In the case of backdoor port being zero, a port number is assigned by
# listen(). In any case, pull the port number out here.
port = sock.getsockname()[1]
LOG.info(
_LI('Eventlet backdoor listening on %(port)s for process %(pid)d') % {
'port': port,
'pid': os.getpid()
})
eventlet.spawn_n(eventlet.backdoor.backdoor_server,
sock,
locals=backdoor_locals)
return port
def _wait_for_exit_or_signal(self, ready_callback=None):
status = None
signo = 0
LOG.debug('Full set of CONF:')
CONF.log_opt_values(LOG, logging.DEBUG)
try:
if ready_callback:
ready_callback()
super(ServiceLauncher, self).wait()
except SignalExit as exc:
signame = _signo_to_signame(exc.signo)
LOG.info(_LI('Caught %s, exiting'), signame)
status = exc.code
signo = exc.signo
except SystemExit as exc:
status = exc.code
finally:
self.stop()
return status, signo
def _wait_for_exit_or_signal(self, ready_callback=None):
status = None
signo = 0
LOG.debug('Full set of CONF:')
CONF.log_opt_values(LOG, std_logging.DEBUG)
try:
if ready_callback:
ready_callback()
super(ServiceLauncher, self).wait()
except SignalExit as exc:
signame = _signo_to_signame(exc.signo)
LOG.info(_LI('Caught %s, exiting'), signame)
status = exc.code
signo = exc.signo
except SystemExit as exc:
status = exc.code
finally:
self.stop()
return status, signo
def _child_wait_for_exit_or_signal(self, launcher):
status = 0
signo = 0
# NOTE(johannes): All exceptions are caught to ensure this
# doesn't fallback into the loop spawning children. It would
# be bad for a child to spawn more children.
try:
launcher.wait()
except SignalExit as exc:
signame = _signo_to_signame(exc.signo)
LOG.info(_LI('Child caught %s, exiting'), signame)
status = exc.code
signo = exc.signo
except SystemExit as exc:
status = exc.code
except BaseException:
LOG.exception(_LE('Unhandled exception'))
status = 2
finally:
launcher.stop()
return status, signo
def _child_wait_for_exit_or_signal(self, launcher):
status = 0
signo = 0
# NOTE(johannes): All exceptions are caught to ensure this
# doesn't fallback into the loop spawning children. It would
# be bad for a child to spawn more children.
try:
launcher.wait()
except SignalExit as exc:
signame = _signo_to_signame(exc.signo)
LOG.info(_LI('Child caught %s, exiting'), signame)
status = exc.code
signo = exc.signo
except SystemExit as exc:
status = exc.code
except BaseException:
LOG.exception(_LE('Unhandled exception'))
status = 2
finally:
launcher.stop()
return status, signo
def sync_pending_row(self, exit_after_run=False):
# Block until all pending rows are processed
session = neutron_db_api.get_session()
while not self.event.is_set():
self.event.wait()
# Clear the event and go back to waiting after
# the sync block exits
self.event.clear()
while True:
LOG.debug("Thread walking database")
row = db.get_oldest_pending_db_row_with_lock(session)
if not row:
LOG.debug("No rows to sync")
break
# Validate the operation
validate_func = db.VALIDATION_MAP[row.object_type]
valid = validate_func(session, row.object_uuid, row.operation,
row.data)
if not valid:
LOG.info(
_LI("%(operation)s %(type)s %(uuid)s is not a "
"valid operation yet, skipping for now"), {
'operation': row.operation,
'type': row.object_type,
'uuid': row.object_uuid
})
continue
LOG.info(
_LI("Syncing %(operation)s %(type)s %(uuid)s"), {
'operation': row.operation,
'type': row.object_type,
'uuid': row.object_uuid
})
# Add code to sync this to ODL
method, urlpath, to_send = self._json_data(row)
try:
self.client.sendjson(method, urlpath, to_send)
db.update_processing_db_row_passed(session, row)
except exceptions.ConnectionError as e:
# Don't raise the retry count, just log an error
LOG.error(
_LE("Cannot connect to the Opendaylight "
"Controller"))
# Set row back to pending
db.update_db_row_pending(session, row)
# Break our of the loop and retry with the next
# timer interval
break
except Exception as e:
LOG.error(
_LE("Error syncing %(type)s %(operation)s,"
" id %(uuid)s Error: %(error)s"), {
'type': row.object_type,
'uuid': row.object_uuid,
'operation': row.operation,
'error': e.message
})
db.update_pending_db_row_retry(session, row,
self._row_retry_count)
LOG.debug("Clearing sync thread event")
if exit_after_run:
# Permanently waiting thread model breaks unit tests
# Adding this arg to exit here only for unit tests
break
def launch_service(self, service, workers=1):
wrap = ServiceWrapper(service, workers)
LOG.info(_LI('Starting %d workers'), wrap.workers)
while self.running and len(wrap.children) < wrap.workers:
self._start_child(wrap)
def launch_service(self, service, workers=1):
wrap = ServiceWrapper(service, workers)
LOG.info(_LI('Starting %d workers'), wrap.workers)
while self.running and len(wrap.children) < wrap.workers:
self._start_child(wrap)
