def __init__(self, count):
self._executor = futurist.GreenThreadPoolExecutor(max_workers=1000)
self.count = count
if 'consoleserver_host' in os.environ:
self.host = os.environ["consoleserver_host"]
else:
self.host = 'http://localhost:12429'
def executor():
"""Return the current futures executor."""
global _EXECUTOR
if _EXECUTOR is None:
_EXECUTOR = futurist.GreenThreadPoolExecutor(
max_workers=CONF.max_concurrency)
return _EXECUTOR
def init_host(self, admin_context=None):
"""Initialize the agent host.
:param admin_context: the admin context to pass to periodic tasks.
:raises: RuntimeError when agent is already running.
"""
if self._started:
raise RuntimeError(
_('Attempt to start an already running '
'agent manager'))
rejection_func = rejection.reject_when_reached(64)
# CONF.conductor.workers_pool_size)
self._executor = futurist.GreenThreadPoolExecutor(
64, check_and_reject=rejection_func)
# JK max_workers=CONF.conductor.workers_pool_size,
"""Executor for performing tasks async."""
# Collect driver-specific periodic tasks.
# Conductor periodic tasks accept context argument,
LOG.info('Collecting periodic tasks')
self._periodic_task_callables = []
self._collect_periodic_tasks(self, (admin_context, ))
self._periodic_tasks = periodics.PeriodicWorker(
self._periodic_task_callables,
executor_factory=periodics.ExistingExecutor(self._executor))
# Start periodic tasks
self._periodic_tasks_worker = self._executor.submit(
self._periodic_tasks.start, allow_empty=True)
self._periodic_tasks_worker.add_done_callback(
self._on_periodic_tasks_stop)
self._started = True
def _parallel_create(func, nodes, total, split_num):
import eventlet
eventlet.monkey_patch(os=False)
import futurist
from futurist import waiters
data = []
i = 0
per_split = total / split_num
while i < split_num - 1:
data.append({'nodes': nodes[i * per_split:(i + 1) * per_split]})
i += 1
data.append({'nodes': nodes[i * per_split:]})
_executor = futurist.GreenThreadPoolExecutor(max_workers=split_num)
j = 0
futures = []
while j < split_num:
future = _executor.submit(func, data[j])
futures.append(future)
j += 1
done, not_done = waiters.wait_for_all(futures, 3600)
result = {'nodes': {}}
for r in done:
result['nodes'].update(r.result()['nodes'])
return result
def _parallel_update(func, names, patch, total, split_num):
import eventlet
eventlet.monkey_patch(os=False)
import futurist
from futurist import waiters
patch_dicts = []
patch_dict = {'nodes': [], "patches": patch}
i = 0
per_split = total / split_num
while i < total:
if i != 0 and i % per_split == 0 and i != split_num * per_split:
patch_dicts.append(patch_dict)
patch_dict = {'nodes': [], "patches": patch}
patch_dict['nodes'].append({'name': names[i]})
i += 1
patch_dicts.append(patch_dict)
_executor = futurist.GreenThreadPoolExecutor(max_workers=split_num)
futures = []
for patch_dict in patch_dicts:
future = _executor.submit(func, patch_dict)
futures.append(future)
done, not_done = waiters.wait_for_all(futures, 3600)
result = {'nodes': {}}
for r in done:
result['nodes'].update(r.result()['nodes'])
return result
def list_floatingips(self, context, region=None):
"""
Get floating ips based on the current context from Neutron
"""
endpoints = self._endpoints(
service_catalog=context.service_catalog,
service_type='network',
endpoint_type=CONF['network_api:neutron'].endpoint_type,
config_section='network_api:neutron',
region=region
)
floating_ips = []
with futurist.GreenThreadPoolExecutor(max_workers=5) as executor:
executors = [
executor.submit(
self._get_floating_ips,
context,
endpoint,
region,
project_id=context.project_id
) for endpoint, region in endpoints
]
for future in concurrent.futures.as_completed(executors):
try:
floating_ips.extend(future.result())
except Exception as e:
raise exceptions.NeutronCommunicationFailure(e)
return floating_ips
def get_pool(size):
import futurist
if eventletutils.is_monkey_patched('thread'):
return futurist.GreenThreadPoolExecutor(size)
return futurist.ThreadPoolExecutor(size)
def test_submit_passes_through(self):
self.useFixture(fixtures.SynchronousThreadPoolExecutorFixture())
tester = mock.MagicMock()
executor = futurist.GreenThreadPoolExecutor()
future = executor.submit(tester.function, 'foo', bar='bar')
tester.function.assert_called_once_with('foo', bar='bar')
result = future.result()
self.assertEqual(tester.function.return_value, result)
def _executor(self):
if CONF.taskflow_executor.engine_mode != 'parallel':
yield None
else:
max_workers = CONF.taskflow_executor.max_workers
try:
yield futurist.GreenThreadPoolExecutor(max_workers=max_workers)
except RuntimeError:
yield futurist.ThreadPoolExecutor(max_workers=max_workers)
def default_executor():
thread_count = 5
try:
thread_count = CONF['service:worker'].threads
except Exception:
pass
# return futures.ThreadPoolExecutor(thread_count)
return futurist.GreenThreadPoolExecutor(thread_count)
def _make_engine(self, flow, flow_detail=None, executor=None):
if executor is None:
executor = futurist.GreenThreadPoolExecutor()
self.addCleanup(executor.shutdown)
return taskflow.engines.load(flow,
flow_detail=flow_detail,
backend=self.backend,
engine='parallel',
executor=executor)
def default_executor():
thread_count = 5
try:
thread_count = CONF['service:worker'].threads
except Exception:
pass
# TODO(mugsie): if (when) we move away from eventlet this may have to
# revert back to ThreadPoolExecutor - this is changing due to
# https://bugs.launchpad.net/bugs/1782647 (eventlet + py37 issues)
return futurist.GreenThreadPoolExecutor(thread_count)
def _fetch_an_executor():
if CONF.taskflow_executor.engine_mode != 'parallel':
return None
else:
max_workers = CONF.taskflow_executor.max_workers
try:
return futurist.GreenThreadPoolExecutor(
max_workers=max_workers)
except RuntimeError:
# NOTE(harlowja): I guess eventlet isn't being made
# useable, well just use native threads then (or try to).
return futurist.ThreadPoolExecutor(max_workers=max_workers)
def execute(self, actions):
try:
flow = gf.Flow("watcher_flow")
for a in actions:
task = TaskFlowActionContainer(a, self)
flow.add(task)
executor = futurist.GreenThreadPoolExecutor(max_workers=100)
e = engines.load(flow, engine='parallel', executor=executor)
e.run()
except Exception as e:
raise exception.WorkflowExecutionException(error=e)
finally:
executor.shutdown()
def main():
if len(sys.argv) == 2:
tbl = []
with open(sys.argv[1], 'rb') as fh:
reader = csv.reader(fh)
for row in reader:
tbl.append([float(r) if r else 0.0 for r in row])
else:
# Make some random table out of thin air...
tbl = []
cols = random.randint(1, 100)
rows = random.randint(1, 100)
for _i in compat_range(0, rows):
row = []
for _j in compat_range(0, cols):
row.append(random.random())
tbl.append(row)
# Generate the work to be done.
f = make_flow(tbl)
# Now run it (using the specified executor)...
try:
executor = futurist.GreenThreadPoolExecutor(max_workers=5)
except RuntimeError:
# No eventlet currently active, use real threads instead.
executor = futurist.ThreadPoolExecutor(max_workers=5)
try:
e = engines.load(f, engine='parallel', executor=executor)
for st in e.run_iter():
print(st)
finally:
executor.shutdown()
# Find the old rows and put them into place...
#
# TODO(harlowja): probably easier just to sort instead of search...
computed_tbl = []
for i in compat_range(0, len(tbl)):
for t in f:
if t.index == i:
computed_tbl.append(e.storage.get(t.name))
# Do some basic validation (which causes the return code of this process
# to be different if things were not as expected...)
if len(computed_tbl) != len(tbl):
return 1
else:
return 0
def get_engine(self, flow, **kwargs):
if flow is None:
LOG.error(_LE("Flow is None, build it first"))
return
executor = kwargs.get('executor', None)
engine = kwargs.get('engine', None)
store = kwargs.get('store', None)
if not executor:
executor = futurist.GreenThreadPoolExecutor()
if not engine:
engine = 'parallel'
flow_engine = engines.load(flow,
executor=executor,
engine=engine,
store=store)
return flow_engine
def get_engine(self, flow, **kwargs):
if flow is None:
LOG.error("The flow is None, build it first")
raise exception.InvalidTaskFlowObject(reason=_("The flow is None"))
executor = kwargs.get('executor', None)
engine = kwargs.get('engine', None)
store = kwargs.get('store', None)
if not executor:
executor = futurist.GreenThreadPoolExecutor()
if not engine:
engine = 'parallel'
flow_engine = engines.load(flow,
executor=executor,
engine=engine,
store=store)
return flow_engine
def __init__(self, operation_manager, thread_count=None):
super(ThreadPoolExecutor, self).__init__(operation_manager)
if thread_count is None:
thread_count = CONF.thread_count
self._pool = futurist.GreenThreadPoolExecutor(thread_count)
self._operation_to_run = defaultdict(int)
self._operation_to_cancel = set()
self._lock = RLock()
self._check_functions = {
self._CHECK_ITEMS['is_waiting']:
lambda op_id: (op_id in self._operation_to_run),
self._CHECK_ITEMS['is_canceled']:
lambda op_id: (op_id in self._operation_to_cancel),
}
def main():
# the behavior is specific to GreenThreadPoolExecutor
threadpool = futurist.GreenThreadPoolExecutor(max_workers=4)
rrwlock = ReentrantReadWriteLock()
futures = []
futures.append(threadpool.submit(get_write, rrwlock))
futures.append(threadpool.submit(get_read, rrwlock))
# Get the results and verify only one of the calls succeeded
# assert that the other call is still pending
# this call will block indefinitely, it should not block state of threads
# in threadpool should not be able to cause wait_for_any to block
# indefinitely.
results = waiters.wait_for_any(futures)
# these statements will never be reached
print(results[0].pop().result == True)
print(len(results[1]))
def test_green_executor_creation(self):
with futurist.GreenThreadPoolExecutor(1) as e:
eng = self._create_engine(executor=e)
self.assertIsInstance(eng._task_executor,
executor.ParallelThreadTaskExecutor)
with eu.get_backend() as backend:
# Try to find a previously passed in tracking id...
try:
book_id, flow_id = sys.argv[2].split("+", 1)
if not uuidutils.is_uuid_like(book_id):
book_id = None
if not uuidutils.is_uuid_like(flow_id):
flow_id = None
except (IndexError, ValueError):
book_id = None
flow_id = None
# Set up how we want our engine to run, serial, parallel...
try:
executor = futurist.GreenThreadPoolExecutor(max_workers=5)
except RuntimeError:
# No eventlet installed, just let the default be used instead.
executor = None
# Create/fetch a logbook that will track the workflows work.
book = None
flow_detail = None
if all([book_id, flow_id]):
# Try to find in a prior logbook and flow detail...
with contextlib.closing(backend.get_connection()) as conn:
try:
book = conn.get_logbook(book_id)
flow_detail = book.find(flow_id)
except exc.NotFound:
pass
def __init__(self, url, method, count):
self._executor = futurist.GreenThreadPoolExecutor(max_workers=1000)
self.url = url
self.method = method
self.count = count
def __init__(self):
self.amount_workers = CONF.watcher_decision_engine.max_general_workers
self._threadpool = futurist.GreenThreadPoolExecutor(
max_workers=self.amount_workers)
def __init__(self, applier_manager):
self.applier_manager = applier_manager
workers = CONF.watcher_applier.workers
self.executor = futurist.GreenThreadPoolExecutor(max_workers=workers)
def init_host(self, admin_context=None):
"""Initialize the conductor host.
:param admin_context: the admin context to pass to periodic tasks.
:raises: RuntimeError when conductor is already running.
:raises: NoDriversLoaded when no drivers are enabled on the conductor.
:raises: DriverNotFound if a driver is enabled that does not exist.
:raises: DriverLoadError if an enabled driver cannot be loaded.
:raises: DriverNameConflict if a classic driver and a dynamic driver
are both enabled and have the same name.
:raises: ConfigInvalid if required config options for connection with
radosgw are missing while storing config drive.
"""
if self._started:
raise RuntimeError(
_('Attempt to start an already running '
'conductor manager'))
self._shutdown = False
self.dbapi = dbapi.get_instance()
self._keepalive_evt = threading.Event()
"""Event for the keepalive thread."""
# TODO(dtantsur): make the threshold configurable?
rejection_func = rejection.reject_when_reached(
CONF.conductor.workers_pool_size)
self._executor = futurist.GreenThreadPoolExecutor(
max_workers=CONF.conductor.workers_pool_size,
check_and_reject=rejection_func)
"""Executor for performing tasks async."""
self.ring_manager = hash_ring.HashRingManager()
"""Consistent hash ring which maps drivers to conductors."""
_check_enabled_interfaces()
# NOTE(deva): these calls may raise DriverLoadError or DriverNotFound
# NOTE(vdrok): Instantiate network and storage interface factory on
# startup so that all the interfaces are loaded at the very
# beginning, and failures prevent the conductor from starting.
drivers = driver_factory.drivers()
hardware_types = driver_factory.hardware_types()
driver_factory.NetworkInterfaceFactory()
driver_factory.StorageInterfaceFactory()
# NOTE(jroll) this is passed to the dbapi, which requires a list, not
# a generator (which keys() returns in py3)
driver_names = list(drivers)
hardware_type_names = list(hardware_types)
# check that at least one driver is loaded, whether classic or dynamic
if not driver_names and not hardware_type_names:
msg = ("Conductor %s cannot be started because no drivers "
"were loaded. This could be because no classic drivers "
"were specified in the 'enabled_drivers' config option "
"and no dynamic drivers were specified in the "
"'enabled_hardware_types' config option.")
LOG.error(msg, self.host)
raise exception.NoDriversLoaded(conductor=self.host)
# check for name clashes between classic and dynamic drivers
name_clashes = set(driver_names).intersection(hardware_type_names)
if name_clashes:
name_clashes = ', '.join(name_clashes)
msg = ("Conductor %(host)s cannot be started because there is "
"one or more name conflicts between classic drivers and "
"dynamic drivers (%(names)s). Check any external driver "
"plugins and the 'enabled_drivers' and "
"'enabled_hardware_types' config options.")
LOG.error(msg, {'host': self.host, 'names': name_clashes})
raise exception.DriverNameConflict(names=name_clashes)
# Collect driver-specific periodic tasks.
# Conductor periodic tasks accept context argument, driver periodic
# tasks accept this manager and context. We have to ensure that the
# same driver interface class is not traversed twice, otherwise
# we'll have several instances of the same task.
LOG.debug('Collecting periodic tasks')
self._periodic_task_callables = []
periodic_task_classes = set()
self._collect_periodic_tasks(self, (admin_context, ))
for driver_obj in drivers.values():
for iface_name in driver_obj.all_interfaces:
iface = getattr(driver_obj, iface_name, None)
if iface and iface.__class__ not in periodic_task_classes:
self._collect_periodic_tasks(iface, (self, admin_context))
periodic_task_classes.add(iface.__class__)
if (len(self._periodic_task_callables) >
CONF.conductor.workers_pool_size):
LOG.warning(
'This conductor has %(tasks)d periodic tasks '
'enabled, but only %(workers)d task workers '
'allowed by [conductor]workers_pool_size option', {
'tasks': len(self._periodic_task_callables),
'workers': CONF.conductor.workers_pool_size
})
self._periodic_tasks = periodics.PeriodicWorker(
self._periodic_task_callables,
executor_factory=periodics.ExistingExecutor(self._executor))
# Check for required config options if object_store_endpoint_type is
# radosgw
if (CONF.deploy.configdrive_use_object_store
and CONF.deploy.object_store_endpoint_type == "radosgw"):
if (None in (CONF.swift.auth_url, CONF.swift.username,
CONF.swift.password)):
msg = _("Parameters missing to make a connection with "
"radosgw. Ensure that [swift]/auth_url, "
"[swift]/username, and [swift]/password are all "
"configured.")
raise exception.ConfigInvalid(msg)
# clear all target_power_state with locks by this conductor
self.dbapi.clear_node_target_power_state(self.host)
# clear all locks held by this conductor before registering
self.dbapi.clear_node_reservations_for_conductor(self.host)
try:
# Register this conductor with the cluster
self.conductor = objects.Conductor.register(
admin_context, self.host, driver_names)
except exception.ConductorAlreadyRegistered:
# This conductor was already registered and did not shut down
# properly, so log a warning and update the record.
LOG.warning(
"A conductor with hostname %(hostname)s was "
"previously registered. Updating registration",
{'hostname': self.host})
self.conductor = objects.Conductor.register(admin_context,
self.host,
driver_names,
update_existing=True)
# register hardware types and interfaces supported by this conductor
# and validate them against other conductors
try:
self._register_and_validate_hardware_interfaces(hardware_types)
except (exception.DriverLoadError, exception.DriverNotFound,
exception.ConductorHardwareInterfacesAlreadyRegistered,
exception.InterfaceNotFoundInEntrypoint,
exception.NoValidDefaultForInterface) as e:
with excutils.save_and_reraise_exception():
LOG.error('Failed to register hardware types. %s', e)
self.del_host()
# Start periodic tasks
self._periodic_tasks_worker = self._executor.submit(
self._periodic_tasks.start, allow_empty=True)
self._periodic_tasks_worker.add_done_callback(
self._on_periodic_tasks_stop)
# NOTE(lucasagomes): If the conductor server dies abruptly
# mid deployment (OMM Killer, power outage, etc...) we
# can not resume the deployment even if the conductor
# comes back online. Cleaning the reservation of the nodes
# (dbapi.clear_node_reservations_for_conductor) is not enough to
# unstick it, so let's gracefully fail the deployment so the node
# can go through the steps (deleting & cleaning) to make itself
# available again.
filters = {'reserved': False, 'provision_state': states.DEPLOYING}
last_error = (_("The deployment can't be resumed by conductor "
"%s. Moving to fail state.") % self.host)
self._fail_if_in_state(ironic_context.get_admin_context(),
filters,
states.DEPLOYING,
'provision_updated_at',
last_error=last_error)
# Start consoles if it set enabled in a greenthread.
try:
self._spawn_worker(self._start_consoles,
ironic_context.get_admin_context())
except exception.NoFreeConductorWorker:
LOG.warning('Failed to start worker for restarting consoles.')
# Spawn a dedicated greenthread for the keepalive
try:
self._spawn_worker(self._conductor_service_record_keepalive)
LOG.info(
'Successfully started conductor with hostname '
'%(hostname)s.', {'hostname': self.host})
except exception.NoFreeConductorWorker:
with excutils.save_and_reraise_exception():
LOG.critical('Failed to start keepalive')
self.del_host()
self._started = True
def _create_executor(self, max_workers=None):
if max_workers is None:
max_workers = self.DEFAULT_WORKERS
return futurist.GreenThreadPoolExecutor(max_workers=max_workers)
world_chorus.add(PrinterTask("%s@world" % name,
inject={'output': world}))
# The composition starts with the conductor and then runs in sequence with
# the chorus running in parallel, but no matter what the 'hello' chorus must
# always run before the 'world' chorus (otherwise the world will fall apart).
song.add(PrinterTask("conductor@begin",
show_name=False, inject={'output': "*ding*"}),
hi_chorus,
world_chorus,
PrinterTask("conductor@end",
show_name=False, inject={'output': "*dong*"}))
# Run in parallel using eventlet green threads...
try:
executor = futurist.GreenThreadPoolExecutor()
except RuntimeError:
# No eventlet currently active, skip running with it...
pass
else:
print("-- Running in parallel using eventlet --")
with executor:
e = engines.load(song, executor=executor, engine='parallel')
e.run()
# Run in parallel using real threads...
with futurist.ThreadPoolExecutor(max_workers=1) as executor:
print("-- Running in parallel using threads --")
e = engines.load(song, executor=executor, engine='parallel')
e.run()
def executor():
global _executor
if not _executor:
_executor = futurist.GreenThreadPoolExecutor(max_workers=10)
return _executor
def init_host(self, admin_context=None):
"""Initialize the conductor host.
:param admin_context: the admin context to pass to periodic tasks.
:raises: RuntimeError when conductor is already running.
:raises: NoDriversLoaded when no drivers are enabled on the conductor.
:raises: DriverNotFound if a driver is enabled that does not exist.
:raises: DriverLoadError if an enabled driver cannot be loaded.
:raises: DriverNameConflict if a classic driver and a dynamic driver
are both enabled and have the same name.
"""
if self._started:
raise RuntimeError(
_('Attempt to start an already running '
'conductor manager'))
self._shutdown = False
self.dbapi = dbapi.get_instance()
self._keepalive_evt = threading.Event()
"""Event for the keepalive thread."""
# TODO(dtantsur): make the threshold configurable?
rejection_func = rejection.reject_when_reached(
CONF.conductor.workers_pool_size)
self._executor = futurist.GreenThreadPoolExecutor(
max_workers=CONF.conductor.workers_pool_size,
check_and_reject=rejection_func)
"""Executor for performing tasks async."""
# TODO(jroll) delete the use_groups argument and use the default
# in Stein.
self.ring_manager = hash_ring.HashRingManager(
use_groups=self._use_groups())
"""Consistent hash ring which maps drivers to conductors."""
_check_enabled_interfaces()
# NOTE(deva): these calls may raise DriverLoadError or DriverNotFound
# NOTE(vdrok): Instantiate network and storage interface factory on
# startup so that all the interfaces are loaded at the very
# beginning, and failures prevent the conductor from starting.
hardware_types = driver_factory.hardware_types()
driver_factory.NetworkInterfaceFactory()
driver_factory.StorageInterfaceFactory()
# NOTE(jroll) this is passed to the dbapi, which requires a list, not
# a generator (which keys() returns in py3)
hardware_type_names = list(hardware_types)
# check that at least one driver is loaded, whether classic or dynamic
if not hardware_type_names:
msg = ("Conductor %s cannot be started because no hardware types "
"were specified in the 'enabled_hardware_types' config "
"option.")
LOG.error(msg, self.host)
raise exception.NoDriversLoaded(conductor=self.host)
self._collect_periodic_tasks(admin_context)
# clear all target_power_state with locks by this conductor
self.dbapi.clear_node_target_power_state(self.host)
# clear all locks held by this conductor before registering
self.dbapi.clear_node_reservations_for_conductor(self.host)
try:
# Register this conductor with the cluster
self.conductor = objects.Conductor.register(
admin_context, self.host, hardware_type_names,
CONF.conductor.conductor_group)
except exception.ConductorAlreadyRegistered:
# This conductor was already registered and did not shut down
# properly, so log a warning and update the record.
LOG.warning(
"A conductor with hostname %(hostname)s was "
"previously registered. Updating registration",
{'hostname': self.host})
self.conductor = objects.Conductor.register(
admin_context,
self.host,
hardware_type_names,
CONF.conductor.conductor_group,
update_existing=True)
# register hardware types and interfaces supported by this conductor
# and validate them against other conductors
try:
self._register_and_validate_hardware_interfaces(hardware_types)
except (exception.DriverLoadError, exception.DriverNotFound,
exception.ConductorHardwareInterfacesAlreadyRegistered,
exception.InterfaceNotFoundInEntrypoint,
exception.NoValidDefaultForInterface) as e:
with excutils.save_and_reraise_exception():
LOG.error('Failed to register hardware types. %s', e)
self.del_host()
# Start periodic tasks
self._periodic_tasks_worker = self._executor.submit(
self._periodic_tasks.start, allow_empty=True)
self._periodic_tasks_worker.add_done_callback(
self._on_periodic_tasks_stop)
for state in states.STUCK_STATES_TREATED_AS_FAIL:
self._fail_transient_state(
state,
_("The %(state)s state can't be resumed by conductor "
"%(host)s. Moving to fail state.") % {
'state': state,
'host': self.host
})
# Start consoles if it set enabled in a greenthread.
try:
self._spawn_worker(self._start_consoles,
ironic_context.get_admin_context())
except exception.NoFreeConductorWorker:
LOG.warning('Failed to start worker for restarting consoles.')
# Spawn a dedicated greenthread for the keepalive
try:
self._spawn_worker(self._conductor_service_record_keepalive)
LOG.info(
'Successfully started conductor with hostname '
'%(hostname)s.', {'hostname': self.host})
except exception.NoFreeConductorWorker:
with excutils.save_and_reraise_exception():
LOG.critical('Failed to start keepalive')
self.del_host()
# Resume allocations that started before the restart.
try:
self._spawn_worker(self._resume_allocations,
ironic_context.get_admin_context())
except exception.NoFreeConductorWorker:
LOG.warning('Failed to start worker for resuming allocations.')
if CONF.conductor.enable_mdns:
self._publish_endpoint()
self._started = True
def init_host(self, admin_context=None):
"""Initialize the conductor host.
:param admin_context: the admin context to pass to periodic tasks.
:raises: RuntimeError when conductor is already running.
:raises: NoDriversLoaded when no drivers are enabled on the conductor.
:raises: DriverNotFound if a driver is enabled that does not exist.
:raises: DriverLoadError if an enabled driver cannot be loaded.
"""
if self._started:
raise RuntimeError(
_('Attempt to start an already running '
'conductor manager'))
self.dbapi = dbapi.get_instance()
self._keepalive_evt = threading.Event()
"""Event for the keepalive thread."""
# TODO(dtantsur): make the threshold configurable?
rejection_func = rejection.reject_when_reached(
CONF.conductor.workers_pool_size)
self._executor = futurist.GreenThreadPoolExecutor(
max_workers=CONF.conductor.workers_pool_size,
check_and_reject=rejection_func)
"""Executor for performing tasks async."""
self.ring_manager = hash.HashRingManager()
"""Consistent hash ring which maps drivers to conductors."""
# NOTE(deva): these calls may raise DriverLoadError or DriverNotFound
# NOTE(vdrok): Instantiate network and storage interface factory on
# startup so that all the interfaces are loaded at the very
# beginning, and failures prevent the conductor from starting.
drivers = driver_factory.drivers()
driver_factory.NetworkInterfaceFactory()
driver_factory.StorageInterfaceFactory()
if not drivers:
msg = _LE("Conductor %s cannot be started because no drivers "
"were loaded. This could be because no drivers were "
"specified in 'enabled_drivers' config option.")
LOG.error(msg, self.host)
raise exception.NoDriversLoaded(conductor=self.host)
# NOTE(jroll) this is passed to the dbapi, which requires a list, not
# a generator (which keys() returns in py3)
driver_names = list(drivers)
# Collect driver-specific periodic tasks.
# Conductor periodic tasks accept context argument, driver periodic
# tasks accept this manager and context. We have to ensure that the
# same driver interface class is not traversed twice, otherwise
# we'll have several instances of the same task.
LOG.debug('Collecting periodic tasks')
self._periodic_task_callables = []
periodic_task_classes = set()
self._collect_periodic_tasks(self, (admin_context, ))
for driver_obj in drivers.values():
# TODO(dtantsur): collecting tasks from driver objects is
# deprecated and should be removed in Ocata.
self._collect_periodic_tasks(driver_obj, (self, admin_context))
for iface_name in driver_obj.all_interfaces:
iface = getattr(driver_obj, iface_name, None)
if iface and iface.__class__ not in periodic_task_classes:
self._collect_periodic_tasks(iface, (self, admin_context))
periodic_task_classes.add(iface.__class__)
if (len(self._periodic_task_callables) >
CONF.conductor.workers_pool_size):
LOG.warning(
_LW('This conductor has %(tasks)d periodic tasks '
'enabled, but only %(workers)d task workers '
'allowed by [conductor]workers_pool_size option'), {
'tasks': len(self._periodic_task_callables),
'workers': CONF.conductor.workers_pool_size
})
self._periodic_tasks = periodics.PeriodicWorker(
self._periodic_task_callables,
executor_factory=periodics.ExistingExecutor(self._executor))
# clear all target_power_state with locks by this conductor
self.dbapi.clear_node_target_power_state(self.host)
# clear all locks held by this conductor before registering
self.dbapi.clear_node_reservations_for_conductor(self.host)
try:
# Register this conductor with the cluster
self.conductor = objects.Conductor.register(
admin_context, self.host, driver_names)
except exception.ConductorAlreadyRegistered:
# This conductor was already registered and did not shut down
# properly, so log a warning and update the record.
LOG.warning(
_LW("A conductor with hostname %(hostname)s "
"was previously registered. Updating registration"),
{'hostname': self.host})
self.conductor = objects.Conductor.register(admin_context,
self.host,
driver_names,
update_existing=True)
# Start periodic tasks
self._periodic_tasks_worker = self._executor.submit(
self._periodic_tasks.start, allow_empty=True)
self._periodic_tasks_worker.add_done_callback(
self._on_periodic_tasks_stop)
# NOTE(lucasagomes): If the conductor server dies abruptly
# mid deployment (OMM Killer, power outage, etc...) we
# can not resume the deployment even if the conductor
# comes back online. Cleaning the reservation of the nodes
# (dbapi.clear_node_reservations_for_conductor) is not enough to
# unstick it, so let's gracefully fail the deployment so the node
# can go through the steps (deleting & cleaning) to make itself
# available again.
filters = {'reserved': False, 'provision_state': states.DEPLOYING}
last_error = (_("The deployment can't be resumed by conductor "
"%s. Moving to fail state.") % self.host)
self._fail_if_in_state(ironic_context.get_admin_context(),
filters,
states.DEPLOYING,
'provision_updated_at',
last_error=last_error)
# Start consoles if it set enabled in a greenthread.
try:
self._spawn_worker(self._start_consoles,
ironic_context.get_admin_context())
except exception.NoFreeConductorWorker:
LOG.warning(_LW('Failed to start worker for restarting consoles.'))
# Spawn a dedicated greenthread for the keepalive
try:
self._spawn_worker(self._conductor_service_record_keepalive)
LOG.info(
_LI('Successfully started conductor with hostname '
'%(hostname)s.'), {'hostname': self.host})
except exception.NoFreeConductorWorker:
with excutils.save_and_reraise_exception():
LOG.critical(_LC('Failed to start keepalive'))
self.del_host()
self._started = True
