def __init__(self, clock, name, apiclient, store_query, interval):
self.log = logging.getLogger('executor.controller.%s' % (name,))
self.name = name
self.apiclient = apiclient
self.store_query = store_query
self.interval = interval
self._problematic = True
self._terminated = []
self._containers = {}
self._task = LoopingCall(clock, self._check_status)
self._started = Event()
def build(self, config):
for name, spec in config.iteritems():
self.states[name] = State(time)
if name != self.name:
self.nodes[name] = Node(name, spec['host'], spec['port'],
self.states[name])
self.cluster = Cluster(self.name, self.nodes, self.requests)
self.combined_state = CombinedState(self.states)
self._consume_peer_state_task = LoopingCall(self.clock,
_consume_peer_state,
self.cluster,
self.combined_state)
self._purge_expired_deltas_task = LoopingCall(self.clock,
_purge_expired_deltas,
self.clock, self.states,
self.combined_state,
self._liveness)
rest_api = RestApi(self.states[self.name], self.combined_state)
self.server = WSGIServer(('', self.port), rest_api)
def __init__(self, clock, name, apiclient, store_query, interval):
self.log = logging.getLogger("executor.controller.%s" % (name,))
self.name = name
self.apiclient = apiclient
self.store_query = store_query
self.interval = interval
self._problematic = True
self._terminated = []
self._containers = {}
self._task = LoopingCall(clock, self._check_status)
self._started = Event()
class _ExecutorController(object):
def __init__(self, clock, name, apiclient, store_query, interval):
self.log = logging.getLogger('executor.controller.%s' % (name,))
self.name = name
self.apiclient = apiclient
self.store_query = store_query
self.interval = interval
self._problematic = True
self._terminated = []
self._containers = {}
self._task = LoopingCall(clock, self._check_status)
self._started = Event()
def containers(self):
return self._containers.keys()
def start(self):
self._task.start(self.interval)
self._started.wait()
return self
def dispatch(self, inst):
container = self._handle_error(self.apiclient.create, inst)
self._remember(container.id, container)
def statuses(self, instances):
for instance in instances:
container = self.find(instance)
yield (container.state if container is not None
else 'unknown')
def delete(self, instance):
"""Delete instance."""
container = self.find(instance)
if container is not None:
self._forget(container.id)
try:
self._handle_error(self.apiclient.delete, container.id)
except Exception:
self._terminated.add(container.id)
raise
def restart(self, instance):
container = self.find(instance)
if container is not None:
container = self._handle_error(
self.apiclient.restart, container.id, instance)
self._remember(container.id, container)
def _check_status(self):
try:
containers = self.apiclient.containers()
except Exception:
raise
else:
if self._problematic:
self._reconcile(containers)
self._problematic = False
for id, container in containers.items():
self._remember(id, container)
self._update_state()
self._started.set()
def _update_state(self):
for id, container in self._containers.items():
if container.state == 'error':
inst = self._geti(container)
if inst is not None and inst.state != inst.STATE_LOST:
inst.set_state(inst.STATE_LOST)
def _reconcile(self, containers):
self._reconcile_missing_containers(containers)
self._mark_lost_containers_as_lost(containers)
self._delete_terminated_containers(containers)
def _reconcile_missing_containers(self, containers):
missing = (set(containers)
- set(self._containers)
- set(self._terminated))
self.log.info("reconscile: %s" % (', '.join(missing)))
for cid, state in [(cid, containers[cid]) for cid in missing]:
if self._geti(state) is not None:
self._remember(cid, state)
# FIXME: what to do with containers where the instance
# cannot be found? at what stage do we remove them?
# NOW!? self._delete_container(cid)
def _mark_lost_containers_as_lost(self, containers):
missing = set(self._containers) - set(containers)
self.log.info("mark containers %s as lost" % (
', '.join(missing)))
for state in [self._containers[cid] for cid in missing]:
inst = self._geti(state)
if inst is not None and inst.state != inst.STATE_LOST:
inst.set_state(inst.STATE_LOST)
def _delete_terminated_containers(self, containers):
for cid in self._terminated:
if cid in containers:
self._client.delete(cid)
del self._terminated[:]
def _handle_error(self, m, *args, **kwargs):
if self._problematic:
raise DispatchError("problematic")
try:
return m(*args, **kwargs)
except Exception, err:
self._problematic = True
raise DispatchError(str(err))
class _ExecutorController(object):
def __init__(self, clock, name, apiclient, store_query, state_cache,
interval):
self.log = logging.getLogger('executor.controller.%s' % (name,))
self.name = name
self.apiclient = apiclient
self.store_query = store_query
self.state_cache = state_cache
self.interval = interval
self._problematic = True
self._terminated = []
self._containers = {}
self._task = LoopingCall(clock, self._check_status)
self._started = Event()
def containers(self):
return self._containers.keys()
def start(self):
self._task.start(self.interval)
self._started.wait()
return self
def dispatch(self, inst):
container = self._handle_error(self.apiclient.create, inst)
self._remember(container.id, container)
def statuses(self, instances):
for instance in instances:
container = self.find(instance)
yield (container.state if container is not None
else 'unknown')
def delete(self, instance):
"""Delete instance."""
container = self.find(instance)
if container is not None:
self._forget(container.id)
try:
self._handle_error(self.apiclient.delete, container.id)
except Exception:
self._terminated.add(container.id)
raise
def restart(self, instance):
container = self.find(instance)
if container is not None:
container = self._handle_error(
self.apiclient.restart, container.id, instance)
self._remember(container.id, container)
def _check_status(self):
try:
containers = self.apiclient.containers()
except Exception:
raise
else:
if self._problematic:
self._reconcile(containers)
self._problematic = False
for id, container in containers.items():
self._remember(id, container)
self._started.set()
def _reconcile(self, containers):
self._reconcile_missing_containers(containers)
self._mark_lost_containers_as_lost(containers)
self._delete_terminated_containers(containers)
def _reconcile_missing_containers(self, containers):
missing = (set(containers)
- set(self._containers)
- set(self._terminated))
self.log.info("reconscile: %s" % (', '.join(missing)))
for cid, state in [(cid, containers[cid]) for cid in missing]:
if self._geti(state) is not None:
self._remember(cid, state)
# FIXME: what to do with containers where the instance
# cannot be found? at what stage do we remove them?
# NOW!? self._delete_container(cid)
def _mark_lost_containers_as_lost(self, containers):
missing = set(self._containers) - set(containers)
self.log.info("mark containers %s as lost" % (
', '.join(missing)))
for state in [self._containers[cid] for cid in missing]:
inst = self._geti(state)
if inst is not None and inst.state != inst.STATE_LOST:
inst.set_state(inst.STATE_LOST)
def _delete_terminated_containers(self, containers):
for cid in self._terminated:
if cid in containers:
self._client.delete(cid)
del self._terminated[:]
def _handle_error(self, m, *args, **kwargs):
if self._problematic:
raise DispatchError("problematic")
try:
return m(*args, **kwargs)
except Exception, err:
self._problematic = True
raise DispatchError(str(err))