def waitForCommand(self):
""" Waits for an command notification on the component in etcd. If one is received,
processes it by attempting to update the component.
"""
is_initial_loop = True
sleep_time = 0
while True:
# Sleep and then check again.
time.sleep(sleep_time)
sleep_time = CHECK_SLEEP_TIME
# Check the component's status.
self.logger.debug('Checking state for component %s',
self.component.getName())
state = self.state.getState()
self.logger.debug('Found state %s for component %s', state,
self.component.getName())
# Determine whether we should give initial status messages.
was_initial_loop = is_initial_loop
is_initial_loop = False
# Take actions based on the status requested.
current_status = ComponentState.getStatusOf(state)
sleep_time = self.handleStatus(current_status, state,
was_initial_loop)
def listStatus(self):
""" Lists the status of all components in this project. """
self.getConfig()
self.initialize([c.name for c in self.config.components])
print "%-20s %-20s %-20s" % ('COMPONENT', 'STATUS', 'IMAGE ID')
for component in self.components:
state = ComponentState(self.project_name, component, self.etcd_client).getState()
status = ComponentState.getStatusOf(state)
imageid = ComponentState.getImageIdOf(state)
print "%-20s %-20s %-20s" % (component.getName(), status, imageid)
def listStatus(self):
""" Lists the status of all components in this project. """
self.getConfig()
self.initialize([c.name for c in self.config.components])
print "%-20s %-20s %-20s" % ('COMPONENT', 'STATUS', 'IMAGE ID')
for component in self.components:
state = ComponentState(self.project_name, component,
self.etcd_client).getState()
status = ComponentState.getStatusOf(state)
imageid = ComponentState.getImageIdOf(state)
print "%-20s %-20s %-20s" % (component.getName(), status, imageid)
def monitorComponent(self):
""" Monitors a component by pinging it every MONITOR_SLEEP_TIME seconds or so. If a component
fails, then the system will try to restart it. If that fails, the component is marked
as dead.
"""
while True:
# Wait for the component to be running.
self.monitor_event.wait()
# Sleep MONITOR_SLEEP_TIME seconds.
time.sleep(MONITOR_SLEEP_TIME)
# Check the component.
report(
"Checking in on component",
project=self.project_name,
component=self.component,
level=ReportLevels.BACKGROUND,
)
if not self.component.isHealthy():
self.logger.debug("Component %s is not healty", self.component.getName())
with self.update_lock:
# Just to be sure...
if not self.is_running:
continue
# Ensure that the component is still ready.
state = self.state.getState()
current_status = ComponentState.getStatusOf(state)
if current_status == READY_STATUS:
report(
"Component " + self.component.getName() + " is not healthy. Restarting...",
project=self.project_name,
component=self.component,
)
if not self.component.update():
report(
"Could not restart component " + self.component.getName(),
project=self.project_name,
component=self.component,
level=ReportLevels.IMPORTANT,
)
self.monitor_event.clear()
continue
def monitorComponent(self):
""" Monitors a component by pinging it every MONITOR_SLEEP_TIME seconds or so. If a component
fails, then the system will try to restart it. If that fails, the component is marked
as dead.
"""
while True:
# Wait for the component to be running.
self.monitor_event.wait()
# Sleep MONITOR_SLEEP_TIME seconds.
time.sleep(MONITOR_SLEEP_TIME)
# Check the component.
report('Checking in on component',
project=self.project_name,
component=self.component,
level=ReportLevels.BACKGROUND)
if not self.component.isHealthy():
self.logger.debug('Component %s is not healty',
self.component.getName())
with self.update_lock:
# Just to be sure...
if not self.is_running:
continue
# Ensure that the component is still ready.
state = self.state.getState()
current_status = ComponentState.getStatusOf(state)
if current_status == READY_STATUS:
report('Component ' + self.component.getName() +
' is not healthy. Restarting...',
project=self.project_name,
component=self.component)
if not self.component.update():
report('Could not restart component ' +
self.component.getName(),
project=self.project_name,
component=self.component,
level=ReportLevels.IMPORTANT)
self.monitor_event.clear()
continue
def waitForCommand(self):
""" Waits for an command notification on the component in etcd. If one is received,
processes it by attempting to update the component.
"""
is_initial_loop = True
sleep_time = 0
while True:
# Sleep and then check again.
time.sleep(sleep_time)
sleep_time = CHECK_SLEEP_TIME
# Check the component's status.
self.logger.debug('Checking state for component %s', self.component.getName())
state = self.state.getState()
self.logger.debug('Found state %s for component %s', state, self.component.getName())
# Determine whether we should give initial status messages.
was_initial_loop = is_initial_loop
is_initial_loop = False
# Take actions based on the status requested.
current_status = ComponentState.getStatusOf(state)
sleep_time = self.handleStatus(current_status, state, was_initial_loop)
