def _monitoring_loop(self):
vmuuid = self.get_uuid()
log.info("monitor_loop handles VM %s" % (vmuuid))
start_time = time.time()
docker.wipe_docker_other_config(self)
# keep track of when to wipe other_config to safe CPU-time
while not self._stop_monitoring_request:
try:
docker.update_docker_info(self)
docker.update_docker_version(self)
# if we got past the above, it's about time to delete the
# error message, as all appears to be working again
self._wipe_monitor_error_message_if_needed()
try:
try:
self.__monitor_vm_events()
finally:
docker.wipe_docker_other_config(self)
except (XenAPI.Failure, util.XSContainerException):
log.exception("__monitor_vm_events threw an exception, "
"will retry")
raise
except (XenAPI.Failure, util.XSContainerException):
passed_time = time.time() - start_time
if (not self._error_message
and passed_time >= MONITOR_TIMEOUT_WARNING_S):
self._send_monitor_error_message()
log.info("Could not connect to VM %s, will retry" % (vmuuid))
if not self._stop_monitoring_request:
time.sleep(MONITORRETRYSLEEPINS)
# Make sure that we don't leave back error messsages for VMs that are
# not monitored anymore
self._wipe_monitor_error_message_if_needed()
log.info("monitor_loop returns from handling vm %s" % (vmuuid))
def _monitoring_loop(self):
vmuuid = self.get_uuid()
log.info("monitor_loop handles VM %s" % (vmuuid))
start_time = time.time()
docker.wipe_docker_other_config(self)
# keep track of when to wipe other_config to safe CPU-time
while not self._stop_monitoring_request:
try:
docker.update_docker_info(self)
docker.update_docker_version(self)
# if we got past the above, it's about time to delete the
# error message, as all appears to be working again
self._wipe_monitor_error_message_if_needed()
try:
try:
self.__monitor_vm_events()
finally:
docker.wipe_docker_other_config(self)
except (XenAPI.Failure, util.XSContainerException):
log.exception("__monitor_vm_events threw an exception, "
"will retry")
raise
except (XenAPI.Failure, util.XSContainerException):
passed_time = time.time() - start_time
if (not self._error_message
and passed_time >= MONITOR_TIMEOUT_WARNING_S):
self._send_monitor_error_message()
log.info("Could not connect to VM %s, will retry" % (vmuuid))
if not self._stop_monitoring_request:
time.sleep(MONITORRETRYSLEEPINS)
# Make sure that we don't leave back error messsages for VMs that are
# not monitored anymore
self._wipe_monitor_error_message_if_needed()
log.info("monitor_loop returns from handling vm %s" % (vmuuid))
def monitor_host():
global DOCKER_MONITOR
session = None
host = None
# container monitoring can get a smaller slice of the CPU time
os.nice(10)
signal.signal(signal.SIGTERM, interrupt_handler)
signal.signal(signal.SIGINT, interrupt_handler)
while True:
try:
session = api_helper.get_local_api_session()
client = api_helper.LocalXenAPIClient()
# need to refresh the host, in case we just joined a pool
host = api_helper.Host(client,
api_helper.get_this_host_ref(session))
if not DOCKER_MONITOR:
DOCKER_MONITOR = DockerMonitor(host)
else:
DOCKER_MONITOR.set_host(host)
log.info("Monitoring host %s" % (host.get_id()))
try:
# Avoid race conditions - get a current event token
event_from = session.xenapi.event_from(["vm"], '', 0.0)
token_from = event_from['token']
# Now load the VMs that are enabled for monitoring
DOCKER_MONITOR.refresh()
while True:
event_from = session.xenapi.event_from(
["vm"], token_from, EVENT_FROM_TIMEOUT_S)
token_from = event_from['token']
events = event_from['events']
for event in events:
if (event['operation'] == 'mod'
and 'snapshot' in event):
# At this point the monitor may need to
# refresh it's monitoring state of a particular
# vm.
DOCKER_MONITOR.process_vmrecord(event['ref'],
event['snapshot'])
finally:
try:
session.xenapi.session.logout()
except XenAPI.Failure:
log.exception("Failed when trying to logout")
except (socket.error, XenAPI.Failure, xmlrpclib.ProtocolError):
if session is not None:
log.error("Could not connect to XAPI - Is XAPI running? " +
"Will retry in %d" % (XAPIRETRYSLEEPINS))
else:
log.exception("Recovering from XAPI failure - Is XAPI " +
"restarting? Will retry in %d."
% (XAPIRETRYSLEEPINS))
time.sleep(XAPIRETRYSLEEPINS)
def monitor_host():
global DOCKER_MONITOR
session = None
host = None
# container monitoring can get a smaller slice of the CPU time
os.nice(10)
signal.signal(signal.SIGTERM, interrupt_handler)
signal.signal(signal.SIGINT, interrupt_handler)
while True:
try:
session = api_helper.get_local_api_session()
client = api_helper.LocalXenAPIClient()
# need to refresh the host, in case we just joined a pool
host = api_helper.Host(client,
api_helper.get_this_host_ref(session))
if not DOCKER_MONITOR:
DOCKER_MONITOR = DockerMonitor(host)
else:
DOCKER_MONITOR.set_host(host)
log.info("Monitoring host %s" % (host.get_id()))
try:
# Avoid race conditions - get a current event token
event_from = session.xenapi.event_from(["vm"], '', 0.0)
token_from = event_from['token']
# Now load the VMs that are enabled for monitoring
DOCKER_MONITOR.refresh()
while True:
event_from = session.xenapi.event_from(
["vm"], token_from, EVENT_FROM_TIMEOUT_S)
token_from = event_from['token']
events = event_from['events']
for event in events:
if (event['operation'] == 'mod'
and 'snapshot' in event):
# At this point the monitor may need to
# refresh it's monitoring state of a particular
# vm.
DOCKER_MONITOR.process_vmrecord(event['ref'],
event['snapshot'])
finally:
try:
session.xenapi.session.logout()
except XenAPI.Failure:
log.exception("Failed when trying to logout")
except (socket.error, XenAPI.Failure, xmlrpclib.ProtocolError):
if session is not None:
log.error("Could not connect to XAPI - Is XAPI running? " +
"Will retry in %d" % (XAPIRETRYSLEEPINS))
else:
log.exception("Recovering from XAPI failure - Is XAPI " +
"restarting? Will retry in %d."
% (XAPIRETRYSLEEPINS))
time.sleep(XAPIRETRYSLEEPINS)
def handle_docker_event(self, event):
if 'status' in event:
if event['status'] in ['create', 'destroy', 'die',
'kill', 'pause', 'restart',
'start', 'stop', 'unpause']:
try:
docker.update_docker_ps(self)
except util.XSContainerException as exception:
# This can happen, when the docker daemon stops
log.exception(exception)
elif event['status'] in ['create', 'destroy', 'delete']:
try:
docker.update_docker_info(self)
except util.XSContainerException as exception:
# This can happen, when the docker daemon stops
log.exception(exception)
def handle_docker_event(self, event):
if 'status' in event:
if event['status'] in ['create', 'destroy', 'die',
'kill', 'pause', 'restart',
'start', 'stop', 'unpause']:
try:
docker.update_docker_ps(self)
except util.XSContainerException, exception:
# This can happen, when the docker daemon stops
log.exception(exception)
elif event['status'] in ['create', 'destroy', 'delete']:
try:
docker.update_docker_info(self)
except util.XSContainerException, exception:
# This can happen, when the docker daemon stops
log.exception(exception)
def remove_if_refcount_less_or_equal(session, tls_secret_uuid,
refcount_threshold):
""" removes TLS secrets if there is fewer VMs using a secret as specified
in refcount_threshold """
refcount = _get_refcount(session, tls_secret_uuid)
if refcount > refcount_threshold:
log.info("refcount for secret uuid %s is larger than threshold with %d"
% (tls_secret_uuid, refcount))
# There's still more references than the threshold - keep
return
try:
tls_secret_ref = session.xenapi.secret.get_by_uuid(tls_secret_uuid)
session.xenapi.secret.destroy(tls_secret_ref)
log.info("Deleted secret uuid %s with refcount %d"
% (tls_secret_uuid, refcount))
except XenAPI.Failure:
log.exception("Failed to delete secret uuid %s, moving on..."
% (tls_secret_uuid))
def remove_if_refcount_less_or_equal(session, tls_secret_uuid,
refcount_threshold):
""" removes TLS secrets if there is fewer VMs using a secret as specified
in refcount_threshold """
refcount = _get_refcount(session, tls_secret_uuid)
if refcount > refcount_threshold:
log.info(
"refcount for secret uuid %s is larger than threshold with %d" %
(tls_secret_uuid, refcount))
# There's still more references than the threshold - keep
return
try:
tls_secret_ref = session.xenapi.secret.get_by_uuid(tls_secret_uuid)
session.xenapi.secret.destroy(tls_secret_ref)
log.info("Deleted secret uuid %s with refcount %d" %
(tls_secret_uuid, refcount))
except XenAPI.Failure:
log.exception("Failed to delete secret uuid %s, moving on..." %
(tls_secret_uuid))
def execute_docker_data_listen(session, vm_uuid, request,
stop_monitoring_request):
host = api_helper.get_suitable_vm_ip(session, vm_uuid, DOCKER_TLS_PORT)
log.info("tls.execute_docker_listen_charbychar for VM %s, via %s"
% (vm_uuid, host))
asocket = _get_socket(session, vm_uuid)
try:
asocket.connect((host, DOCKER_TLS_PORT))
if hasattr(asocket, 'version'):
# Newer python versions provide the TLS version
log.info("Connected VM %s using %s" % (vm_uuid,
asocket.version()))
asocket.send(request)
asocket.setblocking(0)
while not stop_monitoring_request:
rlist, _, _ = select.select(
[asocket.fileno()], [], [],
constants.MONITOR_EVENTS_POLL_INTERVAL)
if not rlist:
continue
try:
read_data = asocket.recv(1024)
if read_data == "":
break
yield read_data
except IOError as exception:
if exception[0] not in (errno.EAGAIN, errno.EINTR):
raise
sys.exc_clear()
continue
except ssl.SSLError as exception:
raise TlsException("Failed to communicate with Docker via TLS: %s"
% exception, (sys.exc_info()[2]))
except socket.error as exception:
raise TlsException("The connection failed: %s"
% exception, (sys.exc_info()[2]))
finally:
try:
asocket.close()
except Exception:
log.exception("Failed to close socket. Moving on.")
def execute_docker(session, vm_uuid, request):
host = api_helper.get_suitable_vm_ip(session, vm_uuid, DOCKER_TLS_PORT)
log.info("tls.execute_docker for VM %s, via %s" % (vm_uuid, host))
asocket = _get_socket(session, vm_uuid)
try:
asocket.connect((host, DOCKER_TLS_PORT))
asocket.send(request)
result = ""
while len(result) < constants.MAX_BUFFER_SIZE:
result_iteration = asocket.recv(
constants.MAX_BUFFER_SIZE - len(result))
if result_iteration == "":
break
result += result_iteration
except ssl.SSLError as exception:
raise TlsException("Failed to communicate with Docker via TLS: %s"
% exception, (sys.exc_info()[2]))
finally:
try:
asocket.close()
except Exception:
log.exception("Failed to close socket. Moving on.")
return result
def execute_docker(session, vm_uuid, request):
host = api_helper.get_suitable_vm_ip(session, vm_uuid, DOCKER_TLS_PORT)
log.info("tls.execute_docker for VM %s, via %s" % (vm_uuid, host))
asocket = _get_socket(session, vm_uuid)
try:
asocket.connect((host, DOCKER_TLS_PORT))
asocket.send(request)
result = asocket.recv(constants.MAX_BUFFER_SIZE)
except ssl.SSLError, exception:
raise TlsException("Failed to communicate with Docker via TLS: %s"
% exception, (sys.exc_info()[2]))
finally:
try:
asocket.close()
except Exception:
log.exception("Failed to close socket. Moving on.")
return result
def execute_docker_data_listen(session, vm_uuid, request,
stop_monitoring_request):
host = api_helper.get_suitable_vm_ip(session, vm_uuid, DOCKER_TLS_PORT)
log.info("tls.execute_docker_listen_charbychar for VM %s, via %s"
% (vm_uuid, host))
asocket = _get_socket(session, vm_uuid)
try:
asocket.connect((host, DOCKER_TLS_PORT))
if hasattr(asocket, 'version'):
# Newer python versions provide the TLS version
log.info("Connected VM %s using %s" % (vm_uuid,
asocket.version()))
result = ""
while len(result) < constants.MAX_BUFFER_SIZE:
result_iteration = asocket.recv(constants.MAX_BUFFER_SIZE -
len(result))
if result_iteration == "":
break
result += result_iteration
except ssl.SSLError, exception:
raise TlsException(
"Failed to communicate with Docker via TLS: %s" % exception,
(sys.exc_info()[2]))
finally:
try:
asocket.close()
except Exception:
log.exception("Failed to close socket. Moving on.")
return result
def execute_docker_data_listen(session, vm_uuid, request,
stop_monitoring_request):
host = api_helper.get_suitable_vm_ip(session, vm_uuid, DOCKER_TLS_PORT)
log.info("tls.execute_docker_listen_charbychar for VM %s, via %s" %
(vm_uuid, host))
asocket = _get_socket(session, vm_uuid)
try:
asocket.connect((host, DOCKER_TLS_PORT))
if hasattr(asocket, 'version'):
# Newer python versions provide the TLS version
log.info("Connected VM %s using %s" % (vm_uuid, asocket.version()))
asocket.send(request)