def run(self):
"""Setup directories' watches and start the re-scan ticker.
"""
# Start idle
self._is_active = False
# Setup the watchdog
watchdog_lease = self.tm_env.watchdogs.create(
name='svc-{svc_name}'.format(svc_name=self.name),
timeout='{hb:d}s'.format(hb=_WATCHDOG_TIMEOUT_SEC),
content='Service %r failed' % self.name
)
watch = dirwatch.DirWatcher(self.tm_env.cache_dir)
watch.on_created = self._on_created
watch.on_modified = self._on_modified
watch.on_deleted = self._on_deleted
# Start the timer
watchdog_lease.heartbeat()
while True:
if watch.wait_for_events(timeout=_HEARTBEAT_SEC):
watch.process_events(max_events=5)
else:
if self._is_active is True:
cached_files = glob.glob(
os.path.join(self.tm_env.cache_dir, '*')
)
running_links = glob.glob(
os.path.join(self.tm_env.running_dir, '*')
)
# Calculate the container names from every event file
cached_containers = {
appcfg.eventfile_unique_name(filename)
for filename in cached_files
}
# Calculate the instance names from every event running
# link
running_instances = {
os.path.basename(linkname)
for linkname in running_links
}
_LOGGER.debug('content of %r and %r: %r <-> %r',
self.tm_env.cache_dir,
self.tm_env.running_dir,
cached_containers,
running_instances)
else:
_LOGGER.info('Still inactive during heartbeat event.')
watchdog_lease.heartbeat()
# Graceful shutdown.
_LOGGER.info('service shutdown.')
watchdog_lease.remove()
def _synchronize(self):
"""Synchronize cache/ instances with running/ instances.
We need to revalidate three things:
- All running instances must have an equivalent entry in cache or be
terminated.
- All event files in the cache that do not have running link must be
started.
- The cached entry and the running link must be for the same
container (equal unique name). Otherwise, terminate it.
"""
cached_files = glob.glob(os.path.join(self.tm_env.cache_dir, '*'))
running_links = glob.glob(os.path.join(self.tm_env.running_dir, '*'))
# Calculate the instance names from every event file
cached_instances = {
os.path.basename(filename)
for filename in cached_files
}
# Calculate the container names from every event file
cached_containers = {
appcfg.eventfile_unique_name(filename)
for filename in cached_files
}
# Calculate the instance names from every event running link
running_instances = {
os.path.basename(linkname)
for linkname in running_links
if os.path.islink(linkname)
}
removed_instances = set()
added_instances = set()
_LOGGER.info('running %r', running_instances)
# If instance is extra, remove the symlink and force svscan to
# re-evaluate
for instance_link in running_links:
# Skip any files and directories (created by svscan). The footprint
# that defines the app is the symlink:
# - running/<instance_name> -> apps/<container_name>
#
# Iterate over all symlinks, skipping other files and directories.
if not os.path.islink(instance_link):
continue
instance_name = os.path.basename(instance_link)
_LOGGER.info('checking %r', instance_link)
if not os.path.exists(instance_link):
# Broken symlink, something went wrong.
_LOGGER.warning('broken link %r', instance_link)
os.unlink(instance_link)
removed_instances.add(instance_name)
continue
elif instance_name not in cached_instances:
self._terminate(instance_name)
removed_instances.add(instance_name)
else:
# Now check that the link match the intended container
container_dir = self._resolve_running_link(instance_link)
container_name = os.path.basename(container_dir)
if container_name not in cached_containers:
self._terminate(instance_name)
removed_instances.add(instance_name)
# For all new apps, read the manifest and configure the app. When all
# apps are configured, force rescan again.
for instance_name in cached_instances:
instance_link = os.path.join(
self.tm_env.running_dir,
instance_name
)
if self._configure(instance_name):
added_instances.add(instance_name)
_LOGGER.debug('End resuld: %r / %r - %r + %r',
cached_containers,
running_instances,
removed_instances,
added_instances)
running_instances -= removed_instances
running_instances |= added_instances
_LOGGER.info('running post cleanup: %r', running_instances)
self._refresh_supervisor(instance_names=running_instances)
def _synchronize(self):
"""Synchronize apps to running/cleanup.
We need to re-validate three things on startup:
- All configured apps should have an associated cache entry.
Otherwise, create a link to cleanup.
- All configured apps with a cache entry and with a cleanup file
should be linked to cleanup. Otherwise, link to running.
- Additional cache entries should be configured to run.
On restart we need to validate another three things:
- All configured apps that have a running link should be checked
if in the cache. If not then terminate the app.
- All configured apps that have a cleanup link should be left
alone as this is handled.
- Additional cache entries should be configured to run.
On startup run.sh will clear running and cleanup which simplifies
the logic for us as we can check running/cleanup first. Then check
startup conditions and finally non-configured apps that are in cache.
NOTE: a link cannot exist in running and cleanup at the same time.
"""
# Disable R0912(too-many-branches)
# pylint: disable=R0912
configured = {
os.path.basename(filename)
for filename in glob.glob(os.path.join(self.tm_env.apps_dir, '*'))
}
cached = {
os.path.basename(filename): appcfg.eventfile_unique_name(filename)
for filename in glob.glob(os.path.join(self.tm_env.cache_dir, '*'))
}
for container in configured:
appname = appcfg.app_name(container)
if os.path.exists(os.path.join(self.tm_env.running_dir, appname)):
# App already running.. check if in cache.
# No need to check if needs cleanup as that is handled
if appname not in cached or cached[appname] != container:
self._terminate(appname)
else:
_LOGGER.info('Ignoring %s as it is running', appname)
cached.pop(appname, None)
elif os.path.exists(os.path.join(self.tm_env.cleanup_dir,
appname)):
# Already in the process of being cleaned up
_LOGGER.info('Ignoring %s as it is in cleanup', appname)
cached.pop(appname, None)
else:
needs_cleanup = True
if appname in cached and cached[appname] == container:
data_dir = os.path.join(self.tm_env.apps_dir, container,
'data')
for cleanup_file in ['exitinfo', 'aborted', 'oom']:
path = os.path.join(data_dir, cleanup_file)
if os.path.exists(path):
_LOGGER.debug('Found cleanup file %r', path)
break
else:
if self._configure(appname):
needs_cleanup = False
_LOGGER.debug('Added existing app %r', appname)
cached.pop(appname, None)
if needs_cleanup:
fs.symlink_safe(
os.path.join(self.tm_env.cleanup_dir, appname),
os.path.join(self.tm_env.apps_dir, container))
_LOGGER.debug('Removed %r', appname)
for appname in six.iterkeys(cached):
if self._configure(appname):
_LOGGER.debug('Added new app %r', appname)
self._refresh_supervisor()