def _shutdown_container(tm_env, instance_id):
"""Shutdown a container.
"""
container_dir = os.path.join(tm_env.apps_dir, instance_id)
utils.touch(os.path.join(container_dir, 'data', 'oom'))
linux_runtime = runtime.get_runtime('linux', tm_env, container_dir)
linux_runtime.kill()
def _prepare_ldpreload(container_dir, app):
"""Add mandatory ldpreloads to the container environment.
"""
etc_dir = os.path.join(container_dir, 'overlay', 'etc')
fs.mkdir_safe(etc_dir)
new_ldpreload = os.path.join(etc_dir, 'ld.so.preload')
try:
shutil.copyfile('/etc/ld.so.preload', new_ldpreload)
except IOError as err:
if err.errno != errno.ENOENT:
raise
_LOGGER.info('/etc/ld.so.preload not found, creating empty.')
utils.touch(new_ldpreload)
ldpreloads = []
if app.ephemeral_ports.tcp or app.ephemeral_ports.udp:
treadmill_bind_preload = subproc.resolve('treadmill_bind_preload.so')
ldpreloads.append(treadmill_bind_preload)
if not ldpreloads:
return
_LOGGER.info('Configuring /etc/ld.so.preload: %r', ldpreloads)
with open(new_ldpreload, 'a') as f:
f.write('\n'.join(ldpreloads) + '\n')
def test_on_created_same_host(self, connection):
"""Test gmsa.HostGroupWatch._on_created_placement."""
# Access protected module
# pylint: disable=W0212
server1_path = os.path.join(self.placement_dir, 'server1.ad.com')
os.mkdir(server1_path)
with io.open(os.path.join(self.servers_dir, 'server1.ad.com'),
'w') as f:
yaml.dump({
servers.DC_KEY: 'dc.ad.com',
servers.DN_KEY: 'CN=server1,DC=AD,DC=COM',
'partition': 'partition1'
}, f)
mock_connection = mock.MagicMock()
connection.return_value = mock_connection
type(mock_connection).result = mock.PropertyMock(side_effect={
'result': 0
})
type(mock_connection).response = mock.PropertyMock(return_value=[
{'attributes': {
'samAccountName': 'proid1-gmsa-hosts',
'member': []
}}
])
watch = gmsa.HostGroupWatch(self.root, 'partition1',
'OU=test,DC=ad,DC=com', '{}-gmsa-hosts')
watch._sync()
self.assertEqual(watch._proids, {
'proid1': {},
})
placement_path1 = os.path.join(server1_path,
'proid1.app#0000000000001')
utils.touch(placement_path1)
watch._on_created_placement(placement_path1)
self.assertEqual(watch._proids, {
'proid1': {'CN=server1,DC=AD,DC=COM': 1},
})
placement_path2 = os.path.join(server1_path,
'proid1.app#0000000000001')
utils.touch(placement_path2)
watch._on_created_placement(placement_path2)
self.assertEqual(watch._proids, {
'proid1': {'CN=server1,DC=AD,DC=COM': 2},
})
mock_connection.modify.assert_has_calls([
mock.call('CN=proid1-gmsa-hosts,OU=test,DC=ad,DC=com',
{'member': [(ldap3.MODIFY_ADD,
['CN=server1,DC=AD,DC=COM'])]}),
])
self.assertEqual(mock_connection.modify.call_count, 1)
def ensure_exists(self, path):
"""Ensure storage path exists."""
fpath = _fpath(self.fsroot, path)
try:
fs.mkdir_safe(os.path.dirname(fpath))
utils.touch(fpath)
except OSError:
raise backend.ObjectNotFoundError()
def _create_sysrun(sys_dir, name, command, down=False):
"""Create system script."""
fs.mkdir_safe(os.path.join(sys_dir, name))
utils.create_script(os.path.join(sys_dir, name, 'run'),
'supervisor.run_sys',
cmd=command)
_create_logrun(os.path.join(sys_dir, name))
if down:
utils.touch(os.path.join(sys_dir, name, 'down'))
def _init_log_file(log_file, link_path):
"""Generate log file and provide hard link in other place
"""
utils.touch(log_file)
try:
os.link(log_file, link_path)
except OSError as err:
if err.errno == errno.EEXIST:
_LOGGER.debug("Linked log file already exists.")
else:
raise
def _on_add_trace_db(zk2fs_sync, zkpath, sow_dir, tmp_dir):
"""Called when new trace DB snapshot is added."""
_LOGGER.info('Added trace db snapshot: %s', zkpath)
data, _metadata = zk2fs_sync.zkclient.get(zkpath)
with tempfile.NamedTemporaryFile(delete=False, mode='wb',
dir=tmp_dir) as trace_db:
trace_db.write(zlib.decompress(data))
db_name = os.path.basename(zkpath)
os.rename(trace_db.name, os.path.join(sow_dir, db_name))
utils.touch(zk2fs_sync.fpath(zkpath))
def test_finish_oom_event(self):
"""Test posting oom event.
"""
app_unique_name = 'proid.myapp-001-0000000ID1234'
app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name)
data_dir = os.path.join(app_dir, 'data')
fs.mkdir_safe(data_dir)
utils.touch(os.path.join(data_dir, 'oom'))
app_finish.finish(self.tm_env, app_dir)
args, _kwargs = treadmill.appevents.post.call_args
_events_dir, event = args
self.assertIsInstance(event, events.KilledTraceEvent)
def touch_file(zkpath):
"""Process immutable trace callback.
Touch file and set modified time to match timestamp of the trace event.
"""
fpath = zk2fs_sync.fpath(zkpath)
event = os.path.basename(fpath)
try:
timestamp, _rest = event.split(',', 1)
utils.touch(fpath)
timestamp = int(float(timestamp))
os.utime(fpath, (timestamp, timestamp))
except ValueError:
_LOGGER.warn('Incorrect trace format: %s', zkpath)
zkutils.ensure_deleted(zk2fs_sync.zkclient, zkpath, recursive=True)
def test_terminated_no_event(self):
"""Test that event won't be posted if container is terminated/evicted.
"""
app_unique_name = 'proid.myapp-001-0000000ID1234'
app_dir = os.path.join(self.tm_env.apps_dir, app_unique_name)
data_dir = os.path.join(app_dir, 'data')
fs.mkdir_safe(data_dir)
utils.touch(os.path.join(data_dir, 'terminated'))
# exitinfo file should be ignored.
with io.open(os.path.join(data_dir, 'exitinfo'), 'w') as f:
f.write('{"service": "web_server", "return_code": 0, "signal": 0}')
app_finish.finish(self.tm_env, app_dir)
treadmill.appevents.post.assert_not_called()
def _terminate(self, instance_name):
"""Removes application from the supervised running list.
Move the link from running directory to the cleanup directory.
:param instance_name:
Name of the instance to configure
:type instance_name:
``str``
"""
# In case the node is deleted from zookeeper (app is terminated),
# the "services" directory still exists, and it is the job of
# svscan to terminate the process.
#
# If services directory does not exist, app finished on it's own, and
# the zk node was deleted by the cleanup script.
#
instance_run_link = os.path.join(
self.tm_env.running_dir,
instance_name
)
container_dir = self._resolve_running_link(instance_run_link)
_LOGGER.info('terminating %sfinished %r (%r)',
('not ' if os.path.exists(container_dir) else ''),
instance_name, container_dir)
container_name = os.path.basename(container_dir)
container_cleanup_link = os.path.join(
self.tm_env.cleanup_dir,
container_name
)
try:
fs.replace(instance_run_link, container_cleanup_link)
utils.touch(os.path.join(container_dir, 'data', 'terminated'))
except OSError as err:
# It is OK if the symlink is already removed (race with app own
# cleanup). Everything else is an error.
if err.errno == errno.ENOENT:
pass
else:
raise
def _on_add_trace_db(zk2fs_sync, zkpath, sow_db):
"""Called when new trace DB snapshot is added."""
_LOGGER.info('Added trace db snapshot: %s', zkpath)
data, _metadata = zk2fs_sync.zkclient.get(zkpath)
with tempfile.NamedTemporaryFile(delete=False, mode='wb', dir=sow_db) as f:
f.write(zlib.decompress(data))
db_path = os.path.join(sow_db, os.path.basename(zkpath))
os.rename(f.name, db_path)
# Now that sow is up to date, cleanup records from file system.
with sqlite3.connect(db_path) as conn:
cursor = conn.cursor()
for row in cursor.execute('SELECT path FROM trace'):
fpath = os.path.join(zk2fs_sync.fsroot, row[0][1:])
fs.rm_safe(fpath)
fs.rm_safe(f.name)
utils.touch(zk2fs_sync.fpath(zkpath))
def test_configure_restart(self):
"""Test monitor run loop.
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
config_dir = os.path.join(self.root, 'config')
watch_dir1 = os.path.join(self.root, 'watch', '1')
fs.mkdir_safe(config_dir)
fs.mkdir_safe(watch_dir1)
event_file = os.path.join(watch_dir1, 'test,12345.123,1,0')
utils.touch(event_file)
with io.open(os.path.join(config_dir, 'default'), 'w') as f:
f.writelines(['{};plugin1\n'.format(watch_dir1)])
impl1 = mock.Mock()
impl1.execute.return_value = True
# W0613(unused-argument)
def _handler1(tm_env, params): # pylint: disable=W0613
return impl1
treadmill.plugin_manager.load.side_effect = [_handler1]
mock_dirwatch = mock.Mock()
treadmill.dirwatch.DirWatcher.return_value = mock_dirwatch
mock_dirwatch.wait_for_events.side_effect = [False, StopIteration()]
mon = monitor.Monitor(tm_env={}, config_dir=config_dir)
self.assertRaises(StopIteration, mon.run)
impl1.execute.assert_called_with({
'return_code': 1,
'id': 'test',
'signal': 0,
'timestamp': 12345.123,
})
self.assertFalse(os.path.exists(event_file))
def setUp(self):
self.tmroot = tempfile.mkdtemp()
server_init_log = os.path.join('init', 'server_init', 'log')
server_init_log_current = os.path.join(server_init_log, 'current')
os.makedirs(os.path.join(self.tmroot, server_init_log))
utils.touch(os.path.join(self.tmroot, server_init_log_current))
self.service_log = os.path.join('running', 'foo', 'data', 'sys', 'bar',
'data', 'log')
self.service_log_current = os.path.join(self.service_log, 'current')
os.makedirs(os.path.join(self.tmroot, self.service_log))
utils.touch(os.path.join(self.tmroot, self.service_log_current))
if sys.platform.startswith('linux'):
os.makedirs(os.path.join(self.tmroot, 'localdisk_svc'))
os.makedirs(os.path.join(self.tmroot, 'presence_svc'))
os.makedirs(os.path.join(self.tmroot, 'network_svc'))
os.makedirs(os.path.join(self.tmroot, 'cgroup_svc'))
os.makedirs(os.path.join(self.tmroot, 'postmortem'))
def setUp(self):
self.tmroot = tempfile.mkdtemp()
self.archive_root = tempfile.mkdtemp()
self.tmp_dir = os.path.join(tempfile.gettempdir(),
'postmortem-%d' % os.getpid())
self.server_init_log = os.path.join('init', 'server_init', 'log')
self.server_init_log_current = os.path.join(self.server_init_log,
'current')
os.makedirs(os.path.join(self.tmroot, self.server_init_log))
utils.touch(os.path.join(self.tmroot, self.server_init_log_current))
self.service_log = os.path.join('running', 'foo', 'data', 'sys', 'bar',
'data', 'log')
self.service_log_current = os.path.join(self.service_log, 'current')
os.makedirs(os.path.join(self.tmroot, self.service_log))
utils.touch(os.path.join(self.tmroot, self.service_log_current))
if sys.platform.startswith('linux'):
os.makedirs('%s/localdisk_svc' % self.tmroot)
os.makedirs('%s/network_svc' % self.tmroot)
os.makedirs('%s/cgroup_svc' % self.tmroot)
def _side_effect():
utils.touch(event_file)
return True
def test_wait_for_ready(self):
"""Test wait for ready
"""
modified = os.path.join(self.root, '.modified')
utils.touch(modified)
self.assertEqual(zksync_utils.wait_for_ready(self.root), modified)
def sync_children(self,
zkpath,
watch_data=False,
on_add=None,
on_del=None,
need_watch_predicate=None,
cont_watch_predicate=None):
"""Sync children of zkpath to fpath.
need_watch_predicate decides if the watch is needed based on the
zkpath alone.
cont_watch_prediacate decides if the watch is needed based on content
of zkpath children.
To avoid race condition, both need to return False, if one of them
returns True, watch will be set.
"""
_LOGGER.info('sync children: zk = %s, watch_data: %s', zkpath,
watch_data)
fpath = self.fpath(zkpath)
fs.mkdir_safe(fpath)
done_file = os.path.join(fpath, '.done')
if os.path.exists(done_file):
_LOGGER.info('Found done file: %s, nothing to watch.', done_file)
return
if not on_del:
on_del = self._default_on_del
if not on_add:
on_add = self._default_on_add
need_watch = True
if need_watch_predicate:
need_watch = need_watch_predicate(zkpath)
_LOGGER.info('Need watch on %s: %s', zkpath, need_watch)
if need_watch:
self._make_children_watch(
zkpath,
watch_data,
on_add,
on_del,
cont_watch_predicate=cont_watch_predicate)
else:
try:
children = self.zkclient.get_children(zkpath)
except kazoo.client.NoNodeError:
children = []
need_watch = self._children_watch(
zkpath,
children,
watch_data,
on_add,
on_del,
cont_watch_predicate=cont_watch_predicate,
)
if need_watch:
self._make_children_watch(
zkpath,
watch_data,
on_add,
on_del,
cont_watch_predicate=cont_watch_predicate)
self._update_last()
if not need_watch:
utils.touch(done_file)
def test_sync(self, connection):
"""Test gmsa.HostGroupWatch.sync."""
# Access protected module
# pylint: disable=W0212
server1_path = os.path.join(self.placement_dir, 'server1.ad.com')
os.mkdir(server1_path)
utils.touch(os.path.join(server1_path, 'proid1.app#0000000000001'))
server2_path = os.path.join(self.placement_dir, 'server2.ad.com')
os.mkdir(server2_path)
utils.touch(os.path.join(server2_path, 'proid4.app#0000000000004'))
server3_path = os.path.join(self.placement_dir, 'server3.ad.com')
os.mkdir(server3_path)
server4_path = os.path.join(self.placement_dir, 'server4.ad.com')
os.mkdir(server4_path)
utils.touch(os.path.join(server4_path, 'proid3.app#0000000000003'))
server5_path = os.path.join(self.placement_dir, 'server5.ad.com')
os.mkdir(server5_path)
utils.touch(os.path.join(server5_path, 'proid5.app#0000000000005'))
utils.touch(os.path.join(server5_path, 'proid5.app#0000000000006'))
with io.open(os.path.join(self.servers_dir, 'server1.ad.com'),
'w') as f:
yaml.dump({
servers.DC_KEY: 'dc.ad.com',
servers.DN_KEY: 'CN=server1,DC=AD,DC=COM',
'partition': 'partition1'
}, f)
with io.open(os.path.join(self.servers_dir, 'server2.ad.com'),
'w') as f:
yaml.dump({
servers.DC_KEY: 'dc.ad.com',
servers.DN_KEY: 'CN=server2,DC=AD,DC=COM',
'partition': 'partition1'
}, f)
with io.open(os.path.join(self.servers_dir, 'server3.ad.com'),
'w') as f:
yaml.dump({
servers.DC_KEY: 'dc.ad.com',
servers.DN_KEY: 'CN=server3,DC=AD,DC=COM',
'partition': 'partition1'
}, f)
with io.open(os.path.join(self.servers_dir, 'server5.ad.com'),
'w') as f:
yaml.dump({
servers.DC_KEY: 'dc.ad.com',
servers.DN_KEY: 'CN=server5,DC=AD,DC=COM',
'partition': 'partition1'
}, f)
mock_connection = mock.MagicMock()
connection.return_value = mock_connection
type(mock_connection).result = mock.PropertyMock(side_effect={
'result': 0
})
type(mock_connection).response = mock.PropertyMock(return_value=[
{'attributes': {
'samAccountName': 'proid1-gmsa-hosts',
'member': ['CN=server1,DC=AD,DC=COM']
}},
{'attributes': {
'samAccountName': 'proid2-gmsa-hosts',
'member': ['CN=server3,DC=AD,DC=COM']
}},
{'attributes': {
'samAccountName': 'proid3-gmsa-hosts',
'member': []
}},
{'attributes': {
'samAccountName': 'proid4-gmsa-hosts',
'member': []
}},
{'attributes': {
'samAccountName': 'proid5-gmsa-hosts',
'member': []
}}
])
watch = gmsa.HostGroupWatch(self.root, 'partition1',
'OU=test,DC=ad,DC=com', '{}-gmsa-hosts')
watch._sync()
self.assertEqual(watch._proids, {
'proid1': {'CN=server1,DC=AD,DC=COM': 1},
'proid2': {},
'proid3': {},
'proid4': {'CN=server2,DC=AD,DC=COM': 1},
'proid5': {'CN=server5,DC=AD,DC=COM': 2},
})
mock_connection.modify.assert_has_calls(
[
mock.call('CN=proid2-gmsa-hosts,OU=test,DC=ad,DC=com',
{'member': [(ldap3.MODIFY_DELETE,
['CN=server3,DC=AD,DC=COM'])]}),
mock.call('CN=proid4-gmsa-hosts,OU=test,DC=ad,DC=com',
{'member': [(ldap3.MODIFY_ADD,
['CN=server2,DC=AD,DC=COM'])]}),
mock.call('CN=proid5-gmsa-hosts,OU=test,DC=ad,DC=com',
{'member': [(ldap3.MODIFY_ADD,
['CN=server5,DC=AD,DC=COM'])]}),
],
any_order=True
)
def _update_last(self):
"""Update .modified timestamp to indicate changes were made."""
if self.ready:
modified_file = os.path.join(self.fsroot, '.modified')
utils.touch(modified_file)
os.utime(modified_file, (time.time(), time.time()))
def configure(tm_env, event):
"""Creates directory necessary for starting the application.
This operation is idem-potent (it can be repeated).
The directory layout is::
- (treadmill root)
- apps
- (app unique name)
- app.yml
run
finish
The 'run' script is responsible for creating container environment
and starting svscan inside the container.
The 'finish' script is invoked when container terminates and will
deallocate any resources (NAT rules, etc) that were allocated for the
container.
"""
# R0915: Need to refactor long function into smaller pieces.
#
# pylint: disable=R0915
# Load the app from the event
try:
manifest_data = app_manifest.load(tm_env, event)
except IOError:
# File is gone. Nothing to do.
_LOGGER.exception("No event to load: %r", event)
return
# Freeze the app data into a namedtuple object
app = utils.to_obj(manifest_data)
# Check the identity we are going to run as. It needs to exists on the host
# or we will fail later on as we try to seteuid.
try:
pwd.getpwnam(app.proid)
except KeyError:
_LOGGER.exception('Unable to find proid %r in passwd database.',
app.proid)
raise
# Generate a unique name for the app
uniq_name = appmgr.app_unique_name(app)
# Create the app's running directory
container_dir = os.path.join(tm_env.apps_dir, uniq_name)
# We assume it is a 'resume' if the container directory already exists.
is_resume = False
try:
os.makedirs(container_dir)
except OSError as err:
if err.errno == errno.EEXIST:
_LOGGER.info('Resuming container %r', uniq_name)
is_resume = True
else:
raise
# Copy the event as 'manifest.yml' in the container dir
shutil.copyfile(
event,
os.path.join(container_dir, 'manifest.yml')
)
# Setup the service clients
cgroup_client = tm_env.svc_cgroup.make_client(
os.path.join(container_dir, 'cgroups')
)
localdisk_client = tm_env.svc_localdisk.make_client(
os.path.join(container_dir, 'localdisk')
)
network_client = tm_env.svc_network.make_client(
os.path.join(container_dir, 'network')
)
# Store the app int the container_dir
app_yml = os.path.join(container_dir, _APP_YML)
with open(app_yml, 'w') as f:
yaml.dump(manifest_data, stream=f)
# Generate resources requests
# Cgroup
cgroup_req = {
'memory': app.memory,
'cpu': app.cpu,
}
# Local Disk
localdisk_req = {
'size': app.disk,
}
# Network
network_req = {
'environment': app.environment,
}
if not is_resume:
cgroup_client.create(uniq_name, cgroup_req)
localdisk_client.create(uniq_name, localdisk_req)
else:
cgroup_client.update(uniq_name, cgroup_req)
localdisk_client.update(uniq_name, localdisk_req)
if not app.shared_network:
if not is_resume:
network_client.create(uniq_name, network_req)
else:
network_client.update(uniq_name, network_req)
# Mark the container as defaulting to down state
utils.touch(os.path.join(container_dir, 'down'))
# Generate the supervisor's run script
app_run_cmd = ' '.join([
os.path.join(treadmill.TREADMILL, 'bin', 'treadmill'),
'sproc', 'run', container_dir
])
run_out_file = os.path.join(container_dir, 'run.out')
utils.create_script(os.path.join(container_dir, 'run'),
'supervisor.run_no_log',
log_out=run_out_file,
cmd=app_run_cmd)
_init_log_file(run_out_file,
os.path.join(tm_env.apps_dir, "%s.run.out" % uniq_name))
# Unique name for the link, based on creation time.
cleanup_link = os.path.join(tm_env.cleanup_dir, uniq_name)
finish_cmd = '/bin/ln -snvf %s %s' % (container_dir, cleanup_link)
utils.create_script(os.path.join(container_dir, 'finish'),
'supervisor.finish',
service=app.name, proid=None,
cmds=[finish_cmd])
appevents.post(
tm_env.app_events_dir,
events.ConfiguredTraceEvent(
instanceid=app.name,
uniqueid=app.uniqueid
)
)
return container_dir
def create_ready_file(fsroot):
"""Create zksync ready file
"""
modified_file = os.path.join(fsroot, '.modified')
utils.touch(modified_file)
return modified_file
def configure(tm_env, event):
"""Creates directory necessary for starting the application.
This operation is idem-potent (it can be repeated).
The directory layout is::
- (treadmill root)
- apps
- (app unique name)
- app.yml
run
finish
log/run
The 'run' script is responsible for creating container environment
and starting svscan inside the container.
The 'finish' script is invoked when container terminates and will
deallocate any resources (NAT rules, etc) that were allocated for the
container.
"""
# R0915: Need to refactor long function into smaller pieces.
#
# pylint: disable=R0915
# Load the app from the event
try:
manifest_data = app_manifest.load(tm_env, event)
except IOError:
# File is gone. Nothing to do.
_LOGGER.exception("No event to load: %r", event)
return
# Freeze the app data into a namedtuple object
app = utils.to_obj(manifest_data)
# Check the identity we are going to run as
_check_identity(app.proid)
# Generate a unique name for the app
uniq_name = appcfg.app_unique_name(app)
# Create the app's running directory
container_dir = os.path.join(tm_env.apps_dir, uniq_name)
if not fs.mkdir_safe(container_dir):
_LOGGER.info('Resuming container %r', uniq_name)
# Copy the event as 'manifest.yml' in the container dir
shutil.copyfile(event, os.path.join(container_dir, 'manifest.yml'))
# Store the app int the container_dir
app_yml = os.path.join(container_dir, _APP_YML)
with open(app_yml, 'w') as f:
yaml.dump(manifest_data, stream=f)
# Mark the container as defaulting to down state
utils.touch(os.path.join(container_dir, 'down'))
# Generate the supervisor's run script
app_run_cmd = ' '.join(
[treadmill.TREADMILL_BIN, 'sproc', 'run', container_dir])
utils.create_script(os.path.join(container_dir, 'run'),
'supervisor.run_no_log',
cmd=app_run_cmd)
fs.mkdir_safe(os.path.join(container_dir, 'log'))
utils.create_script(os.path.join(container_dir, 'log', 'run'),
'logger.run')
# Unique name for the link, based on creation time.
cleanup_link = os.path.join(tm_env.cleanup_dir, uniq_name)
if os.name == 'nt':
finish_cmd = '%%COMSPEC%% /C mklink /D %s %s' % \
(cleanup_link, container_dir)
else:
finish_cmd = '/bin/ln -snvf %s %s' % (container_dir, cleanup_link)
utils.create_script(os.path.join(container_dir, 'finish'),
'supervisor.finish',
service=app.name,
proid=None,
cmds=[finish_cmd])
appevents.post(
tm_env.app_events_dir,
events.ConfiguredTraceEvent(instanceid=app.name,
uniqueid=app.uniqueid))
return container_dir