def _write_temp(temp):
if payload is None:
pass
elif isinstance(payload, six.string_types):
temp.write(payload)
else:
yaml.dump(payload, stream=temp)
def format(item):
"""Return pretty-formatted item."""
list_schema = [
('id', None, None),
('type', None, None),
]
item_schema = [
('id', None, None),
('type', None, None),
('groups', '_ipa',
lambda ipa: '\n'.join(ipa.get('memberof_group', []))),
('indirect-groups', '_ipa',
lambda ipa: '\n'.join(ipa.get('memberofindirect_group', []))),
('hbac-rule', '_ipa',
lambda ipa: '\n'.join(ipa.get('memberofindirect_hbacrule', []))),
('sudo-rule', '_ipa',
lambda ipa: '\n'.join(ipa.get('memberofindirect_sudorule', []))),
('iam-user', '_iam', lambda iam: yaml.dump(iam['user'])),
('iam-role', '_iam', lambda iam: yaml.dump(iam['role'])),
]
format_item = tablefmt.make_dict_to_table(item_schema)
format_list = tablefmt.make_list_to_table(list_schema)
if isinstance(item, list):
return format_list(item)
else:
return format_item(item)
def put(self, rsrc_id, rsrc_data):
"""Request creation/update of a resource.
:param `str` rsrc_id:
Unique identifier for the requested resource.
:param `str` rsrc_data:
(New) Parameters for the requested resource.
"""
req_dir = self._req_dirname(rsrc_id)
fs.mkdir_safe(req_dir)
with io.open(os.path.join(req_dir, REQ_FILE), 'w') as f:
if os.name == 'posix':
os.fchmod(f.fileno(), 0o644)
yaml.dump(rsrc_data,
explicit_start=True,
explicit_end=True,
default_flow_style=False,
stream=f)
req_uuid_file = os.path.join(req_dir, self._REQ_UID_FILE)
try:
with io.open(req_uuid_file) as f:
svc_req_uuid = f.read().strip()
except IOError as err:
if err.errno == errno.ENOENT:
svc_req_uuid = None
else:
raise
with lc.LogContext(_LOGGER, rsrc_id):
if svc_req_uuid is None:
try:
# New request
svc_req_uuid = self._serviceinst.clt_new_request(
rsrc_id, req_dir)
# Write down the UUID
with io.open(req_uuid_file, 'w') as f:
f.write(svc_req_uuid)
os.fchmod(f.fileno(), 0o644)
except OSError:
# Error registration failed, delete the request.
_LOGGER.exception('Unable to submit request')
fs.rmtree_safe(req_dir)
else:
self._serviceinst.clt_update_request(svc_req_uuid)
def configure_apps(ctx, apps):
"""Configure cell API."""
admin_app = admin.Application(context.GLOBAL.ldap.conn)
# For apps that need write access to LDAP. The context LDAP must have
# write access because this is what we use to write manifests here.
write_uri = admin_app.admin.write_uri
ctx.obj.admin_ldap_url = ','.join(write_uri) if write_uri else None
if not apps:
apps = _CELL_APPS
# Configure apps identity groups
identity_groups = _ident_groups(ctx)
for groupname, count in six.iteritems(identity_groups):
masterapi.update_identity_group(
context.GLOBAL.zk.conn,
groupname,
count
)
# Configure apps
for appname in apps:
fullname, app = _render_app(appname, ctx)
print(fullname)
print(yaml.dump(app))
try:
admin_app.create(fullname, app)
except ldap_exceptions.LDAPEntryAlreadyExistsResult:
admin_app.replace(fullname, app)
def print_yaml(obj):
"""Print yaml wih correct options."""
cli.out(
yaml.dump(obj,
default_flow_style=False,
explicit_start=True,
explicit_end=True))
def _cache(self, zkclient, app):
"""Reads the manifest from Zk and stores it as YAML in <cache>/<app>.
"""
appnode = z.path.scheduled(app)
placement_node = z.path.placement(self._hostname, app)
manifest_file = None
try:
manifest = zkutils.get(zkclient, appnode)
# TODO: need a function to parse instance id from name.
manifest['task'] = app[app.index('#') + 1:]
placement_info = zkutils.get(zkclient, placement_node)
if placement_info is not None:
manifest.update(placement_info)
manifest_file = os.path.join(self.tm_env.cache_dir, app)
fs.write_safe(manifest_file,
lambda f: yaml.dump(manifest, stream=f),
prefix='.%s-' % app,
mode='w',
permission=0o644)
_LOGGER.info('Created cache manifest: %s', manifest_file)
except kazoo.exceptions.NoNodeError:
_LOGGER.warning('App %r not found', app)
def test_to_yaml(self):
"""Tests conversion of dict to yaml representation."""
obj = {
'xxx': u'abcd'
}
self.assertEqual(yaml.dump(obj), u'{xxx: abcd}\n')
def mock_get(zkpath, watch=None):
"""Traverse data recursively, return the node content."""
path = zkpath.split('/')
path.pop(0)
content = zk_content
while path:
path_component = path.pop(0)
if path_component not in content:
raise kazoo.client.NoNodeError()
content = content[path_component]
# Content is a copy of the zk data
content = copy.copy(content)
# Setup a fake metadata values
meta_dict = {}
if isinstance(content, dict):
meta_values = content.pop('.metadata', {})
meta_dict.update(meta_values)
data = content.pop('.data', yaml.dump(content))
# Allow override for mock testing, if not set, derive
# children_count from num of children from the mock data.
if 'children_count' not in meta_dict:
meta_dict['children_count'] = len(content)
else:
data = content
# Generate the final readonly metadata
metadata = MockZookeeperMetadata.from_dict(meta_dict)
# Setup the watch
watches[(zkpath, states.EventType.CHANGED)] = watch
return (data, metadata)
def test_watch_placement_yaml(self):
"""Test loading placement stored as yaml, for backward compatibility.
"""
cell_state = state.CellState()
cell_state.running = ['foo.bar#0000000001']
zkclient_mock = _create_zkclient_mock(
yaml.dump([
['foo.bar#0000000001', 'baz', 12345.67890, 'baz', 12345.67890],
['foo.bar#0000000002', 'baz', 12345.67890, 'baz', 12345.67890],
['foo.bar#0000000003', None, None, None, None],
]).encode())
state.watch_placement(zkclient_mock, cell_state)
self.assertEqual(
cell_state.placement, {
'foo.bar#0000000001': {
'state': 'running',
'expires': 12345.6789,
'host': 'baz'
},
'foo.bar#0000000002': {
'state': 'scheduled',
'expires': 12345.6789,
'host': 'baz'
},
'foo.bar#0000000003': {
'state': 'pending',
'expires': None,
'host': None
},
})
def _publish_status(self, status_socket, status_info):
"""Publish service status on the incomming connection on socket
"""
with contextlib.closing(status_socket.accept()[0]) as clt:
clt_stream = clt.makefile(mode='w')
try:
yaml.dump(status_info,
explicit_start=True, explicit_end=True,
default_flow_style=False,
stream=clt_stream)
clt_stream.flush()
except socket.error as err:
if err.errno == errno.EPIPE:
pass
else:
raise
def restart_apps(apps, wait, cors_origin, krb_realm, dry_run):
"""Restart system apps."""
ctx = cell_admin.CellCtx(cors=cors_origin, krb_realm=krb_realm)
cell_apps = cell_admin.get_apps(ctx)
if not apps:
apps = list(cell_apps)
instance_api = instance.API(plugins=['aws-proid-env'])
for appname in apps:
fullname = cell_apps[appname]['fullname']
app = cell_admin.render_template(appname, ctx)
count = cell_apps[appname].get('monitors')
if count is None:
continue
cli.echo_green('Restarting app %s:', fullname)
cli.out(yaml.dump(app, explicit_start=True))
if dry_run:
continue
for idx in range(0, count):
instance_ids = instance_api.create(fullname, app, 1)
for inst_id in instance_ids:
cli.echo_green(inst_id)
if idx <= count - 1 and wait:
time.sleep(wait)
def setUp(self):
"""Setup common test variables"""
self.runner = click.testing.CliRunner()
self.configure = plugin_manager.load('treadmill.cli',
'configure').init()
with tempfile.NamedTemporaryFile(delete=False) as f:
yaml.dump(
{
'memory': '128M',
'cpu': '5%',
'disk': '100M',
'identity_group': 'ident-group',
},
f,
encoding='utf-8')
self.manifest = f.name
def format(obj): # pylint: disable=W0622
"""Returns raw yaml representation of the object."""
if isinstance(obj, dict):
obj.pop('_id', None)
return yaml.dump(obj,
default_flow_style=False,
explicit_start=True,
explicit_end=True)
def put(self, path, value):
"""Store object at a given path."""
fpath = _fpath(self.fsroot, path)
try:
fs.mkdir_safe(os.path.dirname(fpath))
with io.open(fpath, 'w') as node:
node.write(yaml.dump(value))
except OSError:
raise backend.ObjectNotFoundError()
def test_run_withmanifest(self):
"""Test cli.run no manifest."""
manifest = {
'memory': '1G',
'disk': '1G',
'cpu': '100%',
}
with tempfile.NamedTemporaryFile(delete=False, mode='w') as f:
yaml.dump(manifest, stream=f)
result = self.runner.invoke(self.cli, [
'--cell',
'xx',
'proid.app',
'--manifest',
f.name,
])
self.assertEqual(result.exit_code, 0)
treadmill.restclient.post.assert_called_with(
['http://xxx:1234'],
'/instance/proid.app?count=1',
payload=manifest)
# Test manifest parameter override
manifest['memory'] = '333M'
result = self.runner.invoke(self.cli, [
'--cell',
'xx',
'proid.app',
'--memory',
'333M',
'--manifest',
f.name,
])
self.assertEqual(result.exit_code, 0)
treadmill.restclient.post.assert_called_with(
['http://xxx:1234'],
'/instance/proid.app?count=1',
payload=manifest)
def _payload(data):
"""Converts payload to serialized bytes.
"""
payload = b''
if data is not None:
if isinstance(data, bytes):
payload = data
elif isinstance(data, six.string_types) and hasattr(data, 'encode'):
payload = data.encode()
else:
payload = yaml.dump(data).encode()
return payload
def _cache(self, zkclient, app, check_existing=False):
"""Read the manifest and placement data from Zk and store it as YAML in
<cache>/<app>.
:param ``str`` app:
Instance name.
:param ``bool`` check_existing:
Whether to check if the file already exists and is up to date.
"""
placement_node = z.path.placement(self._hostname, app)
try:
placement_data, placement_metadata = zkutils.get_with_metadata(
zkclient, placement_node
)
placement_time = placement_metadata.ctime / 1000.0
except kazoo.exceptions.NoNodeError:
_LOGGER.info('Placement %s/%s not found', self._hostname, app)
return
manifest_file = os.path.join(self.tm_env.cache_dir, app)
if check_existing:
try:
manifest_time = os.stat(manifest_file).st_ctime
except FileNotFoundError:
manifest_time = None
if manifest_time and manifest_time >= placement_time:
_LOGGER.info('%s is up to date', manifest_file)
return
app_node = z.path.scheduled(app)
try:
manifest = zkutils.get(zkclient, app_node)
# TODO: need a function to parse instance id from name.
manifest['task'] = app[app.index('#') + 1:]
if placement_data is not None:
manifest.update(placement_data)
fs.write_safe(
manifest_file,
lambda f: yaml.dump(manifest, stream=f),
prefix='.%s-' % app,
mode='w',
permission=0o644
)
_LOGGER.info('Created cache manifest: %s', manifest_file)
except kazoo.exceptions.NoNodeError:
_LOGGER.info('App %s not found', app)
def restart_apps(ctx, wait, apps):
"""Restart cell API."""
instance_api = instance.API(plugins=['aws-proid-env'])
monitors = _monitors(ctx)
for name, count in six.iteritems(monitors):
_, appname, _ = name.split('.')
if apps and appname not in apps:
continue
_, app = _render_app(appname, ctx)
print(name)
print(yaml.dump(app))
for idx in range(0, count):
instance_ids = instance_api.create(name, app, 1)
for inst_id in instance_ids:
print(inst_id)
if idx <= count - 1 and wait:
time.sleep(wait)
def configure_apps(apps, cors_origin, krb_realm, dry_run):
"""Configure system apps."""
ctx = cell_admin.CellCtx(cors=cors_origin, krb_realm=krb_realm)
cell_apps = cell_admin.get_apps(ctx)
if not apps:
apps = list(cell_apps)
admin_app = admin.Application(context.GLOBAL.ldap.conn)
# For apps that need write access to LDAP. The context LDAP must have
# write access because this is what we use to write manifests here.
write_uri = context.GLOBAL.ldap.write_url
ctx.admin_ldap_url = ','.join(write_uri) if write_uri else None
# Configure apps identity groups
identity_groups = cell_admin.get_identity_groups(ctx)
for groupname, count in identity_groups.items():
cli.echo_green('Configuring identity group %s: %d', groupname,
count)
if not dry_run:
masterapi.update_identity_group(context.GLOBAL.zk.conn,
groupname, count)
# Configure apps
for appname in apps:
fullname = cell_apps[appname]['fullname']
app = cell_admin.render_template(appname, ctx)
cli.echo_green('Configuring app %s:', fullname)
cli.out(yaml.dump(app, explicit_start=True))
if not dry_run:
try:
admin_app.create(fullname, app)
except admin_exceptions.AlreadyExistsResult:
admin_app.replace(fullname, app)
def _on_created(self, impl, filepath):
"""Private handler for request creation events.
"""
# Avoid triggering on changes to the service directory itself.
if filepath == self._rsrc_dir:
return False
req_id = os.path.basename(filepath)
# Avoid triggerring on temporary files
if req_id[0] == '.':
return False
req_file = os.path.join(filepath, REQ_FILE)
rep_file = os.path.join(filepath, REP_FILE)
try:
with io.open(req_file) as f:
req_data = yaml.load(stream=f)
except IOError as err:
if (err.errno == errno.ENOENT or
err.errno == errno.ENOTDIR):
_LOGGER.exception('Removing invalid request: %r', req_id)
try:
fs.rm_safe(filepath)
except OSError as rm_err:
if rm_err.errno == errno.EISDIR:
fs.rmtree_safe(filepath)
else:
raise
return False
raise
# TODO: We should also validate the req_id format
with lc.LogContext(_LOGGER, req_id,
adapter_cls=lc.ContainerAdapter) as log:
log.debug('created %r: %r', req_id, req_data)
try:
# TODO: We should also validate the req_id format
utils.validate(req_data, impl.PAYLOAD_SCHEMA)
res = impl.on_create_request(req_id, req_data)
except exc.InvalidInputError as err:
log.error('Invalid request data: %r: %s', req_data, err)
res = {'_error': {'input': req_data, 'why': str(err)}}
except Exception as err: # pylint: disable=W0703
log.exception('Unable to process request: %r %r:',
req_id, req_data)
res = {'_error': {'input': req_data, 'why': str(err)}}
if res is None:
# Request was not actioned
return False
fs.write_safe(
rep_file,
lambda f: yaml.dump(
res, explicit_start=True, explicit_end=True,
default_flow_style=False, stream=f
),
mode='w',
permission=0o644
)
# Return True if there were no error
return not bool(res.get('_error', False))
def format(obj): # pylint: disable=W0622
"""Returns yaml representation of the object."""
return yaml.dump(obj,
default_flow_style=False,
explicit_start=True,
explicit_end=True)
