def set_attributes(self, message):
"""Helper function to set all this objects attributes from message"""
schema = [
('cell', True, unicode),
('pattern', True, unicode),
('endpoint', False, unicode),
('deltas', False, bool),
]
try:
utils.validate(message, schema)
except exc.InvalidInputError, err:
self.send_error_msg('Invalid data: %s: %s' % (message, err))
return
def test_validate(self):
"""Tests dictionary validation."""
schema = [
('required', True, str),
('optional', False, str),
]
struct = {'required': 'foo'}
utils.validate(struct, schema)
self.assertIn('optional', struct)
struct = {'required': 'foo', 'optional': 'xxx'}
utils.validate(struct, schema)
struct = {'required': 'foo', 'optional': 1234}
self.assertRaises(Exception, utils.validate, struct, schema)
schema = [
('required', True, list),
('optional', False, list),
]
struct = {'required': ['foo']}
utils.validate(struct, schema)
struct = {'required': 'foo'}
self.assertRaises(Exception, utils.validate, struct, schema)
def _on_created(self, impl, filepath):
"""Private handler for request creation events.
"""
# Avoid triggering on changes to the service directory itself.
if filepath == self._rsrc_dir:
return False
req_id = os.path.basename(filepath)
# Avoid triggerring on temporary files
if req_id[0] == '.':
return False
req_file = os.path.join(filepath, REQ_FILE)
rep_file = os.path.join(filepath, REP_FILE)
try:
with io.open(req_file) as f:
req_data = yaml.load(stream=f)
except IOError as err:
if (err.errno == errno.ENOENT or
err.errno == errno.ENOTDIR):
_LOGGER.exception('Removing invalid request: %r', req_id)
try:
fs.rm_safe(filepath)
except OSError as rm_err:
if rm_err.errno == errno.EISDIR:
fs.rmtree_safe(filepath)
else:
raise
return False
raise
# TODO: We should also validate the req_id format
with lc.LogContext(_LOGGER, req_id,
adapter_cls=lc.ContainerAdapter) as log:
log.debug('created %r: %r', req_id, req_data)
try:
# TODO: We should also validate the req_id format
utils.validate(req_data, impl.PAYLOAD_SCHEMA)
res = impl.on_create_request(req_id, req_data)
except exc.InvalidInputError as err:
log.error('Invalid request data: %r: %s', req_data, err)
res = {'_error': {'input': req_data, 'why': str(err)}}
except Exception as err: # pylint: disable=W0703
log.exception('Unable to process request: %r %r:',
req_id, req_data)
res = {'_error': {'input': req_data, 'why': str(err)}}
if res is None:
# Request was not actioned
return False
fs.write_safe(
rep_file,
lambda f: yaml.dump(
res, explicit_start=True, explicit_end=True,
default_flow_style=False, stream=f
),
mode='w',
permission=0o644
)
# Return True if there were no error
return not bool(res.get('_error', False))
def load(event):
"""Loads the app event file, ensuring it is in valid format, and supplement
it into a full Treadmill manifest.
:param event:
Full path to the application node event in the zookeeper cache.
:type event:
``str``
:return:
Application manifest object
:rtype:
``dict``
"""
# pylint: disable=too-many-statements
#
# TODO: need better input validation / setting defaults process.
name = os.path.basename(event)
manifest = read(event, 'yaml')
utils.validate(manifest, [('image', False, str)])
app_type = appcfg.AppType.get_app_type(manifest.get('image'))
schema = [
('proid', True, str),
('environment', True, str),
('services', app_type == appcfg.AppType.NATIVE, list),
('command', False, str),
('args', False, list),
('endpoints', False, list),
('environ', False, list),
('cpu', True, str),
('memory', True, str),
('disk', True, str),
('keytabs', False, list),
]
utils.validate(manifest, schema)
manifest['system_services'] = []
manifest['name'] = name
manifest['app'] = appcfg.appname_basename(name)
manifest['type'] = app_type.value
manifest['uniqueid'] = appcfg.gen_uniqueid(event)
if manifest['environment'] not in ['dev', 'qa', 'uat', 'prod']:
_LOGGER.warning('Unrecognized environment: %s',
manifest['environment'])
raise Exception('Invalid environment: ' + manifest['environment'])
if manifest['cpu'].endswith('%'):
manifest['cpu'] = int(manifest['cpu'][:-1])
# By default network is private.
if 'shared_network' not in manifest:
manifest['shared_network'] = False
else:
manifest['shared_network'] = bool(manifest['shared_network'])
# By default host IP is not shared, not used in the container.
if 'shared_ip' not in manifest:
manifest['shared_ip'] = False
else:
manifest['shared_ip'] = bool(manifest['shared_ip'])
# Check archive
manifest['archive'] = list(manifest.get('archive', []))
if manifest['archive'] is None:
manifest['archive'] = []
# Adds cell specific information to the loaded manifest.
manifest['cell'] = context.GLOBAL.cell
manifest['zookeeper'] = context.GLOBAL.zk.url
def _set_default(attr, value, obj=None):
"""Set default manifest attribute if it is not present."""
if obj is None:
obj = manifest
if attr not in obj:
obj[attr] = value
_set_default('command', None)
_set_default('args', [])
_set_default('environ', [])
_set_default('endpoints', [])
_set_default('passthrough', [])
_set_default('services', [])
_set_default('vring', {})
_set_default('cells', [], manifest['vring'])
_set_default('identity_group', None)
_set_default('identity', None)
_set_default('data_retention_timeout', None)
_set_default('lease', None)
_set_default('keytabs', [])
# Normalize optional and port information
manifest['endpoints'] = [{
'name': endpoint['name'],
'port': int(endpoint['port']),
'type': endpoint.get('type', None),
'proto': endpoint.get('proto', 'tcp'),
} for endpoint in manifest.get('endpoints', [])]
# TODO: need better way to normalize.
if 'ephemeral_ports' not in manifest:
manifest['ephemeral_ports'] = {'tcp': 0, 'udp': 0}
if 'tcp' not in manifest['ephemeral_ports']:
manifest['ephemeral_ports']['tcp'] = 0
else:
manifest['ephemeral_ports']['tcp'] = int(
manifest['ephemeral_ports']['tcp'])
if 'udp' not in manifest['ephemeral_ports']:
manifest['ephemeral_ports']['udp'] = 0
else:
manifest['ephemeral_ports']['udp'] = int(
manifest['ephemeral_ports']['udp'])
return manifest
def vring_cmd(approot, manifest):
"""Run vring manager."""
context.GLOBAL.zk.conn.add_listener(zkutils.exit_on_disconnect)
tm_env = appenv.AppEnvironment(approot)
app = yaml.load(stream=manifest)
with lc.LogContext(_LOGGER, app['name'], lc.ContainerAdapter) as log:
# TODO(boysson): Remove all validation from here.
utils.validate(app, [('vring', True, dict)])
ring = app['vring']
utils.validate(ring, [('rules', True, list),
('cells', True, list)])
if context.GLOBAL.cell not in ring['cells']:
log.critical('cell %s not listed in vring.',
context.GLOBAL.cell)
sys.exit(-1)
rules = ring['rules']
for rule in rules:
utils.validate(rule, [('pattern', True, str),
('endpoints', True, list)])
# Create translation for endpoint name to expected port #.
routing = {}
for endpoint in app.get('endpoints', []):
routing[endpoint['name']] = {
'port': endpoint['port'],
'proto': endpoint['proto']
}
# Check that all ring endpoints are listed in the manifest.
vring_endpoints = set()
for rule in rules:
for rule_endpoint in rule['endpoints']:
if rule_endpoint not in routing:
log.critical(
'vring references non-existing endpoint: [%s]',
rule_endpoint)
sys.exit(-1)
vring_endpoints.add(rule_endpoint)
# TODO: discovery is limited to one rule for now.
if len(rules) != 1:
log.critical('(TODO): multiple rules are not supported.')
sys.exit(-1)
pattern = rules[0]['pattern']
app_unique_name = appcfg.manifest_unique_name(app)
app_discovery = discovery.Discovery(context.GLOBAL.zk.conn,
pattern, '*')
app_discovery.sync()
# Restore default signal mask disabled by python spawning new
# thread for Zk connection.
#
# TODO: should this be done as part of ZK connect?
for sig in range(1, signal.NSIG):
try:
signal.signal(sig, signal.SIG_DFL)
except RuntimeError:
pass
vring.run(
routing,
vring_endpoints,
app_discovery,
tm_env.rules,
app['network']['vip'],
app_unique_name,
)
def vring_cmd(approot, manifest):
"""Run vring manager."""
context.GLOBAL.zk.conn.add_listener(zkutils.exit_on_disconnect)
tm_env = appenv.AppEnvironment(approot)
with io.open(manifest, 'r') as fd:
app = json.load(fd)
with lc.LogContext(_LOGGER, app['name'], lc.ContainerAdapter) as log:
# TODO(boysson): Remove all validation from here.
utils.validate(app, [('vring', True, dict)])
ring = app['vring']
utils.validate(ring, [('rules', True, list), ('cells', True,
list)])
if context.GLOBAL.cell not in ring['cells']:
log.critical('cell %s not listed in vring.',
context.GLOBAL.cell)
sys.exit(-1)
rules = ring['rules']
for rule in rules:
utils.validate(rule, [('pattern', True, str),
('endpoints', True, list)])
# Create translation for endpoint name to expected port #.
routing = {}
for endpoint in app.get('endpoints', []):
routing[endpoint['name']] = {
'port': endpoint['port'],
'proto': endpoint['proto']
}
# Check that all ring endpoints are listed in the manifest.
vring_endpoints = set()
for rule in rules:
for rule_endpoint in rule['endpoints']:
if rule_endpoint not in routing:
log.critical(
'vring references non-existing endpoint: [%s]',
rule_endpoint)
sys.exit(-1)
vring_endpoints.add(rule_endpoint)
patterns = [rule['pattern'] for rule in rules]
app_discovery = discovery.Discovery(context.GLOBAL.zk.conn,
patterns, '*')
app_discovery.sync()
# Restore default signal mask disabled by python spawning new
# thread for Zk connection.
#
# TODO: should this be done as part of ZK connect?
utils.restore_signals()
app_unique_name = appcfg.manifest_unique_name(app)
vring.run(
routing,
vring_endpoints,
app_discovery,
tm_env.rules,
app['network']['vip'],
app_unique_name,
)
