def set_attributes(self, message): """Helper function to set all this objects attributes from message""" schema = [ ('cell', True, unicode), ('pattern', True, unicode), ('endpoint', False, unicode), ('deltas', False, bool), ] try: utils.validate(message, schema) except exc.InvalidInputError, err: self.send_error_msg('Invalid data: %s: %s' % (message, err)) return
def test_validate(self): """Tests dictionary validation.""" schema = [ ('required', True, str), ('optional', False, str), ] struct = {'required': 'foo'} utils.validate(struct, schema) self.assertIn('optional', struct) struct = {'required': 'foo', 'optional': 'xxx'} utils.validate(struct, schema) struct = {'required': 'foo', 'optional': 1234} self.assertRaises(Exception, utils.validate, struct, schema) schema = [ ('required', True, list), ('optional', False, list), ] struct = {'required': ['foo']} utils.validate(struct, schema) struct = {'required': 'foo'} self.assertRaises(Exception, utils.validate, struct, schema)
def _on_created(self, impl, filepath): """Private handler for request creation events. """ # Avoid triggering on changes to the service directory itself. if filepath == self._rsrc_dir: return False req_id = os.path.basename(filepath) # Avoid triggerring on temporary files if req_id[0] == '.': return False req_file = os.path.join(filepath, REQ_FILE) rep_file = os.path.join(filepath, REP_FILE) try: with io.open(req_file) as f: req_data = yaml.load(stream=f) except IOError as err: if (err.errno == errno.ENOENT or err.errno == errno.ENOTDIR): _LOGGER.exception('Removing invalid request: %r', req_id) try: fs.rm_safe(filepath) except OSError as rm_err: if rm_err.errno == errno.EISDIR: fs.rmtree_safe(filepath) else: raise return False raise # TODO: We should also validate the req_id format with lc.LogContext(_LOGGER, req_id, adapter_cls=lc.ContainerAdapter) as log: log.debug('created %r: %r', req_id, req_data) try: # TODO: We should also validate the req_id format utils.validate(req_data, impl.PAYLOAD_SCHEMA) res = impl.on_create_request(req_id, req_data) except exc.InvalidInputError as err: log.error('Invalid request data: %r: %s', req_data, err) res = {'_error': {'input': req_data, 'why': str(err)}} except Exception as err: # pylint: disable=W0703 log.exception('Unable to process request: %r %r:', req_id, req_data) res = {'_error': {'input': req_data, 'why': str(err)}} if res is None: # Request was not actioned return False fs.write_safe( rep_file, lambda f: yaml.dump( res, explicit_start=True, explicit_end=True, default_flow_style=False, stream=f ), mode='w', permission=0o644 ) # Return True if there were no error return not bool(res.get('_error', False))
def load(event): """Loads the app event file, ensuring it is in valid format, and supplement it into a full Treadmill manifest. :param event: Full path to the application node event in the zookeeper cache. :type event: ``str`` :return: Application manifest object :rtype: ``dict`` """ # pylint: disable=too-many-statements # # TODO: need better input validation / setting defaults process. name = os.path.basename(event) manifest = read(event, 'yaml') utils.validate(manifest, [('image', False, str)]) app_type = appcfg.AppType.get_app_type(manifest.get('image')) schema = [ ('proid', True, str), ('environment', True, str), ('services', app_type == appcfg.AppType.NATIVE, list), ('command', False, str), ('args', False, list), ('endpoints', False, list), ('environ', False, list), ('cpu', True, str), ('memory', True, str), ('disk', True, str), ('keytabs', False, list), ] utils.validate(manifest, schema) manifest['system_services'] = [] manifest['name'] = name manifest['app'] = appcfg.appname_basename(name) manifest['type'] = app_type.value manifest['uniqueid'] = appcfg.gen_uniqueid(event) if manifest['environment'] not in ['dev', 'qa', 'uat', 'prod']: _LOGGER.warning('Unrecognized environment: %s', manifest['environment']) raise Exception('Invalid environment: ' + manifest['environment']) if manifest['cpu'].endswith('%'): manifest['cpu'] = int(manifest['cpu'][:-1]) # By default network is private. if 'shared_network' not in manifest: manifest['shared_network'] = False else: manifest['shared_network'] = bool(manifest['shared_network']) # By default host IP is not shared, not used in the container. if 'shared_ip' not in manifest: manifest['shared_ip'] = False else: manifest['shared_ip'] = bool(manifest['shared_ip']) # Check archive manifest['archive'] = list(manifest.get('archive', [])) if manifest['archive'] is None: manifest['archive'] = [] # Adds cell specific information to the loaded manifest. manifest['cell'] = context.GLOBAL.cell manifest['zookeeper'] = context.GLOBAL.zk.url def _set_default(attr, value, obj=None): """Set default manifest attribute if it is not present.""" if obj is None: obj = manifest if attr not in obj: obj[attr] = value _set_default('command', None) _set_default('args', []) _set_default('environ', []) _set_default('endpoints', []) _set_default('passthrough', []) _set_default('services', []) _set_default('vring', {}) _set_default('cells', [], manifest['vring']) _set_default('identity_group', None) _set_default('identity', None) _set_default('data_retention_timeout', None) _set_default('lease', None) _set_default('keytabs', []) # Normalize optional and port information manifest['endpoints'] = [{ 'name': endpoint['name'], 'port': int(endpoint['port']), 'type': endpoint.get('type', None), 'proto': endpoint.get('proto', 'tcp'), } for endpoint in manifest.get('endpoints', [])] # TODO: need better way to normalize. if 'ephemeral_ports' not in manifest: manifest['ephemeral_ports'] = {'tcp': 0, 'udp': 0} if 'tcp' not in manifest['ephemeral_ports']: manifest['ephemeral_ports']['tcp'] = 0 else: manifest['ephemeral_ports']['tcp'] = int( manifest['ephemeral_ports']['tcp']) if 'udp' not in manifest['ephemeral_ports']: manifest['ephemeral_ports']['udp'] = 0 else: manifest['ephemeral_ports']['udp'] = int( manifest['ephemeral_ports']['udp']) return manifest
def vring_cmd(approot, manifest): """Run vring manager.""" context.GLOBAL.zk.conn.add_listener(zkutils.exit_on_disconnect) tm_env = appenv.AppEnvironment(approot) app = yaml.load(stream=manifest) with lc.LogContext(_LOGGER, app['name'], lc.ContainerAdapter) as log: # TODO(boysson): Remove all validation from here. utils.validate(app, [('vring', True, dict)]) ring = app['vring'] utils.validate(ring, [('rules', True, list), ('cells', True, list)]) if context.GLOBAL.cell not in ring['cells']: log.critical('cell %s not listed in vring.', context.GLOBAL.cell) sys.exit(-1) rules = ring['rules'] for rule in rules: utils.validate(rule, [('pattern', True, str), ('endpoints', True, list)]) # Create translation for endpoint name to expected port #. routing = {} for endpoint in app.get('endpoints', []): routing[endpoint['name']] = { 'port': endpoint['port'], 'proto': endpoint['proto'] } # Check that all ring endpoints are listed in the manifest. vring_endpoints = set() for rule in rules: for rule_endpoint in rule['endpoints']: if rule_endpoint not in routing: log.critical( 'vring references non-existing endpoint: [%s]', rule_endpoint) sys.exit(-1) vring_endpoints.add(rule_endpoint) # TODO: discovery is limited to one rule for now. if len(rules) != 1: log.critical('(TODO): multiple rules are not supported.') sys.exit(-1) pattern = rules[0]['pattern'] app_unique_name = appcfg.manifest_unique_name(app) app_discovery = discovery.Discovery(context.GLOBAL.zk.conn, pattern, '*') app_discovery.sync() # Restore default signal mask disabled by python spawning new # thread for Zk connection. # # TODO: should this be done as part of ZK connect? for sig in range(1, signal.NSIG): try: signal.signal(sig, signal.SIG_DFL) except RuntimeError: pass vring.run( routing, vring_endpoints, app_discovery, tm_env.rules, app['network']['vip'], app_unique_name, )
def vring_cmd(approot, manifest): """Run vring manager.""" context.GLOBAL.zk.conn.add_listener(zkutils.exit_on_disconnect) tm_env = appenv.AppEnvironment(approot) with io.open(manifest, 'r') as fd: app = json.load(fd) with lc.LogContext(_LOGGER, app['name'], lc.ContainerAdapter) as log: # TODO(boysson): Remove all validation from here. utils.validate(app, [('vring', True, dict)]) ring = app['vring'] utils.validate(ring, [('rules', True, list), ('cells', True, list)]) if context.GLOBAL.cell not in ring['cells']: log.critical('cell %s not listed in vring.', context.GLOBAL.cell) sys.exit(-1) rules = ring['rules'] for rule in rules: utils.validate(rule, [('pattern', True, str), ('endpoints', True, list)]) # Create translation for endpoint name to expected port #. routing = {} for endpoint in app.get('endpoints', []): routing[endpoint['name']] = { 'port': endpoint['port'], 'proto': endpoint['proto'] } # Check that all ring endpoints are listed in the manifest. vring_endpoints = set() for rule in rules: for rule_endpoint in rule['endpoints']: if rule_endpoint not in routing: log.critical( 'vring references non-existing endpoint: [%s]', rule_endpoint) sys.exit(-1) vring_endpoints.add(rule_endpoint) patterns = [rule['pattern'] for rule in rules] app_discovery = discovery.Discovery(context.GLOBAL.zk.conn, patterns, '*') app_discovery.sync() # Restore default signal mask disabled by python spawning new # thread for Zk connection. # # TODO: should this be done as part of ZK connect? utils.restore_signals() app_unique_name = appcfg.manifest_unique_name(app) vring.run( routing, vring_endpoints, app_discovery, tm_env.rules, app['network']['vip'], app_unique_name, )