def validate(conf):
validate_presto_conf(conf)
if 'coordinator' not in conf['config.properties']:
raise ConfigurationError('Must specify coordinator=true in '
'coordinator\'s config.properties')
if conf['config.properties']['coordinator'] != 'true':
raise ConfigurationError('Coordinator cannot be false in the '
'coordinator\'s config.properties.')
return conf
def validate_workers(workers):
if not isinstance(workers, list):
raise ConfigurationError('Workers must be of type list. Found ' +
str(type(workers)) + '.')
if len(workers) < 1:
raise ConfigurationError('Must specify at least one worker')
for worker in workers:
validate_host(worker)
return workers
def validate_port(port):
try:
port_int = int(port)
except TypeError:
raise ConfigurationError('Port must be of type string, but '
'found ' + str(type(port)) + '.')
except ValueError:
raise ConfigurationError('Invalid port number ' + port +
': port must be a number between 1 and 65535')
if not port_int > 0 or not port_int < 65535:
raise ConfigurationError('Invalid port number ' + port +
': port must be a number between 1 and 65535')
return port_int
def validate(conf):
validate_presto_conf(conf)
if conf['config.properties']['coordinator'] != 'false':
raise ConfigurationError('Coordinator must be false in the '
'worker\'s config.properties')
uri = urlparse.urlparse(conf['config.properties']['discovery.uri'])
if islocalhost(uri.hostname) and len(env.roledefs['all']) > 1:
raise ConfigurationError(
'discovery.uri should not be localhost in a '
'multi-node cluster, but found ' + urlparse.urlunparse(uri) +
'. You may have encountered this error by '
'choosing a coordinator that is localhost and a worker that '
'is not. The default discovery-uri is '
'http://<coordinator>:8080')
return conf
def get_mode(validate=True):
mode_config = _load_mode_config()
mode = mode_config.get(MODE_KEY)
if validate and mode is None:
raise ConfigurationError(
'Required key %s not found in configuration file %s' %
(MODE_KEY, MODE_CONF_PATH))
if validate and not validate_mode(mode):
raise ConfigurationError(
'Invalid mode %s in configuration file %s. Valid modes are %s' %
(mode, MODE_CONF_PATH, ' '.join(VALID_MODES)))
return mode
def _update_env(default_options, non_default_options):
# Fill in the state with the default values
for opt, value in default_options.__dict__.items():
state.env[opt] = value
# Load the values from the topology file, if it exists
load_topology()
_set_arbitrary_env_vars(non_default_options)
# Go back through and add the non-default values (e.g. the values that
# were set on the CLI)
for opt, value in non_default_options.__dict__.items():
# raise error if hosts not in topology
if opt == 'hosts':
command_hosts = set(value.split(','))
topology_hosts = set(state.env.hosts)
if not command_hosts.issubset(topology_hosts):
raise ConfigurationError('Hosts defined in --hosts/-H must be '
'in the topology file.')
state.env[opt] = value
# Handle --hosts, --roles, --exclude-hosts (comma separated string =>
# list)
for key in ['hosts', 'roles', 'exclude_hosts']:
if key in state.env and isinstance(state.env[key], basestring):
state.env[key] = state.env[key].split(',')
state.output['running'] = False
state.output['status'] = False
update_output_levels(show=state.env.show, hide=state.env.hide)
state.env.skip_bad_hosts = True
def validate(conf):
for key in conf.keys():
if key not in PRESTO_ADMIN_PROPERTIES:
raise ConfigurationError('Invalid property: ' + key)
try:
username = conf['username']
except KeyError:
pass
else:
validate_username(username)
try:
coordinator = conf['coordinator']
except KeyError:
pass
else:
validate_coordinator(coordinator)
try:
workers = conf['workers']
except KeyError:
pass
else:
validate_workers(workers)
try:
ssh_port = conf['ssh-port']
except KeyError:
pass
else:
validate_port(ssh_port)
return conf
def mock_error_topology():
@patch('tests.unit.test_main.StandaloneConfig._get_conf_from_file')
@patch('prestoadmin.standalone.config.validate',
side_effect=ConfigurationError())
def loader(load_config_callback, validate_mock, get_conf_mock):
return load_config_callback()
return loader
def split_to_pair(line):
split_line = re.split(r'\s*(?<!\\):\s*|\s*(?<!\\)=\s*|(?<!\\)\s+', line,
maxsplit=1)
if len(split_line) != 2:
raise ConfigurationError(
line + " is not in the expected format: <property>=<value>, "
"<property>:<value> or <property> <value>")
return tuple(split_line)
def validate_presto_conf(conf):
for required in REQUIRED_FILES:
if required not in conf:
raise ConfigurationError("Missing configuration for required "
"file: " + required)
expect_object_msg = "%s must be an object with key-value property pairs"
if not isinstance(conf["node.properties"], dict):
raise ConfigurationError(expect_object_msg % "node.properties")
if not isinstance(conf["jvm.config"], list):
raise ConfigurationError("jvm.config must contain a json array of jvm "
"arguments ([arg1, arg2, arg3])")
if not isinstance(conf["config.properties"], dict):
raise ConfigurationError(expect_object_msg % "config.properties")
return conf
def default_config(self, filename):
try:
conf = copy.deepcopy(self.DEFAULT_PROPERTIES[filename])
except KeyError:
raise ConfigurationError('Invalid configuration file name: %s' %
filename)
if filename == 'config.properties':
coordinator = util.get_coordinator_role()[0]
conf['discovery.uri'] = 'http://%s:8080' % coordinator
return conf
def get_conf_from_json_file(path):
try:
with open(path, 'r') as conf_file:
if os.path.getsize(conf_file.name) == 0:
return {}
return json.load(conf_file)
except IOError:
raise ConfigFileNotFoundError("Missing configuration file at " +
repr(path))
except ValueError as e:
raise ConfigurationError(e)
def validate_host(host):
try:
socket.inet_pton(socket.AF_INET, host)
return host
except TypeError:
raise ConfigurationError('Host must be of type string. Found '
+ str(type(host)) + '.')
except socket.error:
pass
try:
socket.inet_pton(socket.AF_INET6, host)
return host
except socket.error:
pass
if not is_valid_hostname(host):
raise ConfigurationError(repr(host) + ' is not a valid '
'ip address or host name.')
return host
def validate_hosts(cli_hosts, config_path):
# If there's no config file to validate against, don't. This would happen
# in the case of a task that doesn't define a callback that loads config.
if config_path is None:
return
# At this point, state.env.conf_hosts contains the hosts that we loaded
# from the configuration, if any.
cli_host_set = set(cli_hosts.split(','))
if 'conf_hosts' in state.env:
conf_hosts = set(state.env.conf_hosts)
if not cli_host_set.issubset(conf_hosts):
raise ConfigurationError('Hosts defined in --hosts/-H must be '
'present in %s' % (config_path))
else:
raise ConfigurationError(
'Hosts cannot be defined with --hosts/-H when no hosts are listed '
'in the configuration file %s. Correct the configuration file or '
'run the command again without the --hosts or -H option.' %
config_path)
def lookup_in_config(config_key, config_file, host):
with settings(hide('stdout', 'warnings', 'aborts')):
config_value = execute(run,
'grep %s= %s' % (config_key, config_file),
warn_only=True,
host=host)[host]
if isinstance(config_value, Exception) or config_value.return_code == 2:
raise ConfigurationError('Configuration file %s does not exist on '
'host %s' % (config_file, host))
return config_value
def validate_presto_conf(conf):
for required in REQUIRED_FILES:
if required not in conf:
raise ConfigurationError("Missing configuration for required "
"file: " + required)
expect_object_msg = "%s must be an object with key-value property pairs"
if not isinstance(conf["node.properties"], dict):
raise ConfigurationError(expect_object_msg % "node.properties")
if not isinstance(conf["jvm.config"], list):
raise ConfigurationError("jvm.config must contain a json array of jvm "
"arguments ([arg1, arg2, arg3])")
if not isinstance(conf["config.properties"], dict):
raise ConfigurationError(expect_object_msg % "config.properties")
if 'discovery.uri' not in conf['config.properties']:
raise ConfigurationError('Must have discovery.uri defined in '
'config.properties.')
discovery_uri = conf['config.properties']['discovery.uri']
if not discovery_uri.startswith('http://'):
raise ConfigurationError('discovery.uri must start with http://, '
'current URI is: %s' % discovery_uri)
return conf
def default_config(self, filename):
try:
conf = copy.deepcopy(self.DEFAULT_PROPERTIES[filename])
except KeyError:
raise ConfigurationError('Invalid configuration file name: %s' %
filename)
if filename == 'config.properties':
coordinator = env.roledefs['coordinator'][0]
workers = env.roledefs['worker']
if coordinator in workers:
conf['node-scheduler.include-coordinator'] = 'true'
conf['discovery.uri'] = 'http://%s:8080' % coordinator
return conf
def lookup_in_config(config_key, config_file, host):
with settings(hide('stdout', 'warnings', 'aborts')):
config_value = execute(sudo,
'grep %s= %s' % (config_key, config_file),
user='******',
warn_only=True,
host=host)[host]
if isinstance(config_value, Exception) or config_value.return_code == 2:
raise ConfigurationError('Could not access config file %s on '
'host %s' % (config_file, host))
return config_value
def validate(filenames):
for name in filenames:
file_path = os.path.join(constants.CONNECTORS_DIR, name)
_LOGGER.info('Validating connector configuration: ' + str(name))
try:
with open(file_path) as f:
file_content = f.read()
if 'connector.name' not in file_content:
message = ('Catalog configuration %s does not contain '
'connector.name' % name)
raise ConfigurationError(message)
except IOError, e:
fabric.utils.error(message='Error validating ' + file_path,
exception=e)
return False
def test_interactive_install(self, get_conf_mock, mock_set_interactive):
env.topology_config_not_found = ConfigurationError()
get_conf_mock.return_value = {
'username': '******',
'port': '225',
'coordinator': 'master',
'workers': ['slave1', 'slave2']
}
topology.set_topology_if_missing()
self.assertEqual(env.user, 'bob'),
self.assertEqual(env.port, '225')
self.assertEqual(env.hosts, ['master', 'slave1', 'slave2'])
self.assertEqual(env.roledefs['all'], ['master', 'slave1', 'slave2'])
self.assertEqual(env.roledefs['coordinator'], ['master'])
self.assertEqual(env.roledefs['worker'], ['slave1', 'slave2'])
self.assertFalse(env.topology_config_not_found)
def validate(conf):
for key in conf.keys():
if key not in PRESTO_ADMIN_PROPERTIES:
raise ConfigurationError('Invalid property: ' + key)
try:
username = conf['username']
except KeyError:
pass
else:
conf['username'] = validate_username(username)
try:
java_home = conf['java_home']
except KeyError:
pass
else:
conf['java_home'] = validate_java_home(java_home)
try:
coordinator = conf['coordinator']
except KeyError:
pass
else:
conf['coordinator'] = validate_coordinator(coordinator)
try:
workers = conf['workers']
except KeyError:
pass
else:
workers = [h for host in workers for h in _expand_host(host)]
conf['workers'] = validate_workers(workers)
try:
port = conf['port']
except KeyError:
pass
else:
conf['port'] = validate_port(port)
return conf
def lookup_port(host):
"""
Get the http port from config.properties http-server.http.port property
if available.
If the property is missing return default port 8080.
If the file is missing or cannot parse the port number,
throw ConfigurationError
:param host:
:return:
"""
port = lookup_in_config('http-server.http.port', GENERAL_CONFIG_FILE, host)
if not port:
_LOGGER.info('Could not find property http-server.http.port.'
'Defaulting to 8080.')
return 8080
try:
port = port.split('=', 1)[1]
port = prestoadmin.util.validators.validate_port(port)
_LOGGER.info('Looked up port ' + str(port) + ' on host ' + host)
return port
except ConfigurationError as e:
raise ConfigurationError(e.message + ' for property '
'http-server.http.port on host ' + host + '.')
def test_no_warn_if_port_lookup_fail(self, mock_warn, mock_port):
e = ConfigurationError()
mock_port.side_effect = e
env.host = 'any_host'
self.assertFalse(server.is_port_in_use(env.host))
self.assertEqual(False, mock_warn.called)
def validate_username(username):
if not isinstance(username, basestring):
raise ConfigurationError('Username must be of type string.')
return username
def test_load_topology_failure(self, get_conf_mock):
e = ConfigurationError()
get_conf_mock.side_effect = e
self.assertRaises(ConfigurationError, main.load_topology)
def wrapper(*args, **kwargs):
if not config_instance.is_config_loaded():
raise ConfigurationError('Required config not loaded at task '
'execution time.')
return func(*args, **kwargs)
