def __init__(self, hmac_key, peers, callback, port=None, logger=None):
"""Constructor for ConfdClient
@type hmac_key: string
@param hmac_key: hmac key to talk to confd
@type peers: list
@param peers: list of peer nodes
@type callback: f(L{ConfdUpcallPayload})
@param callback: function to call when getting answers
@type port: integer
@param port: confd port (default: use GetDaemonPort)
@type logger: logging.Logger
@param logger: optional logger for internal conditions
"""
if not callable(callback):
raise errors.ProgrammerError("callback must be callable")
self.UpdatePeerList(peers)
self._SetPeersAddressFamily()
self._hmac_key = hmac_key
self._socket = ConfdAsyncUDPClient(self, self._family)
self._callback = callback
self._confd_port = port
self._logger = logger
self._requests = {}
if self._confd_port is None:
self._confd_port = netutils.GetDaemonPort(constants.CONFD)
def ModifyGroupSshPort(ipt_rules, group, nodes, ssh_port):
"""Modifies the node group settings and sets up iptable rules.
For each pair of nodes add two rules that affect SSH connections from one
to the other one.
The first one redirects port 22 to some unused port so that connecting
through 22 fails. The second redirects port `ssh_port` to port 22.
Together this results in master seeing the SSH daemons on the nodes on
`ssh_port` instead of 22.
"""
default_ssh_port = netutils.GetDaemonPort(constants.SSH)
all_nodes = qa_config.get("nodes")
AssertCommand(["gnt-group", "modify",
"--node-parameters=ssh_port=" + str(ssh_port),
group])
for node in nodes:
ipt_rules.RedirectPort(node.primary, "localhost",
default_ssh_port, 65535)
ipt_rules.RedirectPort(node.primary, "localhost",
ssh_port, default_ssh_port)
for node2 in all_nodes:
ipt_rules.RedirectPort(node2.primary, node.primary,
default_ssh_port, 65535)
ipt_rules.RedirectPort(node2.primary, node.primary,
ssh_port, default_ssh_port)
def RunCustomSshPortTests():
"""Test accessing nodes with custom SSH ports.
This requires removing nodes, adding them to a new group, and then undoing
the change.
"""
if not qa_config.TestEnabled("group-custom-ssh-port"):
return
std_port = netutils.GetDaemonPort(constants.SSH)
port = 211
master = qa_config.GetMasterNode()
with qa_config.AcquireManyNodesCtx(1, exclude=master) as nodes:
# Checks if the node(s) could be contacted through IPv6.
# If yes, better skip the whole test.
for node in nodes:
if qa_utils.UsesIPv6Connection(node.primary, std_port):
print(
"Node %s is likely to be reached using IPv6,"
"skipping the test" % (node.primary, ))
return
for node in nodes:
qa_node.NodeRemove(node)
with qa_iptables.RulesContext() as r:
with qa_group.NewGroupCtx() as group:
qa_group.ModifyGroupSshPort(r, group, nodes, port)
for node in nodes:
qa_node.NodeAdd(node, group=group)
# Make sure that the cluster doesn't have any pre-existing problem
qa_cluster.AssertClusterVerify()
# Create and allocate instances
instance1 = qa_instance.TestInstanceAddWithPlainDisk(nodes)
try:
instance2 = qa_instance.TestInstanceAddWithPlainDisk(nodes)
try:
# cluster-verify checks that disks are allocated correctly
qa_cluster.AssertClusterVerify()
# Remove instances
qa_instance.TestInstanceRemove(instance2)
qa_instance.TestInstanceRemove(instance1)
finally:
instance2.Release()
finally:
instance1.Release()
for node in nodes:
qa_node.NodeRemove(node)
for node in nodes:
qa_node.NodeAdd(node)
qa_cluster.AssertClusterVerify()
def __init__(self, resolver, encoder_fn, lock_monitor_cb=None,
_req_process_fn=None):
"""Initializes this class.
"""
proc = _RpcProcessor(resolver,
netutils.GetDaemonPort(constants.NODED),
lock_monitor_cb=lock_monitor_cb)
self._proc = compat.partial(proc, _req_process_fn=_req_process_fn)
self._encoder = compat.partial(self._EncodeArg, encoder_fn)
def ConfigureGroups():
"""Configures groups and nodes for tests such as custom SSH ports.
"""
defgroup = GetDefaultGroup()
nodes = qa_config.get("nodes")
options = qa_config.get("options", {})
# Clear any old configuration
qa_iptables.CleanRules(nodes)
# Custom SSH ports:
ssh_port = options.get("ssh-port")
default_ssh_port = netutils.GetDaemonPort(constants.SSH)
if (ssh_port is not None) and (ssh_port != default_ssh_port):
ModifyGroupSshPort(qa_iptables.GLOBAL_RULES, defgroup, nodes, ssh_port)
def RunSshCmdWithStdin(cluster_name,
node,
basecmd,
port,
data,
debug=False,
verbose=False,
use_cluster_key=False,
ask_key=False,
strict_host_check=False,
ensure_version=False):
"""Runs a command on a remote machine via SSH and provides input in stdin.
@type cluster_name: string
@param cluster_name: Cluster name
@type node: string
@param node: Node name
@type basecmd: string
@param basecmd: Base command (path on the remote machine)
@type port: int
@param port: The SSH port of the remote machine or None for the default
@param data: JSON-serializable input data for script (passed to stdin)
@type debug: bool
@param debug: Enable debug output
@type verbose: bool
@param verbose: Enable verbose output
@type use_cluster_key: bool
@param use_cluster_key: See L{ssh.SshRunner.BuildCmd}
@type ask_key: bool
@param ask_key: See L{ssh.SshRunner.BuildCmd}
@type strict_host_check: bool
@param strict_host_check: See L{ssh.SshRunner.BuildCmd}
"""
cmd = [basecmd]
# Pass --debug/--verbose to the external script if set on our invocation
if debug:
cmd.append("--debug")
if verbose:
cmd.append("--verbose")
if ensure_version:
all_cmds = _EnsureCorrectGanetiVersion(cmd)
else:
all_cmds = [cmd]
if port is None:
port = netutils.GetDaemonPort(constants.SSH)
srun = SshRunner(cluster_name)
scmd = srun.BuildCmd(node,
constants.SSH_LOGIN_USER,
utils.ShellQuoteArgs(
utils.ShellCombineCommands(all_cmds)),
batch=False,
ask_key=ask_key,
quiet=False,
strict_host_check=strict_host_check,
use_cluster_key=use_cluster_key,
port=port)
tempfh = tempfile.TemporaryFile()
try:
tempfh.write(serializer.DumpJson(data))
tempfh.seek(0)
result = utils.RunCmd(scmd, interactive=True, input_fd=tempfh)
finally:
tempfh.close()
if result.failed:
raise errors.OpExecError("Command '%s' failed: %s" %
(result.cmd, result.fail_reason))
def GenericMain(daemon_name,
optionparser,
check_fn,
prepare_fn,
exec_fn,
multithreaded=False,
console_logging=False,
default_ssl_cert=None,
default_ssl_key=None,
warn_breach=False):
"""Shared main function for daemons.
@type daemon_name: string
@param daemon_name: daemon name
@type optionparser: optparse.OptionParser
@param optionparser: initialized optionparser with daemon-specific options
(common -f -d options will be handled by this module)
@type check_fn: function which accepts (options, args)
@param check_fn: function that checks start conditions and exits if they're
not met
@type prepare_fn: function which accepts (options, args)
@param prepare_fn: function that is run before forking, or None;
it's result will be passed as the third parameter to exec_fn, or
if None was passed in, we will just pass None to exec_fn
@type exec_fn: function which accepts (options, args, prepare_results)
@param exec_fn: function that's executed with the daemon's pid file held, and
runs the daemon itself.
@type multithreaded: bool
@param multithreaded: Whether the daemon uses threads
@type console_logging: boolean
@param console_logging: if True, the daemon will fall back to the system
console if logging fails
@type default_ssl_cert: string
@param default_ssl_cert: Default SSL certificate path
@type default_ssl_key: string
@param default_ssl_key: Default SSL key path
@type warn_breach: bool
@param warn_breach: issue a warning at daemon launch time, before
daemonizing, about the possibility of breaking parameter privacy
invariants through the otherwise helpful debug logging.
"""
optionparser.add_option("-f",
"--foreground",
dest="fork",
help="Don't detach from the current terminal",
default=True,
action="store_false")
optionparser.add_option("-d",
"--debug",
dest="debug",
help="Enable some debug messages",
default=False,
action="store_true")
optionparser.add_option("--syslog",
dest="syslog",
help="Enable logging to syslog (except debug"
" messages); one of 'no', 'yes' or 'only' [%s]" %
constants.SYSLOG_USAGE,
default=constants.SYSLOG_USAGE,
choices=["no", "yes", "only"])
family = ssconf.SimpleStore().GetPrimaryIPFamily()
# family will default to AF_INET if there is no ssconf file (e.g. when
# upgrading a cluster from 2.2 -> 2.3. This is intended, as Ganeti clusters
# <= 2.2 can not be AF_INET6
if daemon_name in constants.DAEMONS_PORTS:
default_bind_address = constants.IP4_ADDRESS_ANY
if family == netutils.IP6Address.family:
default_bind_address = constants.IP6_ADDRESS_ANY
default_port = netutils.GetDaemonPort(daemon_name)
# For networked daemons we allow choosing the port and bind address
optionparser.add_option("-p",
"--port",
dest="port",
help="Network port (default: %s)" %
default_port,
default=default_port,
type="int")
optionparser.add_option("-b",
"--bind",
dest="bind_address",
help=("Bind address (default: '%s')" %
default_bind_address),
default=default_bind_address,
metavar="ADDRESS")
optionparser.add_option("-i",
"--interface",
dest="bind_interface",
help=("Bind interface"),
metavar="INTERFACE")
if default_ssl_key is not None and default_ssl_cert is not None:
optionparser.add_option("--no-ssl",
dest="ssl",
help="Do not secure HTTP protocol with SSL",
default=True,
action="store_false")
optionparser.add_option("-K",
"--ssl-key",
dest="ssl_key",
help=("SSL key path (default: %s)" %
default_ssl_key),
default=default_ssl_key,
type="string",
metavar="SSL_KEY_PATH")
optionparser.add_option("-C",
"--ssl-cert",
dest="ssl_cert",
help=("SSL certificate path (default: %s)" %
default_ssl_cert),
default=default_ssl_cert,
type="string",
metavar="SSL_CERT_PATH")
# Disable the use of fork(2) if the daemon uses threads
if multithreaded:
utils.DisableFork()
options, args = optionparser.parse_args()
if getattr(options, "bind_interface", None) is not None:
if options.bind_address != default_bind_address:
msg = (
"Can't specify both, bind address (%s) and bind interface (%s)"
% (options.bind_address, options.bind_interface))
print(msg, file=sys.stderr)
sys.exit(constants.EXIT_FAILURE)
interface_ip_addresses = \
netutils.GetInterfaceIpAddresses(options.bind_interface)
if family == netutils.IP6Address.family:
if_addresses = interface_ip_addresses[constants.IP6_VERSION]
else:
if_addresses = interface_ip_addresses[constants.IP4_VERSION]
if len(if_addresses) < 1:
msg = "Failed to find IP for interface %s" % options.bind_interace
print(msg, file=sys.stderr)
sys.exit(constants.EXIT_FAILURE)
options.bind_address = if_addresses[0]
if getattr(options, "ssl", False):
ssl_paths = {
"certificate": options.ssl_cert,
"key": options.ssl_key,
}
for name, path in ssl_paths.items():
if not os.path.isfile(path):
print("SSL %s file '%s' was not found" % (name, path),
file=sys.stderr)
sys.exit(constants.EXIT_FAILURE)
# TODO: By initiating http.HttpSslParams here we would only read the files
# once and have a proper validation (isfile returns False on directories)
# at the same time.
result, running_uid, expected_uid = _VerifyDaemonUser(daemon_name)
if not result:
msg = ("%s started using wrong user ID (%d), expected %d" %
(daemon_name, running_uid, expected_uid))
print(msg, file=sys.stderr)
sys.exit(constants.EXIT_FAILURE)
if check_fn is not None:
check_fn(options, args)
log_filename = constants.DAEMONS_LOGFILES[daemon_name]
# node-daemon logging in lib/http/server.py, _HandleServerRequestInner
if options.debug and warn_breach:
sys.stderr.write(constants.DEBUG_MODE_CONFIDENTIALITY_WARNING %
daemon_name)
if options.fork:
# Newer GnuTLS versions (>= 3.3.0) use a library constructor for
# initialization and open /dev/urandom on library load time, way before we
# fork(). Closing /dev/urandom causes subsequent ganeti.http.client
# requests to fail and the process to receive a SIGABRT. As we cannot
# reliably detect GnuTLS's socket, we work our way around this by keeping
# all fds referring to /dev/urandom open.
noclose_fds = []
for fd in os.listdir("/proc/self/fd"):
try:
if os.readlink(os.path.join("/proc/self/fd",
fd)) == "/dev/urandom":
noclose_fds.append(int(fd))
except EnvironmentError:
# The fd might have disappeared (although it shouldn't as we're running
# single-threaded).
continue
utils.CloseFDs(noclose_fds=noclose_fds)
(wpipe, stdio_reopen_fn) = utils.Daemonize(logfile=log_filename)
else:
(wpipe, stdio_reopen_fn) = (None, None)
log_reopen_fn = \
utils.SetupLogging(log_filename, daemon_name,
debug=options.debug,
stderr_logging=not options.fork,
multithreaded=multithreaded,
syslog=options.syslog,
console_logging=console_logging)
# Reopen log file(s) on SIGHUP
signal.signal(
signal.SIGHUP,
compat.partial(_HandleSigHup, [log_reopen_fn, stdio_reopen_fn]))
try:
utils.WritePidFile(utils.DaemonPidFileName(daemon_name))
except errors.PidFileLockError as err:
print("Error while locking PID file:\n%s" % err, file=sys.stderr)
sys.exit(constants.EXIT_FAILURE)
try:
try:
logging.info("%s daemon startup", daemon_name)
if callable(prepare_fn):
prep_results = prepare_fn(options, args)
else:
prep_results = None
except Exception as err:
utils.WriteErrorToFD(wpipe, _BeautifyError(err))
raise
if wpipe is not None:
# we're done with the preparation phase, we close the pipe to
# let the parent know it's safe to exit
os.close(wpipe)
exec_fn(options, args, prep_results)
finally:
utils.RemoveFile(utils.DaemonPidFileName(daemon_name))
def RunNodeSetupCmd(cluster_name, node, basecmd, debug, verbose,
use_cluster_key, ask_key, strict_host_check,
port, data):
"""Runs a command to configure something on a remote machine.
@type cluster_name: string
@param cluster_name: Cluster name
@type node: string
@param node: Node name
@type basecmd: string
@param basecmd: Base command (path on the remote machine)
@type debug: bool
@param debug: Enable debug output
@type verbose: bool
@param verbose: Enable verbose output
@type use_cluster_key: bool
@param use_cluster_key: See L{ssh.SshRunner.BuildCmd}
@type ask_key: bool
@param ask_key: See L{ssh.SshRunner.BuildCmd}
@type strict_host_check: bool
@param strict_host_check: See L{ssh.SshRunner.BuildCmd}
@type port: int
@param port: The SSH port of the remote machine or None for the default
@param data: JSON-serializable input data for script (passed to stdin)
"""
cmd = [basecmd]
# Pass --debug/--verbose to the external script if set on our invocation
if debug:
cmd.append("--debug")
if verbose:
cmd.append("--verbose")
logging.debug("Node setup command: %s", cmd)
version = constants.DIR_VERSION
all_cmds = [["test", "-d", os.path.join(pathutils.PKGLIBDIR, version)]]
if constants.HAS_GNU_LN:
all_cmds.extend([["ln", "-s", "-f", "-T",
os.path.join(pathutils.PKGLIBDIR, version),
os.path.join(pathutils.SYSCONFDIR, "ganeti/lib")],
["ln", "-s", "-f", "-T",
os.path.join(pathutils.SHAREDIR, version),
os.path.join(pathutils.SYSCONFDIR, "ganeti/share")]])
else:
all_cmds.extend([["rm", "-f",
os.path.join(pathutils.SYSCONFDIR, "ganeti/lib")],
["ln", "-s", "-f",
os.path.join(pathutils.PKGLIBDIR, version),
os.path.join(pathutils.SYSCONFDIR, "ganeti/lib")],
["rm", "-f",
os.path.join(pathutils.SYSCONFDIR, "ganeti/share")],
["ln", "-s", "-f",
os.path.join(pathutils.SHAREDIR, version),
os.path.join(pathutils.SYSCONFDIR, "ganeti/share")]])
all_cmds.append(cmd)
if port is None:
port = netutils.GetDaemonPort(constants.SSH)
family = ssconf.SimpleStore().GetPrimaryIPFamily()
srun = ssh.SshRunner(cluster_name,
ipv6=(family == netutils.IP6Address.family))
scmd = srun.BuildCmd(node, constants.SSH_LOGIN_USER,
utils.ShellQuoteArgs(
utils.ShellCombineCommands(all_cmds)),
batch=False, ask_key=ask_key, quiet=False,
strict_host_check=strict_host_check,
use_cluster_key=use_cluster_key,
port=port)
tempfh = tempfile.TemporaryFile()
try:
tempfh.write(serializer.DumpJson(data))
tempfh.seek(0)
result = utils.RunCmd(scmd, interactive=True, input_fd=tempfh)
finally:
tempfh.close()
if result.failed:
raise errors.OpExecError("Command '%s' failed: %s" %
(result.cmd, result.fail_reason))
_WaitForSshDaemon(node, port, family)
def GenericMain(daemon_name,
optionparser,
check_fn,
prepare_fn,
exec_fn,
multithreaded=False,
console_logging=False,
default_ssl_cert=None,
default_ssl_key=None):
"""Shared main function for daemons.
@type daemon_name: string
@param daemon_name: daemon name
@type optionparser: optparse.OptionParser
@param optionparser: initialized optionparser with daemon-specific options
(common -f -d options will be handled by this module)
@type check_fn: function which accepts (options, args)
@param check_fn: function that checks start conditions and exits if they're
not met
@type prepare_fn: function which accepts (options, args)
@param prepare_fn: function that is run before forking, or None;
it's result will be passed as the third parameter to exec_fn, or
if None was passed in, we will just pass None to exec_fn
@type exec_fn: function which accepts (options, args, prepare_results)
@param exec_fn: function that's executed with the daemon's pid file held, and
runs the daemon itself.
@type multithreaded: bool
@param multithreaded: Whether the daemon uses threads
@type console_logging: boolean
@param console_logging: if True, the daemon will fall back to the system
console if logging fails
@type default_ssl_cert: string
@param default_ssl_cert: Default SSL certificate path
@type default_ssl_key: string
@param default_ssl_key: Default SSL key path
"""
optionparser.add_option("-f",
"--foreground",
dest="fork",
help="Don't detach from the current terminal",
default=True,
action="store_false")
optionparser.add_option("-d",
"--debug",
dest="debug",
help="Enable some debug messages",
default=False,
action="store_true")
optionparser.add_option("--syslog",
dest="syslog",
help="Enable logging to syslog (except debug"
" messages); one of 'no', 'yes' or 'only' [%s]" %
constants.SYSLOG_USAGE,
default=constants.SYSLOG_USAGE,
choices=["no", "yes", "only"])
family = ssconf.SimpleStore().GetPrimaryIPFamily()
# family will default to AF_INET if there is no ssconf file (e.g. when
# upgrading a cluster from 2.2 -> 2.3. This is intended, as Ganeti clusters
# <= 2.2 can not be AF_INET6
if daemon_name in constants.DAEMONS_PORTS:
default_bind_address = constants.IP4_ADDRESS_ANY
if family == netutils.IP6Address.family:
default_bind_address = constants.IP6_ADDRESS_ANY
default_port = netutils.GetDaemonPort(daemon_name)
# For networked daemons we allow choosing the port and bind address
optionparser.add_option("-p",
"--port",
dest="port",
help="Network port (default: %s)" %
default_port,
default=default_port,
type="int")
optionparser.add_option("-b",
"--bind",
dest="bind_address",
help=("Bind address (default: '%s')" %
default_bind_address),
default=default_bind_address,
metavar="ADDRESS")
optionparser.add_option("-i",
"--interface",
dest="bind_interface",
help=("Bind interface"),
metavar="INTERFACE")
if default_ssl_key is not None and default_ssl_cert is not None:
optionparser.add_option("--no-ssl",
dest="ssl",
help="Do not secure HTTP protocol with SSL",
default=True,
action="store_false")
optionparser.add_option("-K",
"--ssl-key",
dest="ssl_key",
help=("SSL key path (default: %s)" %
default_ssl_key),
default=default_ssl_key,
type="string",
metavar="SSL_KEY_PATH")
optionparser.add_option("-C",
"--ssl-cert",
dest="ssl_cert",
help=("SSL certificate path (default: %s)" %
default_ssl_cert),
default=default_ssl_cert,
type="string",
metavar="SSL_CERT_PATH")
# Disable the use of fork(2) if the daemon uses threads
if multithreaded:
utils.DisableFork()
options, args = optionparser.parse_args()
if getattr(options, "bind_interface", None) is not None:
if options.bind_address != default_bind_address:
msg = (
"Can't specify both, bind address (%s) and bind interface (%s)"
% (options.bind_address, options.bind_interface))
print >> sys.stderr, msg
sys.exit(constants.EXIT_FAILURE)
interface_ip_addresses = \
netutils.GetInterfaceIpAddresses(options.bind_interface)
if family == netutils.IP6Address.family:
if_addresses = interface_ip_addresses[constants.IP6_VERSION]
else:
if_addresses = interface_ip_addresses[constants.IP4_VERSION]
if len(if_addresses) < 1:
msg = "Failed to find IP for interface %s" % options.bind_interace
print >> sys.stderr, msg
sys.exit(constants.EXIT_FAILURE)
options.bind_address = if_addresses[0]
if getattr(options, "ssl", False):
ssl_paths = {
"certificate": options.ssl_cert,
"key": options.ssl_key,
}
for name, path in ssl_paths.iteritems():
if not os.path.isfile(path):
print >> sys.stderr, "SSL %s file '%s' was not found" % (name,
path)
sys.exit(constants.EXIT_FAILURE)
# TODO: By initiating http.HttpSslParams here we would only read the files
# once and have a proper validation (isfile returns False on directories)
# at the same time.
result, running_uid, expected_uid = _VerifyDaemonUser(daemon_name)
if not result:
msg = ("%s started using wrong user ID (%d), expected %d" %
(daemon_name, running_uid, expected_uid))
print >> sys.stderr, msg
sys.exit(constants.EXIT_FAILURE)
if check_fn is not None:
check_fn(options, args)
log_filename = constants.DAEMONS_LOGFILES[daemon_name]
if options.fork:
utils.CloseFDs()
(wpipe, stdio_reopen_fn) = utils.Daemonize(logfile=log_filename)
else:
(wpipe, stdio_reopen_fn) = (None, None)
log_reopen_fn = \
utils.SetupLogging(log_filename, daemon_name,
debug=options.debug,
stderr_logging=not options.fork,
multithreaded=multithreaded,
syslog=options.syslog,
console_logging=console_logging)
# Reopen log file(s) on SIGHUP
signal.signal(
signal.SIGHUP,
compat.partial(_HandleSigHup, [log_reopen_fn, stdio_reopen_fn]))
try:
utils.WritePidFile(utils.DaemonPidFileName(daemon_name))
except errors.PidFileLockError, err:
print >> sys.stderr, "Error while locking PID file:\n%s" % err
sys.exit(constants.EXIT_FAILURE)
