def node_cmd(host, host_uuid, task, cmd, args, opts):
"""
Runs command via ssh if host is not local
"""
localdir = is_host_local(host_uuid)
# this is so to avoid deleting the ssh keys on local node which otherwise
# cause ssh password prompts on the console (race conditions)
# mode_delete() should be cleaning up the session tree
if localdir and task == "delete":
return
pem_key_path = get_pem_key_path(args.session, args.volume)
if not localdir:
# prefix with ssh command if not local node
cmd = ["ssh",
"-i", pem_key_path,
"root@%s" % host] + cmd
execute(cmd, exit_msg="%s - %s failed" % (host, task), logger=logger)
if opts.get("copy_outfile", False):
cmd_copy = ["scp",
"-i", pem_key_path,
"root@%s:/%s" % (host, opts.get("node_outfile")),
os.path.dirname(opts.get("node_outfile"))]
execute(cmd_copy, exit_msg="%s - Copy command failed" % host,
logger=logger)
def node_cmd(host, host_uuid, task, cmd, args, opts):
"""
Runs command via ssh if host is not local
"""
localdir = is_host_local(host_uuid)
# this is so to avoid deleting the ssh keys on local node which otherwise
# cause ssh password prompts on the console (race conditions)
# mode_delete() should be cleaning up the session tree
if localdir and task == "delete":
return
pem_key_path = get_pem_key_path(args.session, args.volume)
if not localdir:
# prefix with ssh command if not local node
cmd = ["ssh", "-i", pem_key_path, "root@%s" % host] + cmd
execute(cmd, exit_msg="%s - %s failed" % (host, task), logger=logger)
if opts.get("copy_outfile", False):
cmd_copy = [
"scp", "-i", pem_key_path,
"root@%s:/%s" % (host, opts.get("node_outfile")),
os.path.dirname(opts.get("node_outfile"))
]
execute(cmd_copy,
exit_msg="%s - Copy command failed" % host,
logger=logger)
def node_cmd(host, host_uuid, task, cmd, args, opts):
"""
Runs command via ssh if host is not local
"""
try:
localdir = is_host_local(host_uuid)
# this is so to avoid deleting the ssh keys on local node which
# otherwise cause ssh password prompts on the console (race conditions)
# mode_delete() should be cleaning up the session tree
if localdir and task == "delete":
return
pem_key_path = get_pem_key_path(args.session, args.volume)
if not localdir:
# prefix with ssh command if not local node
cmd = [
"ssh",
"-oNumberOfPasswordPrompts=0",
"-oStrictHostKeyChecking=no",
# We force TTY allocation (-t -t) so that Ctrl+C is handed
# through; see:
# https://bugzilla.redhat.com/show_bug.cgi?id=1382236
# Note that this turns stderr of the remote `cmd`
# into stdout locally.
"-t",
"-t",
"-i",
pem_key_path,
"root@%s" % host
] + cmd
(returncode, err, out) = execute(cmd, logger=logger)
if returncode != 0:
# Because the `-t -t` above turns the remote stderr into
# local stdout, we need to log both stderr and stdout
# here to print all error messages.
fail("%s - %s failed; stdout (including remote stderr):\n"
"%s\n"
"stderr:\n"
"%s" % (host, task, out, err),
returncode,
logger=logger)
if opts.get("copy_outfile", False) and not localdir:
cmd_copy = [
"scp", "-oNumberOfPasswordPrompts=0",
"-oStrictHostKeyChecking=no", "-i", pem_key_path,
"root@%s:/%s" % (host, opts.get("node_outfile")),
os.path.dirname(opts.get("node_outfile"))
]
execute(cmd_copy,
exit_msg="%s - Copy command failed" % host,
logger=logger)
except KeyboardInterrupt:
sys.exit(2)
def node_cmd(host, host_uuid, task, cmd, args, opts):
"""
Runs command via ssh if host is not local
"""
try:
localdir = is_host_local(host_uuid)
# this is so to avoid deleting the ssh keys on local node which
# otherwise cause ssh password prompts on the console (race conditions)
# mode_delete() should be cleaning up the session tree
if localdir and task == "delete":
return
pem_key_path = get_pem_key_path(args.session, args.volume)
if not localdir:
# prefix with ssh command if not local node
cmd = ["ssh",
"-oNumberOfPasswordPrompts=0",
"-oStrictHostKeyChecking=no",
# We force TTY allocation (-t -t) so that Ctrl+C is handed
# through; see:
# https://bugzilla.redhat.com/show_bug.cgi?id=1382236
# Note that this turns stderr of the remote `cmd`
# into stdout locally.
"-t",
"-t",
"-i", pem_key_path,
"root@%s" % host] + cmd
(returncode, err, out) = execute(cmd, logger=logger)
if returncode != 0:
# Because the `-t -t` above turns the remote stderr into
# local stdout, we need to log both stderr and stdout
# here to print all error messages.
fail("%s - %s failed; stdout (including remote stderr):\n"
"%s\n"
"stderr:\n"
"%s" % (host, task, out, err),
returncode,
logger=logger)
if opts.get("copy_outfile", False) and not localdir:
cmd_copy = ["scp",
"-oNumberOfPasswordPrompts=0",
"-oStrictHostKeyChecking=no",
"-i", pem_key_path,
"root@%s:/%s" % (host, opts.get("node_outfile")),
os.path.dirname(opts.get("node_outfile"))]
execute(cmd_copy, exit_msg="%s - Copy command failed" % host,
logger=logger)
except KeyboardInterrupt:
sys.exit(2)
def node_cleanup(host, args):
localdir = is_host_local(host)
# CHANGE_DETECTOR <SESSION> <VOLUME> <BRICK> <OUTFILE> <START> --debug
# --gfidpath <TYPE>
cmd = [conf.get_opt("nodecleanup"), args.session, args.volume
] + (["--debug"] if args.debug else [])
if not localdir:
# prefix with ssh command if not local node
cmd = ["ssh", "-i", conf.get_opt("secret_pem"), "root@%s" % host] + cmd
execute(cmd, exit_msg="%s - Cleanup failed" % host, logger=logger)
def node_run(volume, host, path, start, outfile, args, fallback=False):
"""
If host is local node, execute the command locally. If not local
execute the CHANGE_DETECTOR command via ssh and copy the output file from
remote node using scp.
"""
localdir = is_host_local(host)
pem_key_path = get_pem_key_path(args.session, args.volume)
# If Full backup is requested or start time is zero, use brickfind
change_detector = conf.get_change_detector(args.change_detector)
if ((start == 0 or args.full) and args.change_detector == "changelog") or \
fallback:
change_detector = conf.get_change_detector("brickfind")
# CHANGE_DETECTOR <SESSION> <VOLUME> <BRICK> <OUTFILE> <START> --debug
# --gfidpath <TYPE>
cmd = [change_detector,
args.session,
volume,
path,
outfile,
str(start),
"--output-prefix",
args.output_prefix] + \
(["--debug"] if args.debug else []) + \
(["--full"] if args.full else [])
if not localdir:
# prefix with ssh command if not local node
cmd = ["ssh",
"-i", pem_key_path,
"root@%s" % host] + cmd
rc, out, err = execute(cmd, logger=logger)
if rc == 2:
# Partial History Fallback
logger.info("%s %s Fallback to brickfind" % (host, err.strip()))
# Exit only from process, handled in main.
sys.exit(rc)
elif rc != 0:
fail("%s - Change detection failed" % host, logger=logger)
if not localdir:
cmd_copy = ["scp",
"-i", pem_key_path,
"root@%s:/%s" % (host, outfile),
os.path.dirname(outfile)]
execute(cmd_copy, exit_msg="%s - Copy command failed" % host,
logger=logger)
def node_run(volume, host, path, start, outfile, args, fallback=False):
"""
If host is local node, execute the command locally. If not local
execute the CHANGE_DETECTOR command via ssh and copy the output file from
remote node using scp.
"""
localdir = is_host_local(host)
# If Full backup is requested or start time is zero, use brickfind
change_detector = conf.get_change_detector(args.change_detector)
if ((start == 0 or args.full) and args.change_detector == "changelog") or \
fallback:
change_detector = conf.get_change_detector("brickfind")
# CHANGE_DETECTOR <SESSION> <VOLUME> <BRICK> <OUTFILE> <START> --debug
# --gfidpath <TYPE>
cmd = [change_detector,
args.session,
volume,
path,
outfile,
str(start),
"--output-prefix",
args.output_prefix] + \
(["--debug"] if args.debug else []) + \
(["--full"] if args.full else [])
if not localdir:
# prefix with ssh command if not local node
cmd = ["ssh", "-i", conf.get_opt("secret_pem"), "root@%s" % host] + cmd
rc, out, err = execute(cmd, logger=logger)
if rc == 2:
# Partial History Fallback
logger.info("%s %s Fallback to brickfind" % (host, err.strip()))
# Exit only from process, handled in main.
sys.exit(rc)
elif rc != 0:
fail("%s - Change detection failed" % host, logger=logger)
if not localdir:
cmd_copy = [
"scp", "-i",
conf.get_opt("secret_pem"),
"root@%s:/%s" % (host, outfile),
os.path.dirname(outfile)
]
execute(cmd_copy,
exit_msg="%s - Copy command failed" % host,
logger=logger)
def node_cleanup(host, args):
localdir = is_host_local(host)
# CHANGE_DETECTOR <SESSION> <VOLUME> <BRICK> <OUTFILE> <START> --debug
# --gfidpath <TYPE>
cmd = [conf.get_opt("nodecleanup"),
args.session,
args.volume] + (["--debug"] if args.debug else [])
if not localdir:
# prefix with ssh command if not local node
cmd = ["ssh",
"-i", conf.get_opt("secret_pem"),
"root@%s" % host] + cmd
execute(cmd, exit_msg="%s - Cleanup failed" % host, logger=logger)
def is_local(self):
"""
Returns true if the connector is running locally.
Raises a ConfigurationError if called on a cluster.
"""
if self._uri_wrapper.is_cluster_uri():
raise ConfigurationError("Cannot call is_local() on '%s' because"
" it is a cluster" % self)
try:
server_host = self.address.split(":")[0]
return server_host is None or is_host_local(server_host)
except Exception, e:
logger.error("Unable to resolve address for server '%s'."
" Cause: %s" % (self, e))
def is_local(self):
"""
Returns true if the connector is running locally.
Raises a ConfigurationError if called on a cluster.
"""
if self._uri_wrapper.is_cluster_uri():
raise ConfigurationError("Cannot call is_local() on '%s' because"
" it is a cluster" % self)
try:
server_host = self.host
return server_host is None or is_host_local(server_host)
except Exception, e:
logger.error("Unable to resolve address for server '%s'."
" Cause: %s" % (self, e))
def node_cmd(host, host_uuid, task, cmd, args, opts):
"""
Runs command via ssh if host is not local
"""
localdir = is_host_local(host_uuid)
pem_key_path = get_pem_key_path(args.session, args.volume)
if not localdir:
# prefix with ssh command if not local node
cmd = ["ssh",
"-i", pem_key_path,
"root@%s" % host] + cmd
execute(cmd, exit_msg="%s - %s failed" % (host, task), logger=logger)
if opts.get("copy_outfile", False):
cmd_copy = ["scp",
"-i", pem_key_path,
"root@%s:/%s" % (host, opts.get("node_outfile")),
os.path.dirname(opts.get("node_outfile"))]
execute(cmd_copy, exit_msg="%s - Copy command failed" % host,
logger=logger)
def node_cmd(host, host_uuid, task, cmd, args, opts):
"""
Runs command via ssh if host is not local
"""
localdir = is_host_local(host_uuid)
pem_key_path = get_pem_key_path(args.session, args.volume)
if not localdir:
# prefix with ssh command if not local node
cmd = ["ssh", "-i", pem_key_path, "root@%s" % host] + cmd
execute(cmd, exit_msg="%s - %s failed" % (host, task), logger=logger)
if opts.get("copy_outfile", False):
cmd_copy = [
"scp", "-i", pem_key_path,
"root@%s:/%s" % (host, opts.get("node_outfile")),
os.path.dirname(opts.get("node_outfile"))
]
execute(cmd_copy,
exit_msg="%s - Copy command failed" % host,
logger=logger)
def run_cmd_nodes(task, args, **kwargs):
global node_outfiles
nodes = get_nodes(args.volume)
pool = []
for num, node in enumerate(nodes):
host, brick = node[1].split(":")
host_uuid = node[0]
cmd = []
opts = {}
# tmpfilename is valid only for tasks: pre, query and cleanup
tmpfilename = kwargs.get("tmpfilename", "BADNAME")
node_outfile = os.path.join(conf.get_opt("working_dir"), args.session,
args.volume, tmpfilename,
"tmp_output_%s" % num)
if task == "pre":
if vol_statusStr != "Started":
fail("Volume %s is not online" % args.volume, logger=logger)
# If Full backup is requested or start time is zero, use brickfind
change_detector = conf.get_change_detector("changelog")
tag = None
if args.full:
change_detector = conf.get_change_detector("brickfind")
tag = args.tag_for_full_find.strip()
if tag == "":
tag = '""' if not is_host_local(host_uuid) else ""
node_outfiles.append(node_outfile)
# remote file will be copied into this directory
mkdirp(os.path.dirname(node_outfile),
exit_on_err=True,
logger=logger)
FS = args.field_separator
if not is_host_local(host_uuid):
FS = "'" + FS + "'"
cmd = [change_detector,
args.session,
args.volume,
host,
brick,
node_outfile] + \
([str(kwargs.get("start")), str(kwargs.get("end"))]
if not args.full else []) + \
([tag] if tag is not None else []) + \
["--output-prefix", args.output_prefix] + \
(["--debug"] if args.debug else []) + \
(["--no-encode"] if args.no_encode else []) + \
(["--only-namespace-changes"] if args.only_namespace_changes
else []) + \
(["--field-separator", FS] if args.full else [])
opts["node_outfile"] = node_outfile
opts["copy_outfile"] = True
elif task == "query":
# If Full backup is requested or start time is zero, use brickfind
tag = None
change_detector = conf.get_change_detector("changelog")
if args.full:
change_detector = conf.get_change_detector("brickfind")
tag = args.tag_for_full_find.strip()
if tag == "":
tag = '""' if not is_host_local(host_uuid) else ""
node_outfiles.append(node_outfile)
# remote file will be copied into this directory
mkdirp(os.path.dirname(node_outfile),
exit_on_err=True,
logger=logger)
FS = args.field_separator
if not is_host_local(host_uuid):
FS = "'" + FS + "'"
cmd = [change_detector,
args.session,
args.volume,
host,
brick,
node_outfile] + \
([str(kwargs.get("start")), str(kwargs.get("end"))]
if not args.full else []) + \
([tag] if tag is not None else []) + \
["--only-query"] + \
["--output-prefix", args.output_prefix] + \
(["--debug"] if args.debug else []) + \
(["--no-encode"] if args.no_encode else []) + \
(["--only-namespace-changes"]
if args.only_namespace_changes else []) + \
(["--field-separator", FS] if args.full else [])
opts["node_outfile"] = node_outfile
opts["copy_outfile"] = True
elif task == "cleanup":
# After pre/query run, cleanup the working directory and other
# temp files. Remove the directory to which node_outfile has
# been copied in main node
try:
os.remove(node_outfile)
except (OSError, IOError):
logger.warn("Failed to cleanup temporary file %s" %
node_outfile)
pass
cmd = [conf.get_opt("nodeagent"),
"cleanup",
args.session,
args.volume,
os.path.dirname(node_outfile)] + \
(["--debug"] if args.debug else [])
elif task == "create":
if vol_statusStr != "Started":
fail("Volume %s is not online" % args.volume, logger=logger)
# When glusterfind create, create session directory in
# each brick nodes
cmd = [conf.get_opt("nodeagent"),
"create",
args.session,
args.volume,
brick,
kwargs.get("time_to_update")] + \
(["--debug"] if args.debug else []) + \
(["--reset-session-time"] if args.reset_session_time
else [])
elif task == "post":
# Rename pre status file to actual status file in each node
cmd = [conf.get_opt("nodeagent"),
"post",
args.session,
args.volume,
brick] + \
(["--debug"] if args.debug else [])
elif task == "delete":
# When glusterfind delete, cleanup all the session files/dirs
# from each node.
cmd = [conf.get_opt("nodeagent"),
"delete",
args.session,
args.volume] + \
(["--debug"] if args.debug else [])
if cmd:
p = Process(target=node_cmd,
args=(host, host_uuid, task, cmd, args, opts))
p.start()
pool.append(p)
for num, p in enumerate(pool):
p.join()
if p.exitcode != 0:
logger.warn("Command %s failed in %s" % (task, nodes[num][1]))
if task in ["create", "delete"]:
fail("Command %s failed in %s" % (task, nodes[num][1]))
elif task == "pre" and args.disable_partial:
sys.exit(1)
def run_cmd_nodes(task, args, **kwargs):
global node_outfiles
nodes = get_nodes(args.volume)
pool = []
for num, node in enumerate(nodes):
host, brick = node[1].split(":")
host_uuid = node[0]
cmd = []
opts = {}
# tmpfilename is valid only for tasks: pre, query and cleanup
tmpfilename = kwargs.get("tmpfilename", "BADNAME")
node_outfile = os.path.join(conf.get_opt("working_dir"),
args.session, args.volume,
tmpfilename,
"tmp_output_%s" % num)
if task == "pre":
if vol_statusStr != "Started":
fail("Volume %s is not online" % args.volume,
logger=logger)
# If Full backup is requested or start time is zero, use brickfind
change_detector = conf.get_change_detector("changelog")
tag = None
if args.full:
change_detector = conf.get_change_detector("brickfind")
tag = args.tag_for_full_find.strip()
if tag == "":
tag = '""' if not is_host_local(host_uuid) else ""
node_outfiles.append(node_outfile)
# remote file will be copied into this directory
mkdirp(os.path.dirname(node_outfile),
exit_on_err=True, logger=logger)
FS = args.field_separator
if not is_host_local(host_uuid):
FS = "'" + FS + "'"
cmd = [change_detector,
args.session,
args.volume,
host,
brick,
node_outfile] + \
([str(kwargs.get("start")), str(kwargs.get("end"))]
if not args.full else []) + \
([tag] if tag is not None else []) + \
["--output-prefix", args.output_prefix] + \
(["--debug"] if args.debug else []) + \
(["--no-encode"] if args.no_encode else []) + \
(["--only-namespace-changes"] if args.only_namespace_changes
else []) + \
(["--type", args.type]) + \
(["--field-separator", FS] if args.full else [])
opts["node_outfile"] = node_outfile
opts["copy_outfile"] = True
elif task == "query":
# If Full backup is requested or start time is zero, use brickfind
tag = None
change_detector = conf.get_change_detector("changelog")
if args.full:
change_detector = conf.get_change_detector("brickfind")
tag = args.tag_for_full_find.strip()
if tag == "":
tag = '""' if not is_host_local(host_uuid) else ""
node_outfiles.append(node_outfile)
# remote file will be copied into this directory
mkdirp(os.path.dirname(node_outfile),
exit_on_err=True, logger=logger)
FS = args.field_separator
if not is_host_local(host_uuid):
FS = "'" + FS + "'"
cmd = [change_detector,
args.session,
args.volume,
host,
brick,
node_outfile] + \
([str(kwargs.get("start")), str(kwargs.get("end"))]
if not args.full else []) + \
([tag] if tag is not None else []) + \
["--only-query"] + \
["--output-prefix", args.output_prefix] + \
(["--debug"] if args.debug else []) + \
(["--no-encode"] if args.no_encode else []) + \
(["--only-namespace-changes"]
if args.only_namespace_changes else []) + \
(["--type", args.type]) + \
(["--field-separator", FS] if args.full else [])
opts["node_outfile"] = node_outfile
opts["copy_outfile"] = True
elif task == "cleanup":
# After pre/query run, cleanup the working directory and other
# temp files. Remove the directory to which node_outfile has
# been copied in main node
try:
os.remove(node_outfile)
except (OSError, IOError):
logger.warn("Failed to cleanup temporary file %s" %
node_outfile)
pass
cmd = [conf.get_opt("nodeagent"),
"cleanup",
args.session,
args.volume,
os.path.dirname(node_outfile)] + \
(["--debug"] if args.debug else [])
elif task == "create":
if vol_statusStr != "Started":
fail("Volume %s is not online" % args.volume,
logger=logger)
# When glusterfind create, create session directory in
# each brick nodes
cmd = [conf.get_opt("nodeagent"),
"create",
args.session,
args.volume,
brick,
kwargs.get("time_to_update")] + \
(["--debug"] if args.debug else []) + \
(["--reset-session-time"] if args.reset_session_time
else [])
elif task == "post":
# Rename pre status file to actual status file in each node
cmd = [conf.get_opt("nodeagent"),
"post",
args.session,
args.volume,
brick] + \
(["--debug"] if args.debug else [])
elif task == "delete":
# When glusterfind delete, cleanup all the session files/dirs
# from each node.
cmd = [conf.get_opt("nodeagent"),
"delete",
args.session,
args.volume] + \
(["--debug"] if args.debug else [])
if cmd:
p = Process(target=node_cmd,
args=(host, host_uuid, task, cmd, args, opts))
p.start()
pool.append(p)
for num, p in enumerate(pool):
p.join()
if p.exitcode != 0:
logger.warn("Command %s failed in %s" % (task, nodes[num][1]))
if task in ["create", "delete"]:
fail("Command %s failed in %s" % (task, nodes[num][1]))
elif task == "pre" and args.disable_partial:
sys.exit(1)