def testDisplayRegroup(self):
"""test CLI.Display (regroup)"""
parser = OptionParser("dummy")
parser.install_display_options(verbose_options=True)
options, _ = parser.parse_args(["-r"])
mtree = MsgTree()
mtree.add("localhost", "message0")
mtree.add("localhost", "message1")
disp = Display(options)
self.assertEqual(disp.regroup, True)
disp.out = open("/dev/null", "w")
disp.err = open("/dev/null", "w")
self.assert_(disp != None)
self.assertEqual(disp.line_mode, False)
f = makeTestFile("""
# A comment
[Main]
default: local
[local]
map: echo localhost
#all:
list: echo all
#reverse:
""")
res = GroupResolverConfig(f.name)
ns = NodeSet("localhost", resolver=res)
# nodeset.regroup() is performed by print_gather()
disp.print_gather(ns, list(mtree.walk())[0][0])
def testDisplayRegroup(self):
"""test CLI.Display (regroup)"""
parser = OptionParser("dummy")
parser.install_display_options(verbose_options=True)
options, _ = parser.parse_args(["-r"])
mtree = MsgTree()
mtree.add("localhost", "message0")
mtree.add("localhost", "message1")
disp = Display(options)
self.assertEqual(disp.regroup, True)
disp.out = open("/dev/null", "w")
disp.err = open("/dev/null", "w")
self.assert_(disp != None)
self.assertEqual(disp.line_mode, False)
f = makeTestFile("""
# A comment
[Main]
default: local
[local]
map: echo localhost
#all:
list: echo all
#reverse:
""")
res = GroupResolverConfig(f.name)
ns = NodeSet("localhost", resolver=res)
# nodeset.regroup() is performed by print_gather()
disp.print_gather(ns, list(mtree.walk())[0][0])
def __init__(self, cmd, result_iter, rng, pretty_str, canceller=None):
self._result_iter = result_iter
self._rng = rng
self._pretty_str = pretty_str
self._errors = {}
self._res_tree = MsgTree()
self._canceller = canceller
self._cmd = cmd
class ParallelAgentResult(object):
"""
Manages the results for parallel client commands
"""
def __init__(self, cmd, result_iter, rng, pretty_str, canceller=None):
self._result_iter = result_iter
self._rng = rng
self._pretty_str = pretty_str
self._errors = {}
self._res_tree = MsgTree()
self._canceller = canceller
self._cmd = cmd
def cancel(self):
self._canceller()
def iterate_all(self, print_results=False, keep_results=True):
for _, _ in self.iterate(print_results, keep_results):
pass
self.raise_errors()
def iterate(self,
yield_results=False,
print_results=False,
keep_results=True):
for key, res in self._result_iter:
if isinstance(res, Exception):
self._errors.setdefault(key, list()).append(res)
yield key, res
else:
if keep_results:
self._res_tree.add(str(key), self._pretty_str(res))
if yield_results:
yield key, res
def raise_errors(self):
if not self._errors:
return
raise AgentCommandError(
self._cmd, "{} failed on {}".format(
self._cmd,
NodeSet.fromlist(
["vm{}".format(i) for i in self.errors.keys()])),
self._errors)
@property
def errors(self):
return self._errors
def __str__(self):
return '\n'.join([
"{0}: {1}".format(NodeSet.fromlist(map(lambda x: "vm" + x, keys)),
m) for m, keys in self._res_tree.walk()
])
def clubak():
"""script subroutine"""
# Argument management
parser = OptionParser("%prog [options]")
parser.install_display_options(verbose_options=True,
separator_option=True,
dshbak_compat=True,
msgtree_mode=True)
options = parser.parse_args()[0]
if options.interpret_keys == THREE_CHOICES[-1]: # auto?
enable_nodeset_key = None # AUTO
else:
enable_nodeset_key = (options.interpret_keys == THREE_CHOICES[1])
# Create new message tree
if options.trace_mode:
tree_mode = MODE_TRACE
else:
tree_mode = MODE_DEFER
tree = MsgTree(mode=tree_mode)
fast_mode = options.fast_mode
if fast_mode:
if tree_mode != MODE_DEFER or options.line_mode:
parser.error("incompatible tree options")
preload_msgs = {}
# Feed the tree from standard input lines
for line in sys.stdin:
try:
linestripped = line.rstrip('\r\n')
if options.verbose or options.debug:
print "INPUT %s" % linestripped
key, content = linestripped.split(options.separator, 1)
key = key.strip()
if not key:
raise ValueError("no node found")
if enable_nodeset_key is False: # interpret-keys=never?
keyset = [ key ]
else:
try:
keyset = NodeSet(key)
except NodeSetParseError:
if enable_nodeset_key: # interpret-keys=always?
raise
enable_nodeset_key = False # auto => switch off
keyset = [ key ]
if fast_mode:
for node in keyset:
preload_msgs.setdefault(node, []).append(content)
else:
for node in keyset:
tree.add(node, content)
except ValueError, ex:
raise ValueError("%s (\"%s\")" % (ex, linestripped))
def clubak():
"""script subroutine"""
# Argument management
parser = OptionParser("%prog [options]")
parser.install_display_options(verbose_options=True,
separator_option=True,
dshbak_compat=True,
msgtree_mode=True)
options = parser.parse_args()[0]
if options.interpret_keys == THREE_CHOICES[-1]: # auto?
enable_nodeset_key = None # AUTO
else:
enable_nodeset_key = (options.interpret_keys == THREE_CHOICES[1])
# Create new message tree
if options.trace_mode:
tree_mode = MODE_TRACE
else:
tree_mode = MODE_DEFER
tree = MsgTree(mode=tree_mode)
fast_mode = options.fast_mode
if fast_mode:
if tree_mode != MODE_DEFER or options.line_mode:
parser.error("incompatible tree options")
preload_msgs = {}
# Feed the tree from standard input lines
for line in sys.stdin:
try:
linestripped = line.rstrip('\r\n')
if options.verbose or options.debug:
print "INPUT %s" % linestripped
key, content = linestripped.split(options.separator, 1)
key = key.strip()
if not key:
raise ValueError("no node found")
if enable_nodeset_key is False: # interpret-keys=never?
keyset = [key]
else:
try:
keyset = NodeSet(key)
except NodeSetParseError:
if enable_nodeset_key: # interpret-keys=always?
raise
enable_nodeset_key = False # auto => switch off
keyset = [key]
if fast_mode:
for node in keyset:
preload_msgs.setdefault(node, []).append(content)
else:
for node in keyset:
tree.add(node, content)
except ValueError, ex:
raise ValueError("%s (\"%s\")" % (ex, linestripped))
def _run_actions(self):
"""
Start actions run-loop.
It clears all previous proxy errors and starts task run-loop. This
launches all FSProxyAction prepared before by example.
"""
self.proxy_errors = MsgTree()
task_self().set_default("stderr_msgtree", False)
task_self().set_info('connect_timeout',
Globals().get_ssh_connect_timeout())
task_self().resume()
def __init__(self, fs_name, event_handler=None):
self.fs_name = fs_name
self.hdlr = event_handler or EventHandler()
self.proxy_errors = MsgTree()
# All FS components (MGT, MDT, OST, Clients, ...)
self.components = ComponentGroup()
# file system MGT
self.mgt = None
# Local server reference
self.local_server = None
self.debug = False
self.logger = self._setup_logging()
def testDisplay(self):
"""test CLI.Display"""
parser = OptionParser("dummy")
parser.install_display_options(verbose_options=True)
options, _ = parser.parse_args([])
ns = NodeSet("hostfoo")
mtree = MsgTree()
mtree.add("hostfoo", "message0")
mtree.add("hostfoo", "message1")
for whencolor in WHENCOLOR_CHOICES: # test whencolor switch
for label in [True, False]: # test no-label switch
options.label = label
options.whencolor = whencolor
disp = Display(options)
# inhibit output
disp.out = StringIO()
disp.err = StringIO()
# test print_* methods...
disp.print_line(ns, "foo bar")
disp.print_line_error(ns, "foo bar")
disp.print_gather(ns, list(mtree.walk())[0][0])
# test also string nodeset as parameter
disp.print_gather("hostfoo", list(mtree.walk())[0][0])
# test line_mode property
self.assertEqual(disp.line_mode, False)
disp.line_mode = True
self.assertEqual(disp.line_mode, True)
disp.print_gather("hostfoo", list(mtree.walk())[0][0])
disp.line_mode = False
self.assertEqual(disp.line_mode, False)
def ev_read(self, worker):
# Read new line from node
node = worker.current_node
self._nodecnt[node] += 1
cnt = self._nodecnt[node]
if len(self._mtreeq) < cnt:
self._mtreeq.append(MsgTree())
self._mtreeq[cnt - self._offload - 1].add(node, worker.current_msg)
self._live_line(worker)
def ev_read(self, worker, node, sname, msg):
if sname != worker.SNAME_STDOUT:
GatherOutputHandler.ev_read(self, worker, node, sname, msg)
return
# Read new line from node
self._nodecnt[node] += 1
cnt = self._nodecnt[node]
if len(self._mtreeq) < cnt:
self._mtreeq.append(MsgTree())
self._mtreeq[cnt - self._offload - 1].add(node, msg)
self._live_line(worker)
def testDisplay(self):
"""test CLI.Display"""
parser = OptionParser("dummy")
parser.install_display_options(verbose_options=True)
options, _ = parser.parse_args([])
ns = NodeSet("hostfoo")
mtree = MsgTree()
mtree.add("hostfoo", "message0")
mtree.add("hostfoo", "message1")
for whencolor in WHENCOLOR_CHOICES: # test whencolor switch
for label in [True, False]: # test no-label switch
options.label = label
options.whencolor = whencolor
disp = Display(options)
# inhibit output
disp.out = StringIO()
disp.err = StringIO()
# test print_* methods...
disp.print_line(ns, "foo bar")
disp.print_line_error(ns, "foo bar")
disp.print_gather(ns, list(mtree.walk())[0][0])
# test also string nodeset as parameter
disp.print_gather("hostfoo", list(mtree.walk())[0][0])
# test line_mode property
self.assertEqual(disp.line_mode, False)
disp.line_mode = True
self.assertEqual(disp.line_mode, True)
disp.print_gather("hostfoo", list(mtree.walk())[0][0])
disp.line_mode = False
self.assertEqual(disp.line_mode, False)
def __init__(self, fs, action, nodes, debug, comps=None, **kwargs):
CommonAction.__init__(self)
self.progpath = os.path.abspath(sys.argv[0])
self.fs = fs
self.action = action
self.nodes = nodes
self.debug = debug
self._comps = comps
self.options = {}
for optname in ('addopts', 'failover', 'mountdata', 'fanout',
'dryrun'):
self.options[optname] = kwargs.get(optname)
self._outputs = MsgTree()
self._errpickle = MsgTree()
self._silentnodes = NodeSet() # Error nodes without output
if self.fs.debug:
print "FSProxyAction %s on %s" % (action, nodes)
def clubak():
"""script subroutine"""
# Argument management
parser = OptionParser("%prog [options]")
parser.install_display_options(separator_option=True,
dshbak_compat=True,
msgtree_mode=True)
options = parser.parse_args()[0]
# Create new message tree
if options.trace_mode:
tree_mode = MODE_TRACE
else:
tree_mode = MODE_DEFER
tree = MsgTree(mode=tree_mode)
fast_mode = options.fast_mode
if fast_mode:
if tree_mode != MODE_DEFER:
parser.error("incompatible tree options")
preload_msgs = {}
# Feed the tree from standard input lines
for line in sys.stdin:
try:
linestripped = line.rstrip('\r\n')
node, content = linestripped.split(options.separator, 1)
node = node.strip()
if not node:
raise ValueError("no node found")
if fast_mode:
preload_msgs.setdefault(node, []).append(content)
else:
tree.add(node, content)
except ValueError, ex:
raise ValueError("%s (\"%s\")" % (ex, linestripped))
def testDisplay(self):
"""test CLI.Display"""
parser = OptionParser("dummy")
parser.install_display_options(verbose_options=True)
options, _ = parser.parse_args([])
ns = NodeSet("hostfoo")
mtree = MsgTree()
mtree.add("hostfoo", b"message0")
mtree.add("hostfoo", b"message1")
list_env_vars = []
list_env_vars.append(dict())
list_env_vars.append(dict(NO_COLOR='0'))
list_env_vars.append(dict(CLICOLOR='0'))
list_env_vars.append(dict(CLICOLOR='1'))
list_env_vars.append(dict(CLICOLOR='0', CLICOLOR_FORCE='0'))
list_env_vars.append(dict(CLICOLOR_FORCE='1'))
for env_vars in list_env_vars:
for var_name in env_vars:
var_value = env_vars[var_name]
os.environ[var_name] = var_value
for whencolor in THREE_CHOICES: # test whencolor switch
if whencolor == "":
options.whencolor = None
else:
options.whencolor = whencolor
for label in [True, False]: # test no-label switch
options.label = label
disp = Display(options)
# inhibit output
disp.out = BytesIO()
disp.err = BytesIO()
# test print_* methods...
disp.print_line(ns, b"foo bar")
disp.print_line_error(ns, b"foo bar")
disp.print_gather(ns, list(mtree.walk())[0][0])
# test also string nodeset as parameter
disp.print_gather("hostfoo", list(mtree.walk())[0][0])
# test line_mode property
self.assertEqual(disp.line_mode, False)
disp.line_mode = True
self.assertEqual(disp.line_mode, True)
disp.print_gather("hostfoo", list(mtree.walk())[0][0])
disp.line_mode = False
self.assertEqual(disp.line_mode, False)
for var_name in env_vars:
os.environ.pop(var_name)
def __init__(self, fs, action, nodes, debug, comps=None, addopts=None,
failover=None, mountdata=None):
CommonAction.__init__(self)
self.progpath = os.path.abspath(sys.argv[0])
self.fs = fs
self.action = action
self.nodes = nodes
self.debug = debug
self._comps = comps
self.addopts = addopts
self.failover = failover
self.mountdata = mountdata
self._outputs = MsgTree()
self._errpickle = MsgTree()
self._silentnodes = NodeSet() # Error nodes without output
if self.fs.debug:
print "FSProxyAction %s on %s" % (action, nodes)
def clubak():
"""script subroutine"""
# Argument management
parser = OptionParser("%prog [options]")
parser.install_groupsconf_option()
parser.install_display_options(verbose_options=True,
separator_option=True,
dshbak_compat=True,
msgtree_mode=True)
options = parser.parse_args()[0]
set_std_group_resolver_config(options.groupsconf)
if options.interpret_keys == THREE_CHOICES[-1]: # auto?
enable_nodeset_key = None # AUTO
else:
enable_nodeset_key = (options.interpret_keys == THREE_CHOICES[1])
# Create new message tree
if options.trace_mode:
tree_mode = MODE_TRACE
else:
tree_mode = MODE_DEFER
tree = MsgTree(mode=tree_mode)
fast_mode = options.fast_mode
if fast_mode:
if tree_mode != MODE_DEFER or options.line_mode:
parser.error("incompatible tree options")
preload_msgs = {}
separator = options.separator.encode()
# Feed the tree from standard input lines
for line in sys_stdin():
try:
linestripped = line.rstrip(b'\r\n')
if options.verbose or options.debug:
sys_stdout().write(b'INPUT ' + linestripped + b'\n')
key, content = linestripped.split(separator, 1)
key = key.strip().decode() # NodeSet requires encoded string
if not key:
raise ValueError("no node found")
if enable_nodeset_key is False: # interpret-keys=never?
keyset = [key]
else:
try:
keyset = NodeSet(key)
except NodeSetParseError:
if enable_nodeset_key: # interpret-keys=always?
raise
enable_nodeset_key = False # auto => switch off
keyset = [key]
if fast_mode:
for node in keyset:
preload_msgs.setdefault(node, []).append(content)
else:
for node in keyset:
tree.add(node, content)
except ValueError as ex:
raise ValueError('%s: "%s"' % (ex, linestripped.decode()))
if fast_mode:
# Messages per node have been aggregated, now add to tree one
# full msg per node
for key, wholemsg in preload_msgs.items():
tree.add(key, b'\n'.join(wholemsg))
# Display results
try:
disp = Display(options)
except ValueError as exc:
parser.error("option mismatch (%s)" % exc)
return
if options.debug:
std_group_resolver().set_verbosity(1)
print("clubak: line_mode=%s gather=%s tree_depth=%d" %
(bool(options.line_mode), bool(disp.gather), tree._depth()),
file=sys.stderr)
display(tree, disp, disp.gather or disp.regroup, \
options.trace_mode, enable_nodeset_key is not False)
class FileSystem:
"""
The Lustre FileSystem abstract class.
"""
def __init__(self, fs_name, event_handler=None):
self.fs_name = fs_name
self.hdlr = event_handler or EventHandler()
self.proxy_errors = MsgTree()
# All FS components (MGT, MDT, OST, Clients, ...)
self.components = ComponentGroup()
# file system MGT
self.mgt = None
# Local server reference
self.local_server = None
self.debug = False
self.logger = self._setup_logging()
def set_debug(self, debug):
self.debug = debug
def _setup_logging(self):
"""Setup logging configuration for the whole filesystem."""
# XXX: This is only here for fsck, currently.
logger = logging.getLogger('Shine.Lustre')
# Level
if self.debug:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
if logger.handlers:
# If some handlers already exist, the logger singleton is already
# configured, so just return it to avoid duplicate handlers.
return logger
# Formatter
formatter = logging.Formatter(
datefmt="%Y-%m-%d %X", fmt='%(name)s %(levelname)s %(message)s')
try:
# Handler
handler = logging.handlers.SysLogHandler(address='/dev/log')
logger.addHandler(handler)
handler.setFormatter(formatter)
except socket.error:
logging.raiseExceptions = False
msg = "Error connecting to syslog, disabling logging."
print("WARNING: %s" % msg, file=sys.stderr)
return logger
def get_mgs_nids(self):
return self.mgt.get_nids()
#
# file system event handling
#
def local_event(self, evtype, **params):
# Currently, all event callbacks need a node.
# When localy called, add the current node
self.hdlr.local_event(evtype, **params)
def distant_event(self, evtype, node, **params):
# Update the local component instance with the provided instance
# if one is available in params.
if evtype == 'comp':
other = params['info'].elem
other.fs = self
try:
# Special hack for Journal object as they are not put in
# components list.
if other.TYPE == Journal.TYPE:
other.target.fs = self
target = self.components[other.target.uniqueid()]
target.journal.update(other)
comp = target.journal
else:
comp = self.components[other.uniqueid()]
# comp.update() updates the component state
# and disk information if the component is a target.
# These information don't need to be updated unless
# we are on a completion event.
if params['status'] not in ('start', 'progress'):
# ensure other.server is the actual distant server
other.server = comp.allservers().select(
NodeSet(node))[0]
# update target from remote one
comp.update(other)
# substitute target parameter by local one
params['comp'] = comp
except KeyError as error:
print("ERROR: Component update failed (%s)" % str(error),
file=sys.stderr)
self.hdlr.event_callback(evtype, node=node, **params)
def _handle_shine_proxy_error(self, nodes, message):
"""
Store error messages, for later processing.
Hostnames are replaced by 'THIS_SHINE_HOST' to allow message grouping.
Grouping outputs which only differ by the host name.
"""
message = message.replace(str(nodes), 'THIS_SHINE_HOST')
self.proxy_errors.add(NodeSet(nodes), message)
#
# file system construction
#
def _attach_component(self, comp):
if comp.TYPE == MGT.TYPE:
if self.mgt and len(self.mgt.get_nids()) > 0:
raise FSError("A Lustre filesystem has only one MGT.")
self.mgt = comp
self.components.add(comp)
def new_target(self,
server,
type,
index,
dev,
jdev=None,
group=None,
tag=None,
enabled=True,
mode='managed',
network=None,
active='yes'):
"""
Create a new attached target.
"""
TYPE_CLASSES = {MGT.TYPE: MGT, MDT.TYPE: MDT, OST.TYPE: OST}
if type not in TYPE_CLASSES:
raise FSError("Unrecognized target type \"%s\"" % type)
target = TYPE_CLASSES[type](fs=self,
server=server,
index=index,
dev=dev,
jdev=jdev,
group=group,
tag=tag,
enabled=enabled,
mode=mode,
network=network,
active=active)
self._attach_component(target)
return target
def new_client(self,
server,
mount_path,
mount_options=None,
subdir=None,
enabled=True):
"""
Create a new attached client.
"""
client = Client(self, server, mount_path, mount_options, subdir,
enabled)
self._attach_component(client)
return client
def new_router(self, server, enabled=True):
"""
Create a new attached router.
"""
router = Router(self, server, enabled)
self._attach_component(router)
return router
#
# Task management.
#
def _proxy_action(self, action, servers, comps=None, **kwargs):
"""Create a proxy action to remotely run a shine action."""
assert isinstance(servers, NodeSet)
assert comps is None or isinstance(comps, ComponentGroup)
return FSProxyAction(self, action, servers, self.debug, comps,
**kwargs)
def _run_actions(self):
"""
Start actions run-loop.
It clears all previous proxy errors and starts task run-loop. This
launches all FSProxyAction prepared before by example.
"""
self.proxy_errors = MsgTree()
task_self().set_default("stderr_msgtree", False)
task_self().set_info('connect_timeout',
Globals().get_ssh_connect_timeout())
task_self().resume()
def _check_errors(self, expected_states, components=None, actions=None):
"""
This verifies that executed tasks were successfull.
It verifies all provided components (Target, Clients, ...) have
expected state. If not, it returns the most incoherent state.
If there is no error, it returns the expected state.
"""
assert isinstance(expected_states, list)
result = set()
if actions and actions.status() == ACT_ERROR:
result.add(TARGET_ERROR)
# If a component list is provided, check that all components from it
# have expected state.
for comp in components or []:
# This should never happen but it is convenient for debugging if
# there is some uncatched bug somewhere.
# (ie: cannot unpickle due to ClusterShell version mismatch)
if comp.state is None:
msg = "WARNING: no state report from node %s" % comp.server
print(msg, file=sys.stderr)
comp.state = RUNTIME_ERROR
if comp.state not in expected_states:
result.add(comp.state)
# Compute component's server.
# Although not the best place semantically speaking to perform this
# task, update_server() is meaningful only when all the component
# states have been filled, and here, we are sure it is the case.
# So, waiting for a better solution, _check_errors() is the
# best place to compute the component server.
if comp.update_server() is False:
msg = "WARNING: %s is mounted multiple times" % comp.label
self._handle_shine_proxy_error(str(comp.server.hostname), msg)
# if empty set, add expected_states[0]
if not result:
result.add(expected_states[0])
return result
def _distant_action_by_server(self, action_class, servers, **kwargs):
# filter local server
distant_servers = Server.distant_servers(servers)
# perform action on distant servers
if len(distant_servers) > 0:
action = action_class(nodes=distant_servers, fs=self, **kwargs)
action.launch()
self._run_actions()
if action.status() == ACT_ERROR:
err_code = None
if task_self().num_timeout():
err_code = -1
elif task_self().max_retcode():
err_code = task_self().max_retcode()
# FSRemoteError is limited and cannot handle more than 1 error
msg, nodes = list(self.proxy_errors.walk())[0]
nodes = NodeSet.fromlist(nodes)
msg = str(msg).replace('THIS_SHINE_HOST', str(nodes))
raise FSRemoteError(nodes, err_code, msg)
def install(self, fs_config_file, servers=None, **kwargs):
"""
Install filesystem configuration file on its servers.
Server list is built from enabled targets and enabled clients only.
"""
# Get all possible servers
servers = (servers or self.components.managed().allservers())
self._distant_action_by_server(Install,
servers,
config_file=fs_config_file,
**kwargs)
def remove(self, servers=None, **kwargs):
"""
Remove FS config files.
"""
result = 0
if servers is None:
# Get all possible servers
servers = self.components.managed().allservers()
# filter local server
distant_servers = Server.distant_servers(servers)
# If size is different, we have a local server in the list
if len(distant_servers) < len(servers):
# remove local fs configuration file
fs_file = os.path.join(Globals().get_conf_dir(),
"%s.xmf" % self.fs_name)
if os.path.exists(fs_file):
self.hdlr.log('detail', msg='[DEL] %s' % fs_file)
if kwargs.get('dryrun', False):
result = 0
else:
result = os.remove(fs_file)
if len(distant_servers) > 0:
# Perform the remove operations on all targets for these nodes.
self._proxy_action('remove', distant_servers, **kwargs).launch()
# Run local actions and FSProxyAction
self._run_actions()
if len(self.proxy_errors) > 0:
return RUNTIME_ERROR
return result
def _prepare(self,
action,
comps=None,
groupby=None,
reverse=False,
need_unload=False,
tunings=None,
allservers=False,
**kwargs):
"""
Instanciate all actions for the component list and but them in a graph
of ActionGroup().
Action could be local or proxy actions.
Components list is filtered, based on action name.
"""
graph = ActionGroup()
comps = comps or self.components
first_comps = None
last_comps = None
localsrv = None
modules = set()
localcomps = None
if groupby:
iterable = comps.groupby(attr=groupby, reverse=reverse)
else:
iterable = [(None, comps)]
# Iterate over targets, grouping them by start order and server.
for _order, comps in iterable:
graph.add(ActionGroup())
compgrp = ActionGroup()
proxygrp = ActionGroup()
for srv, comps in comps.groupbyserver(allservers=allservers):
if srv.action_enabled is True:
if srv.is_local():
localsrv = srv
localcomps = comps
for comp in comps:
compgrp.add(getattr(comp, action)(**kwargs))
else:
act = self._proxy_action(action, srv.hostname, comps,
**kwargs)
if tunings and tunings.filename:
copy = Install(srv.hostname,
self,
tunings.filename,
comps=comps,
**kwargs)
act.depends_on(copy)
proxygrp.add(copy)
proxygrp.add(act)
if len(compgrp) > 0:
graph[-1].add(compgrp)
# Keep track of first comp group
if first_comps is None:
first_comps = compgrp
first_comps.parent = graph[-1]
# Keep track of last comp group
last_comps = compgrp
last_comps.parent = graph[-1]
# Build module loading list, if needed
for comp_action in compgrp:
modules.update(comp_action.needed_modules())
if len(proxygrp) > 0:
graph[-1].add(proxygrp)
# Add module loading, if needed.
if first_comps is not None and len(modules) > 0:
modgrp = ActionGroup()
for module in modules:
modgrp.add(localsrv.load_modules(modname=module, **kwargs))
# Serialize modules loading actions
modgrp.sequential()
first_comps.parent.add(modgrp)
first_comps.depends_on(modgrp)
# Apply tuning to last component group, if needed
if tunings is not None and last_comps is not None:
tune = localsrv.tune(tunings, localcomps, self.fs_name, **kwargs)
last_comps.parent.add(tune)
tune.depends_on(last_comps)
# Add module unloading to last component group, if needed.
if need_unload and last_comps is not None:
unload = localsrv.unload_modules(**kwargs)
last_comps.parent.add(unload)
unload.depends_on(last_comps)
# Join the different part together
graph.sequential()
return graph
def format(self, comps=None, **kwargs):
"""Format filesystem targets."""
comps = (comps or self.components).managed(supports='format')
actions = self._prepare('format', comps, **kwargs)
actions.launch()
self._run_actions()
# Check for errors and return OFFLINE or error code
return self._check_errors([OFFLINE], comps, actions)
def tunefs(self, comps=None, **kwargs):
"""Modify component option set at format."""
comps = (comps or self.components).managed(supports='tunefs')
actions = self._prepare('tunefs', comps, **kwargs)
actions.launch()
self._run_actions()
# Check for errors and return OFFLINE or error code
return self._check_errors([OFFLINE], comps, actions)
def fsck(self, comps=None, **kwargs):
"""Check component filesystem coherency."""
comps = (comps or self.components).managed(supports='fsck')
actions = self._prepare('fsck', comps, **kwargs)
actions.launch()
self._run_actions()
# Check for errors and return OFFLINE or error code
return self._check_errors([OFFLINE], comps, actions)
def status(self, comps=None, **kwargs):
"""Get status of filesystem."""
comps = (comps or self.components).managed(supports='status')
actions = self._prepare('status', comps, allservers=True, **kwargs)
actions.launch()
self._run_actions()
# Here we check MOUNTED but in fact, any status is OK.
return self._check_errors([MOUNTED], comps)
def start(self, comps=None, **kwargs):
"""Start Lustre file system servers."""
comps = (comps or self.components).managed(supports='start')
# What starting order to use?
key = lambda t: t.TYPE == MDT.TYPE
mdt_comps = comps.filter(key=key)
if mdt_comps:
# Found enabled MDT(s): perform writeconf check.
self.status(comps=mdt_comps)
for target in mdt_comps:
if target.has_first_time_flag() or target.has_writeconf_flag():
MDT.START_ORDER, OST.START_ORDER = OST.START_ORDER, MDT.START_ORDER
break
actions = self._prepare('start',
comps,
groupby='START_ORDER',
**kwargs)
actions.launch()
self._run_actions()
return self._check_errors([MOUNTED, RECOVERING], comps, actions)
def stop(self, comps=None, **kwargs):
"""Stop file system."""
comps = (comps or self.components).managed(supports='stop')
actions = self._prepare('stop',
comps,
groupby='START_ORDER',
reverse=True,
need_unload=True,
**kwargs)
actions.launch()
self._run_actions()
return self._check_errors([OFFLINE], comps)
def mount(self, comps=None, **kwargs):
"""Mount FS clients."""
comps = (comps or self.components).managed(supports='mount')
actions = self._prepare('mount', comps, **kwargs)
actions.launch()
self._run_actions()
# Ok, workers have completed, perform late status check...
return self._check_errors([MOUNTED], comps, actions)
def umount(self, comps=None, **kwargs):
"""Unmount FS clients."""
comps = (comps or self.components).managed(supports='umount')
actions = self._prepare('umount', comps, need_unload=True, **kwargs)
actions.launch()
self._run_actions()
# Ok, workers have completed, perform late status check...
return self._check_errors([OFFLINE], comps)
def execute(self, comps=None, **kwargs):
"""Execute custom command."""
comps = (comps or self.components).managed(supports='execute')
actions = self._prepare('execute', comps, **kwargs)
actions.launch()
self._run_actions()
# Here we check MOUNTED but in fact, any status is OK.
# XXX: Is that ok, to check MOUNTED here?
return self._check_errors([MOUNTED], comps, actions)
def tune(self, tuning_model, comps=None, **kwargs):
"""Tune server."""
comps = (comps or self.components).managed()
actions = ActionGroup()
for server, srvcomps in comps.groupbyserver():
if server.is_local():
actions.add(
server.tune(tuning_model, srvcomps, self.fs_name,
**kwargs))
else:
act = self._proxy_action('tune', server.hostname, srvcomps,
**kwargs)
if tuning_model.filename:
copy = Install(server.hostname,
self,
tuning_model.filename,
comps=srvcomps,
**kwargs)
act.depends_on(copy)
actions.add(copy)
actions.add(act)
# Run local actions and FSProxyAction
actions.launch()
self._run_actions()
# Check actions status and return MOUNTED if no error
return self._check_errors([MOUNTED], None, actions)
class FSProxyAction(CommonAction):
"""
Generic file system command proxy action class.
"""
NAME = 'proxy'
def __init__(self, fs, action, nodes, debug, comps=None, **kwargs):
CommonAction.__init__(self)
self.progpath = os.path.abspath(sys.argv[0])
self.fs = fs
self.action = action
self.nodes = nodes
self.debug = debug
self._comps = comps
self.options = {}
for optname in ('addopts', 'failover', 'mountdata', 'fanout',
'dryrun'):
self.options[optname] = kwargs.get(optname)
self._outputs = MsgTree()
self._errpickle = MsgTree()
self._silentnodes = NodeSet() # Error nodes without output
if self.fs.debug:
print "FSProxyAction %s on %s" % (action, nodes)
def info(self):
return ActionInfo(self, description='Proxy action')
def _prepare_cmd(self):
"""Create the command line base on proxy properties."""
command = ["%s" % self.progpath]
command.append(self.action)
command.append("-f %s" % self.fs.fs_name)
command.append("-R")
if self.debug:
command.append("-d")
if self._comps:
command.append("-l %s" % self._comps.labels())
if self.options['addopts']:
command.append("-o '%s'" % self.options['addopts'])
if self.options['failover']:
command.append("-F '%s'" % self.options['failover'])
if self.options['fanout'] is not None:
command.append('--fanout=%d' % self.options['fanout'])
if self.options['dryrun']:
command.append('--dry-run')
# To be compatible with older clients in most cases, do not set the
# option when it is its default value.
if self.options['mountdata'] not in (None, 'auto'):
command.append('--mountdata=%s' % self.options['mountdata'])
return command
def _launch(self):
"""Launch FS proxy command."""
command = self._prepare_cmd()
# Schedule cluster command.
self.task.shell(' '.join(command), nodes=self.nodes, handler=self)
# Launch events
self._actions_start()
def _actions_start(self):
"""
Raise 'proxy' events for all components related to this ProxyAction.
"""
# Add a 'proxy' running action for each component.
if self._comps:
for comp in self._comps:
# This special event is raised to keep track of undergoing
# actions. Maybe this could be dropped is such tracking is no
# more needed.
comp.action_event(self, 'start')
def ev_read(self, worker):
node = worker.current_node
buf = worker.current_msg
try:
data = shine_msg_unpack(buf)
# COMPAT: Prior to 1.4, 'comp'+'action' was used.
# 1.4+ uses ActionInfo
if 'comp' in data:
action = Action()
action.NAME = data.pop('action')
comp = data.pop('comp')
comp.fs = self.fs
desc = "%s of %s" % (action.NAME, comp.longtext())
data['info'] = ActionInfo(action, comp, desc)
evtype = 'comp'
else:
evtype = data.pop('evtype')
self.fs.distant_event(evtype, node=node, **data)
except ProxyActionUnpickleError, exp:
# Maintain a standalone list of unpickling errors.
# Node could have unpickling error but still exit with 0
msg = str(exp)
if msg not in self._errpickle.get(node, ""):
self._errpickle.add(node, msg)
except AttributeError, exp:
msg = "Cannot read message (check Shine and ClusterShell " \
"version): %s" % str(exp)
if msg not in self._errpickle.get(node, ""):
self._errpickle.add(node, msg)
class FSProxyAction(CommonAction):
"""
Generic file system command proxy action class.
"""
NAME = 'proxy'
def __init__(self, fs, action, nodes, debug, comps=None, addopts=None,
failover=None, mountdata=None):
CommonAction.__init__(self)
self.progpath = os.path.abspath(sys.argv[0])
self.fs = fs
self.action = action
self.nodes = nodes
self.debug = debug
self._comps = comps
self.addopts = addopts
self.failover = failover
self.mountdata = mountdata
self._outputs = MsgTree()
self._errpickle = MsgTree()
self._silentnodes = NodeSet() # Error nodes without output
if self.fs.debug:
print "FSProxyAction %s on %s" % (action, nodes)
def _prepare_cmd(self):
"""Create the command line base on proxy properties."""
command = ["%s" % self.progpath]
command.append(self.action)
command.append("-f %s" % self.fs.fs_name)
command.append("-R")
if self.debug:
command.append("-d")
if self._comps:
command.append("-l %s" % self._comps.labels())
if self.addopts:
command.append("-o '%s'" % self.addopts)
if self.failover:
command.append("-F '%s'" % self.failover)
# To be compatible with older clients in most cases, do not set the
# option when it is its default value.
if self.mountdata is not None and self.mountdata != 'auto':
command.append('--mountdata=%s' % self.mountdata)
return command
def _launch(self):
"""Launch FS proxy command."""
command = self._prepare_cmd()
# Schedule cluster command.
self.task.shell(' '.join(command), nodes=self.nodes, handler=self)
# Launch events
self._actions_start()
def _actions_start(self):
"""
Raise 'proxy' events for all components related to this ProxyAction.
"""
# Add a 'proxy' running action for each component.
if self._comps:
for comp in self._comps:
# Warning: there is no clean call at the end of the action.
# cleaning is done by hand.
comp.action_start('proxy')
def ev_read(self, worker):
node = worker.current_node
buf = worker.current_msg
try:
data = shine_msg_unpack(buf)
compname = data.pop('compname')
action = data.pop('action')
status = data.pop('status')
self.fs.distant_event(compname, action, status, node=node, **data)
except ProxyActionUnpickleError, exp:
# Maintain a standalone list of unpickling errors.
# Node could have unpickling error but still exit with 0
msg = str(exp)
if msg not in self._errpickle.get(node, ""):
self._errpickle.add(node, msg)
except AttributeError, exp:
msg = "Cannot read message (check Shine and ClusterShell " \
"version): %s" % str(exp)
if msg not in self._errpickle.get(node, ""):
self._errpickle.add(node, msg)
def clubak():
"""script subroutine"""
# Argument management
parser = OptionParser("%prog [options]")
parser.install_groupsconf_option()
parser.install_display_options(verbose_options=True,
separator_option=True,
dshbak_compat=True,
msgtree_mode=True)
options = parser.parse_args()[0]
set_std_group_resolver_config(options.groupsconf)
if options.interpret_keys == THREE_CHOICES[-1]: # auto?
enable_nodeset_key = None # AUTO
else:
enable_nodeset_key = (options.interpret_keys == THREE_CHOICES[1])
# Create new message tree
if options.trace_mode:
tree_mode = MODE_TRACE
else:
tree_mode = MODE_DEFER
tree = MsgTree(mode=tree_mode)
fast_mode = options.fast_mode
if fast_mode:
if tree_mode != MODE_DEFER or options.line_mode:
parser.error("incompatible tree options")
preload_msgs = {}
separator = options.separator.encode()
# Feed the tree from standard input lines
for line in sys_stdin():
try:
linestripped = line.rstrip(b'\r\n')
if options.verbose or options.debug:
sys_stdout().write(b'INPUT ' + linestripped + b'\n')
key, content = linestripped.split(separator, 1)
key = key.strip().decode() # NodeSet requires encoded string
if not key:
raise ValueError("no node found")
if enable_nodeset_key is False: # interpret-keys=never?
keyset = [ key ]
else:
try:
keyset = NodeSet(key)
except NodeSetParseError:
if enable_nodeset_key: # interpret-keys=always?
raise
enable_nodeset_key = False # auto => switch off
keyset = [ key ]
if fast_mode:
for node in keyset:
preload_msgs.setdefault(node, []).append(content)
else:
for node in keyset:
tree.add(node, content)
except ValueError as ex:
raise ValueError('%s: "%s"' % (ex, linestripped.decode()))
if fast_mode:
# Messages per node have been aggregated, now add to tree one
# full msg per node
for key, wholemsg in preload_msgs.items():
tree.add(key, b'\n'.join(wholemsg))
# Display results
try:
disp = Display(options)
if options.debug:
std_group_resolver().set_verbosity(1)
print("clubak: line_mode=%s gather=%s tree_depth=%d"
% (bool(options.line_mode), bool(disp.gather), tree._depth()),
file=sys.stderr)
display(tree, disp, disp.gather or disp.regroup, \
options.trace_mode, enable_nodeset_key is not False)
except ValueError as exc:
parser.error("option mismatch (%s)" % exc)
class FSProxyAction(CommonAction):
"""
Generic file system command proxy action class.
"""
NAME = 'proxy'
def __init__(self, fs, action, nodes, debug, comps=None, **kwargs):
CommonAction.__init__(self)
self.progpath = os.path.abspath(sys.argv[0])
self.fs = fs
self.action = action
self.nodes = nodes
self.debug = debug
self._comps = comps
self.options = {}
for optname in ('addopts', 'failover', 'mountdata', 'fanout',
'dryrun'):
self.options[optname] = kwargs.get(optname)
self._outputs = MsgTree()
self._errpickle = MsgTree()
self._silentnodes = NodeSet() # Error nodes without output
if self.fs.debug:
print("FSProxyAction %s on %s" % (action, nodes))
def info(self):
return ActionInfo(self, description='Proxy action')
def _prepare_cmd(self):
"""Create the command line base on proxy properties."""
command = ["%s" % self.progpath]
command.append(self.action)
command.append("-f %s" % self.fs.fs_name)
command.append("-R")
if self.debug:
command.append("-d")
if self._comps:
command.append("-l %s" % self._comps.labels())
if self.options['addopts']:
command.append("-o '%s'" % self.options['addopts'])
if self.options['failover']:
command.append("-F '%s'" % self.options['failover'])
if self.options['fanout'] is not None:
command.append('--fanout=%d' % self.options['fanout'])
if self.options['dryrun']:
command.append('--dry-run')
# To be compatible with older clients in most cases, do not set the
# option when it is its default value.
if self.options['mountdata'] not in (None, 'auto'):
command.append('--mountdata=%s' % self.options['mountdata'])
return command
def _launch(self):
"""Launch FS proxy command."""
command = self._prepare_cmd()
# Schedule cluster command.
self.task.shell(' '.join(command), nodes=self.nodes, handler=self)
# Launch events
self._actions_start()
def _actions_start(self):
"""
Raise 'proxy' events for all components related to this ProxyAction.
"""
# Add a 'proxy' running action for each component.
if self._comps:
for comp in self._comps:
# This special event is raised to keep track of undergoing
# actions. Maybe this could be dropped is such tracking is no
# more needed.
comp.action_event(self, 'start')
def ev_read(self, worker):
node = worker.current_node
buf = worker.current_msg
try:
data = shine_msg_unpack(buf)
# COMPAT: Prior to 1.4, 'comp'+'action' was used.
# 1.4+ uses ActionInfo
if 'comp' in data:
action = Action()
action.NAME = data.pop('action')
comp = data.pop('comp')
comp.fs = self.fs
desc = "%s of %s" % (action.NAME, comp.longtext())
data['info'] = ActionInfo(action, comp, desc)
evtype = 'comp'
else:
evtype = data.pop('evtype')
self.fs.distant_event(evtype, node=node, **data)
except ProxyActionUnpickleError as exp:
# Maintain a standalone list of unpickling errors.
# Node could have unpickling error but still exit with 0
msg = str(exp)
if msg not in self._errpickle.get(node, ""):
self._errpickle.add(node, msg)
except AttributeError as exp:
msg = "Cannot read message (check Shine and ClusterShell " \
"version): %s" % str(exp)
if msg not in self._errpickle.get(node, ""):
self._errpickle.add(node, msg)
except ProxyActionUnpackError:
# Store output that is not a shine message
self._outputs.add(node, buf)
def ev_hup(self, worker):
"""Keep a list of node, without output, with a return code != 0"""
# If this node was on error
if worker.current_rc != 0:
# If there is no known outputs
if self._outputs.get(worker.current_node) is None:
self._silentnodes.add(worker.current_node)
def ev_close(self, worker):
"""End of proxy command."""
Action.ev_close(self, worker)
# Before all, we must check if shine command ran without bugs, node
# crash, etc...
# So we need to verify all node retcodes and change the component state
# on the bad nodes.
# Action timed out
if worker.did_timeout():
self.set_status(ACT_ERROR)
return
status = ACT_OK
# Remove the 'proxy' running action for each component.
if self._comps:
for comp in self._comps:
# This special event helps to keep track of undergoing actions
# (see ev_start())
comp.action_event(self, 'done')
comp.sanitize_state(nodes=worker.nodes)
# Gather nodes by return code
for rc, nodes in worker.iter_retcodes():
# Remote command returns only RUNTIME_ERROR (See RemoteCommand)
# some common remote errors:
# rc 127 = command not found
# rc 126 = found but not executable
# rc 1 = python failure...
if rc != 0:
# If there is at least one error, the action is on error.
status = ACT_ERROR
# Gather these nodes by buffer
key = nodes.__contains__
for buffers, nodes in self._outputs.walk(match=key):
# Handle proxy command error
nodes = NodeSet.fromlist(nodes)
msg = "Remote action %s failed: %s\n" % \
(self.action, buffers)
self.fs._handle_shine_proxy_error(nodes, msg)
# Raise errors for each unpickling error,
# which could happen mostly when Shine exits with 0.
for buffers, nodes in self._errpickle.walk():
nodes = NodeSet.fromlist(nodes)
self.fs._handle_shine_proxy_error(nodes, str(buffers))
# Raise an error for nodes without output
if len(self._silentnodes) > 0:
msg = "Remote action %s failed: No response" % self.action
self.fs._handle_shine_proxy_error(self._silentnodes, msg)
self.set_status(status)
