def process_rules(self):
"""
Process string from firewall-cmd application output on EL7, matches ANY source/dest address rule specifications
Note: the expected input is line separated <port>/<proto> pairs, this will give any explicitly allowed rules
added by IML and other applications to the default zone but will not list named services enabled in firewalld
Empty string is a valid input indicating no explicitly added port/proto rules
:return: None on success/valid input, error string otherwise
"""
result = self.remote_access_func(self.address, self.firewall_list_cmd)
if result.rc != 0:
from chroma_common.lib.shell import Shell
raise RuntimeError(
'''process_rules(): remote shell command failed unexpectedly (%s), is firewall-cmd running? (%s) (%s)
systemctl status firewalld:
%s
systemctl status polkit:
%s
journalctl -n 100:
%s''' % (result.rc, result.stdout, result.stderr,
Shell.run(['systemctl', 'status', 'firewalld']).stdout,
Shell.run(['systemctl', 'status', 'polkit'
]).stdout, Shell.run(['journalctl', '-n', '100']).stdout))
if result.stdout.strip() == '':
return None
# handle output separated by either new-line chars, spaces, or both
tokens = [
token for token in result.stdout.replace('\n', ' ').split()
if token
]
# as we are reading in a new firewall table content, reset 'rules' member list
self.rules = []
index = 0
while index < len(tokens):
match = re.search('(?P<port>\d+)\/(?P<proto>\w+)', tokens[index])
if match:
# Note: because we are uncertain about the command used to obtain the input string, assume persist=False
self.rules.append(
self.firewall_rule(match.group('port'),
match.group('proto')))
else:
raise RuntimeError(
'process_rules(): "%s" command output contains unexpected firewall-cmd output'
% self.firewall_list_cmd)
index += 1
return None
def collect_logs(self):
"""
Collect the logs from the target
:return: empty list on success or error messages that can be used for diagnosing what went wrong.
"""
if Shell.run(['rm', '-rf', "%s/*.log" % destination_path]).rc:
return "Fail to clear out destination path for logs collection: %s" % destination_path
errors = []
for test_runner in self.test_runners:
errors.append(
self.fetch_log(test_runner, '/var/log/chroma_test.log',
"%s-chroma_test.log" % test_runner))
errors.append(
self.fetch_log(test_runner, '/var/log/messages',
"%s-messages.log" % test_runner))
for server in self.chroma_managers + self.lustre_servers:
errors.append(
self.fetch_log(server, '/var/log/yum.log',
"%s-yum.log" % server))
errors.extend(self.fetch_chroma_diagnostics(server))
return [error for error in errors if error]
def run_command(args, exit_on_error=True, silent=False):
result = Shell.run(['time'] + args)
if result.rc and exit_on_error:
print "%s exited with error code %s, stdout %s" % (' '.join(args), result.rc, result.stderr)
sys.exit(result.rc)
if not silent:
print "%s returned %s:%s:%s" % (" ".join(args), result.rc, result.stdout, result.stderr)
return result
def fetch_log_dir(self, server, dir):
"""
Collect the log directory from the target
:return: None on success or error message that can be used for diagnosing what went wrong.
"""
logs = Shell.run(['ssh', server, "ls %s | xargs -n1 basename" % dir])
if logs.rc:
return "Failed fecthing log dir %s from %s" % (server, dir)
for log in logs.stdout.split():
destination_log_filename = "%s-%s-%s" % (
server, dir.strip('/').split('/')[-1], log)
self.fetch_log(server, "%s/%s" % (dir, log),
destination_log_filename)
return None
def fetch_log(self, server, source_log_path, destination_log_filename):
"""
Collect the log from the target
:return: None on success or error message that can be used for diagnosing what went wrong.
"""
action = "Fetching %s from %s to %s/%s" % (source_log_path, server,
self.destination_path,
destination_log_filename)
print action
if Shell.run([
'scp',
"%s:%s" % (server, source_log_path),
"%s/%s" % (self.destination_path, destination_log_filename)
]).rc:
error = "Failed %s" % action
with open(destination_log_filename, "w+") as f:
f.write(error)
return error
return None
def fetch_chroma_diagnostics(self, server):
"""
Collect the chroma diagnostics from the target
:return: empty list on success or list of error messages that can be used for diagnosing what went wrong.
"""
# Check that chroma-diagnostics is installed. May not be if installation failed, etc.
if Shell.run(['ssh', server, 'which chroma-diagnostics']).rc:
return [
"chroma-diagnostics not installed on %s. skipping." % server
]
# Generate the diagnostics from the server
result = Shell.run(
['ssh', server, 'chroma-diagnostics', '-v', '-v', '-v'])
if result.timeout:
return ["Chroma Diagnostics timed-out"]
# Find the diagnostics filename from the chroma-diagnostics output
cd_out = result.stderr.decode('utf8')
match = re.compile('/var/log/(diagnostics_.*\.tar\..*)').search(cd_out)
if not match:
return [
"Did not find diagnostics filepath in chroma-diagnostics output:\nstderr:\n%s\nstdout:\n%s"
% (cd_out, result.stdout.decode('utf8'))
]
diagnostics = match.group(1).strip()
errors = []
# Copy and expand the diagnostics locally, so they will be able to be read in browser in Jenkins.
errors.append(self.fetch_log(server, "/var/log/%s" % diagnostics, ''))
errors.append(
self.fetch_log(server, "chroma-diagnostics.log",
'%s-chroma-diagnostics.log' % server))
if diagnostics.endswith('tar.lzma'):
if Shell.run([
'tar', '--lzma', '-xvf',
"%s/%s" % (self.destination_path, diagnostics), '-C',
self.destination_path
]).rc:
errors.append("Error tar --lzma the chroma diagnostics file")
elif diagnostics.endswith('tar.gz'):
if Shell.run([
'tar', '-xvzf',
"%s/%s" % (self.destination_path, diagnostics), '-C',
self.destination_path
]).rc:
errors.append("Error tar -xvzf the chroma diagnostics file")
else:
errors = "Didn't recognize chroma-diagnostics file format"
if Shell.run(
['rm', '-f',
"%s/%s" % (self.destination_path, diagnostics)]).rc:
errors.append("Unable to remove the diagnostics %s/%s" %
(self.destination_path, diagnostics))
return errors
if Shell.run(
['rm', '-f',
"%s/%s" % (self.destination_path, diagnostics)]).rc:
errors.append("Unable to remove the diagnostics %s/%s" %
(self.destination_path, diagnostics))
return errors
if __name__ == '__main__':
destination_path = sys.argv[1]
cluster_cfg_path = sys.argv[2]
Shell.run(['mkdir', '-p', destination_path])
Shell.run(['rm', '-f', "%s.tgz" % destination_path])
cluster_cfg_json = open(cluster_cfg_path)
cluster_cfg = json.loads(cluster_cfg_json.read())
chroma_managers = []
for chroma_manager in cluster_cfg['chroma_managers']:
chroma_managers.append("root@%s" % chroma_manager['address'])
lustre_servers = []
for lustre_server in cluster_cfg['lustre_servers']:
if not lustre_server['distro'] == 'mock':
lustre_servers.append("root@%s" % lustre_server['address'])
test_runners = []
for test_runner in cluster_cfg['test_runners']: