def grepCount(self,
strs,
ignore_str,
unique,
file,
is_and=False,
is_casesensitive=True):
search_str = self.get_grep_string(strs, file, is_and, is_casesensitive)
if not search_str:
return []
if ignore_str:
if unique:
out, err = shell_command([
search_str, '| grep -v', ignore_str,
'| sort -k 9 | uniq -f 8 | wc -l'
])
return out
else:
out, err = shell_command(
[search_str, '| grep -v', ignore_str, '| wc -l'])
return out
else:
if unique:
out, err = shell_command(
[search_str, '| sort -k 9 | uniq -f 8 | wc -l'])
return out
else:
out, err = shell_command([search_str, '| wc -l'])
return out
def collect_sys(self,line):
lsof_cmd='sudo lsof|grep `sudo ps aux|grep -v grep|grep -E \'asd|cld\'|awk \'{print $2}\'` 2>/dev/null'
print util.shell_command([lsof_cmd])
print platform.platform()
smd_home = '/opt/aerospike/smd'
if os.path.isdir(smd_home):
smd_files = [ f for f in os.listdir(smd_home) if os.path.isfile(os.path.join(smd_home, f)) ]
for f in smd_files:
smd_f = os.path.join(smd_home, f)
print smd_f
smd_fp = open(smd_f, 'r')
print smd_fp.read()
smd_fp.close()
def collect_sys(self, line):
lsof_cmd = 'sudo lsof|grep `sudo ps aux|grep -v grep|grep -E \'asd|cld\'|awk \'{print $2}\'` 2>/dev/null'
print util.shell_command([lsof_cmd])
print platform.platform()
smd_home = '/opt/aerospike/smd'
if os.path.isdir(smd_home):
smd_files = [
f for f in os.listdir(smd_home)
if os.path.isfile(os.path.join(smd_home, f))
]
for f in smd_files:
smd_f = os.path.join(smd_home, f)
print smd_f
smd_fp = open(smd_f, 'r')
print smd_fp.read()
smd_fp.close()
def _do_default(self, line):
command = line
out, err = util.shell_command(command)
if err:
print err
if out:
print out
def _do_default(self, line):
command = line
out, err = util.shell_command(command)
if err:
print err
if out:
print out
def is_cluster_log_file(self, file=""):
if not file:
return False
for search_string in self.cluster_log_file_identifier:
try:
out, err = shell_command(
['grep -m 1 %s "%s"' % (search_string, file)])
except:
return False
if err or not out:
return False
return True
def grep(self,
strs,
ignore_str,
unique,
file,
is_and=False,
is_casesensitive=True):
if not strs:
return []
if isinstance(strs, str):
strs = [strs]
if not isinstance(strs, list):
return []
search_str = self.get_grep_string(strs, file, is_and, is_casesensitive)
if not search_str:
return []
if ignore_str:
if unique:
out, err = shell_command([
search_str, '|', 'grep -v ', ignore_str,
'| sort -k 9 | uniq -f 8 | sort -k 1,4'
])
return out
else:
out, err = shell_command(
[search_str, '|', 'grep -v ', ignore_str])
return out
else:
if unique:
out, err = shell_command(
[search_str, '| sort -k 9 | uniq -f 8 | sort -k 1,4'])
return out
else:
out, err = shell_command([search_str])
return out
def collectinfo_content(self, func, parm=''):
name = ''
capture_stdout = util.capture_stdout
sep = "\n=======ASCOLLECTINFO(" + output_time + ")======\n"
try:
name = func.func_name
except Exception:
pass
info_line = "[INFO] Data collection for " + name +str(parm) + " in progress.."
print info_line
sep += info_line+"\n"
if func == 'shell':
o,e = util.shell_command(parm)
elif func == 'cluster':
o = self.cluster.info(parm)
else:
o = capture_stdout(func,parm)
self.write_log(sep+str(o))
return ''
def is_server_log_file(self, file=""):
if not file:
return False
try:
out, err = shell_command(['head -n 10 "%s"' % (file)])
except:
return False
if err or not out:
return False
lines = out.strip().split('\n')
matched_count = 0
for line in lines:
try:
if re.search(self.server_log_file_identifier_pattern, line):
matched_count += 1
except:
pass
if matched_count == len(lines):
return True
return False
def collectinfo_content(self, func, parm=''):
name = ''
capture_stdout = util.capture_stdout
sep = "\n=======ASCOLLECTINFO(" + output_time + ")======\n"
try:
name = func.func_name
except Exception:
pass
info_line = "[INFO] Data collection for " + name + str(
parm) + " in progress.."
print info_line
sep += info_line + "\n"
if func == 'shell':
o, e = util.shell_command(parm)
elif func == 'cluster':
o = self.cluster.info(parm)
else:
o = capture_stdout(func, parm)
self.write_log(sep + str(o))
return ''
def main_collectinfo(self, line):
# Unfortunately timestamp can not be printed in Centos with dmesg,
# storing dmesg logs without timestamp for this particular OS.
if 'centos' == (platform.linux_distribution()[0]).lower():
cmd_dmesg = 'dmesg'
else:
cmd_dmesg = 'dmesg -T'
collect_output = time.strftime("%Y-%m-%d %H:%M:%S UTC\n",
time.gmtime())
global aslogdir, aslogfile, output_time
output_time = time.strftime("%Y%m%d_%H%M%S", time.gmtime())
aslogdir = '/tmp/collectInfo_' + output_time
as_sysinfo_logdir = os.path.join(aslogdir, 'sysInformation')
as_logfile_prefix = aslogdir + '/' + output_time + '_'
info_params = ['network', 'service', 'namespace', 'xdr', 'sindex']
show_params = ['config', 'distribution', 'latency', 'statistics']
cluster_params = [
'service',
'services',
'xdr-min-lastshipinfo:',
'dump-fabric:',
'dump-hb:',
'dump-migrates:',
'dump-msgs:',
'dump-paxos:',
'dump-smd:',
'dump-wr:',
]
shell_cmds = [
'date',
'hostname',
'ip addr',
'ip -s link',
'uptime',
'uname -a',
'lsb_release -a',
'ls /etc|grep release|xargs -I f cat /etc/f',
'rpm -qa|grep -E "citrus|aero"',
'dpkg -l|grep -E "citrus|aero"',
'free -m',
'df -h',
'ls /sys/block/{sd*,xvd*}/queue/rotational |xargs -I f sh -c "echo f; cat f;"',
'ls /sys/block/{sd*,xvd*}/device/model |xargs -I f sh -c "echo f; cat f;"',
'lsof',
cmd_dmesg,
'iostat -x 1 10',
'vmstat -s',
'vmstat -m',
'iptables -L',
]
cpu_stat = [
'top -n3 -b', 'iostat -x 1 10', 'ss -pant', 'sar -n DEV',
'sar -n EDEV'
]
_ip = ((util.shell_command(["hostname -I"])[0]).split(' ')[0].strip())
if 'all' in line:
namespaces = self.parse_namespace(
self.cluster._callNodeMethod([_ip], "info", "namespaces"))
for ns in namespaces:
cluster_params.append('dump-wb:ns=' + ns)
cluster_params.append('dump-wb-summary:ns=' + ns)
if 'ubuntu' == (platform.linux_distribution()[0]).lower():
cmd_dmesg = 'cat /var/log/syslog'
else:
cmd_dmesg = 'cat /var/log/messages'
terminal.enable_color(False)
os.makedirs(as_sysinfo_logdir)
try:
aslogfile = as_logfile_prefix + 'asadmCmd.log'
cinfo = InfoController()
for info_param in info_params:
self.collectinfo_content(cinfo, [info_param])
do_show = ShowController()
for show_param in show_params:
self.collectinfo_content(do_show, [show_param])
except Exception as e:
self.write_log(str(e))
sys.stdout = sys.__stdout__
try:
aslogfile = as_logfile_prefix + 'clusterCmd.log'
for cluster_param in cluster_params:
self.collectinfo_content('cluster', cluster_param)
except Exception as e:
self.write_log(str(e))
sys.stdout = sys.__stdout__
aslogfile = os.path.join(as_sysinfo_logdir,
output_time + '_' + 'sysCmdOutput.log')
for cmd in shell_cmds:
self.collectinfo_content('shell', [cmd])
aslogfile = os.path.join(as_sysinfo_logdir,
output_time + '_' + 'cpu_stat.log')
for _cmd in cpu_stat:
self.collectinfo_content('shell', [_cmd])
aslogfile = os.path.join(as_sysinfo_logdir,
output_time + '_' + 'dmesg.log')
self.collectinfo_content('shell', [cmd_dmesg])
if 'True' in self.cluster.isXDREnabled().values():
aslogfile = as_logfile_prefix + 'xdr.log'
self.collectinfo_content('shell',
['tail -n 10000 /var/log/*xdr.log'])
try:
as_version = self.cluster._callNodeMethod([_ip], "info",
"build").popitem()[1]
log_location = self.cluster._callNodeMethod(
[_ip], "info", "logs").popitem()[1].split(':')[1]
# Comparing with this version because prior to this it was citrusleaf.conf & citrusleaf.log
if StrictVersion(as_version) > StrictVersion("3.0.0"):
aslogfile = as_logfile_prefix + 'aerospike.conf'
self.collectinfo_content('shell',
['cat /etc/aerospike/aerospike.conf'])
else:
aslogfile = as_logfile_prefix + 'citrusleaf.conf'
self.collectinfo_content(
'shell', ['cat /etc/citrusleaf/citrusleaf.conf'])
self.collect_local_file(log_location, aslogdir)
except Exception as e:
self.write_log(str(e))
sys.stdout = sys.__stdout__
aslogfile = as_logfile_prefix + 'collectSys.log'
self.collectinfo_content(self.collect_sys)
aslogfile = as_logfile_prefix + 'awsData.log'
self.collectinfo_content(self.get_awsdata)
self.archive_log(aslogdir)
def archive_log(self, logdir):
self.zip_files(logdir)
util.shell_command(["tar -czvf " + logdir + ".tgz " + aslogdir])
sys.stderr.write("\x1b[2J\x1b[H")
print "\n\n\nFiles in " + logdir + " and " + logdir + ".tgz saved. "
print "END OF ASCOLLECTINFO"
def main_collectinfo(self, line):
# Unfortunately timestamp can not be printed in Centos with dmesg,
# storing dmesg logs without timestamp for this particular OS.
if 'centos' == (platform.linux_distribution()[0]).lower():
cmd_dmesg = 'dmesg'
else:
cmd_dmesg = 'dmesg -T'
collect_output = time.strftime("%Y-%m-%d %H:%M:%S UTC\n", time.gmtime())
global aslogdir, aslogfile, output_time
output_time = time.strftime("%Y%m%d_%H%M%S", time.gmtime())
aslogdir = '/tmp/collectInfo_' + output_time
as_sysinfo_logdir = os.path.join(aslogdir, 'sysInformation')
as_logfile_prefix = aslogdir + '/' + output_time + '_'
info_params = ['network','service', 'namespace', 'xdr', 'sindex']
show_params = ['config', 'distribution', 'latency', 'statistics']
cluster_params = ['service',
'services',
'xdr-min-lastshipinfo:',
'dump-fabric:',
'dump-hb:',
'dump-migrates:',
'dump-msgs:',
'dump-paxos:',
'dump-smd:',
'dump-wr:',
]
shell_cmds = ['date',
'hostname',
'ip addr',
'ip -s link',
'uptime',
'uname -a',
'lsb_release -a',
'ls /etc|grep release|xargs -I f cat /etc/f',
'rpm -qa|grep -E "citrus|aero"',
'dpkg -l|grep -E "citrus|aero"',
'free -m',
'df -h',
'ls /sys/block/{sd*,xvd*}/queue/rotational |xargs -I f sh -c "echo f; cat f;"',
'ls /sys/block/{sd*,xvd*}/device/model |xargs -I f sh -c "echo f; cat f;"',
'lsof',
cmd_dmesg,
'iostat -x 1 10',
'vmstat -s',
'vmstat -m',
'iptables -L',
]
cpu_stat = ['top -n3 -b', 'iostat -x 1 10', 'ss -pant', 'sar -n DEV', 'sar -n EDEV']
_ip = ((util.shell_command(["hostname -I"])[0]).split(' ')[0].strip())
if 'all' in line:
namespaces = self.parse_namespace(self.cluster._callNodeMethod([_ip], "info", "namespaces"))
for ns in namespaces:
cluster_params.append('dump-wb:ns=' + ns)
cluster_params.append('dump-wb-summary:ns=' + ns)
if 'ubuntu' == (platform.linux_distribution()[0]).lower():
cmd_dmesg = 'cat /var/log/syslog'
else:
cmd_dmesg = 'cat /var/log/messages'
terminal.enable_color(False)
os.makedirs(as_sysinfo_logdir)
try:
aslogfile = as_logfile_prefix + 'asadmCmd.log'
cinfo = InfoController()
for info_param in info_params:
self.collectinfo_content(cinfo,[info_param])
do_show = ShowController()
for show_param in show_params:
self.collectinfo_content(do_show,[show_param])
except Exception as e:
self.write_log(str(e))
sys.stdout = sys.__stdout__
try:
aslogfile = as_logfile_prefix + 'clusterCmd.log'
for cluster_param in cluster_params:
self.collectinfo_content('cluster',cluster_param)
except Exception as e:
self.write_log(str(e))
sys.stdout = sys.__stdout__
aslogfile = os.path.join(as_sysinfo_logdir, output_time + '_' + 'sysCmdOutput.log')
for cmd in shell_cmds:
self.collectinfo_content('shell',[cmd])
aslogfile = os.path.join(as_sysinfo_logdir, output_time + '_' + 'cpu_stat.log')
for _cmd in cpu_stat:
self.collectinfo_content('shell',[_cmd])
aslogfile = os.path.join(as_sysinfo_logdir, output_time + '_' + 'dmesg.log')
self.collectinfo_content('shell',[cmd_dmesg])
if 'True' in self.cluster.isXDREnabled().values():
aslogfile = as_logfile_prefix + 'xdr.log'
self.collectinfo_content('shell',['tail -n 10000 /var/log/*xdr.log'])
try:
as_version = self.cluster._callNodeMethod([_ip], "info", "build").popitem()[1]
log_location = self.cluster._callNodeMethod([_ip], "info",
"logs").popitem()[1].split(':')[1]
# Comparing with this version because prior to this it was citrusleaf.conf & citrusleaf.log
if StrictVersion(as_version) > StrictVersion("3.0.0"):
aslogfile = as_logfile_prefix + 'aerospike.conf'
self.collectinfo_content('shell',['cat /etc/aerospike/aerospike.conf'])
else:
aslogfile = as_logfile_prefix + 'citrusleaf.conf'
self.collectinfo_content('shell',['cat /etc/citrusleaf/citrusleaf.conf'])
self.collect_local_file(log_location, aslogdir)
except Exception as e:
self.write_log(str(e))
sys.stdout = sys.__stdout__
aslogfile = as_logfile_prefix + 'collectSys.log'
self.collectinfo_content(self.collect_sys)
aslogfile = as_logfile_prefix + 'awsData.log'
self.collectinfo_content(self.get_awsdata)
self.archive_log(aslogdir)
def archive_log(self,logdir):
self.zip_files(logdir)
util.shell_command(["tar -czvf " + logdir + ".tgz " + aslogdir])
sys.stderr.write("\x1b[2J\x1b[H")
print "\n\n\nFiles in " + logdir + " and " + logdir + ".tgz saved. "
print "END OF ASCOLLECTINFO"
