def process_json(self, content, filename):
log.debug('process_json()')
if not content:
log.warning("blank content passed to process_json for contents of file '%s'", filename)
if isJson(content):
print(json.dumps(json.loads(content)))
return True
elif self.permit_single_quotes:
log.debug('checking for single quoted JSON')
# check if it's regular single quoted JSON a la MongoDB
json_single_quoted = self.convert_single_quoted(content)
if self.process_json_single_quoted(json_single_quoted, filename):
return True
log.debug('single quoted JSON check failed, trying with pre-escaping double quotes')
# check if it's single quoted JSON with double quotes that aren't escaped,
# by pre-escaping them before converting single quotes to doubles for processing
json_single_quoted_escaped = self.convert_single_quoted_escaped(content)
if self.process_json_single_quoted(json_single_quoted_escaped, filename):
log.debug("processed single quoted json with non-escaped double quotes in '%s'", filename)
return True
log.debug('single quoted JSON check failed even with pre-escaping any double quotes')
self.failed = True
log.error("invalid json detected in '%s':", filename)
printerr(content)
if not self.continue_on_error:
sys.exit(ERRORS['CRITICAL'])
return False
def run_hosts(self):
for host in self.host_list:
url = 'http://{host}:{port}/jmx'.format(host=host, port=self.port)
if not self.since_uptime:
url += '?qry=Hadoop:service=HBase,name=RegionServer,sub=Server'
try:
self.run_host(host, url)
except (CriticalError, UnknownError, requests.RequestException) as _:
printerr("ERROR querying JMX stats for host '{}': {}".format(host, _), )
def construct_msg(self):
# user = os.getenv('USER', '').strip()
user = getpass.getuser()
if not isUser(user):
# print("invalid user '%s' determined from environment variable $USER, failed regex validation" % user)
print(
"invalid user '%s' returned by getpass.getuser(), failed regex validation"
% user)
sys.exit(ERRORS['CRITICAL'])
user = self.case_user(user)
msg = 'Welcome %s - ' % user
last = ''
if which("last"):
_ = os.popen('last -100')
_.readline()
re_skip = re.compile(r'^(?:reboot|wtmp)|^\s*$')
last = ''
for line in _:
last = line.rstrip('\n')
if re_skip.match(last):
last = ''
continue
break
_.close()
else:
printerr(
"WARNING: 'last' command not found, will not be able to get last login information"
)
if last:
msg += 'last login was '
last_user = re.sub(r'\s+.*$', '', last)
if last_user == 'root':
last_user = '******'
# strip up to "Day Mon NN" ie "%a %b %e ..."
(last, num_replacements) = re.subn(r'.*(\w{3}\s+\w{3}\s+\d+)',
r'\g<1>', last)
if not num_replacements:
print('failed to find the date format in the last log')
sys.exit(ERRORS['CRITICAL'])
last = re.sub(' *$', '', last)
if last_user == 'ROOT':
msg += 'ROOT'
elif last_user.lower() == user.lower():
msg += 'by you'
else:
msg += 'by %s' % last_user
msg += ' => %s' % last
else:
msg += 'no last login information available!'
return msg
def construct_msg(self): # pylint: disable=no-self-use
# user = os.getenv('USER', '').strip()
user = getpass.getuser()
if not isUser(user):
# print("invalid user '%s' determined from environment variable $USER, failed regex validation" % user)
print("invalid user '%s' returned by getpass.getuser(), failed regex validation" % user)
sys.exit(ERRORS['CRITICAL'])
user = self.case_user(user)
msg = 'Welcome %s - ' % user
last = ''
if which("last"):
_ = os.popen('last -100')
_.readline()
re_skip = re.compile(r'^(?:reboot|wtmp)|^\s*$')
last = ''
for line in _:
last = line.rstrip('\n')
if re_skip.match(last):
last = ''
continue
break
_.close()
else:
printerr("WARNING: 'last' command not found, will not be able to get last login information")
if last:
msg += 'last login was '
last_user = re.sub(r'\s+.*$', '', last)
if last_user == 'root':
last_user = '******'
# strip up to "Day Mon NN" ie "%a %b %e ..."
(last, num_replacements) = re.subn(r'.*(\w{3}\s+\w{3}\s+\d+)', r'\g<1>', last)
if not num_replacements:
print('failed to find the date format in the last log')
sys.exit(ERRORS['CRITICAL'])
last = re.sub(' *$', '', last)
if last_user == 'ROOT':
msg += 'ROOT'
elif last_user.lower() == user.lower():
msg += 'by you'
else:
msg += 'by %s' % last_user
msg += ' => %s' % last
else:
msg += 'no last login information available!'
return msg
def process_file(self, filename, file_handle):
for line in file_handle:
# log.debug(line)
match = self.re_line.match(line)
if not match:
err_msg = "ERROR in file '{0}' on line: {1}".format(
filename, line)
if not self.skip_errors:
die(err_msg)
printerr()
log.warn(err_msg)
continue
metric = match.group(1)
timestamp = match.group(2)
# don't have a need for this right now
# value = match.group(3)
tags = match.group(4)
key = metric
if self.include_timestamps:
timestamp = int(timestamp)
# remove millis
if len(str(timestamp)) >= 15:
timestamp = round(timestamp / 1000)
hour = time.strftime('%Y-%m-%d %H:00', time.gmtime(timestamp))
key += ' ' + hour
for tag in sorted(tags.split()):
key += ' ' + tag.strip()
if self.prefix_length is None:
prefix = key
else:
prefix = key[0:min(self.prefix_length, len(key))]
# prefix = self.bytes_to_str(prefix)
if not self.keys.get(prefix):
self.keys[prefix] = {'count': 0}
self.keys[prefix]['count'] += 1
self.total_keys += 1
if self.verbose < 2 and self.total_keys % 10000 == 0:
print('.', file=sys.stderr, end='')
def process_file(self, filename, file_handle):
for line in file_handle:
# log.debug(line)
match = self.re_line.match(line)
if not match:
err_msg = "ERROR in file '{0}' on line: {1}".format(filename, line)
if not self.skip_errors:
die(err_msg)
printerr()
log.warn(err_msg)
continue
metric = match.group(1)
timestamp = match.group(2)
# don't have a need for this right now
# value = match.group(3)
tags = match.group(4)
key = metric
if self.include_timestamps:
timestamp = int(timestamp)
# remove millis
if len(str(timestamp)) >= 15:
timestamp = round(timestamp / 1000)
hour = time.strftime('%Y-%m-%d %H:00', time.gmtime(timestamp))
key += ' ' + hour
for tag in sorted(tags.split()):
key += ' ' + tag.strip()
if self.prefix_length is None:
prefix = key
else:
prefix = key[0:min(self.prefix_length, len(key))]
# prefix = self.bytes_to_str(prefix)
if not self.keys.get(prefix):
self.keys[prefix] = {'count': 0}
self.keys[prefix]['count'] += 1
self.total_keys += 1
if self.verbose < 2 and self.total_keys % 10000 == 0:
print('.', file=sys.stderr, end='')
pyspark_path()
__author__ = 'Hari Sekhon'
__version__ = '0.3.2'
if not isPythonMinVersion(2.7):
warn(
'Python < 2.7 - IPython may not be available on this version of Python '
+ '(supplied auto-build will likely have failed for this module)\n')
try:
# pylint: disable=wrong-import-position
from IPython.lib import passwd
except ImportError as _:
printerr("""failed to import from IPython.lib
Perhaps you need to 'pip install \"ipython[notebook]\"'
Exception message: %s""" % _)
if not isPythonMinVersion(2.7):
printerr(
'Python < 2.7 - the supplied make auto build with this tool probably failed '
+ 'to install IPython because IPython requires Python >= 2.7')
sys.exit(ERRORS['UNKNOWN'])
sys.exit(ERRORS['CRITICAL'])
# Mac now supported
#try:
# linux_only()
#except LinuxOnlyException, e:
# die(e)
def run(self):
for host in self.host_list:
try:
self.print_region_stats(host)
except (CriticalError, UnknownError, requests.RequestException) as _:
printerr("ERROR querying JMX stats for host '{}': {}".format(host, _), )
sys.exit(4)
pyspark_path()
__author__ = 'Hari Sekhon'
__version__ = '0.3.2'
if not isPythonMinVersion(2.7):
warn('Python < 2.7 - IPython may not be available on this version of Python ' +
'(supplied auto-build will likely have failed for this module)\n')
try:
# pylint: disable=wrong-import-position
from IPython.lib import passwd
except ImportError as _:
printerr("""failed to import from IPython.lib
Perhaps you need to 'pip install \"ipython[notebook]\"'
Exception message: %s""" % _)
if not isPythonMinVersion(2.7):
printerr('Python < 2.7 - the supplied make auto build with this tool probably failed ' +
'to install IPython because IPython requires Python >= 2.7')
sys.exit(ERRORS['UNKNOWN'])
sys.exit(ERRORS['CRITICAL'])
# Mac now supported
#try:
# linux_only()
#except LinuxOnlyException, e:
# die(e)
if len(sys.argv) > 1:
