def test_storage_engine_detection(self):
""" LogFile: test if the correct storage engine is detected """
logfile = LogFile(self.file_year_rollover)
assert logfile.storage_engine == None
logfile_path = os.path.join(os.path.dirname(mtools.__file__),
'test/logfiles/', 'mongod_26.log')
mmapv1 = open(logfile_path, 'r')
logfile = LogFile(mmapv1)
assert logfile.storage_engine == 'mmapv1'
# test for 3.0 WT detection
logfile_path = os.path.join(os.path.dirname(mtools.__file__),
'test/logfiles/', 'wiredtiger.log')
wiredtiger = open(logfile_path, 'r')
logfile = LogFile(wiredtiger)
assert logfile.storage_engine == 'wiredTiger'
# test for 3.2 WT detection
logfile_path = os.path.join(os.path.dirname(mtools.__file__),
'test/logfiles/', 'mongod_328.log')
wiredtiger = open(logfile_path, 'r')
logfile = LogFile(wiredtiger)
assert logfile.storage_engine == 'wiredTiger'
def test_shard_csrs(self):
"""LogFile: test if sharded cluster CSRS is detected (MongoDB 3.6+) """
# mongos log
logfile_path = os.path.join(os.path.dirname(mtools.__file__),
'test/logfiles/','sharding_360_mongos.log')
mongos_log = open(logfile_path, 'rb')
logfile = LogFile(mongos_log)
assert logfile.csrs == ('configRepl', 'localhost:27033')
# config log
logfile_path = os.path.join(os.path.dirname(mtools.__file__),
'test/logfiles/','sharding_360_CSRS.log')
csrs_log = open(logfile_path, 'rb')
logfile = LogFile(csrs_log)
assert logfile.csrs == ('configRepl', '[ { _id: 0, host: "localhost:27033",'
' arbiterOnly: false, buildIndexes: true, hidden: false,'
' priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 } ]')
# shard log
logfile_path = os.path.join(os.path.dirname(mtools.__file__),
'test/logfiles/','sharding_360_shard.log')
shard_log = open(logfile_path, 'rb')
logfile = LogFile(shard_log)
assert logfile.csrs == ('configRepl', 'localhost:27033')
def test_shard_info(self):
"""LogFile: test if sharding info is detected (MongoDB 3.6+) """
# mongos log
logfile_path = os.path.join(os.path.dirname(mtools.__file__),
'test/logfiles/','sharding_360_mongos.log')
mongos_log = open(logfile_path, 'rb')
logfile = LogFile(mongos_log)
shards = logfile.shards
for name, repl_set in shards:
assert re.match(r'shard\d+', name)
assert re.match(r'localhost:270\d+', repl_set)
# config log
logfile_path = os.path.join(os.path.dirname(mtools.__file__),
'test/logfiles/','sharding_360_CSRS.log')
csrs_log = open(logfile_path, 'rb')
logfile = LogFile(csrs_log)
shards = logfile.shards
for name, repl_set in shards:
assert re.match(r'shard\d+', name)
assert re.match(r'localhost:270\d+', repl_set)
# shard log
logfile_path = os.path.join(os.path.dirname(mtools.__file__),
'test/logfiles/','sharding_360_shard.log')
shard_log = open(logfile_path, 'rb')
logfile = LogFile(shard_log)
shards = logfile.shards
for name, repl_set in shards:
assert re.match(r'shard\d+', name)
assert re.match(r'localhost:270\d+', repl_set)
def setup(self):
""" startup method to create mloginfo tool. """
self.tool = MLogInfoTool()
# load logfile(s)
self.logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'mongod_225.log')
self.logfile = LogFile(open(self.logfile_path, 'r'))
def __call__(self, string):
try:
# catch filetype and return LogFile object
filehandle = argparse.FileType.__call__(self, string)
return LogFile(filehandle)
except argparse.ArgumentTypeError as e:
raise argparse.ArgumentTypeError("can't open %s" % string)
def test_from_to_26_log(self):
logfile_26_path = os.path.join(os.path.dirname(mtools.__file__),
'test/logfiles/', 'mongod_26.log')
logfile_26 = LogFile(open(logfile_26_path, 'r'))
random_start = random_date(logfile_26.start, logfile_26.end)
random_end = random_date(random_start + timedelta(minutes=1),
logfile_26.end + timedelta(minutes=1))
print random_start, random_end
print logfile_26.start, logfile_26.end
self.tool.run(
'%s --from %s --to %s' %
(logfile_26_path, random_start.strftime("%b %d %H:%M:%S"),
random_end.strftime("%b %d %H:%M:%S")))
output = sys.stdout.getvalue()
assert len(output.splitlines()) > 0
at_least_one = False
for line in output.splitlines():
le = LogEvent(line)
if not le.datetime:
continue
at_least_one = True
assert (le.datetime >= random_start and le.datetime <= random_end)
assert at_least_one
def test_start_end(self):
""" LogFile: test .start and .end property work correctly """
logfile = LogFile(self.file_year_rollover)
assert logfile.start == datetime(2013, 12, 30, 00, 13, 01, 661000, tzutc())
assert logfile.end == datetime(2014, 01, 02, 23, 27, 11, 720000, tzutc())
def test_timezone(self):
logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'mongod_26.log')
mongod_26 = open(logfile_path, 'r')
logfile = LogFile(mongod_26)
assert logfile.timezone == tzoffset(None, -14400)
def __call__(self, string):
"""Open log file or MongoDB database."""
try:
# catch filetype and return LogFile object
filehandle = argparse.FileType.__call__(self, string)
return LogFile(filehandle)
except argparse.ArgumentTypeError:
# not a file, try open as MongoDB database
m = re.match('^(\w+)(?::(\d+))?(?:/([a-zA-Z0-9._-]+))?$',
string)
if m:
hostname, port, namespace = m.groups()
port = int(port) if port else 27017
namespace = namespace or 'test.system.profile'
if '.' in namespace:
database, collection = namespace.split('.', 1)
else:
database = namespace
collection = 'system.profile'
if (hostname == 'localhost'
or re.match('\d+\.\d+\.\d+\.\d+', hostname)):
return ProfileCollection(hostname, port, database,
collection)
raise argparse.ArgumentTypeError("can't open %s as file or "
"MongoDB connection string." %
string)
def setup(self):
""" start up method to create mlaunch tool and find free port. """
self.tool = MLogFilterTool()
# load logfile(s)
self.logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'mongod_225.log')
self.logfile = LogFile(open(self.logfile_path, 'r'))
def __init__(self, multiple_logfiles=False, stdin_allowed=True):
"""Add logfile(s) and stdin option to the argument parser."""
BaseCmdLineTool.__init__(self)
self.multiple_logfiles = multiple_logfiles
self.stdin_allowed = stdin_allowed
arg_opts = {'action': 'store', 'type': InputSourceAction('rb')}
if self.multiple_logfiles:
arg_opts['nargs'] = '*'
arg_opts['help'] = 'logfile(s) to parse'
else:
arg_opts['help'] = 'logfile to parse'
if self.is_stdin:
if not self.stdin_allowed:
raise SystemExit("this tool can't parse input from stdin.")
arg_opts['const'] = LogFile(sys.stdin)
arg_opts['action'] = 'store_const'
if 'type' in arg_opts:
del arg_opts['type']
if 'nargs' in arg_opts:
del arg_opts['nargs']
self.argparser.add_argument('logfile', **arg_opts)
def test_hostname_port(self):
# mongod
logfile_path = os.path.join(os.path.dirname(mtools.__file__),
'test/logfiles/', 'mongod_26.log')
mongod_26 = open(logfile_path, 'r')
logfile = LogFile(mongod_26)
assert logfile.hostname == 'enter.local'
assert logfile.port == '27019'
# mongos
logfile_path = os.path.join(os.path.dirname(mtools.__file__),
'test/logfiles/', 'mongos.log')
mongos = open(logfile_path, 'r')
logfile2 = LogFile(mongos)
print logfile2.hostname
assert logfile2.hostname == 'jimoleary.local'
assert logfile2.port == '27017'
def test_rsinfo(self):
"""LogFile: test if replication info is detected (MongoDB 3.2+) """
logfile_path = os.path.join(os.path.dirname(mtools.__file__),
'test/logfiles/', 'rsinfo_36.log')
rslog = open(logfile_path, 'rb')
logfile = LogFile(rslog)
assert logfile.repl_set == 'replset'
assert logfile.repl_set_version == '1'
assert logfile.repl_set_protocol == '1'
def test_len(self):
""" LogFile: test len() and iteration over LogFile method """
logfile = LogFile(self.file_year_rollover)
length = len(logfile)
i = 0
for i, le in enumerate(logfile):
assert isinstance(le, LogEvent)
assert length == i+1
assert length == 1836
def run(self, arguments=None):
""" Print out useful information about the log file. """
LogFileTool.run(self, arguments)
logfile = LogFile(self.args['logfile'])
print "start of logfile: %s" % (logfile.start.strftime(
"%b %d %H:%M:%S") if logfile.start else "unknown")
print " end of logfile: %s" % (logfile.end.strftime("%b %d %H:%M:%S")
if logfile.start else "unknown")
# get one logline (within first 20 lines) for datetime format
logline = None
for i in range(20):
try:
logline = LogLine(self.args['logfile'].next())
except StopIteration as e:
raise SystemExit(
"no valid log lines found (datetime not available).")
if logline.datetime:
break
# TODO: add timezone if iso8601 format
print " line numbers: %s" % logfile.num_lines
print " binary: %s" % (logfile.binary or "unknown")
version = (' -> '.join(logfile.versions) or "unknown")
# if version is unknown, go by date
if version == 'unknown' and logline:
if logline.datetime_format == 'ctime-pre2.4':
version = '< 2.4 (no milliseconds)'
elif logline.datetime_format == 'ctime':
version = '>= 2.4 (milliseconds present)'
elif logline.datetime_format.startswith('iso8601-'):
version = '>= 2.6 (iso8601 format)'
print " version: %s" % version
# restarts section
if self.args['restarts']:
print
print "RESTARTS"
for version, logline in logfile.restarts:
print " %s version %s" % (
logline.datetime.strftime("%b %d %H:%M:%S"), version)
if len(logfile.restarts) == 0:
print " no restarts found"
def logfile_generator(self):
""" generator method that yields each line of the logfile, or the next line in case of several log files. """
if not self.args["exclude"]:
# ask all filters for a start_limit and fast-forward to the maximum
start_limits = [f.start_limit for f in self.filters if hasattr(f, "start_limit")]
if start_limits:
for logfile in self.args["logfile"]:
lf_info = LogFile(logfile)
lf_info.fast_forward(max(start_limits))
if len(self.args["logfile"]) > 1:
# todo, merge
for logline in self._merge_logfiles():
yield logline
else:
# only one file
for line in self.args["logfile"][0]:
logline = LogLine(line)
if logline.datetime:
logline._datetime = logline.datetime + timedelta(hours=self.args["timezone"][0])
yield logline
def test_shard_chunk_migration_from(self):
"""
LogFile: test if shard chunk migration (from) is detected (MongoDB 3.6+)
"""
logfile_path = os.path.join(os.path.dirname(mtools.__file__),
'test/logfiles/','sharding_360_shard.log')
shard_log = open(logfile_path, 'rb')
logfile = LogFile(shard_log)
chunk_moved_from = logfile.chunks_moved_from[0]
assert len(logfile.chunks_moved_from) == 2
assert isinstance(chunk_moved_from[0], datetime)
assert chunk_moved_from[1] == "min: { sku: MinKey }, max: { sku: 23153496 }"
assert chunk_moved_from[2] == "shard02"
assert chunk_moved_from[3] == "test.products"
assert ('step 6 of 6', '57') in chunk_moved_from[4]
assert chunk_moved_from[5] == "success"
def _test_base(self, filename='mongod_225.log'):
# load logfile(s)
self.logfile_path = os.path.join(os.path.dirname(mtools.__file__),
'test/logfiles/', filename)
self.logfile = LogFile(open(self.logfile_path, 'rb'))
self.current_year = datetime.now().year
def _test_base(self, filename='mongod_225.log'):
# load logfile(s)
self.logfile_path = os.path.join(os.path.dirname(mtools.__file__),
'test/logfiles/', filename)
self.logfile = LogFile(open(self.logfile_path, 'r'))
def test_rollover_detection(self):
""" LogFile: test datetime_format and year_rollover properties """
logfile = LogFile(self.file_year_rollover)
assert logfile.datetime_format == "ctime"
assert logfile.year_rollover == logfile.end
