def test_storage_engine_detection(self): """ LogFile: test if the correct storage engine is detected """ logfile = LogFile(self.file_year_rollover) assert logfile.storage_engine == None logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'mongod_26.log') mmapv1 = open(logfile_path, 'r') logfile = LogFile(mmapv1) assert logfile.storage_engine == 'mmapv1' # test for 3.0 WT detection logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'wiredtiger.log') wiredtiger = open(logfile_path, 'r') logfile = LogFile(wiredtiger) assert logfile.storage_engine == 'wiredTiger' # test for 3.2 WT detection logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'mongod_328.log') wiredtiger = open(logfile_path, 'r') logfile = LogFile(wiredtiger) assert logfile.storage_engine == 'wiredTiger'
def test_shard_csrs(self): """LogFile: test if sharded cluster CSRS is detected (MongoDB 3.6+) """ # mongos log logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/','sharding_360_mongos.log') mongos_log = open(logfile_path, 'rb') logfile = LogFile(mongos_log) assert logfile.csrs == ('configRepl', 'localhost:27033') # config log logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/','sharding_360_CSRS.log') csrs_log = open(logfile_path, 'rb') logfile = LogFile(csrs_log) assert logfile.csrs == ('configRepl', '[ { _id: 0, host: "localhost:27033",' ' arbiterOnly: false, buildIndexes: true, hidden: false,' ' priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 } ]') # shard log logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/','sharding_360_shard.log') shard_log = open(logfile_path, 'rb') logfile = LogFile(shard_log) assert logfile.csrs == ('configRepl', 'localhost:27033')
def test_shard_info(self): """LogFile: test if sharding info is detected (MongoDB 3.6+) """ # mongos log logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/','sharding_360_mongos.log') mongos_log = open(logfile_path, 'rb') logfile = LogFile(mongos_log) shards = logfile.shards for name, repl_set in shards: assert re.match(r'shard\d+', name) assert re.match(r'localhost:270\d+', repl_set) # config log logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/','sharding_360_CSRS.log') csrs_log = open(logfile_path, 'rb') logfile = LogFile(csrs_log) shards = logfile.shards for name, repl_set in shards: assert re.match(r'shard\d+', name) assert re.match(r'localhost:270\d+', repl_set) # shard log logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/','sharding_360_shard.log') shard_log = open(logfile_path, 'rb') logfile = LogFile(shard_log) shards = logfile.shards for name, repl_set in shards: assert re.match(r'shard\d+', name) assert re.match(r'localhost:270\d+', repl_set)
def setup(self): """ startup method to create mloginfo tool. """ self.tool = MLogInfoTool() # load logfile(s) self.logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'mongod_225.log') self.logfile = LogFile(open(self.logfile_path, 'r'))
def __call__(self, string): try: # catch filetype and return LogFile object filehandle = argparse.FileType.__call__(self, string) return LogFile(filehandle) except argparse.ArgumentTypeError as e: raise argparse.ArgumentTypeError("can't open %s" % string)
def test_from_to_26_log(self): logfile_26_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'mongod_26.log') logfile_26 = LogFile(open(logfile_26_path, 'r')) random_start = random_date(logfile_26.start, logfile_26.end) random_end = random_date(random_start + timedelta(minutes=1), logfile_26.end + timedelta(minutes=1)) print random_start, random_end print logfile_26.start, logfile_26.end self.tool.run( '%s --from %s --to %s' % (logfile_26_path, random_start.strftime("%b %d %H:%M:%S"), random_end.strftime("%b %d %H:%M:%S"))) output = sys.stdout.getvalue() assert len(output.splitlines()) > 0 at_least_one = False for line in output.splitlines(): le = LogEvent(line) if not le.datetime: continue at_least_one = True assert (le.datetime >= random_start and le.datetime <= random_end) assert at_least_one
def test_start_end(self): """ LogFile: test .start and .end property work correctly """ logfile = LogFile(self.file_year_rollover) assert logfile.start == datetime(2013, 12, 30, 00, 13, 01, 661000, tzutc()) assert logfile.end == datetime(2014, 01, 02, 23, 27, 11, 720000, tzutc())
def test_timezone(self): logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'mongod_26.log') mongod_26 = open(logfile_path, 'r') logfile = LogFile(mongod_26) assert logfile.timezone == tzoffset(None, -14400)
def __call__(self, string): """Open log file or MongoDB database.""" try: # catch filetype and return LogFile object filehandle = argparse.FileType.__call__(self, string) return LogFile(filehandle) except argparse.ArgumentTypeError: # not a file, try open as MongoDB database m = re.match('^(\w+)(?::(\d+))?(?:/([a-zA-Z0-9._-]+))?$', string) if m: hostname, port, namespace = m.groups() port = int(port) if port else 27017 namespace = namespace or 'test.system.profile' if '.' in namespace: database, collection = namespace.split('.', 1) else: database = namespace collection = 'system.profile' if (hostname == 'localhost' or re.match('\d+\.\d+\.\d+\.\d+', hostname)): return ProfileCollection(hostname, port, database, collection) raise argparse.ArgumentTypeError("can't open %s as file or " "MongoDB connection string." % string)
def setup(self): """ start up method to create mlaunch tool and find free port. """ self.tool = MLogFilterTool() # load logfile(s) self.logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'mongod_225.log') self.logfile = LogFile(open(self.logfile_path, 'r'))
def __init__(self, multiple_logfiles=False, stdin_allowed=True): """Add logfile(s) and stdin option to the argument parser.""" BaseCmdLineTool.__init__(self) self.multiple_logfiles = multiple_logfiles self.stdin_allowed = stdin_allowed arg_opts = {'action': 'store', 'type': InputSourceAction('rb')} if self.multiple_logfiles: arg_opts['nargs'] = '*' arg_opts['help'] = 'logfile(s) to parse' else: arg_opts['help'] = 'logfile to parse' if self.is_stdin: if not self.stdin_allowed: raise SystemExit("this tool can't parse input from stdin.") arg_opts['const'] = LogFile(sys.stdin) arg_opts['action'] = 'store_const' if 'type' in arg_opts: del arg_opts['type'] if 'nargs' in arg_opts: del arg_opts['nargs'] self.argparser.add_argument('logfile', **arg_opts)
def test_hostname_port(self): # mongod logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'mongod_26.log') mongod_26 = open(logfile_path, 'r') logfile = LogFile(mongod_26) assert logfile.hostname == 'enter.local' assert logfile.port == '27019' # mongos logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'mongos.log') mongos = open(logfile_path, 'r') logfile2 = LogFile(mongos) print logfile2.hostname assert logfile2.hostname == 'jimoleary.local' assert logfile2.port == '27017'
def test_rsinfo(self): """LogFile: test if replication info is detected (MongoDB 3.2+) """ logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'rsinfo_36.log') rslog = open(logfile_path, 'rb') logfile = LogFile(rslog) assert logfile.repl_set == 'replset' assert logfile.repl_set_version == '1' assert logfile.repl_set_protocol == '1'
def test_len(self): """ LogFile: test len() and iteration over LogFile method """ logfile = LogFile(self.file_year_rollover) length = len(logfile) i = 0 for i, le in enumerate(logfile): assert isinstance(le, LogEvent) assert length == i+1 assert length == 1836
def run(self, arguments=None): """ Print out useful information about the log file. """ LogFileTool.run(self, arguments) logfile = LogFile(self.args['logfile']) print "start of logfile: %s" % (logfile.start.strftime( "%b %d %H:%M:%S") if logfile.start else "unknown") print " end of logfile: %s" % (logfile.end.strftime("%b %d %H:%M:%S") if logfile.start else "unknown") # get one logline (within first 20 lines) for datetime format logline = None for i in range(20): try: logline = LogLine(self.args['logfile'].next()) except StopIteration as e: raise SystemExit( "no valid log lines found (datetime not available).") if logline.datetime: break # TODO: add timezone if iso8601 format print " line numbers: %s" % logfile.num_lines print " binary: %s" % (logfile.binary or "unknown") version = (' -> '.join(logfile.versions) or "unknown") # if version is unknown, go by date if version == 'unknown' and logline: if logline.datetime_format == 'ctime-pre2.4': version = '< 2.4 (no milliseconds)' elif logline.datetime_format == 'ctime': version = '>= 2.4 (milliseconds present)' elif logline.datetime_format.startswith('iso8601-'): version = '>= 2.6 (iso8601 format)' print " version: %s" % version # restarts section if self.args['restarts']: print print "RESTARTS" for version, logline in logfile.restarts: print " %s version %s" % ( logline.datetime.strftime("%b %d %H:%M:%S"), version) if len(logfile.restarts) == 0: print " no restarts found"
def logfile_generator(self): """ generator method that yields each line of the logfile, or the next line in case of several log files. """ if not self.args["exclude"]: # ask all filters for a start_limit and fast-forward to the maximum start_limits = [f.start_limit for f in self.filters if hasattr(f, "start_limit")] if start_limits: for logfile in self.args["logfile"]: lf_info = LogFile(logfile) lf_info.fast_forward(max(start_limits)) if len(self.args["logfile"]) > 1: # todo, merge for logline in self._merge_logfiles(): yield logline else: # only one file for line in self.args["logfile"][0]: logline = LogLine(line) if logline.datetime: logline._datetime = logline.datetime + timedelta(hours=self.args["timezone"][0]) yield logline
def test_shard_chunk_migration_from(self): """ LogFile: test if shard chunk migration (from) is detected (MongoDB 3.6+) """ logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/','sharding_360_shard.log') shard_log = open(logfile_path, 'rb') logfile = LogFile(shard_log) chunk_moved_from = logfile.chunks_moved_from[0] assert len(logfile.chunks_moved_from) == 2 assert isinstance(chunk_moved_from[0], datetime) assert chunk_moved_from[1] == "min: { sku: MinKey }, max: { sku: 23153496 }" assert chunk_moved_from[2] == "shard02" assert chunk_moved_from[3] == "test.products" assert ('step 6 of 6', '57') in chunk_moved_from[4] assert chunk_moved_from[5] == "success"
def _test_base(self, filename='mongod_225.log'): # load logfile(s) self.logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', filename) self.logfile = LogFile(open(self.logfile_path, 'rb')) self.current_year = datetime.now().year
def _test_base(self, filename='mongod_225.log'): # load logfile(s) self.logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', filename) self.logfile = LogFile(open(self.logfile_path, 'r'))
def test_rollover_detection(self): """ LogFile: test datetime_format and year_rollover properties """ logfile = LogFile(self.file_year_rollover) assert logfile.datetime_format == "ctime" assert logfile.year_rollover == logfile.end