def _get_local_osds(self): if not self.cli_cache['ceph_volume_lvm_list']: return s = FileSearcher() sd = SequenceSearchDef(start=SearchDef(r"^=+\s+osd\.(\d+)\s+=+.*"), body=SearchDef([ r"\s+osd\s+(fsid)\s+(\S+)\s*", r"\s+(devices)\s+([\S]+)\s*" ]), tag="ceph-lvm") s.add_search_term(sd, path=self.cli_cache['ceph_volume_lvm_list']) local_osds = [] for results in s.search().find_sequence_sections(sd).values(): id = None fsid = None dev = None for result in results: if result.tag == sd.start_tag: id = int(result.get(1)) elif result.tag == sd.body_tag: if result.get(1) == "fsid": fsid = result.get(2) elif result.get(1) == "devices": dev = result.get(2) local_osds.append(CephOSD(id, fsid, dev)) return local_osds
def udev_bcache_devs(self): """ If bcache devices exist fetch information and return as a list. """ if self._bcache_devs: return self._bcache_devs udevadm_info = self.cli.udevadm_info_exportdb() if not udevadm_info: return self._bcache_devs s = FileSearcher() sdef = SequenceSearchDef(start=SearchDef(r"^P: .+/(bcache\S+)"), body=SearchDef(r"^S: disk/by-uuid/(\S+)"), tag="bcacheinfo") s.add_search_term(sdef, utils.mktemp_dump('\n'.join(udevadm_info))) results = s.search() devs = [] for section in results.find_sequence_sections(sdef).values(): dev = {} for r in section: if r.tag == sdef.start_tag: dev["name"] = r.get(1) else: dev["by-uuid"] = r.get(1) devs.append(dev) self._bcache_devs = devs return self._bcache_devs
def test_sequence_searcher_multi_sequence(self): """ Test scenario: * search containing multiple seqeunce definitions * data containing 2 results of each where one is incomplete * test that single incomplete result gets removed """ with tempfile.NamedTemporaryFile(mode='w', delete=False) as ftmp: ftmp.write(MULTI_SEQ_TEST) ftmp.close() s = FileSearcher() sdA = SequenceSearchDef(start=SearchDef(r"^sectionA (\d+)"), body=SearchDef(r"\d_\d"), end=SearchDef(r"^section\S+ (\d+)"), tag="seqA-search-test") sdB = SequenceSearchDef(start=SearchDef(r"^sectionB (\d+)"), body=SearchDef(r"\d_\d"), end=SearchDef(r"^section\S+ (\d+)"), tag="seqB-search-test") s.add_search_term(sdA, path=ftmp.name) s.add_search_term(sdB, path=ftmp.name) results = s.search() sections = results.find_sequence_sections(sdA) self.assertEqual(len(sections), 1) sections = results.find_sequence_sections(sdB) self.assertEqual(len(sections), 2) os.remove(ftmp.name)
def __summary_agent_checks(self): # Only run if we think Openstack is installed. if not self.openstack_installed: return s = FileSearcher() checks = [AgentApparmorChecks(searchobj=s), ApacheEventChecks(searchobj=s), NeutronL3HAEventChecks(searchobj=s), NeutronAgentEventChecks(searchobj=s), NovaAgentEventChecks(searchobj=s), OctaviaAgentEventChecks(searchobj=s)] for check in checks: check.load() results = s.search() _final_results = {} for check in checks: check.run(results) check_results = check.raw_output if check_results: _final_results.update(check_results) if _final_results: return _final_results
def test_sequence_searcher_section_start_end_same(self): """ Test scenario: * multiple sections that end with start of the next * start def matches unique start * end def matches any start """ with tempfile.NamedTemporaryFile(mode='w', delete=False) as ftmp: ftmp.write(SEQ_TEST_7) ftmp.close() s = FileSearcher() sd = SequenceSearchDef(start=SearchDef(r"^section (2)"), body=SearchDef(r"\d_\d"), end=SearchDef(r"^section (\d+)"), tag="seq-search-test7") s.add_search_term(sd, path=ftmp.name) results = s.search() sections = results.find_sequence_sections(sd) self.assertEqual(len(sections), 1) for id in sections: for r in sections[id]: if r.tag == sd.start_tag: self.assertEqual(r.get(1), "2") elif r.tag == sd.body_tag: self.assertTrue(r.get(0) in ["2_1"]) os.remove(ftmp.name)
def _create_search_results(self, path, contents): with open(path, 'w') as fd: for line in contents: fd.write(line) s = FileSearcher() s.add_search_term(SearchDef(r'^(\S+) (\S+) .+', tag='all'), path) return s.search().find_by_tag('all')
def _get_vcpu_info(self): nova_config = OpenstackConfig( os.path.join(HotSOSConfig.DATA_ROOT, "etc/nova/nova.conf")) vcpu_info = {} guests = [] s = FileSearcher() if self.nova.instances: for i in self.nova.instances.values(): guests.append(i.name) path = os.path.join(HotSOSConfig.DATA_ROOT, 'etc/libvirt/qemu', "{}.xml".format(i.name)) s.add_search_term(SearchDef(".+vcpus>([0-9]+)<.+", tag=i.name), path) total_vcpus = 0 results = s.search() for guest in guests: for r in results.find_by_tag(guest): vcpus = r.get(1) total_vcpus += int(vcpus) vcpu_info["used"] = total_vcpus sysinfo = SystemBase() if sysinfo.num_cpus is not None: total_cores = sysinfo.num_cpus vcpu_info["system-cores"] = total_cores pinset = nova_config.get("vcpu_pin_set", expand_to_list=True) or [] pinset += nova_config.get("cpu_dedicated_set", expand_to_list=True) or [] pinset += nova_config.get("cpu_shared_set", expand_to_list=True) or [] if pinset: # if pinning is used, reduce total num of cores available # to those included in nova cpu sets. available_cores = len(set(pinset)) else: available_cores = total_cores vcpu_info["available-cores"] = available_cores cpu = CPU() # put this here so that available cores value has # context if cpu.smt is not None: vcpu_info["smt"] = cpu.smt factor = float(total_vcpus) / available_cores vcpu_info["overcommit-factor"] = round(factor, 2) return vcpu_info
def stats(self): """ Get ip link info for the interface. """ counters = self.cache_load() if counters: return counters s = FileSearcher() seqdef = SequenceSearchDef( # match start of interface start=SearchDef(IP_IFACE_NAME_TEMPLATE.format(self.name)), # match body of interface body=SearchDef(r".+"), # match next interface or EOF end=SearchDef([IP_IFACE_NAME, IP_EOF]), tag="ifaces") f_ip_link_show = mktemp_dump(''.join(self.cli_helper.ip_link())) s.add_search_term(seqdef, path=f_ip_link_show) results = s.search() os.unlink(f_ip_link_show) stats_raw = [] for section in results.find_sequence_sections(seqdef).values(): for result in section: if result.tag == seqdef.body_tag: stats_raw.append(result.get(0)) if not stats_raw: return {} # NOTE: we only expect one match counters = {} for i, line in enumerate(stats_raw): ret = re.compile(r"\s+([RT]X):\s+.+").findall(line) if ret: rxtx = ret[0].lower() ret = re.compile(r"\s*([a-z]+)\s*").findall(line) if ret: for j, column in enumerate(ret): value = int(stats_raw[i + 1].split()[j]) if column in ['packets', 'dropped', 'errors', 'overrun']: if rxtx not in counters: counters[rxtx] = {} counters[rxtx][column] = value if counters: self.cache_save(counters) return counters return {}
def _get_version_info(self, daemon_type=None): """ Returns a dict of ceph versions info for the provided daemon type. If no daemon type provided, version info is collected for all types and the resulting dict is keyed by daemon type otherwise it is keyed by version (and only versions for that daemon type.) """ out = self.cli_cache['ceph_versions'] if not out: return versions = {} s = FileSearcher() body = SearchDef(r"\s+\"ceph version (\S+) .+ (\S+) " r"\(\S+\)\":\s+(\d)+,?$") if daemon_type is None: # all/any - start matches any so no seq ending needed sd = SequenceSearchDef(start=SearchDef(r"^\s+\"(\S+)\":\s+{"), body=body, tag='versions') else: start = SearchDef(r"^\s+\"({})\":\s+{{".format(daemon_type)) sd = SequenceSearchDef(start=start, body=body, end=SearchDef(r"^\s+\"\S+\":\s+{"), tag='versions') s.add_search_term(sd, path=self.cli_cache['ceph_versions']) for section in s.search().find_sequence_sections(sd).values(): _versions = {} for result in section: if result.tag == sd.start_tag: _daemon_type = result.get(1) versions[_daemon_type] = _versions elif result.tag == sd.body_tag: version = result.get(1) rname = result.get(2) amount = result.get(3) _versions[version] = { 'release_name': rname, 'count': int(amount) } # If specific daemon_type provided only return version for that type # otherwise all. if daemon_type is not None: versions = versions.get(daemon_type) return versions
def process_events(self, event): ext_output = {} events = {} events_found = {} s = FileSearcher() for result in event.results: instance_id = result.get(1) event_id = result.get(3) events[event_id] = { 'instance_id': instance_id, 'data_source': result.source } for stage in EXT_EVENT_META[event.name]['stages_keys']: expr = (r".+\[instance: {}\]\s+{}\s.*\s?event\s+{}-{}.? " ".+".format(instance_id, stage, event.name, event_id)) tag = "{}_{}_{}".format(instance_id, event_id, stage) sd = SearchDef(expr, tag, hint=event.name) s.add_search_term(sd, result.source) results = s.search() for event_id in events: instance_id = events[event_id]['instance_id'] data_source = events[event_id]['data_source'] stages = self.get_state_dict(event.name) for stage in stages: tag = "{}_{}_{}".format(instance_id, event_id, stage) r = results.find_by_tag(tag, path=data_source) if r: stages[stage] = True if all([stages[stage] for stage in stages]): result = 'succeeded' else: result = 'failed' if result not in ext_output: ext_output[result] = [] info = {'port': event_id, 'instance': instance_id} ext_output[result].append(info) if ext_output: for result in ext_output: events_found[result] = list(ext_output[result]) return events_found
def __init__(self, *args, **kwargs): super().__init__( *args, callback_helper=EVENTCALLBACKS, yaml_defs_group='nova-external-events', searchobj=FileSearcher(), **kwargs, )
def timed_out_plugins(self): timeouts = [] if not os.path.exists(os.path.join(HotSOSConfig.DATA_ROOT, 'sos_logs')): return timeouts searcher = FileSearcher() path = os.path.join(HotSOSConfig.DATA_ROOT, 'sos_logs/ui.log') searcher.add_search_term(SearchDef(r".* Plugin (\S+) timed out.*", tag="timeouts"), path=path) results = searcher.search() for r in results.find_by_tag("timeouts"): plugin = r.get(1) timeouts.append(plugin) return timeouts
def test_filesearcher_network_info(self): filepath = os.path.join(HotSOSConfig.DATA_ROOT, 'sos_commands', 'networking', 'ip_-d_address') filepath2 = os.path.join(HotSOSConfig.DATA_ROOT, 'sos_commands', 'networking', 'ip_-s_-d_link') ip = "10.0.0.128" mac = "22:c2:7b:1c:12:1b" s = FileSearcher() sd = SearchDef(r".+({}).+".format(ip)) s.add_search_term(sd, filepath) sd = SearchDef(r"^\s+link/ether\s+({})\s+.+".format(mac)) s.add_search_term(sd, filepath2) results = s.search() self.assertEqual(set(results.files), set([filepath, filepath2])) self.assertEqual(len(results.find_by_path(filepath)), 1) self.assertEqual(len(results.find_by_path(filepath2)), 2) self.assertEqual(results.find_by_path(filepath)[0].linenumber, 38) for result in results.find_by_path(filepath): self.assertEqual(result.get(1), ip) expected = {52: mac, 141: mac} for result in results.find_by_path(filepath2): ln = result.linenumber self.assertEqual(result.tag, None) self.assertEqual(result.get(1), expected[ln])
def rss(self): """Return memory RSS for a given daemon. NOTE: this assumes we have ps auxwwwm format. """ if self._rss: return self._rss s = FileSearcher() # columns: USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND sd = SearchDef(r"\S+\s+\d+\s+\S+\s+\S+\s+\d+\s+(\d+)\s+.+/ceph-{}\s+" r".+--id\s+{}\s+.+".format(self.daemon_type, self.id)) s.add_search_term(sd, path=self.cli_cache['ps']) rss = 0 # we only expect one result for result in s.search().find_by_path(self.cli_cache['ps']): rss = int(int(result.get(1)) / 1024) break self._rss = "{}M".format(rss) return self._rss
def test_sequence_searcher_overlapping_incomplete(self): with tempfile.NamedTemporaryFile(mode='w', delete=False) as ftmp: ftmp.write(SEQ_TEST_3) ftmp.close() s = FileSearcher() sd = SequenceSearchDef( start=SearchDef(r"^(a\S*) (start\S*) point\S*"), body=SearchDef(r"leads to"), end=SearchDef(r"^an (ending)$"), tag="seq-search-test3") s.add_search_term(sd, path=ftmp.name) results = s.search() sections = results.find_sequence_sections(sd) self.assertEqual(len(sections), 1) for id in sections: for r in sections[id]: if r.tag == sd.start_tag: self.assertEqual(r.get(1), "another") elif r.tag == sd.end_tag: self.assertEqual(r.get(1), "ending") os.remove(ftmp.name)
def test_filesearch_filesort(self): ordered_contents = [] self.maxDiff = None with tempfile.TemporaryDirectory() as dtmp: os.mknod(os.path.join(dtmp, "my-test-agent.log")) ordered_contents.append("my-test-agent.log") os.mknod(os.path.join(dtmp, "my-test-agent.log.1")) ordered_contents.append("my-test-agent.log.1") # add in an erroneous file that does not follow logrotate format os.mknod(os.path.join(dtmp, "my-test-agent.log.tar.gz")) for i in range(2, 100): fname = "my-test-agent.log.{}.gz".format(i) os.mknod(os.path.join(dtmp, fname)) ordered_contents.append(fname) self.assertEqual(FileSearcher().logrotate_file_sort(fname), i) ordered_contents.append("my-test-agent.log.tar.gz") contents = os.listdir(dtmp) self.assertEqual( sorted(contents, key=FileSearcher().logrotate_file_sort), ordered_contents)
def test_sequence_searcher_multiple_sections(self): with tempfile.NamedTemporaryFile(mode='w', delete=False) as ftmp: ftmp.write(SEQ_TEST_5) ftmp.close() s = FileSearcher() sd = SequenceSearchDef( start=SearchDef(r"^(a\S*) (start\S*) point\S*"), body=SearchDef(r"value is (\S+)"), end=SearchDef(r"^$"), tag="seq-search-test5") s.add_search_term(sd, path=ftmp.name) results = s.search() sections = results.find_sequence_sections(sd) self.assertEqual(len(sections), 2) for id in sections: for r in sections[id]: if r.tag == sd.start_tag: self.assertEqual(r.get(1), "another") elif r.tag == sd.body_tag: self.assertTrue(r.get(1) in ["3", "4"]) elif r.tag == sd.end_tag: self.assertEqual(r.get(0), "") os.remove(ftmp.name)
def test_filesearcher_error(self): s = FileSearcher() with mock.patch.object(SearchResult, '__init__') as mock_init: def fake_init(*args, **kwargs): raise EOFError("some error") mock_init.side_effect = fake_init path = os.path.join(HotSOSConfig.DATA_ROOT) s.add_search_term(SearchDef("."), path) s.search()
def test_search_filter_invert_match(self): with tempfile.NamedTemporaryFile(mode='w', delete=False) as ftmp: ftmp.write(FILTER_TEST_1) ftmp.close() s = FileSearcher() fd = FilterDef(r" (ERROR)", invert_match=True) s.add_filter_term(fd, path=ftmp.name) sd = SearchDef(r".+ INFO (.+)") s.add_search_term(sd, path=ftmp.name) results = s.search().find_by_path(ftmp.name) self.assertEqual(len(results), 1) for r in results: self.assertEqual(r.get(1), "blah") os.remove(ftmp.name)
def test_filesearch_glob_filesort(self): dir_contents = [] self.maxDiff = None with tempfile.TemporaryDirectory() as dtmp: dir_contents.append(os.path.join(dtmp, "my-test-agent.0.log")) dir_contents.append(os.path.join(dtmp, "my-test-agent.1.log")) dir_contents.append(os.path.join(dtmp, "my-test-agent.1.log.1")) dir_contents.append(os.path.join(dtmp, "my-test-agent.2.log")) dir_contents.append(os.path.join(dtmp, "my-test-agent.16.log")) dir_contents.append(os.path.join(dtmp, "my-test-agent.49.log")) dir_contents.append(os.path.join(dtmp, "my-test-agent.49.log.1")) dir_contents.append(os.path.join(dtmp, "my-test-agent.77.log")) dir_contents.append(os.path.join(dtmp, "my-test-agent.100.log")) dir_contents.append(os.path.join(dtmp, "my-test-agent.100.log.1")) dir_contents.append(os.path.join(dtmp, "my-test-agent.110.log")) dir_contents.append(os.path.join(dtmp, "my-test-agent.142.log")) dir_contents.append(os.path.join(dtmp, "my-test-agent.183.log")) for e in dir_contents: os.mknod(e) for i in range(2, HotSOSConfig.MAX_LOGROTATE_DEPTH + 10): fname = os.path.join(dtmp, "my-test-agent.1.log.{}.gz".format(i)) os.mknod(fname) if i <= HotSOSConfig.MAX_LOGROTATE_DEPTH: dir_contents.append(fname) for i in range(2, HotSOSConfig.MAX_LOGROTATE_DEPTH + 10): fname = os.path.join(dtmp, "my-test-agent.49.log.{}.gz".format(i)) os.mknod(fname) if i <= HotSOSConfig.MAX_LOGROTATE_DEPTH: dir_contents.append(fname) for i in range(2, HotSOSConfig.MAX_LOGROTATE_DEPTH + 10): fname = os.path.join(dtmp, "my-test-agent.100.log.{}.gz".format(i)) os.mknod(fname) if i <= HotSOSConfig.MAX_LOGROTATE_DEPTH: dir_contents.append(fname) exp = sorted(dir_contents) path = os.path.join(dtmp, 'my-test-agent*.log*') act = sorted(FileSearcher().filtered_paths(glob.glob(path))) self.assertEqual(act, exp)
def __init__(self): # NOTE: we are OpenstackEventChecksBase to get the call structure but # we dont currently use yaml to define out searches. super().__init__(yaml_defs_group='agent-exceptions', searchobj=FileSearcher()) # The following are expected to be WARNING self._agent_warnings = { 'nova': [ 'MessagingTimeout', 'DiskNotFound', ], 'neutron': [ r'OVS is dead', r'MessagingTimeout', ] } # The following are expected to be ERROR. This is typically used to # catch events that are not defined as an exception. self._agent_errors = {'neutron': [r'RuntimeError']}
def test_ordered_multiple(self): start0 = '2021-07-19 09:08:58.498000' end0 = '2021-07-19 09:09:58.498000' start1 = '2021-07-19 09:03:58.498000' end1 = '2021-07-19 09:04:58.498000' expected = { '0': { 'duration': 60.0, 'start': start0, 'end': end0 }, '1': { 'duration': 60.0, 'start': start1, 'end': end1 } } with tempfile.NamedTemporaryFile(mode='w', delete=False) as ftmp: ftmp.write(SEQ_TEST_6) ftmp.close() s = FileSearcher() expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) start' s.add_search_term(SearchDef(expr, tag="eventX-start"), path=ftmp.name) expr = r'^([0-9\-]+) (\S+) iteration:([0-9]+) end' s.add_search_term(SearchDef(expr, tag="eventX-end"), path=ftmp.name) events = analytics.LogEventStats(s.search(), "eventX") os.remove(ftmp.name) events.run() top5 = events.get_top_n_events_sorted(5) self.assertEqual(top5, expected) stats = events.get_event_stats() expected = { 'avg': 60.0, 'incomplete': 2, 'max': 60.0, 'min': 60.0, 'samples': 2, 'stdev': 0.0 } self.assertEqual(stats, expected)
def __init__(self): self._property_cache = {} # save to file so we can search it later cli = host_helpers.CLIHelper() self._f_report = mktemp_dump(''.join(cli.rabbitmqctl_report())) searcher = FileSearcher() searcher.add_search_term(self.connections_searchdef, self._f_report) searcher.add_search_term(self.memory_searchdef, self._f_report) searcher.add_search_term(self.cluster_partition_handling_searchdef, self._f_report) searcher.add_search_term(self.queues_searchdef, self._f_report) self.results = searcher.search()
def _get_interfaces(self, namespaces=False): """ Get all interfaces in ip address show. @param namespaces: if set to True will get interfaces from all namespaces on the host. @return: list of NetworkPort objects for each interface found. """ interfaces = [] interfaces_raw = self.cache_load(namespaces=namespaces) if interfaces_raw: for iface in interfaces_raw: interfaces.append(NetworkPort(**iface)) return interfaces interfaces_raw = [] seq = SequenceSearchDef(start=SearchDef(IP_IFACE_NAME), body=SearchDef([IP_IFACE_V4_ADDR, IP_IFACE_V6_ADDR, IP_IFACE_HW_ADDR, IP_IFACE_VXLAN_INFO]), tag='ip_addr_show') search_obj = FileSearcher() if namespaces: for ns in self.cli.ip_netns(): ns_name = ns.partition(" ")[0] ip_addr = self.cli.ns_ip_addr(namespace=ns_name) path = mktemp_dump('\n'.join(ip_addr)) search_obj.add_search_term(seq, path) else: path = mktemp_dump('\n'.join(self.cli.ip_addr())) search_obj.add_search_term(seq, path) if not search_obj.paths: log.debug("no network info found (namespaces=%s)", namespaces) return [] r = search_obj.search() for path in search_obj.paths: # we no longer need this file so can delete it os.unlink(path) sections = r.find_sequence_sections(seq, path).values() for section in sections: addrs = [] encap_info = None hwaddr = None name = None state = None for result in section: if result.tag == seq.start_tag: name = result.get(1) state = result.get(2) elif result.tag == seq.body_tag: if result.get(1) in ['inet', 'inet6']: addrs.append(result.get(2)) elif result.get(1) in ['vxlan']: encap_info = {result.get(1): { 'id': result.get(2), 'local_ip': result.get(3), 'dev': result.get(4)}} else: hwaddr = result.get(2) interfaces_raw.append({'name': name, 'addresses': addrs, 'hwaddr': hwaddr, 'state': state, 'encap_info': encap_info}) self.cache_save(interfaces_raw, namespaces=namespaces) for iface in interfaces_raw: interfaces.append(NetworkPort(**iface)) return interfaces
def __init__(self): super().__init__(yaml_defs_group='kernlog', searchobj=FileSearcher(), callback_helper=EVENTCALLBACKS) self.cli_helper = CLIHelper() self.hostnet_helper = HostNetworkingHelper()
def _result(self): if self.expr: if self.cache.expr: results = self.cache.expr.results log.debug("check %s - using cached result=%s", self.name, results) else: s = FileSearcher() s.add_search_term(SearchDef(self.expr.value, tag=self.name), self.input.path) results = s.search() self.expr.cache.set('results', results) # The following aggregates results by group/index and stores in # the property cache to make them accessible via # PropertyCacheRefResolver. results_by_idx = {} for item in results: for _result in item[1]: for idx, value in enumerate(_result): if idx not in results_by_idx: results_by_idx[idx] = set() results_by_idx[idx].add(value) for idx in results_by_idx: self.expr.cache.set('results_group_{}'.format(idx), list(results_by_idx[idx])) self.cache.set('expr', self.expr.cache) if not results: log.debug("check %s search has no matches so result=False", self.name) return False results = results.find_by_tag(self.name) parameters = self.check_paramaters if parameters: result_age_hours = parameters.search_result_age_hours results = self.filter_by_age(results, result_age_hours) if results: period_hours = parameters.search_period_hours results = self.filter_by_period(results, period_hours, parameters.min_results) count = len(results) if count >= parameters.min_results: return True else: log.debug("check %s does not have enough matches (%s) to " "satisfy min of %s", self.name, count, parameters.min_results) return False else: log.debug("no check paramaters provided") return len(results) > 0 elif self.requires: if self.cache.requires: result = self.cache.requires.passes log.debug("check %s - using cached result=%s", self.name, result) else: result = self.requires.passes self.cache.set('requires', self.requires.cache) return result else: raise Exception("no supported properties found in check {}".format( self.name))
def __init__(self): super().__init__(yaml_defs_group='mygroup', searchobj=FileSearcher(), callback_helper=EVENTCALLBACKS)
def test_filesearcher_num_cpus_no_override(self, mock_cpu_count): mock_cpu_count.return_value = 3 s = FileSearcher() self.assertEqual(s.num_cpus, 3)
def test_filesearcher_num_cpus_w_override(self, mock_cpu_count): setup_config(MAX_PARALLEL_TASKS=2) mock_cpu_count.return_value = 3 s = FileSearcher() self.assertEqual(s.num_cpus, 2)
def test_filesearcher_logs(self): expected = { 9891: '2022-02-09 22:50:18.131', 9892: '2022-02-09 22:50:19.703' } logs_root = "var/log/neutron/" filepath = os.path.join(HotSOSConfig.DATA_ROOT, logs_root, 'neutron-openvswitch-agent.log.2.gz') globpath = os.path.join(HotSOSConfig.DATA_ROOT, logs_root, 'neutron-l3-agent.log') globpath_file1 = os.path.join(HotSOSConfig.DATA_ROOT, logs_root, 'neutron-l3-agent.log') globpath_file2 = os.path.join(HotSOSConfig.DATA_ROOT, logs_root, 'neutron-l3-agent.log.1.gz') s = FileSearcher() sd = SearchDef(r'^(\S+\s+[0-9:\.]+)\s+.+full sync.+', tag="T1") s.add_search_term(sd, filepath) sd = SearchDef(r'^(\S+\s+[0-9:\.]+)\s+.+ERROR.+', tag="T2") s.add_search_term(sd, filepath) sd = SearchDef((r'^(\S+\s+[0-9:\.]+)\s+.+ INFO .+ Router [0-9a-f\-]+' '.+'), tag="T3") s.add_search_term(sd, globpath) sd = SearchDef(r'non-existant-pattern', tag="T4") # search for something that doesn't exist to test that code path s.add_search_term(sd, globpath) results = s.search() self.assertEqual(set(results.files), set([filepath, globpath])) self.assertEqual(len(results.find_by_path(filepath)), 1220) tag_results = results.find_by_tag("T1", path=filepath) self.assertEqual(len(tag_results), 2) for result in tag_results: ln = result.linenumber self.assertEqual(result.tag, "T1") self.assertEqual(result.get(1), expected[ln]) tag_results = results.find_by_tag("T1") self.assertEqual(len(tag_results), 2) for result in tag_results: ln = result.linenumber self.assertEqual(result.tag, "T1") self.assertEqual(result.get(1), expected[ln]) self.assertEqual(len(results.find_by_path(globpath_file1)), 1) self.assertEqual(len(results.find_by_path(globpath_file2)), 0) # these files have the same content so expect same result from both expected = {5380: '2022-02-10 16:09:22.641'} path_results = results.find_by_path(globpath_file1) for result in path_results: ln = result.linenumber self.assertEqual(result.tag, "T3") self.assertEqual(result.get(1), expected[ln]) path_results = results.find_by_path(globpath_file2) for result in path_results: ln = result.linenumber self.assertEqual(result.tag, "T3") self.assertEqual(result.get(1), expected[ln])