def handle_tokenizer(self, object=None, profile=None, config=None, *args, **kwargs):
cfg = None
if config:
if not os.path.exists(config):
self.die("File not found: %s" % config)
with open(config) as f:
cfg = f.read()
if object:
connect()
mo = ManagedObject.get_by_id(object)
if not mo:
self.die("Managed Object not found")
elif profile:
p = loader.get_profile(profile)
if not p:
self.die("Invalid profile: %s" % profile)
if not cfg:
self.die("Specify config file with --config option")
# Mock up tokenizer
connect()
mo = ManagedObject.mock_object(profile=profile)
else:
self.die("Eigther object or profile must be set")
tokenizer = mo.iter_config_tokens(config=cfg)
for token in tokenizer:
self.print(token)
def test_profile_name(sa_profile):
profile = loader.get_profile(sa_profile)
assert getattr(profile, "name"), "Profile should has name"
req_name = profile.__module__
if req_name.startswith("noc.sa.profiles."):
req_name = req_name[16:]
if req_name == "Generic":
pytest.skip("Generic profile")
assert profile.name == req_name
def handle(self, paths, profile, format, progress=False, *args, **options):
assert profile_loader.get_profile(
profile), "Invalid profile: %s" % profile
t0 = time.time()
ruleset = RuleSet()
ruleset.load()
self.print("Ruleset load in %.2fms" % ((time.time() - t0) * 1000))
reader = getattr(self, "read_%s" % format, None)
assert reader, "Invalid format %s" % format
self.managed_object = ManagedObject(id=1,
name="test",
address="127.0.0.1",
profile_name=profile)
t0 = time.time()
stats = defaultdict(int)
total = 0
for p in paths:
if not os.path.isfile(p):
continue
for f in iter_open(p):
for event in reader(f):
e_vars = event.raw_vars.copy()
if event.source == "SNMP Trap":
e_vars.update(MIB.resolve_vars(event.raw_vars))
rule, r_vars = ruleset.find_rule(event, e_vars)
stats[rule.event_class.name] += 1
total += 1
if progress and total % 1000 == 0:
self.print("%d records processed" % total)
dt = time.time() - t0
self.print("%d events processed in %.2fms (%.fevents/sec)" %
(total, dt * 1000, float(total) / dt))
if stats:
# Prepare statistics
s_data = sorted([(k, stats[k]) for k in stats],
key=operator.itemgetter(1),
reverse=True)
s_total = sum(stats[k] for k in stats if not self.is_ignored(k))
data = [["Events", "%", "Event class"]]
for ecls, qty in s_data:
data += [[
str(qty),
"%3.2f%%" % (float(stats[ecls] * 100) / float(total)), ecls
]]
# Calculate classification quality
data += [[
"",
"%3.2f%%" % (float(s_total * 100) / total),
"Classification Quality"
]]
# Ruleset hit rate
rs_rate = float(metrics["rules_checked"].value) / float(total)
data += [["", "%.2f" % rs_rate, "Rule checks per event"]]
# Dump table
self.print("Event classes summary:")
self.print(format_table([4, 6, 10], data))
def test_profile_name(sa_profile):
profile = loader.get_profile(sa_profile)
assert getattr(profile, "name"), "Profile should has name"
req_name = profile.__module__
if req_name.startswith("noc.sa.profiles."):
req_name = req_name[16:]
if req_name == "Generic.profile":
pytest.skip("Generic profile")
parts = req_name.split(".")
assert 2 <= len(parts) <= 3
req_name = "%s.%s" % (parts[0], parts[1])
assert profile.name == req_name
def handle_syntax(self, path=None, profile=None, *args, **kwargs):
def dump_node(node, level=0, recursive=True):
indent = " " * level
if node.name:
label = "<%s>" % node.name
elif node.token is None:
label = "ANY"
else:
label = node.token
if node.multi:
label = "*%s" % label
self.print("%s%s" % (indent, label))
if recursive and node.children:
for nc in node.children:
dump_node(nc, level + 1)
def find_root(children, rest_path, level=0):
if not rest_path:
return children
if len(children) == 1 and not children[0].token:
dump_node(children[0], level, recursive=False)
return find_root(children[0].children, rest_path[1:], level=level + 1)
p = rest_path[0]
for cc in children:
if cc.token == p:
dump_node(cc, level, recursive=False)
return find_root(cc.children, rest_path[1:], level=level + 1)
from noc.core.confdb.syntax.base import SYNTAX
from noc.core.handler import get_handler
s = SYNTAX
if profile:
p = loader.get_profile(profile)
if not p:
self.die("Invalid profile: %s" % profile)
n_handler, n_config = p.get_config_normalizer(self)
n_cls = get_handler("noc.sa.profiles.%s.confdb.normalizer.%s" % (p.name, n_handler))
s = n_cls.SYNTAX
root = find_root(s, path)
if not root:
return
for c in root:
dump_node(c, level=len(path) if path else 0)
def handle_query(self, object=None, profile=None, config=None, query=None, *args, **kwargs):
cfg = None
if config:
if not os.path.exists(config):
self.die("File not found: %s" % config)
with open(config) as f:
cfg = f.read()
if object:
connect()
from noc.sa.models.managedobject import ManagedObject
mo = ManagedObject.objects.get(name=object)
if not mo:
self.die("Managed Object not found")
elif profile:
p = loader.get_profile(profile)
if not p:
self.die("Invalid profile: %s" % profile)
if not cfg:
self.die("Specify config file with --config option")
# Mock up tokenizer
connect()
from noc.sa.models.managedobject import ManagedObject
mo = ManagedObject.mock_object(profile=profile)
else:
self.die("Eigther object or profile must be set")
confdb = mo.get_confdb()
headers = []
table = []
width = []
for r in confdb.query(query):
row = []
for key in r:
if key not in headers:
headers += [key]
width += [40]
row.insert(headers.count(key), r[key])
table += [row]
if table:
self.print("Result:\n", format_table(width, [headers] + table))
else:
self.print("Result:")
def handle(cls, args, text):
if "syntax" in args:
format = args["syntax"]
else:
format = "text"
if format.startswith("noc."):
profile_name = format[4:]
try:
profile = profile_loader.get_profile(profile_name)
except Exception:
profile = None
format = "text"
if profile:
return profile().highlight_config(text)
try:
lexer = get_lexer_by_name(format)
except Exception:
lexer = get_lexer_by_name("text")
return highlight(text, lexer, NOCHtmlFormatter())
def test_profile(path):
# Open YAML
full_path = PREFIX + (path, )
with open(os.path.join(*full_path)) as f:
test = yaml.load(f.read(), Loader)
# Check test format
assert "config" in test, "Test must have 'config' section"
assert test["config"], "Config section must be non-empty"
assert "result" in test, "Test must have 'result' section"
assert test["result"], "Result section must be non-empty"
assert isinstance(test["result"], list), "Result section must be list"
# Load profile
profile_name = ".".join(path.split(os.sep)[:2])
profile = profile_loader.get_profile(profile_name)
assert profile, "Invalid profile '%s'" % profile_name
# Create mock object
mo = MockManagedObject(profile=profile)
# Setup tokenizer
tokenizer_name, tokenizer_conf = profile.get_config_tokenizer(mo)
tokenizer_cls = tokenizer_loader.get_class(tokenizer_name)
assert tokenizer_cls, "Tokenizer not found"
tokenizer = tokenizer_cls(test["config"], **tokenizer_conf)
# Setup normalizer
normalizer_name, normalizer_conf = profile.get_config_normalizer(mo)
if not normalizer_name.startswith("noc."):
normalizer_name = "noc.sa.profiles.%s.confdb.normalizer.%s" % (
profile.name,
normalizer_name,
)
normalizer_cls = get_handler(normalizer_name)
assert normalizer_cls, "Normalizer not found"
normalizer = normalizer_cls(mo, tokenizer, **normalizer_conf)
# Check result
result = list(normalizer)
expected = [tuple(x) for x in test["result"]]
assert result == expected
def get_profile(self):
return loader.get_profile(self.name)()
def __init__(
self,
service,
credentials,
args=None,
capabilities=None,
version=None,
parent=None,
timeout=None,
name=None,
session=None,
session_idle_timeout=None,
):
self.service = service
self.tos = config.activator.tos
self.pool = config.pool
self.parent = parent
self._motd = None
name = name or self.name
self.logger = PrefixLoggerAdapter(
self.base_logger, "%s] [%s" % (self.name, credentials.get("address", "-"))
)
if self.parent:
self.profile = self.parent.profile
else:
self.profile = profile_loader.get_profile(".".join(name.split(".")[:2]))()
self.credentials = credentials or {}
self.version = version or {}
self.capabilities = capabilities or {}
self.timeout = timeout or self.get_timeout()
self.start_time = None
self._interface = self.interface()
self.args = self.clean_input(args) if args else {}
self.cli_stream = None
self.mml_stream = None
self.rtsp_stream = None
if self.parent:
self.snmp = self.root.snmp
elif self.is_beefed:
self.snmp = BeefSNMP(self)
else:
self.snmp = SNMP(self)
if self.parent:
self.http = self.root.http
else:
self.http = HTTP(self)
self.to_disable_pager = not self.parent and self.profile.command_disable_pager
self.scripts = ScriptsHub(self)
# Store session id
self.session = session
self.session_idle_timeout = session_idle_timeout or self.SESSION_IDLE_TIMEOUT
# Cache CLI and SNMP calls, if set
self.is_cached = False
# Suitable only when self.parent is None.
# Cached results for scripts marked with "cache"
self.call_cache = {}
# Suitable only when self.parent is None
# Cached results of self.cli calls
self.cli_cache = {}
#
self.http_cache = {}
self.partial_result = None
# Tracking
self.to_track = False
self.cli_tracked_data = {} # command -> [packets]
self.cli_tracked_command = None
# state -> [..]
self.cli_fsm_tracked_data = {}
#
if not parent and version and not name.endswith(".get_version"):
self.logger.debug("Filling get_version cache with %s", version)
s = name.split(".")
self.set_cache("%s.%s.get_version" % (s[0], s[1]), {}, version)
# Fill matchers
if not self.name.endswith(".get_version"):
self.apply_matchers()
#
if self.profile.setup_script:
self.profile.setup_script(self)
def test_profile_type(sa_profile):
profile = loader.get_profile(sa_profile)
assert issubclass(profile, BaseProfile)
def test_profile_loading(sa_profile):
profile = loader.get_profile(sa_profile)
assert profile is not None
