def handler(window, config, *args, **kwargs):
"""
Calculate via handler
:param window:
:param config:
:param args:
:param kwargs:
:return:
"""
h = get_handler(config)
if not h:
raise ValueError("Invalid handler %s" % config)
return h(window)
def on_enter_state(self, obj):
"""
Called when object enters state
:param obj:
:return:
"""
# Process on enter handlers
if self.on_enter_handlers:
logger.debug("[%s|%s] Running on_enter_handlers", obj,
obj.state.name)
for hn in self.on_enter_handlers:
try:
h = get_handler(str(hn))
except ImportError as e:
logger.error("Error import on_enter handler: %s" % e)
h = None
if h:
logger.debug("[%s|%s] Running %s", obj, self.name, hn)
h(obj) # @todo: Catch exceptions
else:
logger.debug("[%s|%s] Invalid handler %s, skipping", obj,
self.name, hn)
# Run Job handler when necessary
if self.job_handler:
logger.debug("[%s|%s] Running job handler %s", obj, self.name,
self.job_handler)
try:
h = get_handler(self.job_handler)
except ImportError as e:
logger.error("Error import state job handler: %s" % e)
h = None
if h:
call_later(STATE_JOB,
handler=self.job_handler,
model=get_model_id(obj),
object=str(obj.pk))
else:
logger.debug("[%s|%s] Invalid job handler %s, skipping", obj,
self.name, self.job_handler)
def get_cli_stream(self):
if self.parent:
return self.root.get_cli_stream()
if not self.cli_stream and self.session:
# Try to get cached session's CLI
with self.session_lock:
self.cli_stream = self.session_cli.get(self.session)
if self.cli_stream:
if self.cli_stream.is_closed:
# Stream closed by external reason,
# mark as invalid and start new one
self.cli_stream = None
# Remove stream from pool to prevent cli session hijacking
del self.session_cli[self.session]
if self.cli_stream:
if self.to_reuse_cli_session():
self.logger.debug("Using cached session's CLI")
self.cli_stream.set_script(self)
else:
self.logger.debug(
"Script cannot reuse existing CLI session, starting new one"
)
self.close_cli_stream()
if not self.cli_stream:
protocol = self.credentials.get("cli_protocol", "telnet")
self.logger.debug("Open %s CLI", protocol)
self.cli_stream = get_handler(
self.cli_protocols[protocol]
)(self, tos=self.tos)
# Store to the sessions
if self.session:
with self.session_lock:
self.session_cli[self.session] = self.cli_stream
self.cli_stream.setup_session()
# Disable pager when nesessary
# @todo: Move to CLI
if self.to_disable_pager:
self.logger.debug("Disable paging")
self.to_disable_pager = False
if isinstance(self.profile.command_disable_pager, six.string_types):
self.cli(
self.profile.command_disable_pager,
ignore_errors=True
)
elif isinstance(self.profile.command_disable_pager, list):
for cmd in self.profile.command_disable_pager:
self.cli(cmd, ignore_errors=True)
else:
raise UnexpectedResultError
return self.cli_stream
def get_applicator(cfg):
if isinstance(cfg, six.string_types):
a_handler, a_cfg = cfg, {}
else:
a_handler, a_cfg = cfg
if not a_handler.startswith("noc."):
a_handler = "noc.sa.profiles.%s.confdb.applicator.%s" % (
profile_name, a_handler)
a_cls = get_handler(a_handler)
assert a_cls, "Invalid applicator %s" % a_handler
applicator = a_cls(object, confdb, **a_cfg)
if applicator.can_apply():
return applicator
return None
def run_job(self, job, mo, checks):
if job == "segment":
scheduler = Scheduler("scheduler",
pool=None,
service=ServiceStub())
else:
scheduler = Scheduler("discovery",
pool=mo.pool.name,
service=ServiceStub())
jcls = self.jcls[job]
# Try to dereference job
job_args = scheduler.get_collection().find_one({
Job.ATTR_CLASS: jcls,
Job.ATTR_KEY: mo.id
})
if job_args:
self.print("Job ID: %s" % job_args["_id"])
else:
job_args = {Job.ATTR_ID: "fakeid", Job.ATTR_KEY: mo.id}
job_args["_checks"] = checks
job = get_handler(jcls)(scheduler, job_args)
if job.context_version:
ctx_key = job.get_context_cache_key()
self.print("Loading job context from %s" % ctx_key)
ctx = cache.get(ctx_key, version=job.context_version)
if not ctx:
self.print("Job context is empty")
job.load_context(ctx)
sample = 1 if self.trace else 0
with Span(sample=sample):
job.dereference()
job.handler()
if sample:
spans = get_spans()
self.print("Spans:")
self.print("\n".join(str(s) for s in spans))
if scheduler.service.metrics:
self.print("Collected CH data:")
for t in scheduler.service.metrics:
self.print("Table: %s" % t)
self.print("\n".join(
str(x) for x in scheduler.service.metrics[t]))
if job.context_version and job.context:
self.print("Saving job context to %s" % ctx_key)
scheduler.cache_set(key=ctx_key,
value=job.context,
version=job.context_version)
scheduler.apply_cache_ops()
time.sleep(3)
def list_commands(self):
commands = set()
for root in config.get_customized_paths("commands"):
for f in os.listdir(root):
help = ""
if f.startswith("_") or f.startswith("."):
continue
elif f.endswith(".py"):
try:
if root == "commands":
h = get_handler("noc.commands.%s" % f[:-3])
else:
h = get_handler("noc.custom.commands.%s" % f[:-3])
ha = getattr(h, "Command", "")
if ha:
help = ha.help
except Exception:
help = ""
commands.add((f[:-3], help))
elif f.endswith(".sh"):
commands.add((f[:-3], help))
for cmd in sorted(commands):
self.print("%-20s %s" % cmd)
return 0
def test_profile(interfaces, paths):
ifaces = {}
for iface_name, iface_type in interfaces:
ifaces[iface_name] = MockInterface(iface_name, "", iface_type)
h = get_handler("noc.core.confdb.collator.ifpath.IfPathCollator")
collator = h()
for path, result in paths:
physical_path = [
PathItem(object=MockObject(*p["object"]),
connection=MockObjectConnection(*p["connection"]))
for p in path
]
r = collator.collate(physical_path, ifaces)
assert r == result
def handle_syntax(self, path=None, profile=None, *args, **kwargs):
def dump_node(node, level=0, recursive=True):
indent = " " * level
if node.name:
label = "<%s>" % node.name
elif node.token is None:
label = "ANY"
else:
label = node.token
if node.multi:
label = "*%s" % label
self.print("%s%s" % (indent, label))
if recursive and node.children:
for nc in node.children:
dump_node(nc, level + 1)
def find_root(children, rest_path, level=0):
if not rest_path:
return children
if len(children) == 1 and not children[0].token:
dump_node(children[0], level, recursive=False)
return find_root(children[0].children, rest_path[1:], level=level + 1)
p = rest_path[0]
for cc in children:
if cc.token == p:
dump_node(cc, level, recursive=False)
return find_root(cc.children, rest_path[1:], level=level + 1)
from noc.core.confdb.syntax.base import SYNTAX
from noc.core.handler import get_handler
s = SYNTAX
if profile:
p = loader.get_profile(profile)
if not p:
self.die("Invalid profile: %s" % profile)
n_handler, n_config = p.get_config_normalizer(self)
n_cls = get_handler("noc.sa.profiles.%s.confdb.normalizer.%s" % (p.name, n_handler))
s = n_cls.SYNTAX
root = find_root(s, path)
if not root:
return
for c in root:
dump_node(c, level=len(path) if path else 0)
def setup_middleware(self):
mw_list = self.script.profile.get_http_request_middleware(self.script)
if not mw_list:
return
self.request_middleware = []
for mw_cfg in mw_list:
if isinstance(mw_cfg, tuple):
name, cfg = mw_cfg
else:
name, cfg = mw_cfg, {}
if "." in name:
# Handler
mw_cls = get_handler(name)
assert mw_cls
assert issubclass(mw_cls, BaseMiddleware)
else:
# Middleware name
mw_cls = loader.get_class(name)
self.request_middleware += [mw_cls(self, **cfg)]
def state_job(handler, model, object):
"""
State.job_handler wrapper
:param handler:
:param model:
:param object:
:return:
"""
# Resolve handler
h = get_handler(handler)
if not h:
logger.error("Invalid handler %s", handler)
return
# Resolve object
obj = get_object(model, object)
if not obj:
logger.error("Cannot dereference %s:%s", model, object)
# Call handler
h(obj)
def _get_rule_settings(self, ps, scope):
"""
Process PolicySettings object and returns a list of
(validator class, config)
"""
r = []
for pi in ps.policies:
policy = pi.policy
if not pi.is_active or not policy.is_active:
continue
for ri in policy.rules:
if not ri.is_active:
continue
rule = ri.rule
if rule.is_active and rule.is_applicable_for(self.object):
vc = get_handler(rule.handler)
if vc and bool(vc.SCOPE & scope):
r += [(vc, rule)]
return r
def on_leave_state(self, obj):
"""
Called when object leaves state
:param obj:
:return:
"""
if self.on_leave_handlers:
logger.debug("[%s|%s] Running on_leave_handlers", obj, self.name)
for hn in self.on_leave_handlers:
try:
h = get_handler(str(hn))
except ImportError as e:
logger.error("Error import on_leave_state handler: %s" % e)
h = None
if h:
logger.debug("[%s|%s] Running %s", obj, self.name, hn)
h(obj) # @todo: Catch exceptions
else:
logger.debug("[%s|%s] Invalid handler %s, skipping", obj,
self.name, hn)
def on_transition(self, obj):
"""
Called during transition
:param obj:
:return:
"""
if self.handlers:
logger.debug("[%s|%s|%s] Running transition handlers", obj,
obj.state.name, self.label)
for hn in self.handlers:
try:
h = get_handler(str(hn))
except ImportError as e:
logger.error("Error import handler: %s" % e)
h = None
if h:
logger.debug("[%s|%s|%s] Running %s", obj, obj.state.name,
self.label, hn)
h(obj) # @todo: Catch exceptions
else:
logger.debug("[%s|%s|%s] Invalid handler %s, skipping",
obj, obj.state.name, self.label, hn)
def load_handlers(self):
self.logger.info("Loading handlers")
self.handlers = {}
# Process altered handlers
enabled = defaultdict(list) # event class id -> [handlers]
disabled = defaultdict(list) # event class id -> [handlers]
for ec_id, status, handler in self.alter_handlers:
if status:
if handler in disabled[ec_id]:
disabled[ec_id].remove(handler)
if handler not in enabled[ec_id]:
enabled[ec_id] += [handler]
else:
if handler not in disabled[ec_id]:
disabled[ec_id] += [handler]
if handler in enabled[ec_id]:
enabled[ec_id].remove(handler)
self.alter_handlers = []
# Load handlers
for ec in EventClass.objects.filter():
handlers = (ec.handlers or []) + enabled[ec.id]
if not handlers:
continue
self.logger.debug(" <%s>: %s", ec.name, ", ".join(handlers))
hl = []
for h in handlers:
if h in disabled[ec.id]:
self.logger.debug(" disabling handler %s", h)
continue
# Resolve handler
try:
hh = get_handler(h)
hl += [hh]
except ImportError:
self.logger.error("Failed to load handler '%s'. Ignoring",
h)
if hl:
self.handlers[ec.id] = hl
self.logger.info("Handlers are loaded")
def get_cli_stream(self):
if self.parent:
return self.root.get_cli_stream()
if not self.cli_stream and self.session:
# Try to get cached session's CLI
self.cli_stream = self.cli_session_store.get(self.session)
if self.cli_stream:
if self.to_reuse_cli_session():
self.logger.debug("Using cached session's CLI")
self.cli_stream.set_script(self)
else:
self.logger.debug(
"Script cannot reuse existing CLI session, starting new one"
)
self.close_cli_stream()
if not self.cli_stream:
protocol = self.credentials.get("cli_protocol", "telnet")
self.logger.debug("Open %s CLI", protocol)
self.cli_stream = get_handler(self.cli_protocols[protocol])(
self, tos=self.tos)
# Store to the sessions
if self.session:
self.cli_session_store.put(self.session, self.cli_stream)
self.cli_stream.setup_session()
# Disable pager when necessary
# @todo: Move to CLI
if self.to_disable_pager:
self.logger.debug("Disable paging")
self.to_disable_pager = False
if isinstance(self.profile.command_disable_pager, str):
self.cli(self.profile.command_disable_pager,
ignore_errors=True)
elif isinstance(self.profile.command_disable_pager, list):
for cmd in self.profile.command_disable_pager:
self.cli(cmd, ignore_errors=True)
else:
raise UnexpectedResultError
return self.cli_stream
def get_diag(alarm, cfg, state):
mo = alarm.managed_object
if not mo:
return
result = []
for c in cfg:
if c.get("header"):
result += [c["header"].strip()]
if "script" in c:
logger.info("[%s] Running script %s", alarm.id, c["script"])
try:
g = getattr(mo.scripts, c["script"])
result += [g()]
except Exception as e:
error_report()
result += [str(e)]
if "action" in c:
logger.info("[%s] Running action %s", alarm.id, c["action"])
try:
g = getattr(mo.actions, c["action"])
result += [g()]
except Exception as e:
error_report()
result += [str(e)]
if "handler" in c:
logger.info("[%s] Running handler %s", alarm.id, c["handler"])
try:
h = get_handler(c["handler"])
try:
result += [h(alarm)]
except Exception as e:
error_report()
result += [str(e)]
except ImportError:
result += ["Invalid handler: %s" % c["handler"]]
if result:
AlarmDiagnostic.save_diagnostics(alarm, result, state)
def get_mml_stream(self):
if self.parent:
return self.root.get_mml_stream()
if not self.mml_stream and self.session:
# Try to get cached session's CLI
self.mml_stream = self.mml_session_store.get(self.session)
if self.mml_stream:
if self.to_reuse_cli_session():
self.logger.debug("Using cached session's MML")
self.mml_stream.set_script(self)
else:
self.logger.debug(
"Script cannot reuse existing MML session, starting new one"
)
self.close_mml_stream()
if not self.mml_stream:
protocol = self.credentials.get("cli_protocol", "telnet")
self.logger.debug("Open %s MML", protocol)
self.mml_stream = get_handler(self.mml_protocols[protocol])(
self, tos=self.tos)
# Store to the sessions
if self.session:
self.mml_session_store.put(self.session, self.mml_stream)
return self.mml_stream
def test_profile(path):
# Open YAML
full_path = PREFIX + (path, )
with open(os.path.join(*full_path)) as f:
test = yaml.load(f.read(), Loader)
# Check test format
assert "config" in test, "Test must have 'config' section"
assert test["config"], "Config section must be non-empty"
assert "result" in test, "Test must have 'result' section"
assert test["result"], "Result section must be non-empty"
assert isinstance(test["result"], list), "Result section must be list"
# Load profile
profile_name = ".".join(path.split(os.sep)[:2])
profile = profile_loader.get_profile(profile_name)
assert profile, "Invalid profile '%s'" % profile_name
# Create mock object
mo = MockManagedObject(profile=profile)
# Setup tokenizer
tokenizer_name, tokenizer_conf = profile.get_config_tokenizer(mo)
tokenizer_cls = tokenizer_loader.get_class(tokenizer_name)
assert tokenizer_cls, "Tokenizer not found"
tokenizer = tokenizer_cls(test["config"], **tokenizer_conf)
# Setup normalizer
normalizer_name, normalizer_conf = profile.get_config_normalizer(mo)
if not normalizer_name.startswith("noc."):
normalizer_name = "noc.sa.profiles.%s.confdb.normalizer.%s" % (
profile.name,
normalizer_name,
)
normalizer_cls = get_handler(normalizer_name)
assert normalizer_cls, "Normalizer not found"
normalizer = normalizer_cls(mo, tokenizer, **normalizer_conf)
# Check result
result = list(normalizer)
expected = [tuple(x) for x in test["result"]]
assert result == expected
def get_rtsp_stream(self):
if self.parent:
return self.root.get_rtsp_stream()
if not self.rtsp_stream and self.session:
# Try to get cached session's CLI
self.rtsp_stream = self.rtsp_session_store.get(self.session)
if self.rtsp_stream:
if self.to_reuse_cli_session():
self.logger.debug("Using cached session's RTSP")
self.rtsp_stream.set_script(self)
else:
self.logger.debug(
"Script cannot reuse existing RTSP session, starting new one"
)
self.close_rtsp_stream()
if not self.rtsp_stream:
protocol = "tcp"
self.logger.debug("Open %s RTSP", protocol)
self.rtsp_stream = get_handler(self.rtsp_protocols[protocol])(
self, tos=self.tos)
# Store to the sessions
if self.session:
self.rtsp_session_store.put(self.session, self.rtsp_stream)
return self.rtsp_stream
def load_triggers(self):
self.logger.info("Loading triggers")
self.triggers = {}
n = 0
cn = 0
self.alter_handlers = []
ec = [(c.name, c.id) for c in EventClass.objects.all()]
for t in EventTrigger.objects.all():
self.logger.debug("Trigger '%s' for classes:", t.name)
for c_name, c_id in ec:
if re.search(t.event_class_re, c_name, re.IGNORECASE):
if (t.handler and t.condition == "True"
and t.selector is None and t.time_pattern is None
and t.template is None
and t.notification_group is None):
# Alter handlers
self.alter_handlers += [(c_id, t.is_enabled, t.handler)
]
elif t.is_enabled:
# Register trigger
h = t.handler
if h:
try:
h = get_handler(h)
except ImportError:
self.logger.error(
"Failed to load handler '%s'. Ignoring", h)
h = None
if c_id in self.triggers:
self.triggers[c_id] += [Trigger(t, handler=h)]
else:
self.triggers[c_id] = [Trigger(t, handler=h)]
cn += 1
self.logger.debug(" %s", c_name)
n += 1
self.logger.info("%d triggers has been loaded to %d classes", n, cn)
def test_invalid_format():
# Invalid format
with pytest.raises(ImportError):
get_handler("xxx")
def get_handler(self):
if self.handler:
return get_handler(self.handler)
else:
return None
def get_handler(self):
"""
Get callable from handler
:return:
"""
return get_handler(self.handler)
def dereference(self):
self.object = get_handler(self.attrs[self.ATTR_KEY])
return bool(self.object)
def test_simple_handler():
h = get_handler("noc.tests.test_handler.my_simple_handler")
assert h
assert h() == 1
def test_add_handler():
h = get_handler("noc.tests.test_handler.my_add_handler")
assert h
assert h(1, 2) == 3
def test_callable():
h = get_handler(my_simple_handler)
assert h
assert h == my_simple_handler
def test_invalid_module():
# Invalid module
with pytest.raises(ImportError):
get_handler("invalid.handler")
def test_invalid_attribute():
# Invalid attribute
with pytest.raises(ImportError):
get_handler("noc.tests.test_handler.invalid_handler")
def get_handler(self):
return get_handler(self.handler)
