def run_script(self, oid, script, args, span_id=0, bi_id=None):
with Span(
server="MRT",
service="run_script",
sample=int(config.mrt.enable_command_logging),
in_label=bi_id or oid,
parent=span_id,
client=self.current_user,
) as span:
try:
yield self.write_chunk({"id": str(oid), "running": True})
logger.debug("[%s] Run script %s %s %s", span.context, oid, script, args)
r = yield self.service.sae.script(oid, script, args)
metrics["mrt_success"] += 1
except RPCRemoteError as e:
span.error_code = getattr(e, "remote_code", 1)
span.error_text = str(e)
raise tornado.gen.Return({"id": str(oid), "error": str(e)})
except RPCError as e:
logger.error("RPC Error: %s" % str(e))
span.error_code = getattr(e, "code", 1)
span.error_text = str(e)
raise tornado.gen.Return({"id": str(oid), "error": str(e)})
except Exception as e:
error_report()
metrics["mrt_failed"] += 1
span.error_code = 1
span.error_text = str(e)
raise tornado.gen.Return({"id": str(oid), "error": str(e)})
if r["errors"]:
span.error_code = 1
span.error_text = r["output"]
raise tornado.gen.Return({"id": str(oid), "error": r["output"]})
span.out_label = r["output"]
raise tornado.gen.Return({"id": str(oid), "result": r["output"]})
def handle(self, *args, **options):
#
parsers = []
# Read config
config = SafeConfigParser()
for p in self.get_parsers():
config.read(os.path.join("etc", "address", "%s.defaults" % p))
config.read(os.path.join("etc", "address", "%s.conf" % p))
if config.getboolean(p, "enabled"):
m = __import__("noc.gis.parsers.address.%s" % p, {}, {}, "*")
for l in dir(m):
a = getattr(m, l)
if inspect.isclass(a) and issubclass(
a, AddressParser) and a != AddressParser:
parsers += [a]
else:
print("Parser '%s' is not enabled. Skipping.." % p)
# Initialize parsers
parsers = [p(config, options) for p in parsers]
# Download
if options["download"]:
for p in parsers:
print("Downloading", p.name)
if not p.download():
raise CommandError("Failed to download %s" % p.name)
else:
print("Skipping downloads")
# Sync
try:
for p in parsers:
print("Syncing", p.name)
p.sync()
except Exception:
error_report()
def handle(self, *args, **options):
if len(args) < 1:
print("USAGE: %s <model> <object id> [.. <object id>]" %
sys.argv[0])
sys.exit(1)
m = args[0].replace("-", "_")
connect()
if m not in self.models:
raise CommandError("Invalid model '%s'. Valid models are: %s" %
(m, ", ".join(self.models)))
objects = []
getter = getattr(self, "get_%s" % m)
wiper = getattr(self, "wipe_%s" % m)
# Get objects
for o_id in args[1:]:
o = getter(o_id)
if not o: # Not found
raise CommandError("Object '%s' is not found" % o_id)
objects += [o]
# Wipe objects
from noc.core.debug import error_report
with bulk_datastream_changes():
for o in objects:
with self.log("Wiping '%s':" % unicode(o), True):
try:
wiper(o)
except KeyboardInterrupt:
raise CommandError(
"Interrupted. Wiping is not complete")
except Exception:
error_report()
def call_event_handlers(self, event):
"""
Call handlers associated with event class
:param event:
:return:
"""
if event.event_class.id not in self.handlers:
return False
event_id = event.id # Temporary store id
for h in self.handlers[event.event_class.id]:
try:
h(event)
except Exception:
error_report()
if event.to_drop:
self.logger.info(
"[%s|%s|%s] Dropped by handler",
event.id,
event.managed_object.name,
event.managed_object.address,
)
event.id = event_id # Restore event id
event.delete()
metrics[CR_DELETED] += 1
return True
return False
async def error_handler(self, request, exc) -> Response:
"""
Error handler for ServerErrorMiddleware
:return:
"""
error_report(logger=self.logger)
return PlainTextResponse("Internal Server Error", status_code=500)
def call_event_triggers(self, event):
"""
Call triggers associated with event class
:param event:
:return:
"""
if event.event_class.id not in self.triggers:
return False
event_id = event.id
for t in self.triggers[event.event_class.id]:
try:
t.call(event)
except Exception:
error_report()
if event.to_drop:
# Delete event and stop processing
self.logger.info(
"[%s|%s|%s] Dropped by trigger %s",
event_id,
event.managed_object.name,
event.managed_object.address,
t.name,
)
event.id = event_id # Restore event id
event.delete()
metrics[CR_DELETED] += 1
return True
else:
return False
def call(self, event):
if not self.match(event):
return
logging.debug("Calling trigger '%s'" % self.name)
# Notify if necessary
if self.notification_group and self.template:
subject = {}
body = {}
for lang in self.notification_group.languages:
s = event.subject
b = event.body
subject[lang] = self.template.render_subject(LANG=lang,
event=event,
subject=s,
body=b)
body[lang] = self.template.render_body(LANG=lang,
event=event,
subject=s,
body=b)
self.notification_group.notify(subject=subject, body=body)
# Call Handler
if self.handler:
try:
self.handler(event)
except Exception:
error_report()
def start(self):
"""
Run main server loop
"""
parser = self.create_parser()
self.add_arguments(parser)
options = parser.parse_args(sys.argv[1:])
cmd_options = vars(options)
args = cmd_options.pop("args", ())
# Bootstrap logging with --loglevel
self.setup_logging(cmd_options["loglevel"])
self.log_separator()
# Setup timezone
try:
self.logger.info("Setting timezone to %s", config.timezone)
setup_timezone()
except ValueError as e:
self.die(str(e))
# Setup title
self.set_proc_title()
# Setup signal handlers
self.setup_signal_handlers()
self.on_start()
# Starting IOLoop
self.is_active = True
if self.pooled:
self.logger.warn(
"Running service %s (pool: %s)",
self.name, config.pool
)
else:
self.logger.warn(
"Running service %s", self.name
)
try:
if config.features.use_uvlib:
from tornaduv import UVLoop
self.logger.warn("Using libuv")
tornado.ioloop.IOLoop.configure(UVLoop)
self.ioloop = tornado.ioloop.IOLoop.instance()
# Initialize DCS
self.dcs = get_dcs(cmd_options["dcs"], self.ioloop)
# Activate service
self.ioloop.add_callback(self.activate)
self.logger.warn("Starting IOLoop")
self.ioloop.start()
except KeyboardInterrupt:
self.logger.warn("Interrupted by Ctrl+C")
except self.RegistrationError:
self.logger.info("Registration failed")
except Exception:
error_report()
finally:
if self.ioloop:
self.ioloop.add_callback(self.deactivate())
for cb, args, kwargs in self.close_callbacks:
cb(*args, **kwargs)
self.logger.warn("Service %s has been terminated", self.name)
def run_from_argv(self, argv):
"""
Execute command. Usually from script
if __name__ == "__main__":
import sys
sys.exit(Command.run_from_argv())
"""
parser = self.create_parser()
self.add_default_arguments(parser)
self.add_arguments(parser)
options = parser.parse_args(argv)
cmd_options = vars(options)
args = cmd_options.pop("args", ())
loglevel = cmd_options.pop("loglevel")
if loglevel:
self.setup_logging(loglevel)
enable_profiling = cmd_options.pop("enable_profiling", False)
show_metrics = cmd_options.pop("show_metrics", False)
if enable_profiling:
# Start profiler
import yappi
yappi.start()
try:
return self.handle(*args, **cmd_options) or 0
except CommandError as e:
self.print(str(e))
return 1
except KeyboardInterrupt:
self.print("Ctrl+C")
return 3
except AssertionError as e:
if e.args and e.args[0]:
self.print("ERROR: %s" % e.args[0])
else:
self.print("Assertion error: %s" % e)
return 4
except Exception:
from noc.core.debug import error_report
error_report()
return 2
finally:
if enable_profiling:
i = yappi.get_func_stats()
i.print_all(out=self.stdout,
columns={
0: ("name", 80),
1: ("ncall", 10),
2: ("tsub", 8),
3: ("ttot", 8),
4: ("tavg", 8)
})
if show_metrics:
from noc.core.perf import apply_metrics
d = apply_metrics({})
self.print("Internal metrics:")
for k in d:
self.print("%40s : %s" % (k, d[k]))
def handle(self, *args, **options):
try:
self._handle(*args, **options)
except CommandError as why:
raise CommandError(why)
except SystemExit:
pass
except Exception:
error_report()
def _check(self):
"""
Perform object configuration check
"""
self.logger.info("Checking %s", self.object)
parser = self.object.get_parser()
self.config = self.object.config.read()
if not self.config:
self.logger.error("No config for %s. Giving up", self.object)
return
# Parse facts
self.logger.debug("Parsing facts")
facts = list(parser.parse(self.config))
self.logger.debug("%d facts are extracted", len(facts))
self.interface_ranges = parser.interface_ranges
self.logger.debug("%d interface sections detected",
len(self.interface_ranges))
# Define default templates
self.get_template(Error(None))
self.get_template(Role(None))
# Learn facts
self.logger.debug("Learning facts")
self.learn(facts)
self.logger.debug("Learning complete")
# Install rules
rules = []
for r in self.get_rules():
if r.is_applicable():
self.logger.debug("Using validation rule: %s", r.rule.name)
try:
cfg = r.get_config()
r.prepare(**cfg)
except clips.ClipsError as e:
self.logger.error("CLIPS Error: %s\n%s", e,
clips.ErrorStream.Read())
continue
except Exception:
error_report()
continue
rules += [(r, cfg)]
# Run python validators
for r, cfg in rules:
r.check(**cfg)
# Run CLIPS engine
while True:
self.logger.debug("Running engine")
n = self.run()
self.logger.debug("%d rules matched", n)
break # @todo: Check for commands
# Extract errors
for e in self.iter_errors():
self.logger.info("Error found: %s", e)
# Store object's facts
self.sync_facts()
# Manage related alarms
if self.AC_POLICY_VIOLATION:
self.sync_alarms()
def run_round(self):
self.logger.info("Running checks")
now = time.time()
for c in self.collectors:
if not c.can_run_at(now):
break # Done
with Span(sample=PARENT_SAMPLE, server="selfmon", service=c.name):
try:
c.run()
except Exception:
error_report()
c.schedule_next()
self.reorder()
def dispose_worker(self, message, event_id, event_hint=None):
metrics["alarm_dispose"] += 1
try:
if event_hint:
event = self.get_event_from_hint(event_hint)
else:
event = self.lookup_event(event_id)
if event:
self.dispose_event(event)
except Exception:
metrics["alarm_dispose_error"] += 1
error_report()
self.ioloop.add_callback(message.finish)
def run(self):
if not self.is_enabled():
self.logger.info("Check is disabled. Skipping")
return
if self.has_fatal_error():
self.logger.info(
"Check is disabled due to previous fatal error. Skipping")
return
if not self.has_required_artefacts():
return
with Span(server="discovery",
service=self.name) as span, self.job.check_timer(self.name):
# Check required scripts
if not self.has_required_script():
self.logger.info("%s script is not supported. Skipping",
self.required_script)
return
# Check required capabilities
if not self.has_required_capabilities():
return
# Run check
try:
self.handler()
except RPCRemoteError as e:
self.logger.error("RPC Remote error (%s): %s", e.remote_code,
e)
if e.remote_code:
message = "Remote error code %s" % e.remote_code
else:
message = "Remote error code %s, message: %s" % (
e.remote_code, e)
self.set_problem(alarm_class=self.error_map.get(e.remote_code),
message=message,
fatal=e.remote_code in self.fatal_errors)
span.error_code = e.remote_code
span.error_text = str(e)
except RPCError as e:
self.set_problem(alarm_class=self.error_map.get(
e.default_code),
message="RPC Error: %s" % e,
fatal=e.default_code in self.fatal_errors)
self.logger.error("Terminated due RPC error: %s", e)
span.error_code = e.default_code
span.error_text = str(e)
except Exception as e:
self.set_problem(
alarm_class="Discovery | Error | Unhandled Exception",
message="Unhandled exception: %s" % e)
error_report(logger=self.logger)
span.error_code = ERR_UNKNOWN
span.error_text = str(e)
def get(self, card_type, card_id, *args, **kwargs):
if not self.current_user:
raise tornado.web.HTTPError(404, "Not found")
is_ajax = card_id == "ajax"
tpl = self.CARDS.get(card_type)
if not tpl:
raise tornado.web.HTTPError(404, "Card template not found")
try:
card = tpl(self, card_id)
if is_ajax:
data = card.get_ajax_data()
else:
data = card.render()
except BaseCard.RedirectError as e:
return self.redirect(e.args[0])
except BaseCard.NotFoundError:
raise tornado.web.HTTPError(404, "Not found")
except Exception:
error_report()
raise tornado.web.HTTPError(500, "Internal server error")
if not data:
raise tornado.web.HTTPError(404, "Not found")
self.set_header("Cache-Control", "no-cache, must-revalidate")
if is_ajax:
self.set_header("Content-Type", "text/json")
self.write(
orjson.dumps(data,
option=orjson.OPT_SERIALIZE_NUMPY
| orjson.OPT_NON_STR_KEYS))
else:
self.set_header("Content-Type", "text/html; charset=utf-8")
refresh = self.get_argument("refresh", None)
if refresh:
try:
refresh = int(refresh)
self.set_header("Refresh", str(refresh))
except ValueError:
pass
self.write(self.get_card_template().render({
"card_data":
data,
"card_title":
str(card.object),
"hashed":
self.hashed,
"card_js":
card.card_js,
"card_css":
card.card_css,
}))
def start(self):
"""
Run main server loop
"""
self.startup_ts = perf_counter()
parser = self.create_parser()
self.add_arguments(parser)
options = parser.parse_args(sys.argv[1:])
cmd_options = vars(options)
cmd_options.pop("args", ())
# Bootstrap logging with --loglevel
self.setup_logging(cmd_options["loglevel"])
self.log_separator()
# Setup timezone
try:
self.logger.info("Setting timezone to %s", config.timezone)
setup_timezone()
except ValueError as e:
self.die(str(e))
# Setup title
self.set_proc_title()
# Setup signal handlers
self.setup_signal_handlers()
self.on_start()
# Starting IOLoop
self.is_active = True
if self.pooled:
self.logger.warning("Running service %s (pool: %s)", self.name,
config.pool)
else:
self.logger.warning("Running service %s", self.name)
try:
setup_asyncio()
self.loop = asyncio.get_event_loop()
# Initialize DCS
self.dcs = get_dcs(cmd_options["dcs"])
# Activate service
self.loop.create_task(self.activate())
self.logger.warning("Starting IOLoop")
self.loop.run_forever()
except KeyboardInterrupt:
self.logger.warning("Interrupted by Ctrl+C")
except self.RegistrationError:
self.logger.info("Registration failed")
except Exception:
error_report()
finally:
if self.loop:
self.loop.create_task(self.deactivate())
self.logger.warning("Service %s has been terminated", self.name)
def on_complete(f):
logger.debug("Completion slow operation %s", so.id)
if f.exception():
so.status = cls.STATUS_FAILED
try:
f.result()
except Exception:
error_report()
so.pickled_result = dumps(f.exception(), HIGHEST_PROTOCOL)
else:
so.status = cls.STATUS_COMPLETE
so.pickled_result = dumps(f.result(), HIGHEST_PROTOCOL)
so.duration = time.time() - t0
so.save()
def load(self, extractors=None):
extractors = extractors or self.get_extractors()
error = None
try:
self.get_handler().load(extractors)
except Exception as e:
error_report()
error = str(e)
self.last_load = datetime.datetime.now()
if error:
self.load_error = error
else:
self.last_successful_load = self.last_load
self.save()
def extract(self, extractors=None, quiet=False):
extractors = extractors or self.get_extractors()
error = None
try:
self.get_handler().extract(extractors)
except Exception as e:
if not quiet:
raise e
error_report()
error = str(e)
self.last_extract = datetime.datetime.now()
if not error:
self.last_successful_extract = self.last_extract
self.extract_error = error
self.save()
def send_reports(cls):
"""
Calculate and send all reports for today
:return:
"""
subscriptions = list(ReportSubscription.objects.filter(is_active=True))
for s in subscriptions:
if s.can_run():
try:
path = s.build_report()
except Exception:
path = None
error_report()
s.update_status(bool(path))
if path and s.notification_group:
s.send_report(path)
def correlate(self, r, a):
# Topology RCA
if a.alarm_class.topology_rca:
with self.topology_rca_lock:
self.topology_rca(a)
# Rule-based RCA
with self.rca_lock:
if a.alarm_class.id in self.rca_forward:
# Check alarm is a consequence of existing one
self.set_root_cause(a)
if a.alarm_class.id in self.rca_reverse:
# Check alarm is the root cause for existing ones
self.set_reverse_root_cause(a)
# Call handlers
for h in a.alarm_class.get_handlers():
try:
has_root = bool(a.root)
h(a)
if not has_root and a.root:
self.logger.info(
"[%s|%s|%s] Set root to %s (handler %s)",
a.id,
a.managed_object.name,
a.managed_object.address,
a.root,
h,
)
except: # noqa. Can probable happens anything from handler
error_report()
metrics["error", ("type", "alarm_handler")] += 1
# Call triggers if necessary
if r.alarm_class.id in self.triggers:
for t in self.triggers[r.alarm_class.id]:
try:
t.call(a)
except: # noqa. Can probable happens anything from trigger
error_report()
#
if not a.severity:
# Alarm severity has been reset to 0 by handlers
# Silently drop alarm
self.logger.debug("Alarm severity is 0, dropping")
a.delete()
metrics["alarm_drop"] += 1
return
async def on_dispose_event(self, msg: Message) -> None:
"""
Called on new dispose message
"""
data = orjson.loads(msg.value)
event_id = data["event_id"]
hint = data["event"]
self.logger.info("[%s] Receiving message", event_id)
metrics["alarm_dispose"] += 1
try:
event = ActiveEvent.from_json(hint)
event.timestamp = event.timestamp.replace(tzinfo=None)
await self.dispose_event(event)
except Exception:
metrics["alarm_dispose_error"] += 1
error_report()
finally:
if self.topo_rca_lock:
# Release pending RCA Lock
await self.topo_rca_lock.release()
self.topo_rca_lock = None
def get_diag(alarm, cfg, state):
mo = alarm.managed_object
if not mo:
return
result = []
for c in cfg:
if c.get("header"):
result += [c["header"].strip()]
if "script" in c:
logger.info("[%s] Running script %s", alarm.id, c["script"])
try:
g = getattr(mo.scripts, c["script"])
result += [g()]
except Exception as e:
error_report()
result += [str(e)]
if "action" in c:
logger.info("[%s] Running action %s", alarm.id, c["action"])
try:
g = getattr(mo.actions, c["action"])
result += [g()]
except Exception as e:
error_report()
result += [str(e)]
if "handler" in c:
logger.info("[%s] Running handler %s", alarm.id, c["handler"])
try:
h = get_handler(c["handler"])
try:
result += [h(alarm)]
except Exception as e:
error_report()
result += [str(e)]
except ImportError:
result += ["Invalid handler: %s" % c["handler"]]
if result:
AlarmDiagnostic.save_diagnostics(alarm, result, state)
def run(self):
with Span(server=self.scheduler.name,
service=self.attrs[self.ATTR_CLASS],
sample=self.attrs.get(self.ATTR_SAMPLE, 0),
in_label=self.attrs.get(self.ATTR_KEY, "")):
self.start_time = perf_counter()
if self.is_retries_exceeded():
self.logger.info("[%s|%s] Retries exceeded. Remove job",
self.name, self.attrs[Job.ATTR_ID])
self.remove_job()
return
self.logger.info(
"[%s] Starting at %s (Lag %.2fms)", self.name,
self.scheduler.scheduler_id,
total_seconds(datetime.datetime.now() -
self.attrs[self.ATTR_TS]) * 1000.0)
# Run handler
status = self.E_EXCEPTION
delay = None
with Span(service="job.dereference"):
try:
ds = self.dereference()
can_run = self.can_run()
except Exception as e:
self.logger.error("Unknown error during dereference: %s",
e)
ds = None
can_run = False
if ds:
with Span(service="job.run"):
if can_run:
try:
data = self.attrs.get(self.ATTR_DATA) or {}
result = self.handler(**data)
if tornado.gen.is_future(result):
# Wait for future
result = yield result
status = self.E_SUCCESS
except RetryAfter as e:
self.logger.info("Retry after %ss: %s", e.delay, e)
status = self.E_RETRY
delay = e.delay
except self.failed_exceptions:
status = self.E_FAILED
except Exception:
error_report()
status = self.E_EXCEPTION
else:
self.logger.info("Deferred")
status = self.E_DEFERRED
elif ds is not None:
self.logger.info("Cannot dereference")
status = self.E_DEREFERENCE
self.duration = perf_counter() - self.start_time
self.logger.info("Completed. Status: %s (%.2fms)",
self.STATUS_MAP.get(status, status),
self.duration * 1000)
# Schedule next run
if delay is None:
with Span(service="job.schedule_next"):
self.schedule_next(status)
else:
with Span(service="job.schedule_retry"):
# Retry
if self.context_version:
ctx = self.context or None
ctx_key = self.get_context_cache_key()
else:
ctx = None
ctx_key = None
self.scheduler.set_next_run(
self.attrs[self.ATTR_ID],
status=status,
ts=datetime.datetime.now() +
datetime.timedelta(seconds=delay),
duration=self.duration,
context_version=self.context_version,
context=ctx,
context_key=ctx_key)
def clear_alarm(self, message, ts=None, force=False):
"""
Clear alarm
:param message: Log clearing message
:param ts: Clearing timestamp
:param force: Clear ever if wait_tt seg
"""
ts = ts or datetime.datetime.now()
if self.wait_tt and not force:
# Wait for escalated tt to close
if not self.wait_ts:
self.wait_ts = ts
self.log_message("Waiting for TT to close")
call_later(
"noc.services.escalator.wait_tt.wait_tt",
scheduler="escalator",
pool=self.managed_object.escalator_shard,
alarm_id=self.id
)
return
if self.alarm_class.clear_handlers:
# Process clear handlers
for h in self.alarm_class.get_clear_handlers():
try:
h(self)
except Exception:
error_report()
log = self.log + [AlarmLog(timestamp=ts, from_status="A",
to_status="C", message=message)]
a = ArchivedAlarm(
id=self.id,
timestamp=self.timestamp,
clear_timestamp=ts,
managed_object=self.managed_object,
alarm_class=self.alarm_class,
severity=self.severity,
vars=self.vars,
log=log,
root=self.root,
escalation_ts=self.escalation_ts,
escalation_tt=self.escalation_tt,
escalation_error=self.escalation_error,
escalation_ctx=self.escalation_ctx,
opening_event=self.opening_event,
closing_event=self.closing_event,
discriminator=self.discriminator,
reopens=self.reopens,
direct_services=self.direct_services,
direct_subscribers=self.direct_subscribers,
total_objects=self.total_objects,
total_services=self.total_services,
total_subscribers=self.total_subscribers,
adm_path=self.adm_path,
segment_path=self.segment_path,
container_path=self.container_path,
uplinks=self.uplinks
)
ct = self.alarm_class.get_control_time(self.reopens)
if ct:
a.control_time = datetime.datetime.now() + datetime.timedelta(seconds=ct)
a.save()
# Send notifications
if not a.root and not self.reopens:
a.managed_object.event(a.managed_object.EV_ALARM_CLEARED, {
"alarm": a,
"subject": a.subject,
"body": a.body,
"symptoms": a.alarm_class.symptoms,
"recommended_actions": a.alarm_class.recommended_actions,
"probable_causes": a.alarm_class.probable_causes
})
elif ct:
pass
# Set checks on all consequences
for d in self._get_collection().find({
"root": self.id
}, {"_id": 1, "alarm_class": 1}):
ac = AlarmClass.get_by_id(d["alarm_class"])
if not ac:
continue
t = ac.recover_time
if not t:
continue
call_later(
"noc.services.correlator.check.check_close_consequence",
scheduler="correlator",
pool=self.managed_object.pool.name,
delay=t,
alarm_id=d["_id"]
)
# Clear alarm
self.delete()
# Close TT
# MUST be after .delete() to prevent race conditions
if a.escalation_tt or self.clear_template:
if self.clear_template:
ctx = {
"alarm": a
}
subject = self.clear_template.render_subject(**ctx)
body = self.clear_template.render_body(**ctx)
else:
subject = "Alarm cleared"
body = "Alarm has been cleared"
call_later(
"noc.services.escalator.escalation.notify_close",
scheduler="escalator",
pool=self.managed_object.escalator_shard,
max_runs=ALARM_CLOSE_RETRIES,
alarm_id=self.id,
tt_id=self.escalation_tt,
subject=subject,
body=body,
notification_group_id=self.clear_notification_group.id if self.clear_notification_group else None,
close_tt=self.close_tt
)
# Gather diagnostics
AlarmDiagnosticConfig.on_clear(a)
# Return archived
return a
def handler500(request):
error_report()
return HttpResponseServerError("Internal Server Error")
def check(self, extractors=None):
extractors = extractors or self.get_extractors()
try:
return self.get_handler().check(extractors)
except Exception:
error_report()
def inner(request, *args, **kwargs):
def nq(s):
"""
Convert var[]=xxxx to var=xxxx
"""
if s.endswith("[]"):
return s[:-2]
else:
return s
try:
v = view_map[request.method]
except KeyError:
logger.info("No handler for '%s' method", request.method)
return HttpResponseNotFound("No handler for '%s' method" %
request.method)
if not request.user or not v.access.check(app, request.user):
return HttpResponseForbidden()
to_log_api_call = self.log_api_calls and hasattr(v,
"api") and v.api
app_logger = v.__self__.logger
try:
# Validate requests
if getattr(v, "validate", False):
# Additional validation
errors = None
if isinstance(v.validate, DictParameter):
# Validate via NOC interfaces
if request.method == "GET":
g = {
nq(k): v[0] if len(v) == 1 else v
for k, v in request.GET.lists() if k != "_dc"
}
else:
if self.is_json(request.META.get("CONTENT_TYPE")):
try:
g = orjson.loads(request.body)
except ValueError as e:
logger.error("Unable to decode JSON: %s",
e)
errors = "Unable to decode JSON: %s" % e
else:
g = {
k: v[0] if len(v) == 1 else v
for k, v in request.POST.lists()
}
if not errors:
try:
kwargs.update(v.validate.clean(g))
except InterfaceTypeError as e:
errors = str(e)
if errors:
#
if to_log_api_call:
app_logger.error("ERROR: %s", errors)
# Return error response
ext_format = "__format=ext" in request.META[
"QUERY_STRING"].split("&")
r = orjson.dumps({"status": False, "errors": errors})
status = 200 if ext_format else 400 # OK or BAD_REQUEST
return HttpResponse(
r,
status=status,
content_type="text/json; charset=utf-8")
# Log API call
if to_log_api_call:
a = {}
if request.method in ("POST", "PUT"):
ct = request.META.get("CONTENT_TYPE")
if ct and ("text/json" in ct
or "application/json" in ct):
a = orjson.loads(request.body)
else:
a = {
k: v[0] if len(v) == 1 else v
for k, v in request.POST.lists()
}
elif request.method == "GET":
a = {
k: v[0] if len(v) == 1 else v
for k, v in request.GET.lists()
}
app_logger.debug("API %s %s %s", request.method,
request.path, a)
# Call handler
r = v(request, *args, **kwargs)
# Dump SQL statements
if self.log_sql_statements:
from django.db import connections
tsc = 0
sc = defaultdict(int)
for conn in connections.all():
for q in conn.queries:
stmt = q["sql"].strip().split(" ", 1)[0].upper()
sc[stmt] += 1
tsc += 1
app_logger.debug("SQL %(sql)s %(time)ss" % q)
x = ", ".join("%s: %d" % (k, cv) for k, cv in sc.items())
if x:
x = " (%s)" % x
app_logger.debug("SQL statements: %d%s" % (tsc, x))
except PermissionDenied as e:
return HttpResponseForbidden(e)
except Http404 as e:
return HttpResponseNotFound(e)
except Exception:
# Generate 500
r = HttpResponse(
content=error_report(logger=app_logger),
status=500,
content_type="text/plain; charset=utf-8",
)
# Serialize response when necessary
if not isinstance(r, HttpResponse):
try:
r = HttpResponse(
orjson.dumps(
r,
option=orjson.OPT_SERIALIZE_NUMPY
| orjson.OPT_NON_STR_KEYS,
default=orjson_defaults,
),
content_type="text/json; charset=utf-8",
)
except Exception:
error_report(logger=app_logger)
r = HttpResponse(error_report(), status=500)
r["Pragma"] = "no-cache"
r["Cache-Control"] = "no-cache"
r["Expires"] = "0"
return r
def post(self, *args, **kwargs):
span_ctx = self.request.headers.get("X-NOC-Span-Ctx", 0)
span_id = self.request.headers.get("X-NOC-Span", 0)
sample = 1 if span_ctx and span_id else 0
# Parse JSON
try:
req = ujson.loads(self.request.body)
except ValueError as e:
self.api_error(e)
raise tornado.gen.Return()
# Parse request
id = req.get("id")
params = req.get("params", [])
method = req.get("method")
if not method or not hasattr(self.api_class, method):
self.api_error(
"Invalid method: '%s'" % method,
id=id
)
raise tornado.gen.Return()
api = self.api_class(self.service, self.request, self)
h = getattr(api, method)
if not getattr(h, "api", False):
self.api_error(
"Method is not callable: '%s'" % method,
id=id
)
raise tornado.gen.Return()
calling_service = self.request.headers.get(
self.CALLING_SERVICE_HEADER,
"unknown"
)
self.service.logger.debug(
"[RPC call from %s] %s.%s(%s)",
calling_service, api.name, method, params
)
in_label = None
if config.features.forensic:
lh = getattr(api, "%s_get_label" % method, None)
if lh:
in_label = lh(*params)
with Span(server=self.service.name,
service="api.%s" % method, sample=sample,
parent=span_id, context=span_ctx,
in_label=in_label) as span:
try:
if getattr(h, "executor", ""):
# Threadpool version
executor = self.service.get_executor(h.executor)
result = executor.submit(h, *params)
else:
# Serialized version
result = h(*params)
if tornado.gen.is_future(result):
result = yield result
if isinstance(result, Redirect):
# Redirect protocol extension
self.set_status(307, "Redirect")
self.set_header("Location", result.location)
self.write(ujson.dumps({
"id": id,
"method": result.method,
"params": result.params
}))
else:
# Dump output
self.write(ujson.dumps({
"id": id,
"error": None,
"result": result
}))
except NOCError as e:
span.error_code = e.code
span.error_text = str(e)
self.api_error(
"Failed: %s" % e,
id=id,
code=e.code
)
except Exception as e:
error_report()
span.error_code = ERR_UNKNOWN
span.error_text = str(e)
self.api_error(
"Failed: %s" % e,
id=id
)
