def handle(self, *args, **kwargs):
try:
self._handle(*args, **kwargs)
except CommandError:
raise
except:
error_report()
def run(self):
self.logger.debug("Starting worker thread")
while True:
# Get task from queue
try:
self.set_idle(True)
task = self.queue.get(block=True, timeout=1)
self.set_idle(False)
except Empty:
self.set_idle(False)
continue
if task is None:
break # Shutdown
title, f, args, kwargs = task
# Run job
self.title = title
self.start_time = time.time()
try:
self.can_cancel = True
f(*args, **kwargs)
self.can_cancel = False
except CancelledError:
break
except:
self.can_cancel = False
if not self.cancelled:
error_report()
self.queue.task_done()
# Shutdown
self.queue.task_done()
self.pool.thread_done(self)
self.logger.debug("Stopping worker thread")
def run(self):
self.running = True
try:
self.logger.debug("Running")
result = self.handler(self.probe, **self.config)
self.logger.debug("Result %s", result)
if result is not None:
t = time.time()
if not isinstance(result, dict):
if self.default_metric_type:
result = {self.default_metric_type: result}
else:
self.logger.error(
"Handler must return dict. Returned: %s", result)
raise ValueError("Handler must return dict")
# Feed result
for m in result:
if m in self.mdata:
v = result[m]
if v is not None:
self.mdata[m].set_value(t, v)
self.logger.debug("Done")
except:
error_report()
self.last_run = self.next_run
self.next_run = self.get_next_run()
self.running = False
self.daemon.reschedule(self)
def guarded_socket_call(self, socket, method):
"""
Wrapper for safe call of socket method. Handles and reports
socket errors.
:return: Call status
:rtype: Bool
"""
try:
method()
except Exception:
exc = get_socket_error()
try:
if exc:
socket.on_error(exc)
else:
socket.error("Unhandled exception when calling %s" %
str(method))
error_report()
socket.close()
except Exception:
socket.error("Error when handling error condition")
error_report()
return False
return True
def handle(self, *args, **options):
if len(args) < 1:
print "USAGE: %s <model> <object id> [.. <object id>]" % sys.argv[0]
sys.exit(1)
m = args[0].replace("-", "_")
if m not in self.models:
raise CommandError("Invalid model '%s'. Valid models are: %s" %
(m, ", ".join(self.models)))
objects = []
getter = getattr(self, "get_%s" % m)
wiper = getattr(self, "wipe_%s" % m)
# Get objects
for o_id in args[1:]:
o = getter(o_id)
if not o: # Not found
raise CommandError("Object '%s' is not found" % o_id)
objects += [o]
# Wipe objects
from noc.lib.debug import error_report
for o in objects:
with self.log("Wiping '%s':" % unicode(o), True):
try:
wiper(o)
except KeyboardInterrupt:
raise CommandError("Interrupted. Wiping is not complete")
except:
error_report()
def handle(self, *args, **options):
#
parsers = []
# Read config
config = SafeConfigParser()
for p in self.get_parsers():
config.read(os.path.join("etc", "address", "%s.defaults" % p))
config.read(os.path.join("etc", "address", "%s.conf" % p))
if config.getboolean(p, "enabled"):
m = __import__("noc.gis.parsers.address.%s" % p, {}, {}, "*")
for l in dir(m):
a = getattr(m, l)
if inspect.isclass(a) and issubclass(a, AddressParser) and a != AddressParser:
parsers += [a]
# Initialize parsers
parsers = [p(config, options) for p in parsers]
# Download
if options["download"]:
for p in parsers:
print "Downloading", p.name
if not p.download():
raise CommandError("Failed to download %s" % p.name)
else:
print "Skipping downloads"
# Sync
try:
for p in parsers:
print "Syncing", p.name
p.sync()
except:
error_report()
def get_active(self, timeout):
"""
Returns a tuple of active [readers], [writers]
:return:
"""
try:
events = self.poll.poll(timeout * 1000) # s -> ms
except select.error, why:
if why[0] not in (EINTR, EBADF):
error_report() # non-ignorable errors
return [], []
def run(self):
self.info("Running")
self.on_start()
while not self.to_shutdown:
id, to, subject, body, link = self.queue.get(block=True)
try:
r = self.send(to, subject, body, link)
self.daemon.report_result(id, r)
except Exception:
error_report()
self.daemon.report_result(id, False)
self.on_shutdown()
def handle(self, *args, **kwargs):
try:
self.get_existing_mo()
self.fix_uuids()
self.fix_inv_root()
self.fix_inv_lost_and_found()
self.fix_inv_orphans()
self.fix_metricsettings()
self.fix_fm_outage_orphans()
self.fix_wiping_mo()
except:
error_report()
sys.exit(1)
def on_complete(f):
logger.debug("Completion slow operation %s", so.id)
if f.exception():
so.status = cls.STATUS_FAILED
try:
f.result()
except Exception:
error_report()
so.pickled_result = cPickle.dumps(f.exception())
else:
so.status = cls.STATUS_COMPLETE
so.pickled_result = cPickle.dumps(f.result())
so.duration = time.time() - t0
so.save()
def get_active(self, timeout):
"""
Returns a tuple of active [readers], [writers]
:return:
"""
try:
r, w, x = select.select(self.rset, self.wset, [], timeout)
rset = [self.sockets[x] for x in r if x in self.sockets]
wset = [self.sockets[x] for x in w if x in self.sockets]
return rset, wset
except select.error, why:
if why[0] not in (EINTR, EBADF):
error_report() # non-ignorable errors
return [], []
def run(self):
logger.info("Running FM sender")
while True:
self.data_event.wait()
with self.data_lock:
data = self.data
self.clear_data()
self.data_event.clear()
try:
if not self.send_data(data):
self.restore_data(data)
except:
error_report()
self.restore_data(data)
time.sleep(1) # Prevent CPU hogging
def close(self):
"""
Close socket and unregister from factory
"""
if self.closing:
return
self.logger.debug("Closing socket")
self.closing = True
if self.socket:
self.factory.unregister_socket(self)
if self.socket:
try:
self.socket.close()
except socket.error, why:
if why[0] not in IGNORABLE_CLOSE_ERRORS:
error_report(logger=self.logger)
self.socket = None
self.on_close()
def run(self):
logger.info("Starting configuration thread")
while not self.to_shutdown:
try:
t0 = time.time()
interval = self.interval
config = self.get_config()
if config:
self.apply_config(config)
elif config is None:
interval = self.failed_interval
t = time.time()
nt = t0 + interval
if nt > t:
time.sleep(nt - t)
except:
error_report()
logger.info("Configuration thread stopped")
def guarded_run(self):
"""
Run daemon and catch common exceptions
:return:
"""
if self.start_delay:
self.logger.info("Delaying start for %s seconds", self.start_delay)
time.sleep(self.start_delay)
try:
self.run()
except KeyboardInterrupt:
pass
except MemoryError:
self.logger.error("Out of memory. Exiting.")
except SystemExit:
self.logger.info("Exiting")
except:
error_report()
self.at_exit()
def task_wrapper(self, task):
"""Periodic thread target"""
logging.info(u"Periodic task=%s status=running" % unicode(task))
t = datetime.datetime.now()
cwd = os.getcwd()
try:
if task.periodic_name.startswith("pyrule:"):
status = PyRule.call(task.periodic_name[7:],
timeout=task.timeout)
else:
status = task.periodic(task.timeout).execute()
except:
error_report()
status = False
logging.info(u"Periodic task=%s status=%s" % (unicode(task),
"completed" if status else "failed"))
# Current path may be implicitly changed by periodic. Restore old value
# to prevent further bugs
new_cwd = os.getcwd()
if cwd != new_cwd:
logging.error("CWD changed by periodic '%s' ('%s' -> '%s'). Restoring old cwd" % (unicode(task), cwd, new_cwd))
os.chdir(cwd)
# Mark task results
try:
nt = Schedule.objects.get(id=task.id)
nt.mark_run(t, status)
except Schedule.DoesNotExist:
pass # Scheule deleted during task run
with self.running_lock:
self.running.remove(task.periodic_name)
# Create appropriative FM event
self.write_event([
("source", "system"),
("type", "periodic status"),
("task", unicode(task)),
("status", {True: "success", False: "failure"}[status]),
])
def run(self, run_forever=False):
"""
Socket factory event loop.
:param run_forever: Run event loop forever, when True, else shutdown
fabric when no sockets available
"""
logger.info("Running socket factory (%s)",
self.poller.__class__.__name__)
self.create_pending_sockets()
if run_forever:
cond = lambda: True
else:
if self.write_delay:
cond = lambda: bool(self.sockets)
else:
cond = lambda: len(self.sockets) > 1
# Wait for any socket
while not cond():
time.sleep(1)
last_tick = last_stale = time.time()
while cond() and not self.to_shutdown:
self.loop(1)
t = time.time()
if self.tick_callback and t - last_tick >= 1:
self.metrics.ticks += 1
try:
self.tick_callback()
except Exception:
error_report()
logger.info("Restoring from tick() failure")
last_tick = t
if t - last_stale >= 1:
self.close_stale()
last_stale = t
logger.info("Stopping socket factory")
def rpc_handle_request(self, id, request):
logging.debug("rpc_handle_request: %s" % request.method)
if id in self.transactions:
self.send_error(id, ERR_TRANSACTION_EXISTS,
"Transaction %s is already exists" % id)
return
method = self.service.GetDescriptor().FindMethodByName(request.method)
if method:
req = self.service.GetRequestClass(method)()
req.ParseFromString(request.serialized_request)
# logging.debug("Request accepted:\nid: %s\n%s" % (id, str(req)))
controller = Controller(self)
controller.transaction = self.transactions.begin(
id=id, method=request.method)
try:
self.service.CallMethod(method, controller, req,
self.send_response)
except:
self.send_error(id, ERR_INTERNAL,
"RPC Call to %s failed" % request.method)
error_report()
else:
self.send_error(id, ERR_INVALID_METHOD,
"Invalid method '%s'" % request.method)
#!./bin/python
# -*- coding: utf-8 -*-
##----------------------------------------------------------------------
## noc-discovery daemon
##----------------------------------------------------------------------
## Copyright (C) 2007-2014 The NOC Project
## See LICENSE for details
##----------------------------------------------------------------------
if __name__ == "__main__":
from noc.inv.discovery.daemon import DiscoveryDaemon
from noc.lib.debug import error_report
try:
DiscoveryDaemon().process_command()
except SystemExit:
pass
except Exception:
error_report()
def writer(self):
while True:
try:
self._writer()
except:
error_report()
stmt = q["sql"].strip().split(" ", 1)[0].upper()
sc[stmt] += 1
tsc += 1
app_logger.debug("SQL %(sql)s %(time)ss" % q)
x = ", ".join(
["%s: %d" % (k, v) for k, v in sc.iteritems()])
if x:
x = " (%s)" % x
app_logger.debug("SQL statements: %d%s" % (tsc, x))
except PermissionDenied, why:
return HttpResponseForbidden(why)
except Http404, why:
return HttpResponseNotFound(why)
except:
# Generate 500
r = HttpResponse(content=error_report(logger=app_logger),
status=500,
mimetype="text/plain; charset=utf-8")
# Serialize response when necessary
if not isinstance(r, HttpResponse):
try:
r = HttpResponse(JSONEncoder(ensure_ascii=False).encode(r),
mimetype="text/json; charset=utf-8")
except:
error_report(logger=app_logger)
r = HttpResponse(error_report(), status=500)
r["Pragma"] = "no-cache"
r["Cache-Control"] = "no-cache"
r["Expires"] = "0"
return r
def run_pending(self):
n = 0
self.mrt_overload = False
# Run pending intial submits
if self.initial_submit_next_check:
for jcls in self.initial_submit_next_check:
if jcls.name in self.ignored:
continue
t0 = time.time()
if self.initial_submit_next_check[jcls] <= t0:
# Get existing keys
keys = [x[self.ATTR_KEY] for x in
self.collection.find({
self.ATTR_CLASS: jcls.name
}, [self.ATTR_KEY])]
# Run initial submit
try:
self.logger.info("Running initial submit for %s", jcls.name)
jcls.initial_submit(self, keys)
except Exception:
error_report()
# Reschedule initial submit
self.initial_submit_next_check[jcls] = (
t0 + jcls.initial_submit_interval)
# Check for complete MRT
if self.active_mrt:
complete = [t for t in self.active_mrt if t.complete]
for t in complete:
self.complete_mrt_job(t)
self.active_mrt = dict(
(t, self.active_mrt[t])
for t in self.active_mrt if t not in complete)
# Check for pending persistent tasks
for job_data in self.iter_pending_jobs():
jcls = self.job_classes.get(job_data[self.ATTR_CLASS])
if not jcls:
# Invalid job class. Park job to FAIL state
self.logger.error("Invalid job class: %s", jcls)
self.set_job_status(job_data[self.ATTR_CLASS],
job_data[self.ATTR_KEY], Job.S_FAILED)
continue
job = jcls(self,
job_data[self.ATTR_KEY], job_data[self.ATTR_DATA],
job_data[self.ATTR_SCHEDULE]
)
# Check for late jobs
if (job.max_delay and
job_data[self.ATTR_TS] < datetime.datetime.now() - datetime.timedelta(seconds=job.max_delay)):
job.logger.info("Job scheduled too late")
job.started = time.time()
self._complete_job(job, job.S_LATE, None)
continue
# Check for group limits
group = job.get_group()
if self.can_run(job):
if group is not None:
with self.running_lock:
self.running_count[group] += 1
self.run_job(job)
n += 1
return n
def raise_alarm(self, r, e):
managed_object = self.eval_expression(r.managed_object, event=e)
if not managed_object:
logging.debug("Empty managed object, ignoring")
return
if e.managed_object.id != managed_object.id:
logging.debug("Changing managed object to %s",
managed_object.name)
discriminator, vars = r.get_vars(e)
if r.unique:
assert discriminator is not None
# @todo: unneeded SQL lookup here
a = ActiveAlarm.objects.filter(
managed_object=managed_object.id,
discriminator=discriminator).first()
if not a:
# Try to reopen alarm
a = ArchivedAlarm.objects.filter(
managed_object=managed_object.id,
discriminator=discriminator,
control_time__gte=e.timestamp
).first()
if a:
# Reopen alarm
logging.debug("%s: Event %s(%s) reopens alarm %s(%s)" % (
r.u_name, str(e.id), e.event_class.name,
str(a.id), a.alarm_class.name))
a = a.reopen("Reopened by disposition rule '%s'" % r.u_name)
if a:
# Active alarm found, refresh
logging.debug("%s: Contributing event %s(%s) to active alarm %s(%s)" % (
r.u_name, str(e.id), e.event_class.name,
str(a.id), a.alarm_class.name))
a.contribute_event(e)
return
# Create new alarm
a = ActiveAlarm(
timestamp=e.timestamp,
last_update=e.timestamp,
managed_object=managed_object.id,
alarm_class=r.alarm_class,
severity=r.severity,
vars=vars,
discriminator=discriminator,
log=[
AlarmLog(
timestamp=datetime.datetime.now(),
from_status="A",
to_status="A",
message="Alarm risen from event %s(%s) by rule '%s'" % (
str(e.id), str(e.event_class.name), r.u_name)
)
]
)
a.save()
a.contribute_event(e, open=True)
logging.debug("%s: Event %s (%s) raises alarm %s (%s): %r",
r.u_name, str(e.id), e.event_class.name,
str(a.id), r.alarm_class.name, a.vars)
# RCA
if a.alarm_class.id in self.rca_forward:
# Check alarm is a consequence of existing one
self.set_root_cause(a)
if a.alarm_class.id in self.rca_reverse:
# Check alarm is the root cause for existing ones
self.set_reverse_root_cause(a)
# Call handlers
if a.alarm_class.id in self.handlers:
for h in self.handlers[a.alarm_class.id]:
try:
h(a)
except:
error_report()
# Call triggers if necessary
if r.alarm_class.id in self.triggers:
for t in self.triggers[r.alarm_class.id]:
try:
t.call(a)
except:
error_report()
#
if not a.severity:
# Alarm severity has been reset to 0 by handlers
# Silently drop alarm
logging.debug("Alarm severity is 0, dropping")
a.delete()
return
# Launch jobs when necessary
if a.alarm_class.id in self.alarm_jobs:
for job in self.alarm_jobs[r.alarm_class.id]:
job.submit(a)
# Notify about new alarm
if not a.root:
a.managed_object.event(a.managed_object.EV_ALARM_RISEN, {
"alarm": a,
"subject": a.subject,
"body": a.body,
"symptoms": a.alarm_class.symptoms,
"recommended_actions": a.alarm_class.recommended_actions,
"probable_causes": a.alarm_class.probable_causes
}, delay=a.alarm_class.get_notification_delay())
def save_config(self, data):
if isinstance(data, list):
# Convert list to plain text
r = []
for d in sorted(data, lambda x, y: cmp(x["name"], y["name"])):
r += ["==[ %s ]========================================\n%s" % (d["name"], d["config"])]
data = "\n".join(r)
# Pass data through config filter, if given
if self.config_filter_rule:
data = self.config_filter_rule(
managed_object=self, config=data)
# Pass data through the validation filter, if given
# @todo: Remove
if self.config_validation_rule:
warnings = self.config_validation_rule(
managed_object=self, config=data)
if warnings:
# There are some warnings. Notify responsible persons
self.event(
self.EV_CONFIG_POLICY_VIOLATION,
{
"object": self,
"warnings": warnings
}
)
# Calculate diff
old_data = self.config.read()
is_new = not bool(old_data)
diff = None
if not is_new:
# Calculate diff
if self.config_diff_filter_rule:
# Pass through filters
old_data = self.config_diff_filter_rule(
managed_object=self, config=old_data)
new_data = self.config_diff_filter_rule(
managed_object=self, config=data)
if not old_data and not new_data:
logger.error("[%s] broken config_diff_filter: Returns empty result", self.name)
else:
new_data = data
if old_data == new_data:
return # Nothing changed
diff = "".join(difflib.unified_diff(
old_data.splitlines(True),
new_data.splitlines(True),
fromfile=os.path.join("a", self.name.encode("utf8")),
tofile=os.path.join("b", self.name.encode("utf8"))
))
# Notify changes
self.event(
self.EV_CONFIG_CHANGED,
{
"object": self,
"is_new": is_new,
"config": data,
"diff": diff
}
)
# Save config
self.config.write(data)
# Run config validation
from noc.cm.engine import Engine
engine = Engine(self)
try:
engine.check()
except:
logger.error("Failed to validate config for %s", self.name)
error_report()
