def handle(self, *args, **kwargs): try: self._handle(*args, **kwargs) except CommandError: raise except: error_report()
def run(self): self.logger.debug("Starting worker thread") while True: # Get task from queue try: self.set_idle(True) task = self.queue.get(block=True, timeout=1) self.set_idle(False) except Empty: self.set_idle(False) continue if task is None: break # Shutdown title, f, args, kwargs = task # Run job self.title = title self.start_time = time.time() try: self.can_cancel = True f(*args, **kwargs) self.can_cancel = False except CancelledError: break except: self.can_cancel = False if not self.cancelled: error_report() self.queue.task_done() # Shutdown self.queue.task_done() self.pool.thread_done(self) self.logger.debug("Stopping worker thread")
def run(self): self.running = True try: self.logger.debug("Running") result = self.handler(self.probe, **self.config) self.logger.debug("Result %s", result) if result is not None: t = time.time() if not isinstance(result, dict): if self.default_metric_type: result = {self.default_metric_type: result} else: self.logger.error( "Handler must return dict. Returned: %s", result) raise ValueError("Handler must return dict") # Feed result for m in result: if m in self.mdata: v = result[m] if v is not None: self.mdata[m].set_value(t, v) self.logger.debug("Done") except: error_report() self.last_run = self.next_run self.next_run = self.get_next_run() self.running = False self.daemon.reschedule(self)
def guarded_socket_call(self, socket, method): """ Wrapper for safe call of socket method. Handles and reports socket errors. :return: Call status :rtype: Bool """ try: method() except Exception: exc = get_socket_error() try: if exc: socket.on_error(exc) else: socket.error("Unhandled exception when calling %s" % str(method)) error_report() socket.close() except Exception: socket.error("Error when handling error condition") error_report() return False return True
def handle(self, *args, **options): if len(args) < 1: print "USAGE: %s <model> <object id> [.. <object id>]" % sys.argv[0] sys.exit(1) m = args[0].replace("-", "_") if m not in self.models: raise CommandError("Invalid model '%s'. Valid models are: %s" % (m, ", ".join(self.models))) objects = [] getter = getattr(self, "get_%s" % m) wiper = getattr(self, "wipe_%s" % m) # Get objects for o_id in args[1:]: o = getter(o_id) if not o: # Not found raise CommandError("Object '%s' is not found" % o_id) objects += [o] # Wipe objects from noc.lib.debug import error_report for o in objects: with self.log("Wiping '%s':" % unicode(o), True): try: wiper(o) except KeyboardInterrupt: raise CommandError("Interrupted. Wiping is not complete") except: error_report()
def handle(self, *args, **options): # parsers = [] # Read config config = SafeConfigParser() for p in self.get_parsers(): config.read(os.path.join("etc", "address", "%s.defaults" % p)) config.read(os.path.join("etc", "address", "%s.conf" % p)) if config.getboolean(p, "enabled"): m = __import__("noc.gis.parsers.address.%s" % p, {}, {}, "*") for l in dir(m): a = getattr(m, l) if inspect.isclass(a) and issubclass(a, AddressParser) and a != AddressParser: parsers += [a] # Initialize parsers parsers = [p(config, options) for p in parsers] # Download if options["download"]: for p in parsers: print "Downloading", p.name if not p.download(): raise CommandError("Failed to download %s" % p.name) else: print "Skipping downloads" # Sync try: for p in parsers: print "Syncing", p.name p.sync() except: error_report()
def get_active(self, timeout): """ Returns a tuple of active [readers], [writers] :return: """ try: events = self.poll.poll(timeout * 1000) # s -> ms except select.error, why: if why[0] not in (EINTR, EBADF): error_report() # non-ignorable errors return [], []
def run(self): self.info("Running") self.on_start() while not self.to_shutdown: id, to, subject, body, link = self.queue.get(block=True) try: r = self.send(to, subject, body, link) self.daemon.report_result(id, r) except Exception: error_report() self.daemon.report_result(id, False) self.on_shutdown()
def handle(self, *args, **kwargs): try: self.get_existing_mo() self.fix_uuids() self.fix_inv_root() self.fix_inv_lost_and_found() self.fix_inv_orphans() self.fix_metricsettings() self.fix_fm_outage_orphans() self.fix_wiping_mo() except: error_report() sys.exit(1)
def on_complete(f): logger.debug("Completion slow operation %s", so.id) if f.exception(): so.status = cls.STATUS_FAILED try: f.result() except Exception: error_report() so.pickled_result = cPickle.dumps(f.exception()) else: so.status = cls.STATUS_COMPLETE so.pickled_result = cPickle.dumps(f.result()) so.duration = time.time() - t0 so.save()
def get_active(self, timeout): """ Returns a tuple of active [readers], [writers] :return: """ try: r, w, x = select.select(self.rset, self.wset, [], timeout) rset = [self.sockets[x] for x in r if x in self.sockets] wset = [self.sockets[x] for x in w if x in self.sockets] return rset, wset except select.error, why: if why[0] not in (EINTR, EBADF): error_report() # non-ignorable errors return [], []
def run(self): logger.info("Running FM sender") while True: self.data_event.wait() with self.data_lock: data = self.data self.clear_data() self.data_event.clear() try: if not self.send_data(data): self.restore_data(data) except: error_report() self.restore_data(data) time.sleep(1) # Prevent CPU hogging
def close(self): """ Close socket and unregister from factory """ if self.closing: return self.logger.debug("Closing socket") self.closing = True if self.socket: self.factory.unregister_socket(self) if self.socket: try: self.socket.close() except socket.error, why: if why[0] not in IGNORABLE_CLOSE_ERRORS: error_report(logger=self.logger) self.socket = None self.on_close()
def run(self): logger.info("Starting configuration thread") while not self.to_shutdown: try: t0 = time.time() interval = self.interval config = self.get_config() if config: self.apply_config(config) elif config is None: interval = self.failed_interval t = time.time() nt = t0 + interval if nt > t: time.sleep(nt - t) except: error_report() logger.info("Configuration thread stopped")
def guarded_run(self): """ Run daemon and catch common exceptions :return: """ if self.start_delay: self.logger.info("Delaying start for %s seconds", self.start_delay) time.sleep(self.start_delay) try: self.run() except KeyboardInterrupt: pass except MemoryError: self.logger.error("Out of memory. Exiting.") except SystemExit: self.logger.info("Exiting") except: error_report() self.at_exit()
def task_wrapper(self, task): """Periodic thread target""" logging.info(u"Periodic task=%s status=running" % unicode(task)) t = datetime.datetime.now() cwd = os.getcwd() try: if task.periodic_name.startswith("pyrule:"): status = PyRule.call(task.periodic_name[7:], timeout=task.timeout) else: status = task.periodic(task.timeout).execute() except: error_report() status = False logging.info(u"Periodic task=%s status=%s" % (unicode(task), "completed" if status else "failed")) # Current path may be implicitly changed by periodic. Restore old value # to prevent further bugs new_cwd = os.getcwd() if cwd != new_cwd: logging.error("CWD changed by periodic '%s' ('%s' -> '%s'). Restoring old cwd" % (unicode(task), cwd, new_cwd)) os.chdir(cwd) # Mark task results try: nt = Schedule.objects.get(id=task.id) nt.mark_run(t, status) except Schedule.DoesNotExist: pass # Scheule deleted during task run with self.running_lock: self.running.remove(task.periodic_name) # Create appropriative FM event self.write_event([ ("source", "system"), ("type", "periodic status"), ("task", unicode(task)), ("status", {True: "success", False: "failure"}[status]), ])
def run(self, run_forever=False): """ Socket factory event loop. :param run_forever: Run event loop forever, when True, else shutdown fabric when no sockets available """ logger.info("Running socket factory (%s)", self.poller.__class__.__name__) self.create_pending_sockets() if run_forever: cond = lambda: True else: if self.write_delay: cond = lambda: bool(self.sockets) else: cond = lambda: len(self.sockets) > 1 # Wait for any socket while not cond(): time.sleep(1) last_tick = last_stale = time.time() while cond() and not self.to_shutdown: self.loop(1) t = time.time() if self.tick_callback and t - last_tick >= 1: self.metrics.ticks += 1 try: self.tick_callback() except Exception: error_report() logger.info("Restoring from tick() failure") last_tick = t if t - last_stale >= 1: self.close_stale() last_stale = t logger.info("Stopping socket factory")
def rpc_handle_request(self, id, request): logging.debug("rpc_handle_request: %s" % request.method) if id in self.transactions: self.send_error(id, ERR_TRANSACTION_EXISTS, "Transaction %s is already exists" % id) return method = self.service.GetDescriptor().FindMethodByName(request.method) if method: req = self.service.GetRequestClass(method)() req.ParseFromString(request.serialized_request) # logging.debug("Request accepted:\nid: %s\n%s" % (id, str(req))) controller = Controller(self) controller.transaction = self.transactions.begin( id=id, method=request.method) try: self.service.CallMethod(method, controller, req, self.send_response) except: self.send_error(id, ERR_INTERNAL, "RPC Call to %s failed" % request.method) error_report() else: self.send_error(id, ERR_INVALID_METHOD, "Invalid method '%s'" % request.method)
#!./bin/python # -*- coding: utf-8 -*- ##---------------------------------------------------------------------- ## noc-discovery daemon ##---------------------------------------------------------------------- ## Copyright (C) 2007-2014 The NOC Project ## See LICENSE for details ##---------------------------------------------------------------------- if __name__ == "__main__": from noc.inv.discovery.daemon import DiscoveryDaemon from noc.lib.debug import error_report try: DiscoveryDaemon().process_command() except SystemExit: pass except Exception: error_report()
def writer(self): while True: try: self._writer() except: error_report()
stmt = q["sql"].strip().split(" ", 1)[0].upper() sc[stmt] += 1 tsc += 1 app_logger.debug("SQL %(sql)s %(time)ss" % q) x = ", ".join( ["%s: %d" % (k, v) for k, v in sc.iteritems()]) if x: x = " (%s)" % x app_logger.debug("SQL statements: %d%s" % (tsc, x)) except PermissionDenied, why: return HttpResponseForbidden(why) except Http404, why: return HttpResponseNotFound(why) except: # Generate 500 r = HttpResponse(content=error_report(logger=app_logger), status=500, mimetype="text/plain; charset=utf-8") # Serialize response when necessary if not isinstance(r, HttpResponse): try: r = HttpResponse(JSONEncoder(ensure_ascii=False).encode(r), mimetype="text/json; charset=utf-8") except: error_report(logger=app_logger) r = HttpResponse(error_report(), status=500) r["Pragma"] = "no-cache" r["Cache-Control"] = "no-cache" r["Expires"] = "0" return r
def run_pending(self): n = 0 self.mrt_overload = False # Run pending intial submits if self.initial_submit_next_check: for jcls in self.initial_submit_next_check: if jcls.name in self.ignored: continue t0 = time.time() if self.initial_submit_next_check[jcls] <= t0: # Get existing keys keys = [x[self.ATTR_KEY] for x in self.collection.find({ self.ATTR_CLASS: jcls.name }, [self.ATTR_KEY])] # Run initial submit try: self.logger.info("Running initial submit for %s", jcls.name) jcls.initial_submit(self, keys) except Exception: error_report() # Reschedule initial submit self.initial_submit_next_check[jcls] = ( t0 + jcls.initial_submit_interval) # Check for complete MRT if self.active_mrt: complete = [t for t in self.active_mrt if t.complete] for t in complete: self.complete_mrt_job(t) self.active_mrt = dict( (t, self.active_mrt[t]) for t in self.active_mrt if t not in complete) # Check for pending persistent tasks for job_data in self.iter_pending_jobs(): jcls = self.job_classes.get(job_data[self.ATTR_CLASS]) if not jcls: # Invalid job class. Park job to FAIL state self.logger.error("Invalid job class: %s", jcls) self.set_job_status(job_data[self.ATTR_CLASS], job_data[self.ATTR_KEY], Job.S_FAILED) continue job = jcls(self, job_data[self.ATTR_KEY], job_data[self.ATTR_DATA], job_data[self.ATTR_SCHEDULE] ) # Check for late jobs if (job.max_delay and job_data[self.ATTR_TS] < datetime.datetime.now() - datetime.timedelta(seconds=job.max_delay)): job.logger.info("Job scheduled too late") job.started = time.time() self._complete_job(job, job.S_LATE, None) continue # Check for group limits group = job.get_group() if self.can_run(job): if group is not None: with self.running_lock: self.running_count[group] += 1 self.run_job(job) n += 1 return n
def raise_alarm(self, r, e): managed_object = self.eval_expression(r.managed_object, event=e) if not managed_object: logging.debug("Empty managed object, ignoring") return if e.managed_object.id != managed_object.id: logging.debug("Changing managed object to %s", managed_object.name) discriminator, vars = r.get_vars(e) if r.unique: assert discriminator is not None # @todo: unneeded SQL lookup here a = ActiveAlarm.objects.filter( managed_object=managed_object.id, discriminator=discriminator).first() if not a: # Try to reopen alarm a = ArchivedAlarm.objects.filter( managed_object=managed_object.id, discriminator=discriminator, control_time__gte=e.timestamp ).first() if a: # Reopen alarm logging.debug("%s: Event %s(%s) reopens alarm %s(%s)" % ( r.u_name, str(e.id), e.event_class.name, str(a.id), a.alarm_class.name)) a = a.reopen("Reopened by disposition rule '%s'" % r.u_name) if a: # Active alarm found, refresh logging.debug("%s: Contributing event %s(%s) to active alarm %s(%s)" % ( r.u_name, str(e.id), e.event_class.name, str(a.id), a.alarm_class.name)) a.contribute_event(e) return # Create new alarm a = ActiveAlarm( timestamp=e.timestamp, last_update=e.timestamp, managed_object=managed_object.id, alarm_class=r.alarm_class, severity=r.severity, vars=vars, discriminator=discriminator, log=[ AlarmLog( timestamp=datetime.datetime.now(), from_status="A", to_status="A", message="Alarm risen from event %s(%s) by rule '%s'" % ( str(e.id), str(e.event_class.name), r.u_name) ) ] ) a.save() a.contribute_event(e, open=True) logging.debug("%s: Event %s (%s) raises alarm %s (%s): %r", r.u_name, str(e.id), e.event_class.name, str(a.id), r.alarm_class.name, a.vars) # RCA if a.alarm_class.id in self.rca_forward: # Check alarm is a consequence of existing one self.set_root_cause(a) if a.alarm_class.id in self.rca_reverse: # Check alarm is the root cause for existing ones self.set_reverse_root_cause(a) # Call handlers if a.alarm_class.id in self.handlers: for h in self.handlers[a.alarm_class.id]: try: h(a) except: error_report() # Call triggers if necessary if r.alarm_class.id in self.triggers: for t in self.triggers[r.alarm_class.id]: try: t.call(a) except: error_report() # if not a.severity: # Alarm severity has been reset to 0 by handlers # Silently drop alarm logging.debug("Alarm severity is 0, dropping") a.delete() return # Launch jobs when necessary if a.alarm_class.id in self.alarm_jobs: for job in self.alarm_jobs[r.alarm_class.id]: job.submit(a) # Notify about new alarm if not a.root: a.managed_object.event(a.managed_object.EV_ALARM_RISEN, { "alarm": a, "subject": a.subject, "body": a.body, "symptoms": a.alarm_class.symptoms, "recommended_actions": a.alarm_class.recommended_actions, "probable_causes": a.alarm_class.probable_causes }, delay=a.alarm_class.get_notification_delay())
def save_config(self, data): if isinstance(data, list): # Convert list to plain text r = [] for d in sorted(data, lambda x, y: cmp(x["name"], y["name"])): r += ["==[ %s ]========================================\n%s" % (d["name"], d["config"])] data = "\n".join(r) # Pass data through config filter, if given if self.config_filter_rule: data = self.config_filter_rule( managed_object=self, config=data) # Pass data through the validation filter, if given # @todo: Remove if self.config_validation_rule: warnings = self.config_validation_rule( managed_object=self, config=data) if warnings: # There are some warnings. Notify responsible persons self.event( self.EV_CONFIG_POLICY_VIOLATION, { "object": self, "warnings": warnings } ) # Calculate diff old_data = self.config.read() is_new = not bool(old_data) diff = None if not is_new: # Calculate diff if self.config_diff_filter_rule: # Pass through filters old_data = self.config_diff_filter_rule( managed_object=self, config=old_data) new_data = self.config_diff_filter_rule( managed_object=self, config=data) if not old_data and not new_data: logger.error("[%s] broken config_diff_filter: Returns empty result", self.name) else: new_data = data if old_data == new_data: return # Nothing changed diff = "".join(difflib.unified_diff( old_data.splitlines(True), new_data.splitlines(True), fromfile=os.path.join("a", self.name.encode("utf8")), tofile=os.path.join("b", self.name.encode("utf8")) )) # Notify changes self.event( self.EV_CONFIG_CHANGED, { "object": self, "is_new": is_new, "config": data, "diff": diff } ) # Save config self.config.write(data) # Run config validation from noc.cm.engine import Engine engine = Engine(self) try: engine.check() except: logger.error("Failed to validate config for %s", self.name) error_report()