def handle_show_trials(self): def q_seg(id): ns = NetworkSegment.get_by_id(id) if ns: return ns.name return str(id) def q_mo(id): mo = ManagedObject.get_by_id(id) if mo: return mo.name return str(id) connect() self.print( "ID | Status | Attacker | Target | Attacker Object | Target Object | Outcome | Error" ) for trial in BioSegTrial.objects.all().order_by("ts"): self.print(" | ".join([ str(trial.id), "Processed" if trial.processed else "Pending", q_seg(trial.attacker_id), q_seg(trial.target_id), q_mo(trial.attacker_object_id), q_mo(trial.target_object_id), trial.outcome or "-", trial.error or "-", ]))
def handle(self, *args, **options): ctr = options.get("countries", []) connect() print(ctr) print(options) # Check countries for c in ctr: if c not in self.HEADERS or c not in self.DATA: raise CommandError("Unsupported country: %s" % c) # header = ["LEVEL%d" % d for d in range(self.LEVELS)] header += [ "STREET", "HOUSE_ADDR", "NUM", "NUM2", "NUM_LETTER", "BUILD", "BUILD_LETTER", "STRUCT", "STRUCT2", "STRUCT_LETTER", "POSTAL_CODE", ] for c in ctr: header += self.HEADERS[c] writer = csv.writer(sys.stdout) writer.writerow(header) for d in Division.get_top(): self.dump_division(writer, d, ctr, [])
def handle(self, host=None, port=None, *args, **options): connect() db = get_db() collections = set(db.list_collection_names()) for model_id in iter_model_id(): model = get_model(model_id) if not model: self.die("Invalid model: %s" % model_id) if not is_document(model): continue # Rename collections when necessary legacy_collections = model._meta.get("legacy_collections", []) for old_name in legacy_collections: if old_name in collections: new_name = model._meta["collection"] self.print("[%s] Renaming %s to %s" % (model_id, old_name, new_name)) db[old_name].rename(new_name) break # Ensure only documents with auto_create_index == False if model._meta.get("auto_create_index", True): continue # Index model self.index_model(model_id, model) # Index datastreams self.index_datastreams() # Index GridVCS self.index_gridvcs() # Index mongo cache self.index_cache() # Index datasource cache self.index_datasource_cache() # @todo: Detect changes self.print("OK")
def iter_objects(self, objects): r = set() connect() for x in objects: for o in ManagedObjectSelector.get_objects_from_expression(x): r.add(o) yield from r
def handle(self, devices, *args, **options): devs = set() connect() for d in devices: try: devs |= set( ManagedObjectSelector.get_objects_from_expression(d)) except ManagedObjectSelector.DoesNotExist: self.die("Invalid object '%s'" % d) self.stdout.write("profile,platform,oid,value\n") for o in sorted(devs, key=lambda x: x.name): if "get_snmp_get" not in o.scripts: continue if o.platform: platform = o.platform.full_name else: try: v = o.scripts.get_version() except AttributeError: v = {"platform": "Unknown"} platform = v["platform"] # sysObjectID try: v = o.scripts.get_snmp_get( oid=mib["SNMPv2-MIB::sysObjectID.0"]) except NOCError: continue self.stdout.write( "%s,%s,%s,%s\n" % (o.profile.name, platform, "SNMPv2-MIB::sysObjectID.0", v))
def handle(self, *args, **options): if len(args) < 1: print("USAGE: %s <model> <object id> [.. <object id>]" % sys.argv[0]) sys.exit(1) m = args[0].replace("-", "_") connect() if m not in self.models: raise CommandError("Invalid model '%s'. Valid models are: %s" % (m, ", ".join(self.models))) objects = [] getter = getattr(self, "get_%s" % m) wiper = getattr(self, "wipe_%s" % m) # Get objects for o_id in args[1:]: o = getter(o_id) if not o: # Not found raise CommandError("Object '%s' is not found" % o_id) objects += [o] # Wipe objects from noc.core.debug import error_report with bulk_datastream_changes(): for o in objects: with self.log("Wiping '%s':" % unicode(o), True): try: wiper(o) except KeyboardInterrupt: raise CommandError( "Interrupted. Wiping is not complete") except Exception: error_report()
def handle(self, *args, **options): connect() account_id = options.get("account") if account_id: logins = [] utm = UTM5() #Получаем список сенрвисных связок services = utm.get_slinks_for_account(account_id) if services['Result'] == 'Ok': #Формируем массив логинов for i in range(len(services['data'])): if services['data'][i]['service_type_array'] == 3: service = utm.get_ipslink_data(services['data'][i]['slink_id_array']) if 'iptraffic_login' in service['data']: if service['data']['iptraffic_login'] != '': logins.append(service['data']['iptraffic_login']) if services['data'][i]['service_type_array'] == 5: service = utm.get_dhsslink_data(services['data'][i]['slink_id_array']) if 'login' in service['data']: if service['data']['login'] != '': logins.append(service['data']['login']) if len((logins)) > 0: #Для всех логинов, для всех брасов формируем список комманд и выполняем их from noc.sa.models.action import Action from noc.sa.models.managedobject import ManagedObject action = Action.objects.get(name='clearsession') bras = [ManagedObject.objects.get(id=105), ManagedObject.objects.get(id=360)] for i in range(len(bras)): commands = [str(action.expand(bras[i],username=x)) for x in logins] bras[i].scripts.commands(commands=commands) else: print("Need --account parametr")
def handle_sync(self): connect() for c in Collection.iter_collections(): try: c.sync() except ValueError as e: self.die(str(e))
def handle(self, *args, **options): connect() fname = options.get("fname") olt = options.get("olt") if fname and olt: from noc.sa.models.action import Action from noc.sa.models.managedobject import ManagedObject olt = ManagedObject.objects.get(id=olt) with open(fname, "r") as read_file: data = json.load(read_file) out = open(fname + '.txt', 'w') for item in data: up = 'n' down = 'n' command = [ str('show gpon onu detail-info gpon-onu_' + item['interface'] + ':' + item['id']) ] r = olt.scripts.commands(commands=command) out.write('\n'.join((str(r['output'][0]).split('\n')))) close(out) # downlevel = re.match(r'^\s+down\s+Rx\s+:(?P<onurx>\S+)\s+', i) # print(str(r['output'][0])) # if uplevel: # up = uplevel.group(0) # if downlevel: # down = downlevel.group(0) else: print("Need --file and --olt parametr")
def handle_tokenizer(self, object=None, profile=None, config=None, *args, **kwargs): cfg = None if config: if not os.path.exists(config): self.die("File not found: %s" % config) with open(config) as f: cfg = f.read() if object: connect() mo = ManagedObject.get_by_id(object) if not mo: self.die("Managed Object not found") elif profile: p = loader.get_profile(profile) if not p: self.die("Invalid profile: %s" % profile) if not cfg: self.die("Specify config file with --config option") # Mock up tokenizer connect() mo = ManagedObject.mock_object(profile=profile) else: self.die("Eigther object or profile must be set") tokenizer = mo.iter_config_tokens(config=cfg) for token in tokenizer: self.print(token)
def _handle(self, *args, **options): connect() try: handler = getattr(self, "handle_%s" % options["cmd"]) events = self.get_events(options) handler(options, events) except AttributeError: self.die("Invalid action: %s" % options["action"])
def post(self): connect() code, result = yield self.executor.submit(self.handler) self.set_status(code) if isinstance(result, six.string_types): self.write(result) else: self.write(ujson.dumps(result))
def iter_slot_streams(self) -> Tuple[str, str]: # Common streams for slot_name, stream_name in self.SLOT_STREAMS: yield slot_name, stream_name # Pooled streams connect() for pool in Pool.objects.all(): for slot_mask, stream_mask in self.POOLED_SLOT_STREAMS: yield slot_mask % pool.name, stream_mask % pool.name
def on_start(self): """ Load rules from database just after loading config """ super(CorrelatorService, self).on_start() connect() # use_mongo connect do after on_start. self.load_rules() self.load_triggers() self.load_rca_rules()
def handle(self, *args, **options): self.repo = options["repo"] connect() self.vcs = GridVCS(self.repo) if self.repo in self.clean_int: self.clean_id = lambda y: int(y) else: self.clean_id = lambda y: y return getattr(self, "handle_%s" % options["cmd"])(*args, **options)
def handle(self, cmd, *args, **options): connect() if "mos" in options: moo = options["mos"] else: self.stdout.write("No ManagedObject for proccessed") self.die("No ManagedObject for proccessed") return False return getattr(self, "handle_%s" % cmd)(moo, *args, **options)
def handle_dump(self, object=None, show_hints=False, *args, **kwargs): if object: connect() from noc.sa.models.managedobject import ManagedObject mo = ManagedObject.objects.get(name=object) if not mo: self.die("Managed Object not found") confdb = mo.get_confdb(cleanup=not show_hints) self.print(confdb.dump())
def handle(self, *args, **options): connect() try: self._handle(*args, **options) except CommandError as why: raise CommandError(why) except SystemExit: pass except Exception: error_report()
def handle_find_serial(self, serials): connect() for serial in serials: for obj in Object.objects.filter(data__match={ "interface": "asset", "attr": "serial", "value": serial }): self.print("@@@ Serial %s" % serial) self.dump_object(obj)
def handle(self, backend, user, password, *args, **kwargs): if not password: password = getpass.getpass() connect() backend = BaseAuthBackend.get_backend(backend) auth = backend(None) try: auth.authenticate(user=user, password=password) except backend.LoginError as e: self.die("Failed to login: %s" % e) self.print("Login successful")
def handle(self, *args, **options): from noc.lib.app.site import site connect() # Install service stub site.service = get_service() # Synchronize permissions try: Permission.sync() except ValueError as e: self.die(str(e))
def handle(self, *args, **options): connect() self.olt_id = options.get("olt") self.fname = options.get("file") self.bras_id = options.get("bras") if (not self.olt_id): self.help() return self.getusdata() self.olt = ManagedObject.objects.get(id=self.olt_id) self.bras = ManagedObject.objects.get(id=self.bras_id) self.getonulist()
def handle(self, *args, **options): login = options.get("login") connect() r = url + 'user/' + login + '/session' response = requests.get(r, verify=True, headers=headers) if (response.ok): usession = json.loads(response.content) else: response.raise_for_status() if (usession['Result'] == 'Ok'): print(usession['data']) else: print(usession)
def handle(self, paths, profile, format, progress=False, *args, **options): assert profile_loader.get_profile(profile), "Invalid profile: %s" % profile connect() t0 = time.time() ruleset = RuleSet() ruleset.load() self.print("Ruleset load in %.2fms" % ((time.time() - t0) * 1000)) reader = getattr(self, "read_%s" % format, None) assert reader, "Invalid format %s" % format self.managed_object = ManagedObject( id=1, name="test", address="127.0.0.1", profile_name=profile ) t0 = time.time() stats = defaultdict(int) total = 0 for p in paths: if not os.path.isfile(p): continue for f in iter_open(p): for event in reader(f): e_vars = event.raw_vars.copy() if event.source == "SNMP Trap": e_vars.update(MIB.resolve_vars(event.raw_vars)) rule, r_vars = ruleset.find_rule(event, e_vars) stats[rule.event_class.name] += 1 total += 1 if progress and total % 1000 == 0: self.print("%d records processed" % total) dt = time.time() - t0 self.print( "%d events processed in %.2fms (%.fevents/sec)" % (total, dt * 1000, float(total) / dt) ) if stats: # Prepare statistics s_data = sorted( [(k, stats[k]) for k in stats], key=operator.itemgetter(1), reverse=True ) s_total = sum(stats[k] for k in stats if not self.is_ignored(k)) data = [["Events", "%", "Event class"]] for ecls, qty in s_data: data += [[str(qty), "%3.2f%%" % (float(stats[ecls] * 100) / float(total)), ecls]] # Calculate classification quality data += [["", "%3.2f%%" % (float(s_total * 100) / total), "Classification Quality"]] # Ruleset hit rate rs_rate = float(metrics["rules_checked"].value) / float(total) data += [["", "%.2f" % rs_rate, "Rule checks per event"]] # Dump table self.print("Event classes summary:") self.print(format_table([4, 6, 10], data))
def __init__(self): super(TrapCollectorService, self).__init__() connect() self.messages = [] self.send_callback = None self.mappings_callback = None self.report_invalid_callback = None self.source_configs = {} # id -> SourceConfig self.address_configs = {} # address -> SourceConfig self.invalid_sources = defaultdict(int) # ip -> count self.ignorept = {} ignore = IgnorePattern.objects.filter(source='SNMP Trap') for item in ignore: l = item.pattern.split('|') self.ignorept[l[0]] = re.compile(l[1])
async def activate(self): """ Initialize services before run """ self.logger.warning("Activating service") if self.use_mongo: from noc.core.mongo.connection import connect connect() await self.init_api() # if self.use_telemetry: self.start_telemetry_callback() self.loop.create_task(self.on_register())
def handle_apply(self, fixes=None, *args, **options): if not fixes: return # Connect to mongo from noc.core.mongo.connection import connect connect() # Apply fixes for f in fixes: fix = self.get_fix(f) if not fix: self.die("Invalid fix '%s'" % f) print("Apply %s ..." % f, file=self.stdout) fix() print("... done", file=self.stdout)
def handle_apply(self, host=None, port=None, dry_run=True, *args, **options): connect() read_only = dry_run ch = connection(host, port, read_only=read_only) today = datetime.date.today() # Get partitions parts = self.get_parts(ch) # partition_claimed = [] claimed_bytes = 0 for p in CHPolicy.objects.filter(is_active=True).order_by("table"): table_claimed = 0 if not p.ttl: continue # Disabled deadline = today - datetime.timedelta(days=p.ttl) is_dry = dry_run or p.dry_run self.print("# Table %s deadline %s%s" % (p.table, deadline.isoformat(), " (Dry Run)" if is_dry else "")) for pi in parts[p.table]: if pi.max_date >= deadline: continue self.print( " Removing partition %s (%s -- %s, %d rows, %d bytes)" % (pi.partition, pi.min_date, pi.max_date, pi.rows, pi.bytes)) table_claimed += pi.bytes if not is_dry: partition_claimed += [(p.table, pi.partition)] self.print(" Total %d bytes to be reclaimed" % table_claimed) claimed_bytes += table_claimed if partition_claimed: self.print("Claimed data will be Loss..\n") for i in reversed(range(1, 10)): self.print("%d\n" % i) time.sleep(1) for c in partition_claimed: table, part = c[0], c[1] if table.startswith(".inner"): table = table[7:] ch.execute("ALTER TABLE %s.%s DROP PARTITION '%s'" % (config.clickhouse.db, table, part)) self.print("# Done. %d bytes to be reclaimed" % claimed_bytes)
def handle_rebuild(self, datastream, jobs=0, *args, **kwargs): def update_object(obj_id): ds.update_object(obj_id) return obj_id def grouper(iterable, n): args = [iter(iterable)] * n return itertools.zip_longest(*args) def do_update(bulk): ds.bulk_update(bulk) yield from range(len(bulk)) if not datastream: self.die("--datastream is not set. Set one from list: %s" % ", ".join(self.MODELS)) model = self.get_model(datastream) connect() ds = loader[datastream] if not ds: self.die("Cannot initialize datastream") total = self.get_total(model) STEP = 100 BATCH = 100 n = 1 report_interval = max(total // STEP, 1) next_report = report_interval if jobs: from multiprocessing.pool import ThreadPool pool = ThreadPool(jobs) iterable = pool.imap_unordered(update_object, self.iter_id(model)) else: iterable = (ds.bulk_update([b for b in bulk if b is not None]) for bulk in grouper(self.iter_id(model), BATCH)) if not self.no_progressbar: # Disable logging from noc.core.datastream.base import logger logger.setLevel(logging.CRITICAL) for _ in self.progress(iterable, max_value=total / BATCH): if self.no_progressbar and n == next_report: self.print("[%02d%%]" % ((n * 100) // total)) next_report += report_interval n += 1 self.print("Done")
def handle(self, *args, **options): connect() if len(args) < 1: self._usage() r = args[0].split(".") if len(r) != 2: self._usage() app, model = r load_models() m = apps.get_model(app, model) if not m: return self._usage() print( csv_export(m, queryset=self.get_queryset(m, args[1:]), first_row_only=options.get("template")))