def handle_show_trials(self):
def q_seg(id):
ns = NetworkSegment.get_by_id(id)
if ns:
return ns.name
return str(id)
def q_mo(id):
mo = ManagedObject.get_by_id(id)
if mo:
return mo.name
return str(id)
connect()
self.print(
"ID | Status | Attacker | Target | Attacker Object | Target Object | Outcome | Error"
)
for trial in BioSegTrial.objects.all().order_by("ts"):
self.print(" | ".join([
str(trial.id),
"Processed" if trial.processed else "Pending",
q_seg(trial.attacker_id),
q_seg(trial.target_id),
q_mo(trial.attacker_object_id),
q_mo(trial.target_object_id),
trial.outcome or "-",
trial.error or "-",
]))
def handle(self, *args, **options):
ctr = options.get("countries", [])
connect()
print(ctr)
print(options)
# Check countries
for c in ctr:
if c not in self.HEADERS or c not in self.DATA:
raise CommandError("Unsupported country: %s" % c)
#
header = ["LEVEL%d" % d for d in range(self.LEVELS)]
header += [
"STREET",
"HOUSE_ADDR",
"NUM",
"NUM2",
"NUM_LETTER",
"BUILD",
"BUILD_LETTER",
"STRUCT",
"STRUCT2",
"STRUCT_LETTER",
"POSTAL_CODE",
]
for c in ctr:
header += self.HEADERS[c]
writer = csv.writer(sys.stdout)
writer.writerow(header)
for d in Division.get_top():
self.dump_division(writer, d, ctr, [])
def handle(self, host=None, port=None, *args, **options):
connect()
db = get_db()
collections = set(db.list_collection_names())
for model_id in iter_model_id():
model = get_model(model_id)
if not model:
self.die("Invalid model: %s" % model_id)
if not is_document(model):
continue
# Rename collections when necessary
legacy_collections = model._meta.get("legacy_collections", [])
for old_name in legacy_collections:
if old_name in collections:
new_name = model._meta["collection"]
self.print("[%s] Renaming %s to %s" %
(model_id, old_name, new_name))
db[old_name].rename(new_name)
break
# Ensure only documents with auto_create_index == False
if model._meta.get("auto_create_index", True):
continue
# Index model
self.index_model(model_id, model)
# Index datastreams
self.index_datastreams()
# Index GridVCS
self.index_gridvcs()
# Index mongo cache
self.index_cache()
# Index datasource cache
self.index_datasource_cache()
# @todo: Detect changes
self.print("OK")
def iter_objects(self, objects):
r = set()
connect()
for x in objects:
for o in ManagedObjectSelector.get_objects_from_expression(x):
r.add(o)
yield from r
def handle(self, devices, *args, **options):
devs = set()
connect()
for d in devices:
try:
devs |= set(
ManagedObjectSelector.get_objects_from_expression(d))
except ManagedObjectSelector.DoesNotExist:
self.die("Invalid object '%s'" % d)
self.stdout.write("profile,platform,oid,value\n")
for o in sorted(devs, key=lambda x: x.name):
if "get_snmp_get" not in o.scripts:
continue
if o.platform:
platform = o.platform.full_name
else:
try:
v = o.scripts.get_version()
except AttributeError:
v = {"platform": "Unknown"}
platform = v["platform"]
# sysObjectID
try:
v = o.scripts.get_snmp_get(
oid=mib["SNMPv2-MIB::sysObjectID.0"])
except NOCError:
continue
self.stdout.write(
"%s,%s,%s,%s\n" %
(o.profile.name, platform, "SNMPv2-MIB::sysObjectID.0", v))
def handle(self, *args, **options):
if len(args) < 1:
print("USAGE: %s <model> <object id> [.. <object id>]" %
sys.argv[0])
sys.exit(1)
m = args[0].replace("-", "_")
connect()
if m not in self.models:
raise CommandError("Invalid model '%s'. Valid models are: %s" %
(m, ", ".join(self.models)))
objects = []
getter = getattr(self, "get_%s" % m)
wiper = getattr(self, "wipe_%s" % m)
# Get objects
for o_id in args[1:]:
o = getter(o_id)
if not o: # Not found
raise CommandError("Object '%s' is not found" % o_id)
objects += [o]
# Wipe objects
from noc.core.debug import error_report
with bulk_datastream_changes():
for o in objects:
with self.log("Wiping '%s':" % unicode(o), True):
try:
wiper(o)
except KeyboardInterrupt:
raise CommandError(
"Interrupted. Wiping is not complete")
except Exception:
error_report()
def handle(self, *args, **options):
connect()
account_id = options.get("account")
if account_id:
logins = []
utm = UTM5()
#Получаем список сенрвисных связок
services = utm.get_slinks_for_account(account_id)
if services['Result'] == 'Ok':
#Формируем массив логинов
for i in range(len(services['data'])):
if services['data'][i]['service_type_array'] == 3:
service = utm.get_ipslink_data(services['data'][i]['slink_id_array'])
if 'iptraffic_login' in service['data']:
if service['data']['iptraffic_login'] != '':
logins.append(service['data']['iptraffic_login'])
if services['data'][i]['service_type_array'] == 5:
service = utm.get_dhsslink_data(services['data'][i]['slink_id_array'])
if 'login' in service['data']:
if service['data']['login'] != '':
logins.append(service['data']['login'])
if len((logins)) > 0:
#Для всех логинов, для всех брасов формируем список комманд и выполняем их
from noc.sa.models.action import Action
from noc.sa.models.managedobject import ManagedObject
action = Action.objects.get(name='clearsession')
bras = [ManagedObject.objects.get(id=105), ManagedObject.objects.get(id=360)]
for i in range(len(bras)):
commands = [str(action.expand(bras[i],username=x)) for x in logins]
bras[i].scripts.commands(commands=commands)
else:
print("Need --account parametr")
def handle_sync(self):
connect()
for c in Collection.iter_collections():
try:
c.sync()
except ValueError as e:
self.die(str(e))
def handle(self, *args, **options):
connect()
fname = options.get("fname")
olt = options.get("olt")
if fname and olt:
from noc.sa.models.action import Action
from noc.sa.models.managedobject import ManagedObject
olt = ManagedObject.objects.get(id=olt)
with open(fname, "r") as read_file:
data = json.load(read_file)
out = open(fname + '.txt', 'w')
for item in data:
up = 'n'
down = 'n'
command = [
str('show gpon onu detail-info gpon-onu_' +
item['interface'] + ':' + item['id'])
]
r = olt.scripts.commands(commands=command)
out.write('\n'.join((str(r['output'][0]).split('\n'))))
close(out)
# downlevel = re.match(r'^\s+down\s+Rx\s+:(?P<onurx>\S+)\s+', i)
# print(str(r['output'][0]))
# if uplevel:
# up = uplevel.group(0)
# if downlevel:
# down = downlevel.group(0)
else:
print("Need --file and --olt parametr")
def handle_tokenizer(self, object=None, profile=None, config=None, *args, **kwargs):
cfg = None
if config:
if not os.path.exists(config):
self.die("File not found: %s" % config)
with open(config) as f:
cfg = f.read()
if object:
connect()
mo = ManagedObject.get_by_id(object)
if not mo:
self.die("Managed Object not found")
elif profile:
p = loader.get_profile(profile)
if not p:
self.die("Invalid profile: %s" % profile)
if not cfg:
self.die("Specify config file with --config option")
# Mock up tokenizer
connect()
mo = ManagedObject.mock_object(profile=profile)
else:
self.die("Eigther object or profile must be set")
tokenizer = mo.iter_config_tokens(config=cfg)
for token in tokenizer:
self.print(token)
def _handle(self, *args, **options):
connect()
try:
handler = getattr(self, "handle_%s" % options["cmd"])
events = self.get_events(options)
handler(options, events)
except AttributeError:
self.die("Invalid action: %s" % options["action"])
def post(self):
connect()
code, result = yield self.executor.submit(self.handler)
self.set_status(code)
if isinstance(result, six.string_types):
self.write(result)
else:
self.write(ujson.dumps(result))
def iter_slot_streams(self) -> Tuple[str, str]:
# Common streams
for slot_name, stream_name in self.SLOT_STREAMS:
yield slot_name, stream_name
# Pooled streams
connect()
for pool in Pool.objects.all():
for slot_mask, stream_mask in self.POOLED_SLOT_STREAMS:
yield slot_mask % pool.name, stream_mask % pool.name
def on_start(self):
"""
Load rules from database just after loading config
"""
super(CorrelatorService, self).on_start()
connect() # use_mongo connect do after on_start.
self.load_rules()
self.load_triggers()
self.load_rca_rules()
def handle(self, *args, **options):
self.repo = options["repo"]
connect()
self.vcs = GridVCS(self.repo)
if self.repo in self.clean_int:
self.clean_id = lambda y: int(y)
else:
self.clean_id = lambda y: y
return getattr(self, "handle_%s" % options["cmd"])(*args, **options)
def handle(self, cmd, *args, **options):
connect()
if "mos" in options:
moo = options["mos"]
else:
self.stdout.write("No ManagedObject for proccessed")
self.die("No ManagedObject for proccessed")
return False
return getattr(self, "handle_%s" % cmd)(moo, *args, **options)
def handle_dump(self, object=None, show_hints=False, *args, **kwargs):
if object:
connect()
from noc.sa.models.managedobject import ManagedObject
mo = ManagedObject.objects.get(name=object)
if not mo:
self.die("Managed Object not found")
confdb = mo.get_confdb(cleanup=not show_hints)
self.print(confdb.dump())
def handle(self, *args, **options):
connect()
try:
self._handle(*args, **options)
except CommandError as why:
raise CommandError(why)
except SystemExit:
pass
except Exception:
error_report()
def handle_find_serial(self, serials):
connect()
for serial in serials:
for obj in Object.objects.filter(data__match={
"interface": "asset",
"attr": "serial",
"value": serial
}):
self.print("@@@ Serial %s" % serial)
self.dump_object(obj)
def handle(self, backend, user, password, *args, **kwargs):
if not password:
password = getpass.getpass()
connect()
backend = BaseAuthBackend.get_backend(backend)
auth = backend(None)
try:
auth.authenticate(user=user, password=password)
except backend.LoginError as e:
self.die("Failed to login: %s" % e)
self.print("Login successful")
def handle(self, *args, **options):
from noc.lib.app.site import site
connect()
# Install service stub
site.service = get_service()
# Synchronize permissions
try:
Permission.sync()
except ValueError as e:
self.die(str(e))
def handle(self, *args, **options):
connect()
self.olt_id = options.get("olt")
self.fname = options.get("file")
self.bras_id = options.get("bras")
if (not self.olt_id):
self.help()
return
self.getusdata()
self.olt = ManagedObject.objects.get(id=self.olt_id)
self.bras = ManagedObject.objects.get(id=self.bras_id)
self.getonulist()
def handle(self, *args, **options):
login = options.get("login")
connect()
r = url + 'user/' + login + '/session'
response = requests.get(r, verify=True, headers=headers)
if (response.ok):
usession = json.loads(response.content)
else:
response.raise_for_status()
if (usession['Result'] == 'Ok'):
print(usession['data'])
else:
print(usession)
def handle(self, paths, profile, format, progress=False, *args, **options):
assert profile_loader.get_profile(profile), "Invalid profile: %s" % profile
connect()
t0 = time.time()
ruleset = RuleSet()
ruleset.load()
self.print("Ruleset load in %.2fms" % ((time.time() - t0) * 1000))
reader = getattr(self, "read_%s" % format, None)
assert reader, "Invalid format %s" % format
self.managed_object = ManagedObject(
id=1, name="test", address="127.0.0.1", profile_name=profile
)
t0 = time.time()
stats = defaultdict(int)
total = 0
for p in paths:
if not os.path.isfile(p):
continue
for f in iter_open(p):
for event in reader(f):
e_vars = event.raw_vars.copy()
if event.source == "SNMP Trap":
e_vars.update(MIB.resolve_vars(event.raw_vars))
rule, r_vars = ruleset.find_rule(event, e_vars)
stats[rule.event_class.name] += 1
total += 1
if progress and total % 1000 == 0:
self.print("%d records processed" % total)
dt = time.time() - t0
self.print(
"%d events processed in %.2fms (%.fevents/sec)" % (total, dt * 1000, float(total) / dt)
)
if stats:
# Prepare statistics
s_data = sorted(
[(k, stats[k]) for k in stats], key=operator.itemgetter(1), reverse=True
)
s_total = sum(stats[k] for k in stats if not self.is_ignored(k))
data = [["Events", "%", "Event class"]]
for ecls, qty in s_data:
data += [[str(qty), "%3.2f%%" % (float(stats[ecls] * 100) / float(total)), ecls]]
# Calculate classification quality
data += [["", "%3.2f%%" % (float(s_total * 100) / total), "Classification Quality"]]
# Ruleset hit rate
rs_rate = float(metrics["rules_checked"].value) / float(total)
data += [["", "%.2f" % rs_rate, "Rule checks per event"]]
# Dump table
self.print("Event classes summary:")
self.print(format_table([4, 6, 10], data))
def __init__(self):
super(TrapCollectorService, self).__init__()
connect()
self.messages = []
self.send_callback = None
self.mappings_callback = None
self.report_invalid_callback = None
self.source_configs = {} # id -> SourceConfig
self.address_configs = {} # address -> SourceConfig
self.invalid_sources = defaultdict(int) # ip -> count
self.ignorept = {}
ignore = IgnorePattern.objects.filter(source='SNMP Trap')
for item in ignore:
l = item.pattern.split('|')
self.ignorept[l[0]] = re.compile(l[1])
async def activate(self):
"""
Initialize services before run
"""
self.logger.warning("Activating service")
if self.use_mongo:
from noc.core.mongo.connection import connect
connect()
await self.init_api()
#
if self.use_telemetry:
self.start_telemetry_callback()
self.loop.create_task(self.on_register())
def handle_apply(self, fixes=None, *args, **options):
if not fixes:
return
# Connect to mongo
from noc.core.mongo.connection import connect
connect()
# Apply fixes
for f in fixes:
fix = self.get_fix(f)
if not fix:
self.die("Invalid fix '%s'" % f)
print("Apply %s ..." % f, file=self.stdout)
fix()
print("... done", file=self.stdout)
def handle_apply(self,
host=None,
port=None,
dry_run=True,
*args,
**options):
connect()
read_only = dry_run
ch = connection(host, port, read_only=read_only)
today = datetime.date.today()
# Get partitions
parts = self.get_parts(ch)
#
partition_claimed = []
claimed_bytes = 0
for p in CHPolicy.objects.filter(is_active=True).order_by("table"):
table_claimed = 0
if not p.ttl:
continue # Disabled
deadline = today - datetime.timedelta(days=p.ttl)
is_dry = dry_run or p.dry_run
self.print("# Table %s deadline %s%s" %
(p.table, deadline.isoformat(),
" (Dry Run)" if is_dry else ""))
for pi in parts[p.table]:
if pi.max_date >= deadline:
continue
self.print(
" Removing partition %s (%s -- %s, %d rows, %d bytes)" %
(pi.partition, pi.min_date, pi.max_date, pi.rows,
pi.bytes))
table_claimed += pi.bytes
if not is_dry:
partition_claimed += [(p.table, pi.partition)]
self.print(" Total %d bytes to be reclaimed" % table_claimed)
claimed_bytes += table_claimed
if partition_claimed:
self.print("Claimed data will be Loss..\n")
for i in reversed(range(1, 10)):
self.print("%d\n" % i)
time.sleep(1)
for c in partition_claimed:
table, part = c[0], c[1]
if table.startswith(".inner"):
table = table[7:]
ch.execute("ALTER TABLE %s.%s DROP PARTITION '%s'" %
(config.clickhouse.db, table, part))
self.print("# Done. %d bytes to be reclaimed" % claimed_bytes)
def handle_rebuild(self, datastream, jobs=0, *args, **kwargs):
def update_object(obj_id):
ds.update_object(obj_id)
return obj_id
def grouper(iterable, n):
args = [iter(iterable)] * n
return itertools.zip_longest(*args)
def do_update(bulk):
ds.bulk_update(bulk)
yield from range(len(bulk))
if not datastream:
self.die("--datastream is not set. Set one from list: %s" %
", ".join(self.MODELS))
model = self.get_model(datastream)
connect()
ds = loader[datastream]
if not ds:
self.die("Cannot initialize datastream")
total = self.get_total(model)
STEP = 100
BATCH = 100
n = 1
report_interval = max(total // STEP, 1)
next_report = report_interval
if jobs:
from multiprocessing.pool import ThreadPool
pool = ThreadPool(jobs)
iterable = pool.imap_unordered(update_object, self.iter_id(model))
else:
iterable = (ds.bulk_update([b for b in bulk if b is not None])
for bulk in grouper(self.iter_id(model), BATCH))
if not self.no_progressbar:
# Disable logging
from noc.core.datastream.base import logger
logger.setLevel(logging.CRITICAL)
for _ in self.progress(iterable, max_value=total / BATCH):
if self.no_progressbar and n == next_report:
self.print("[%02d%%]" % ((n * 100) // total))
next_report += report_interval
n += 1
self.print("Done")
def handle(self, *args, **options):
connect()
if len(args) < 1:
self._usage()
r = args[0].split(".")
if len(r) != 2:
self._usage()
app, model = r
load_models()
m = apps.get_model(app, model)
if not m:
return self._usage()
print(
csv_export(m,
queryset=self.get_queryset(m, args[1:]),
first_row_only=options.get("template")))
