def event(request):
path, cfg = request.param
coll = cfg.get("$collection", COLLECTION_NAME)
assert coll == COLLECTION_NAME, "Invalid collection %s" % coll
ec = EventClass.get_by_name(
cfg.get("eventclass__name", DEFAULT_EVENT_CLASS)
)
mo = ManagedObject(
id=MO_ID,
name=MO_NAME,
address=MO_ADDRESS,
profile=Profile.get_by_name(cfg.get("profile__name", DEFAULT_PROFILE))
)
now = datetime.datetime.now()
data = cfg.get("data", {})
source = data.pop("source", "other")
event = ActiveEvent(
timestamp=now,
start_timestamp=now,
managed_object=mo,
source=source,
raw_vars=data,
repeats=1
)
request.fixturename = "events-%s" % cfg.get("uuid")
# request.fspath = path
return event, ec, cfg.get("vars", {})
def handler(self):
self.logger.info("Checking CPEs")
now = datetime.datetime.now()
result = self.object.scripts.get_cpe()
for cpe in result:
if cpe["status"] != "active":
self.logger.debug(
"[%s|%s] CPE status is '%s'. Skipping",
cpe["id"],
cpe["global_id"],
cpe["status"],
)
continue
mo = self.find_cpe(cpe["global_id"])
if mo:
changes = self.update_if_changed(
mo,
{
"controller": self.object,
"local_cpe_id": cpe["id"],
"global_cpe_id": cpe["global_id"],
"address": cpe["ip"],
"last_seen": now,
},
)
if changes:
self.logger.info(
"[%s|%s] Changed: %s",
cpe["id"],
cpe["global_id"],
", ".join("%s='%s'" % c for c in changes),
)
else:
name = cpe.get("name") or "cpe-%s" % cpe["global_id"]
if ManagedObject.objects.filter(name=name).exists():
name = "cpe-%s" % cpe["global_id"]
self.logger.info("[%s|%s] Created CPE %s", cpe["id"],
cpe["global_id"], name)
mo = ManagedObject(
name=name,
pool=self.object.pool,
profile=Profile.get_by_id(
Profile.get_generic_profile_id()),
object_profile=self.object.object_profile.cpe_profile
or self.object.object_profile,
administrative_domain=self.object.administrative_domain,
scheme=self.object.scheme,
segment=self.object.segment,
auth_profile=self.object.object_profile.cpe_auth_profile
or self.object.auth_profile,
address=cpe.get("ip") or "0.0.0.0",
controller=self.object,
last_seen=now,
local_cpe_id=cpe["id"],
global_cpe_id=cpe["global_id"],
)
mo.save()
def handle(self, paths, profile, format, progress=False, *args, **options):
assert profile_loader.get_profile(
profile), "Invalid profile: %s" % profile
t0 = time.time()
ruleset = RuleSet()
ruleset.load()
self.print("Ruleset load in %.2fms" % ((time.time() - t0) * 1000))
reader = getattr(self, "read_%s" % format, None)
assert reader, "Invalid format %s" % format
self.managed_object = ManagedObject(id=1,
name="test",
address="127.0.0.1",
profile_name=profile)
t0 = time.time()
stats = defaultdict(int)
total = 0
for p in paths:
if not os.path.isfile(p):
continue
for f in iter_open(p):
for event in reader(f):
e_vars = event.raw_vars.copy()
if event.source == "SNMP Trap":
e_vars.update(MIB.resolve_vars(event.raw_vars))
rule, r_vars = ruleset.find_rule(event, e_vars)
stats[rule.event_class.name] += 1
total += 1
if progress and total % 1000 == 0:
self.print("%d records processed" % total)
dt = time.time() - t0
self.print("%d events processed in %.2fms (%.fevents/sec)" %
(total, dt * 1000, float(total) / dt))
if stats:
# Prepare statistics
s_data = sorted([(k, stats[k]) for k in stats],
key=operator.itemgetter(1),
reverse=True)
s_total = sum(stats[k] for k in stats if not self.is_ignored(k))
data = [["Events", "%", "Event class"]]
for ecls, qty in s_data:
data += [[
str(qty),
"%3.2f%%" % (float(stats[ecls] * 100) / float(total)), ecls
]]
# Calculate classification quality
data += [[
"",
"%3.2f%%" % (float(s_total * 100) / total),
"Classification Quality"
]]
# Ruleset hit rate
rs_rate = float(metrics["rules_checked"].value) / float(total)
data += [["", "%.2f" % rs_rate, "Rule checks per event"]]
# Dump table
self.print("Event classes summary:")
self.print(format_table([4, 6, 10], data))
def get_objects(self, seg: NetworkSegment) -> List[ManagedObject]:
if not self._objects:
profile = ManagedObjectProfile(name="mock", level=self.LEVEL)
self._objects = [
patch_model(
ManagedObject(name="%d" % (i + 1), segment=self.target, object_profile=profile)
)
for i in range(self.N)
]
objects = [mo for mo in self._objects if mo.segment == seg]
self.set_power(seg, sum(mo.object_profile.level for mo in objects))
return objects
def discovery_object(request):
global _configs
name, address, pool_name, beef_storage_url, beef_path, data = request.param
beef = Beef.load(beef_storage_url, beef_path)
mo = ManagedObject(name=name,
is_managed=True,
administrative_domain=get_ad(pool_name),
segment=get_segment(pool_name),
pool=get_pool(pool_name),
profile=get_profile(beef.box.profile),
object_profile=get_managedobjectprofile(),
scheme=BEEF,
address=name)
mo.save()
# Store beef path
mo._beef_storage = beef_storage_url
mo._beef_path = beef_path
# Shortcut scripts to beef
mo.set_scripts_caller(BeefCallWrapper)
#
_configs[mo.id] = data
return mo
def handle(self, *args, **options):
connect()
if options["verbosity"] >= 2:
self.logger.setLevel(logging.DEBUG)
else:
self.logger.setLevel(logging.INFO)
for h in self.logger.handlers:
h.setFormatter(
logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
if not options["routerdb"]:
raise CommandError("No routerdb given")
if not options["cloginrc"]:
raise CommandError("No cloginrc given")
if not options["hosts"]:
options["hosts"] = ["/etc/hosts"]
if not options["repo"]:
raise CommandError("No CVS repository")
repo_prefix = options.get("repoprefix") or ""
if not options["object_profile"]:
raise CommandError("No object profile set")
try:
object_profile = ManagedObjectProfile.objects.get(
name=options["object_profile"].strip())
except ManagedObjectProfile.DoesNotExist:
raise CommandError("Invalid object profile: %s" %
options["object_profile"])
if not options["domain"]:
raise CommandError("No administrative domain set")
try:
domain = AdministrativeDomain.objects.get(
name=options["domain"].strip())
except AdministrativeDomain.DoesNotExist:
raise CommandError("Invalid administrative domain: %s" %
options["domain"])
if not options["pool"]:
raise CommandError("No pool set")
try:
pool = Pool.objects.get(name=options["domain"].strip())
except Pool.DoesNotExist:
raise CommandError("Invalid pool: %s" % options["pool"])
shard_member = 0
shard_members = 0
if options.get("shard"):
shard = options["shard"]
if "/" not in shard:
raise CommandError("Shard must be <member>/<members>")
shard_member, shard_members = [int(x) for x in shard.split("/")]
tags = []
if options["tags"]:
for t in options["tags"]:
tags += [x.strip() for x in t.split(",")]
self.dry_run = bool(options["dry_run"])
#
if not os.path.isdir(self.TMP):
os.mkdir(self.TMP)
#
revisions = self.index_cvs(options["repo"])
# Read configs
hosts = self.parse_hosts(options["hosts"])
rdb = self.parse_routerdb(options["routerdb"])
login, ldefaults = self.parse_cloginrc(options["cloginrc"])
# Process data
n = 0
count = len(rdb)
for name in sorted(rdb):
if shard_members:
if n % shard_members != shard_member:
n += 1
continue # Processed by other shard
self.logger.debug("[%s/%s] Processing host %s", n, count, name)
n += 1
profile = Profile.get_by_name(rdb[name])
address = hosts.get(name)
if not address:
# @todo: Resolve
self.logger.info("Cannot resolve address for %s. Skipping",
name)
continue
ld = login.get(name, ldefaults)
if not ld:
self.logger.info("No credentials for %s. Skipping", name)
continue
user = ld.get("user")
password = ld.get("password")
if "method" in ld and "ssh" in ld["method"]:
method = "ssh"
else:
method = "telnet"
self.logger.info(
"Managed object found: %s (%s, %s://%s@%s/)",
name,
profile.name,
method,
user,
address,
)
if not self.dry_run:
try:
mo = ManagedObject.objects.get(
Q(name=name) | Q(address=address))
self.logger.info("Already in the database")
except ManagedObject.DoesNotExist:
self.logger.info("Creating managed object %s", name)
mo = ManagedObject(
name=name,
object_profile=object_profile,
administrative_domain=domain,
pool=pool,
scheme=SSH if method == "ssh" else TELNET,
address=address,
profile=profile,
user=user,
password=password,
trap_source_ip=address,
tags=tags,
)
mo.save()
if name not in revisions:
self.logger.error("Cannot find config for %s", name)
continue
if not self.dry_run:
self.import_revisions(options["repo"], repo_prefix, mo, name,
revisions[name])