def handle_clean(self, files=None, ttl=None, dry_run=True, *args, **options):
remote_system = RemoteSystem.get_by_name(options["system"])
if not remote_system:
self.die("Invalid remote system: %s" % options["system"])
if files and files < CLEANUP_SAFE_FILES_COUNT:
self.die("3 is minimal value to save file")
deadline = None
if ttl:
today = datetime.datetime.now()
deadline = today - datetime.timedelta(days=ttl)
self.print("Cleaned files before: %s" % deadline)
if not files and not deadline:
self.die("Set one of policy setting (ttl or files")
clean_paths = []
extractors = set(options.get("extractors", []))
chain = remote_system.get_loader_chain()
for ldr in chain:
if extractors and ldr.name not in extractors:
continue
if os.path.isdir(ldr.archive_dir):
fn = list(
reversed(
sorted(f for f in os.listdir(ldr.archive_dir) if ldr.rx_archive.match(f))
)
)
else:
self.die("No archived dir")
clean_files = []
if files:
# Protect last 3 files
files = max(files, CLEANUP_SAFE_FILES_COUNT)
clean_files = fn[files:]
elif deadline:
for f in fn:
if (
datetime.datetime.strptime(f.split(".", 1)[0], "import-%Y-%m-%d-%H-%M-%S")
> deadline
):
continue
clean_files += [f]
clean_files = list(reversed(sorted(clean_files)))[CLEANUP_SAFE_FILES_COUNT:]
if not clean_files:
self.print("Nothing to remove. Continue...")
else:
clean_paths += [os.path.join(ldr.archive_dir, f) for f in clean_files]
self.print("Cleanup files (%d):\n %s " % (len(clean_files), "\n".join(clean_files)))
if not dry_run:
self.print("Claimed data will be Loss..\n")
for i in reversed(range(1, 10)):
self.print("%d\n" % i)
time.sleep(1)
for path in list(clean_paths):
self.print("Clean %s" % path)
os.unlink(path)
self.print("# Done.")
def handle_load(self, *args, **options):
remote_system = RemoteSystem.get_by_name(options["system"])
if not remote_system:
self.die("Invalid remote system: %s" % options["system"])
remote_system.load(options.get("loaders", []), quiet=options.get("quiet", False))
if not remote_system.load_error:
return 0
return 1
def do_mapping(self,
scope,
local_ids=None,
remote_system=None,
remote_ids=None):
"""
Perform mapping
:param scope: scope name
:param local_ids: List of Local id
:param remote_system: Remote system id
:param remote_ids: List of Id from remote system
:param kwargs: Ignored args
:return:
"""
def format_obj(o):
r = {"scope": scope, "id": str(o.id), "mappings": []}
if o.remote_system:
r["mappings"] += [{
"remote_system": str(o.remote_system.id),
"remote_id": str(o.remote_id)
}]
return r
# Get model to query
model = get_model(self.SCOPES[scope])
if not model:
return 400, self.error_msg("Invalid scope")
# Query remote objects
result = []
if remote_system and remote_ids:
rs = RemoteSystem.get_by_id(remote_system)
if not rs:
return 404, self.error_msg("Remote system not found")
if len(remote_ids) == 1:
qs = model.objects.filter(remote_system=rs.id,
remote_id=remote_ids[0])
else:
qs = model.objects.filter(remote_system=rs.id,
remote_id__in=remote_ids)
result += [format_obj(o) for o in qs]
# Query local objects
seen = set(o["id"] for o in result)
# Skip already collected objects
local_ids = local_ids or []
local_ids = [o for o in local_ids if o not in seen]
if local_ids:
if len(local_ids) == 1:
qs = model.objects.filter(id=local_ids[0])
else:
qs = model.objects.filter(id__in=local_ids)
result += [format_obj(o) for o in qs]
# 404 if no objects found
if not result:
return 404, self.error_msg("Not found")
return 200, list(sorted(result, key=operator.itemgetter("id")))
def extract(self):
containers = ReportContainerData(
sorted(
set(ManagedObject.objects.all().order_by(
"container").values_list("container", flat=True))))
containers = containers.get_dictionary()
for (
mo_id,
bi_id,
name,
address,
profile,
platform,
version,
remote_id,
remote_system,
adm_id,
adm_name,
container,
) in (ManagedObject.objects.all().order_by("id").values_list(
"id",
"bi_id",
"name",
"address",
"profile",
"platform",
"version",
"remote_id",
"remote_system",
"administrative_domain",
"administrative_domain__name",
"container",
)):
yield (
bi_id,
mo_id,
name,
address,
Profile.get_by_id(profile).name if profile else "",
Platform.get_by_id(platform).name if platform else "",
Firmware.get_by_id(version).version if version else "",
remote_id if remote_id else "",
RemoteSystem.get_by_id(remote_system).name
if remote_system else "",
adm_id,
adm_name,
containers.get(container, ("", ))[0] if container else "",
)
def handle_db_diff(self, *args, **options):
from noc.sa.models.managedobject import ManagedObject
remote_system = RemoteSystem.get_by_name(options["system"])
if not remote_system:
self.die("Invalid remote system: %s" % options["system"])
loader = options["extractor"]
include_fields = None
if options.get("fields"):
include_fields = set(options["fields"].split(","))
chain = LoaderChain(remote_system)
line = chain.get_loader(loader)
ls = line.get_new_state()
if not ls:
ls = line.get_current_state()
iter_db = line.model.objects.filter(remote_system=remote_system)
deleted_filter = {}
if line.model is ManagedObject:
# On delete ManagedObject is set is_managed False
self.print("Apply is_managed filter")
deleted_filter = set(
iter_db.filter(is_managed=False).values_list("remote_id",
flat=True))
iter_db = iter_db.filter(is_managed=True)
for o, n in line.diff(
line.iter_jsonl(ls),
self.iter_db(iter_db.values().order_by("remote_id"),
line.data_model),
include_fields=include_fields,
):
if o is None and n:
print("New:", n.id, n.name)
elif o and n is None:
if deleted_filter and o.id in deleted_filter:
continue
print("Deleted:", o.id, o.name)
else:
print(
"Changed:",
o.id,
o.name,
o.dict(include=include_fields),
n.dict(include=include_fields),
)
def handle_diff(self, summary=False, *args, **options):
remote_system = RemoteSystem.get_by_name(options["system"])
if not remote_system:
self.die("Invalid remote system: %s" % options["system"])
diffs = set(options.get("diffs", []))
if summary:
self.stdout.write(self.SUMMARY_MASK %
("Loader", "New", "Updated", "Deleted"))
control_dict = {}
if options["control_dict"]:
try:
control_dict = ujson.loads(options["control_dict"])
except ValueError as e:
self.die("Failed to parse JSON: %s in %s" %
(e, options["control_dict"]))
except TypeError as e:
self.die("Failed to parse JSON: %s in %s" %
(e, options["control_dict"]))
chain = remote_system.get_loader_chain()
for l in chain:
if diffs and l.name not in diffs:
continue
if summary:
i, u, d = l.check_diff_summary()
control_num = control_dict.get(l.name,
options["control_default"])
self.stdout.write(self.SUMMARY_MASK % (l.name, i, u, d))
if control_num:
if sum([i, u, d]) >= control_num:
self.stdout.write(
self.CONTROL_MESSAGE %
(l.name, sum([i, u, d]), control_num))
self.stderr.write(
self.CONTROL_MESSAGE %
(l.name, sum([i, u, d]), control_num))
n_errors = 1
break
else:
l.check_diff()
else:
n_errors = 0
return 1 if n_errors else 0
def get_object_and_interface(self,
object=None,
interface=None,
service=None):
# type: (Optional[Dict[str, Any]], Optional[Dict[str, Any]], Optional[Dict[str, Any]], Optional[Dict[str, Any]]) -> Tuple[ManagedObject, Optional[Interface]]
"""
Process from and to section of request and get object and interface
:param object: request.object
:param interface: request.interface
:param service: request.service
:return: ManagedObject Instance, Optional[Interface Instance]
:raises ValueError:
"""
if object:
if "id" in object:
# object.id
mo = ManagedObject.get_by_id(object["id"])
elif "remote_system" in object:
# object.remote_system/remote_id
rs = RemoteSystem.get_by_id(object["remote_system"])
if not rs:
raise ValueError("Remote System not found")
mo = ManagedObject.objects.filter(
remote_system=rs.id,
remote_id=object["remote_id"]).first()
else:
raise ValueError("Neither id or remote system specified")
if not mo:
raise ValueError("Object not found")
if interface:
# Additional interface restriction
iface = mo.get_interface(interface["name"])
if iface is None:
raise ValueError("Interface not found")
return mo, iface
else:
# No interface restriction
return mo, None
if interface:
iface = Interface.objects.filter(id=interface["id"]).first()
if not iface:
raise ValueError("Interface not found")
return iface.managed_object, iface
if service:
if "id" in service:
svc = Service.objects.filter("id").first()
elif "remote_system" in service:
rs = RemoteSystem.get_by_id(object["remote_system"])
if not rs:
raise ValueError("Remote System not found")
svc = Service.objects.filter(
remote_system=rs.id,
remote_id=service["remote_id"]).first()
else:
raise ValueError("Neither id or remote system specified")
if svc is None:
raise ValueError("Service not found")
iface = Interface.objects.filter(service=svc.id).first()
if not iface:
raise ValueError("Interface not found")
return iface.managed_object, iface
raise ValueError("Invalid search condition")
def handle_check(self, *args, **options):
remote_system = RemoteSystem.get_by_name(options["system"])
if not remote_system:
self.die("Invalid remote system: %s" % options["system"])
n_errors = remote_system.check(self.stdout)
return 1 if n_errors else 0
def handle_extract(self, *args, **options):
remote_system = RemoteSystem.get_by_name(options["system"])
if not remote_system:
self.die("Invalid remote system: %s" % options["system"])
remote_system.extract(options.get("extractors", []))
def handle_load(self, *args, **options):
remote_system = RemoteSystem.get_by_name(options["system"])
if not remote_system:
self.die("Invalid remote system: %s" % options["system"])
remote_system.load(options.get("loaders", []))