def handle_clean(self, files=None, ttl=None, dry_run=True, *args, **options): remote_system = RemoteSystem.get_by_name(options["system"]) if not remote_system: self.die("Invalid remote system: %s" % options["system"]) if files and files < CLEANUP_SAFE_FILES_COUNT: self.die("3 is minimal value to save file") deadline = None if ttl: today = datetime.datetime.now() deadline = today - datetime.timedelta(days=ttl) self.print("Cleaned files before: %s" % deadline) if not files and not deadline: self.die("Set one of policy setting (ttl or files") clean_paths = [] extractors = set(options.get("extractors", [])) chain = remote_system.get_loader_chain() for ldr in chain: if extractors and ldr.name not in extractors: continue if os.path.isdir(ldr.archive_dir): fn = list( reversed( sorted(f for f in os.listdir(ldr.archive_dir) if ldr.rx_archive.match(f)) ) ) else: self.die("No archived dir") clean_files = [] if files: # Protect last 3 files files = max(files, CLEANUP_SAFE_FILES_COUNT) clean_files = fn[files:] elif deadline: for f in fn: if ( datetime.datetime.strptime(f.split(".", 1)[0], "import-%Y-%m-%d-%H-%M-%S") > deadline ): continue clean_files += [f] clean_files = list(reversed(sorted(clean_files)))[CLEANUP_SAFE_FILES_COUNT:] if not clean_files: self.print("Nothing to remove. Continue...") else: clean_paths += [os.path.join(ldr.archive_dir, f) for f in clean_files] self.print("Cleanup files (%d):\n %s " % (len(clean_files), "\n".join(clean_files))) if not dry_run: self.print("Claimed data will be Loss..\n") for i in reversed(range(1, 10)): self.print("%d\n" % i) time.sleep(1) for path in list(clean_paths): self.print("Clean %s" % path) os.unlink(path) self.print("# Done.")
def handle_load(self, *args, **options): remote_system = RemoteSystem.get_by_name(options["system"]) if not remote_system: self.die("Invalid remote system: %s" % options["system"]) remote_system.load(options.get("loaders", []), quiet=options.get("quiet", False)) if not remote_system.load_error: return 0 return 1
def do_mapping(self, scope, local_ids=None, remote_system=None, remote_ids=None): """ Perform mapping :param scope: scope name :param local_ids: List of Local id :param remote_system: Remote system id :param remote_ids: List of Id from remote system :param kwargs: Ignored args :return: """ def format_obj(o): r = {"scope": scope, "id": str(o.id), "mappings": []} if o.remote_system: r["mappings"] += [{ "remote_system": str(o.remote_system.id), "remote_id": str(o.remote_id) }] return r # Get model to query model = get_model(self.SCOPES[scope]) if not model: return 400, self.error_msg("Invalid scope") # Query remote objects result = [] if remote_system and remote_ids: rs = RemoteSystem.get_by_id(remote_system) if not rs: return 404, self.error_msg("Remote system not found") if len(remote_ids) == 1: qs = model.objects.filter(remote_system=rs.id, remote_id=remote_ids[0]) else: qs = model.objects.filter(remote_system=rs.id, remote_id__in=remote_ids) result += [format_obj(o) for o in qs] # Query local objects seen = set(o["id"] for o in result) # Skip already collected objects local_ids = local_ids or [] local_ids = [o for o in local_ids if o not in seen] if local_ids: if len(local_ids) == 1: qs = model.objects.filter(id=local_ids[0]) else: qs = model.objects.filter(id__in=local_ids) result += [format_obj(o) for o in qs] # 404 if no objects found if not result: return 404, self.error_msg("Not found") return 200, list(sorted(result, key=operator.itemgetter("id")))
def extract(self): containers = ReportContainerData( sorted( set(ManagedObject.objects.all().order_by( "container").values_list("container", flat=True)))) containers = containers.get_dictionary() for ( mo_id, bi_id, name, address, profile, platform, version, remote_id, remote_system, adm_id, adm_name, container, ) in (ManagedObject.objects.all().order_by("id").values_list( "id", "bi_id", "name", "address", "profile", "platform", "version", "remote_id", "remote_system", "administrative_domain", "administrative_domain__name", "container", )): yield ( bi_id, mo_id, name, address, Profile.get_by_id(profile).name if profile else "", Platform.get_by_id(platform).name if platform else "", Firmware.get_by_id(version).version if version else "", remote_id if remote_id else "", RemoteSystem.get_by_id(remote_system).name if remote_system else "", adm_id, adm_name, containers.get(container, ("", ))[0] if container else "", )
def handle_db_diff(self, *args, **options): from noc.sa.models.managedobject import ManagedObject remote_system = RemoteSystem.get_by_name(options["system"]) if not remote_system: self.die("Invalid remote system: %s" % options["system"]) loader = options["extractor"] include_fields = None if options.get("fields"): include_fields = set(options["fields"].split(",")) chain = LoaderChain(remote_system) line = chain.get_loader(loader) ls = line.get_new_state() if not ls: ls = line.get_current_state() iter_db = line.model.objects.filter(remote_system=remote_system) deleted_filter = {} if line.model is ManagedObject: # On delete ManagedObject is set is_managed False self.print("Apply is_managed filter") deleted_filter = set( iter_db.filter(is_managed=False).values_list("remote_id", flat=True)) iter_db = iter_db.filter(is_managed=True) for o, n in line.diff( line.iter_jsonl(ls), self.iter_db(iter_db.values().order_by("remote_id"), line.data_model), include_fields=include_fields, ): if o is None and n: print("New:", n.id, n.name) elif o and n is None: if deleted_filter and o.id in deleted_filter: continue print("Deleted:", o.id, o.name) else: print( "Changed:", o.id, o.name, o.dict(include=include_fields), n.dict(include=include_fields), )
def handle_diff(self, summary=False, *args, **options): remote_system = RemoteSystem.get_by_name(options["system"]) if not remote_system: self.die("Invalid remote system: %s" % options["system"]) diffs = set(options.get("diffs", [])) if summary: self.stdout.write(self.SUMMARY_MASK % ("Loader", "New", "Updated", "Deleted")) control_dict = {} if options["control_dict"]: try: control_dict = ujson.loads(options["control_dict"]) except ValueError as e: self.die("Failed to parse JSON: %s in %s" % (e, options["control_dict"])) except TypeError as e: self.die("Failed to parse JSON: %s in %s" % (e, options["control_dict"])) chain = remote_system.get_loader_chain() for l in chain: if diffs and l.name not in diffs: continue if summary: i, u, d = l.check_diff_summary() control_num = control_dict.get(l.name, options["control_default"]) self.stdout.write(self.SUMMARY_MASK % (l.name, i, u, d)) if control_num: if sum([i, u, d]) >= control_num: self.stdout.write( self.CONTROL_MESSAGE % (l.name, sum([i, u, d]), control_num)) self.stderr.write( self.CONTROL_MESSAGE % (l.name, sum([i, u, d]), control_num)) n_errors = 1 break else: l.check_diff() else: n_errors = 0 return 1 if n_errors else 0
def get_object_and_interface(self, object=None, interface=None, service=None): # type: (Optional[Dict[str, Any]], Optional[Dict[str, Any]], Optional[Dict[str, Any]], Optional[Dict[str, Any]]) -> Tuple[ManagedObject, Optional[Interface]] """ Process from and to section of request and get object and interface :param object: request.object :param interface: request.interface :param service: request.service :return: ManagedObject Instance, Optional[Interface Instance] :raises ValueError: """ if object: if "id" in object: # object.id mo = ManagedObject.get_by_id(object["id"]) elif "remote_system" in object: # object.remote_system/remote_id rs = RemoteSystem.get_by_id(object["remote_system"]) if not rs: raise ValueError("Remote System not found") mo = ManagedObject.objects.filter( remote_system=rs.id, remote_id=object["remote_id"]).first() else: raise ValueError("Neither id or remote system specified") if not mo: raise ValueError("Object not found") if interface: # Additional interface restriction iface = mo.get_interface(interface["name"]) if iface is None: raise ValueError("Interface not found") return mo, iface else: # No interface restriction return mo, None if interface: iface = Interface.objects.filter(id=interface["id"]).first() if not iface: raise ValueError("Interface not found") return iface.managed_object, iface if service: if "id" in service: svc = Service.objects.filter("id").first() elif "remote_system" in service: rs = RemoteSystem.get_by_id(object["remote_system"]) if not rs: raise ValueError("Remote System not found") svc = Service.objects.filter( remote_system=rs.id, remote_id=service["remote_id"]).first() else: raise ValueError("Neither id or remote system specified") if svc is None: raise ValueError("Service not found") iface = Interface.objects.filter(service=svc.id).first() if not iface: raise ValueError("Interface not found") return iface.managed_object, iface raise ValueError("Invalid search condition")
def handle_check(self, *args, **options): remote_system = RemoteSystem.get_by_name(options["system"]) if not remote_system: self.die("Invalid remote system: %s" % options["system"]) n_errors = remote_system.check(self.stdout) return 1 if n_errors else 0
def handle_extract(self, *args, **options): remote_system = RemoteSystem.get_by_name(options["system"]) if not remote_system: self.die("Invalid remote system: %s" % options["system"]) remote_system.extract(options.get("extractors", []))
def handle_load(self, *args, **options): remote_system = RemoteSystem.get_by_name(options["system"]) if not remote_system: self.die("Invalid remote system: %s" % options["system"]) remote_system.load(options.get("loaders", []))