def __init__(self, log=None): self.__ssl_context = None dbs = ISEServer.objects.filter(pxgrid_enable=True) c = '{"hostname": [], "nodename": "", "clientcert": "", "clientkey": "", "clientkeypassword": "",' \ '"servercert": "", "password": "", "description": "", "id": null, "verbose": true, "services": null,' \ '"port": 8910, "service_details": null, "service": "com.cisco.ise.config.trustsec",' \ '"topic": "securityGroupTopic,securityGroupAclTopic", "session_dedup": false,' \ '"subscribe_all": false}' self.config = json.loads(c, object_hook=lambda d: Namespace(**d)) if len(dbs) > 0: append_log( log, "pxgrid_monitor::pxgrid_config::Establishing websocket for pxGrid..." ) db = dbs[0] self.config.hostname = [db.pxgrid_ip] self.config.nodename = db.pxgrid_cliname self.config.clientcert = str(db.pxgrid_clicert.file) self.config.clientkey = str(db.pxgrid_clikey.file) self.config.clientkeypassword = db.pxgrid_clipw self.config.servercert = str(db.pxgrid_isecert.file) self.config.id = db.id else: self.config.id = None append_log( log, "pxgrid_monitor::pxgrid_config::No pxGrid servers configured..." )
def read_json_file(in_filename, log): fail_count = 0 while True: try: r = read_file_all(in_filename) out = json.loads(r) return out except Exception as e: fail_count += 1 time.sleep(1) append_log(log, "dashboard_monitor::read_json_file::", fail_count, e)
def process_webhook(request): log = [] if request.method == 'POST': whdata = json.loads(request.body) append_log(log, "webhook post", whdata) dbs = Dashboard.objects.filter(webhook_enable=True) if len(dbs) > 0: db = dbs[0] db.force_rebuild = True db.save() append_log(log, "setting dashboard to force rebuild") db_log("dashboard_webhook", log) return HttpResponse("Send webhooks here as POST requests.")
def job(): log = [] try: ret = run() if ret is not False: scheduler.remove_job("dashboard_webhook") append_log(log, "Webhook Monitor started") else: append_log( log, "Dashboard webhook configuration not present. Will check again..." ) db_log("dashboard_webhook", log) except Exception as e: append_log( log, "#### Dashboard webhooks are not configured: dashboard_webhook", e) db_log("dashboard_webhook", log)
def parse_url(request): log = [] baseurl = "/".join(request.build_absolute_uri().split("/")[:3]) p = request.path.replace("/meraki/api/v1/organizations/", "").replace("/meraki/api/v1/organizations", "") arr = p.split("/") isotime = datetime.datetime.now().isoformat() org_id = arr[0] fixedvals = { "organizations": { "id": "{{id-num:18}}", "url": "{{url}}/o/{{id-mix:7}}/manage/organization/overview" }, "groups": { "groupId": "{{length}}", "versionNum": 1, "createdAt": isotime, "updatedAt": isotime }, "acls": { "aclId": "{{length}}", "versionNum": 1, "createdAt": isotime, "updatedAt": isotime }, "bindings": { "versionNum": 1, "updatedAt": isotime } } postvals = { "organizations": { "name": None }, "groups": { "name": None, "description": None, "value": None, "networkObjectId": None }, "acls": { "name": None, "description": None, "ipVersion": None, "rules": None }, "bindings": { "srcGroupId": None, "dstGroupId": None, "name": None, "description": None, "aclIds": None, "catchAllRule": None, "bindingEnabled": None, "monitorModeEnabled": None } } info = { "organizations": { "id": "id", "unique": [{ "id": [] }] }, "groups": { "id": "groupId", "unique": [{ "value": [], "groupId": [] }] }, "acls": { "id": "aclId", "unique": [{ "name": [], "aclId": [] }] }, "bindings": { "none_as_delete_key": "aclIds", "put_unique": ["srcGroupId", "dstGroupId"], "unique_results": [] } } append_log(log, "dashboard_simulator::", request.path) ret = None try: if len(arr) == 1: file_type = "orgs.json" full_dataset = [] dataset = read_json_file(file_type, log) if arr[0] == "": elem_id = None else: elem_id = arr[0] endpoint = "organizations" else: file_type = arr[2] + ".json" full_dataset = read_json_file(file_type, log) dataset = full_dataset.pop(org_id, []) if len(arr) == 3 or request.method == "POST": elem_id = None else: elem_id = arr[3] endpoint = arr[2] if endpoint == "bindings" and (request.method == "POST" or request.method == "DELETE"): append_log(log, "dashboard_monitor::bindings::Unsupported Method") db_log("dashboard_simulator", log) return HttpResponseBadRequest("Unsupported Method") if request.body: jd = json.loads(request.body) else: jd = None updated_data, ret = handle_request(request.method, jd, baseurl, endpoint, elem_id, dataset, fixedvals, postvals, info) if updated_data: if isinstance(full_dataset, list): write_file(file_type, json.dumps(full_dataset + [updated_data], indent=4)) else: full_dataset[org_id] = updated_data write_file(file_type, json.dumps(full_dataset, indent=4)) except Exception as e: append_log(log, "dashboard_simulator::Exception.", e) db_log("dashboard_simulator", log) return ret
def digest_database_data(sa, log): append_log(log, "dashboard_monitor::digest_database_data::Account -", sa) dashboard = meraki.DashboardAPI(base_url=sa.dashboard.baseurl, api_key=sa.dashboard.apikey, print_console=False, output_log=False, caller=settings.CUSTOM_UA, suppress_logging=True) if not sa.apply_changes: append_log( log, "dashboard_monitor::digest_database_data::sync session not set to apply changes;" ) return tags = TagData.objects.filter(Q(tag__do_sync=True) & Q(update_failed=False)).\ exclude(organization=None) for o in tags: if o.source_id and o.update_dest() == "meraki": if o.tag.push_delete: try: ret = meraki_delete_sgt(dashboard, o.organization.orgid, o.source_id) append_log( log, "dashboard_monitor::digest_database_data::SGT delete", ret) o.delete() except Exception as e: # pragma: no cover append_log( log, "dashboard_monitor::digest_database_data::SGT Delete Exception", e, traceback.format_exc()) o.update_failed = True o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = { "tag": str(o), "error": "Exception: " + str(e) } o.save() else: try: ret = meraki_update_sgt(dashboard, o.organization.orgid, o.source_id, name=o.tag.name, description=o.tag.description, value=o.tag.tag_number) o.last_update_data = json.dumps(ret) if "groupId" in ret: o.last_update_state = "True" o.source_id = ret["groupId"] o.source_data = json.dumps(ret) else: o.last_update_state = "False" o.last_update = make_aware(datetime.datetime.now()) o.save() append_log( log, "dashboard_monitor::digest_database_data::Push SGT update", o.source_id, o.tag.name, o.tag.description, ret) except Exception as e: # pragma: no cover append_log( log, "dashboard_monitor::digest_database_data::SGT Update Exception", e, traceback.format_exc()) o.update_failed = True o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = { "tag": str(o), "error": "Exception: " + str(e) } o.save() elif o.update_dest() == "meraki": try: ret = meraki_create_sgt(dashboard, o.organization.orgid, value=o.tag.tag_number, name=o.tag.name, description=o.tag.description) o.last_update_data = json.dumps(ret) if "groupId" in ret: o.last_update_state = "True" o.source_id = ret["groupId"] o.source_data = json.dumps(ret) else: o.last_update_state = "False" o.last_update = make_aware(datetime.datetime.now()) o.save() append_log( log, "dashboard_monitor::digest_database_data::Push SGT create", o.tag.tag_number, o.tag.name, o.tag.description, ret) except Exception as e: # pragma: no cover append_log( log, "dashboard_monitor::digest_database_data::SGT Create Exception", e, traceback.format_exc()) o.update_failed = True o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = { "tag": str(o), "error": "Exception: " + str(e) } o.save() acls = ACLData.objects.filter(Q(acl__do_sync=True) & Q(update_failed=False)).\ exclude(organization=None) for o in acls: if o.source_id and o.update_dest() == "meraki": if o.acl.push_delete: try: ret = meraki_delete_sgacl(dashboard, o.organization.orgid, o.source_id) append_log( log, "dashboard_monitor::digest_database_data::SGACL delete", ret) o.delete() except Exception as e: # pragma: no cover append_log( log, "dashboard_monitor::digest_database_data::SGACL Delete Exception", e, traceback.format_exc()) o.update_failed = True o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = { "acl": str(o), "error": "Exception: " + str(e) } o.save() else: try: ret = meraki_update_sgacl(dashboard, o.organization.orgid, o.source_id, name=o.acl.name, description=o.acl.description, rules=o.lookup_rules(o), ipVersion=o.lookup_version(o)) o.last_update_data = json.dumps(ret) if "aclId" in ret: o.last_update_state = "True" o.source_id = ret["aclId"] o.source_data = json.dumps(ret) else: o.last_update_state = "False" o.last_update = make_aware(datetime.datetime.now()) o.save() append_log( log, "dashboard_monitor::digest_database_data::Push SGACL update", o.source_id, o.acl.name, o.acl.description, ret) except Exception as e: # pragma: no cover append_log( log, "dashboard_monitor::digest_database_data::SGACL Update Exception", e, traceback.format_exc()) o.update_failed = True o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = { "acl": str(o), "error": "Exception: " + str(e) } o.save() elif o.update_dest() == "meraki": try: ret = meraki_create_sgacl(dashboard, o.organization.orgid, name=o.acl.name, description=o.acl.description, rules=list(o.lookup_rules(o)), ipVersion=o.lookup_version(o)) o.last_update_data = json.dumps(ret) if "aclId" in ret: o.last_update_state = "True" o.source_id = ret["aclId"] o.source_data = json.dumps(ret) else: o.last_update_state = "False" o.last_update = make_aware(datetime.datetime.now()) o.save() append_log( log, "dashboard_monitor::digest_database_data::Push SGACL create", o.acl.name, o.acl.description, ret) except Exception as e: # pragma: no cover append_log( log, "dashboard_monitor::digest_database_data::SGACL Create Exception", e, traceback.format_exc()) o.update_failed = True o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = { "acl": str(o), "error": "Exception: " + str(e) } o.save() policies = PolicyData.objects.filter(Q(policy__do_sync=True) & Q(update_failed=False)).\ exclude(organization=None) for o in policies: if o.policy.push_delete and o.update_dest() == "meraki": try: srcsgt, dstsgt = o.policy.lookup_sgts(o) orgs = sa.dashboard.organization.all() for org in orgs: ret = meraki_update_sgpolicy(dashboard, org.orgid, srcGroupId=srcsgt.source_id, dstGroupId=dstsgt.source_id, aclIds=None, catchAllRule="global") append_log( log, "dashboard_monitor::digest_database_data::Policy delete", ret) o.delete() except Exception as e: # pragma: no cover append_log( log, "dashboard_monitor::digest_database_data::Policy Delete Exception", e, traceback.format_exc()) o.update_failed = True o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = { "policy": str(o), "error": "Exception: " + str(e) } o.save() elif o.update_dest() == "meraki": try: srcsgt, dstsgt = o.lookup_sgt_data(o) sgacl = o.lookup_sgacl_data(o) acls = [] if sgacl: for s in sgacl: acls.append(s.source_id) if not srcsgt or not dstsgt or sgacl is None: o.update_failed = False # was True; disabled for now o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = { "policy": str(o), "error": "Meraki: Unable to locate sgt/acl data;" + str(srcsgt) + ";" + str(dstsgt) + ";" + str(sgacl) } o.save() continue ret = meraki_update_sgpolicy( dashboard, o.organization.orgid, name=o.policy.name, description=o.policy.description, srcGroupId=srcsgt.source_id, dstGroupId=dstsgt.source_id, aclIds=acls, catchAllRule=o.lookup_acl_catchall(o), bindingEnabled=True, monitorModeEnabled=False) o.last_update_data = json.dumps(ret) if "srcGroupId" in ret: o.last_update_state = "True" o.source_id = "s" + str(ret["srcGroupId"]) + "-d" + str( ret["dstGroupId"]) o.source_data = json.dumps(ret) else: o.last_update_state = "False" o.last_update = make_aware(datetime.datetime.now()) o.save() append_log( log, "dashboard_monitor::digest_database_data::Push Policy update", o.source_id, o.policy.name, o.policy.description, ret) except Exception as e: # pragma: no cover append_log( log, "dashboard_monitor::digest_database_data::Policy Update Exception", e, traceback.format_exc()) o.update_failed = True o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = { "policy": str(o), "error": "Exception: " + str(e) } o.save()
def sync_dashboard(): log = [] msg = "SYNC_DASHBOARD-NO_ACTION_REQUIRED" append_log( log, "dashboard_monitor::sync_dashboard::Checking Dashboard Accounts for re-sync..." ) # If we know that something needs to be created, do that first. sss = SyncSession.objects.all() for ss in sss: digest_database_data(ss, log) ss.dashboard.force_rebuild = True ss.save() msg = "SYNC_DASHBOARD-CHANGES_MADE_FORCE_UPDATE" # Ensure that ISE has already completed a sync if it is the source of truth stat = SyncSession.objects.filter( Q(ise_source=False) | (Q(iseserver__last_sync__isnull=False) & Q(dashboard__last_sync__isnull=True)) | (Q(iseserver__last_sync__isnull=False) & Q(iseserver__last_sync__gte=F('dashboard__last_sync')))) if len(stat) <= 0: append_log( log, "dashboard_monitor::sync_dashboard::Skipping sync as ISE is primary and needs to sync first." ) msg = "SYNC_DASHBOARD-ISE_NEEDS_SYNC" else: append_log(log, "dashboard_monitor::sync_dashboard::Running sync") for s in stat: ctime = make_aware(datetime.datetime.now()) - datetime.timedelta( seconds=s.sync_interval) # Perform sync if one of the following conditions is met # 1) The Sync Session is set to Force Rebuild (this one shouldn't be seen here. but just in case...) # 2) The Dashboard Instance is set to Force Rebuild # 3) The timestamp of the Dashboard database object isn't the same as the timestamp of it's last sync # 4) The timestamp of the Dashboard database object's last sync is beyond the configured manual sync timer dbs = SyncSession.objects.filter( Q(dashboard__force_rebuild=True) | Q(force_rebuild=True) | ~Q(dashboard__last_sync=F('dashboard__last_update')) | Q(dashboard__last_sync__lte=ctime)) for d in dbs: # Log the reason(s) for the current sync if d.force_rebuild: # pragma: no cover append_log( log, "dashboard_monitor::sync_dashboard::Sync Session Force Rebuild", d) msg = "SYNC_DASHBOARD-SYNCSESSION_FORCE_REBUILD" d.force_rebuild = False d.save() if d.dashboard.force_rebuild: append_log( log, "dashboard_monitor::sync_dashboard::Dashboard Force Rebuild", d) msg = "SYNC_DASHBOARD-DASHBOARD_FORCE_REBUILD" d.dashboard.force_rebuild = False d.dashboard.save() if d.dashboard.last_sync != d.dashboard.last_update: append_log( log, "dashboard_monitor::sync_dashboard::Database Config / Sync Timestamp Mismatch", d) msg = "SYNC_DASHBOARD-CONFIG_SYNC_TIMESTAMP_MISMATCH" if d.dashboard.last_sync and (d.dashboard.last_sync <= ctime): append_log( log, "dashboard_monitor::sync_dashboard::Past Manual Sync Interval", d) msg = "SYNC_DASHBOARD-PAST_SYNC_INTERVAL" ingest_dashboard_data(dbs, log) # After ingesting data, more updates may be required (Should these have been caught elsewhere?) sss = SyncSession.objects.all() for ss in sss: digest_database_data(ss, log) append_log(log, "dashboard_monitor::sync_dashboard::Done") db_log("dashboard_monitor", log) return msg, log
def ingest_dashboard_data(accounts, log): append_log(log, "dashboard_monitor::ingest_dashboard_data::Accounts -", accounts) dt = make_aware(datetime.datetime.now()) for sa in accounts: if not sa.sync_enabled: append_log( log, "dashboard_monitor::digest_database_data::sync session not set to allow sync;" ) return a = sa.dashboard append_log(log, "dashboard_monitor::ingest_dashboard_data::Resync -", a.description) dashboard = meraki.DashboardAPI(base_url=a.baseurl, api_key=a.apikey, print_console=False, output_log=False, caller=settings.CUSTOM_UA, suppress_logging=True) orgs = a.organization.all() if orgs: for org in orgs: org_id = org.orgid append_log(log, "dashboard_monitor::processing orgid::", org_id) sgts = meraki_read_sgt(dashboard, org_id) sgacls = meraki_read_sgacl(dashboard, org_id) sgpolicies = meraki_read_sgpolicy(dashboard, org_id) append_log( log, "dashboard_monitor::ingest_dashboard_data::SGTs - ", len(sgts)) append_log( log, "dashboard_monitor::ingest_dashboard_data::SGACLs - ", len(sgacls)) append_log( log, "dashboard_monitor::ingest_dashboard_data::Policies - ", len(sgpolicies)) merge_sgts("meraki", sgts, not sa.ise_source, sa, log, org) merge_sgacls("meraki", sgacls, not sa.ise_source, sa, log, org) merge_sgpolicies("meraki", sgpolicies, not sa.ise_source, sa, log, org) clean_sgts("meraki", sgts, not sa.ise_source, sa, log, org) clean_sgacls("meraki", sgacls, not sa.ise_source, sa, log, org) clean_sgpolicies("meraki", sgpolicies, not sa.ise_source, sa, log, org) org.raw_data = json.dumps({ "groups": sgts, "acls": sgacls, "bindings": sgpolicies }) org.force_rebuild = False org.last_sync = dt org.last_update = dt org.skip_sync = True org.save() sa.dashboard.last_sync = dt sa.dashboard.save() else: append_log( log, "dashboard_monitor::ingest_dashboard_data::No OrgId present")
def digest_database_data(sa, log): append_log(log, "ise_monitor::digest_database_data::Account -", sa) ise = ERS(ise_node=sa.iseserver.ipaddress, ers_user=sa.iseserver.username, ers_pass=sa.iseserver.password, verify=False, disable_warnings=True) if not sa.apply_changes: append_log(log, "ise_monitor::digest_database_data::sync session not set to apply changes") return policies = PolicyData.objects.filter(Q(policy__do_sync=True) & Q(update_failed=False)).exclude(iseserver=None) for o in policies: if o.source_id and o.update_dest() == "ise": if o.policy.push_delete: try: ret = ise.delete_egressmatrixcell(o.source_id) append_log(log, "ise_monitor::digest_database_data::Policy delete", ret) o.delete() except Exception as e: # pragma: no cover append_log(log, "ise_monitor::digest_database_data::Policy Delete Exception", e, traceback.format_exc()) o.update_failed = True o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = {"tag": str(o), "error": "Exception: " + str(e)} o.save() append_log(log, "ise_monitor::digest_database_data::Tag check") from django.forms.models import model_to_dict tags = TagData.objects.filter(Q(tag__do_sync=True) & Q(update_failed=False)).exclude(iseserver=None) for o in tags: append_log(log, "ise_monitor::digest_database_data::Tag", o.tag.name, o.tag.do_sync, o.tag.in_sync(), o.update_dest(), model_to_dict(o)) if o.source_id and o.update_dest() == "ise": if o.tag.push_delete: try: ret = ise.delete_sgt(o.source_id) append_log(log, "ise_monitor::digest_database_data::SGT delete", ret) o.delete() except Exception as e: # pragma: no cover append_log(log, "ise_monitor::digest_database_data::SGT Delete Exception", e, traceback.format_exc()) o.update_failed = True o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = {"tag": str(o), "error": "Exception: " + str(e)} o.save() else: try: ret = ise.update_sgt(o.source_id, o.tag.cleaned_name(), o.tag.description, o.tag.tag_number, return_object=True) o.last_update = make_aware(datetime.datetime.now()) o.last_update_data = ret o.last_update_state = str(ret.get("success", False)) if ret["response"] and isinstance(ret["response"], dict): o.source_id = ret["response"]["id"] o.source_data = json.dumps(ret["response"]) append_log(log, "ise_monitor::digest_database_data::Push SGT update", o.source_id, o.tag.cleaned_name(), o.tag.description, o.tag.tag_number, ret) else: append_log(log, "ise_monitor::digest_database_data::SGT Invalid Return", ret) o.update_failed = True o.last_update_state = "False" o.last_update_data = {"tag": str(o), "error": "SGT Invalid return"} o.save() except Exception as e: # pragma: no cover append_log(log, "ise_monitor::digest_database_data::SGT Update Exception", e, traceback.format_exc()) o.update_failed = True o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = {"tag": str(o), "error": "Exception: " + str(e)} o.save() elif o.update_dest() == "ise": try: ret = ise.add_sgt(o.tag.cleaned_name(), o.tag.description, o.tag.tag_number, return_object=True) o.last_update = make_aware(datetime.datetime.now()) o.last_update_data = ret o.last_update_state = str(ret.get("success", False)) if ret["response"] and isinstance(ret["response"], dict): o.source_id = ret["response"]["id"] o.source_data = json.dumps(ret["response"]) append_log(log, "ise_monitor::digest_database_data::Push SGT create", o.tag.cleaned_name(), o.tag.description, o.tag.tag_number, ret) else: # pragma: no cover append_log(log, "ise_monitor::digest_database_data::SGT Invalid Return", ret) o.update_failed = True o.last_update_state = "False" o.last_update_data = {"tag": str(o), "error": "SGT Invalid Return"} o.save() except Exception as e: # pragma: no cover append_log(log, "ise_monitor::digest_database_data::SGT Create Exception", e, traceback.format_exc()) o.update_failed = True o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = {"tag": str(o), "error": "Exception: " + str(e)} o.save() acls = ACLData.objects.filter(Q(acl__do_sync=True) & Q(update_failed=False)).exclude(iseserver=None) for o in acls: if o.source_id and o.update_dest() == "ise": if o.acl.push_delete: try: ret = ise.delete_sgacl(o.source_id) append_log(log, "ise_monitor::digest_database_data::SGACL delete", ret) o.delete() except Exception as e: # pragma: no cover append_log(log, "ise_monitor::digest_database_data::SGACL Delete Exception", e, traceback.format_exc()) o.update_failed = True o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = {"acl": str(o), "error": "Exception: " + str(e)} o.save() else: try: ret = ise.update_sgacl(o.source_id, o.acl.cleaned_name(), o.acl.description, o.lookup_version(o), o.lookup_rules(o).split("\n"), return_object=True) o.last_update = make_aware(datetime.datetime.now()) o.last_update_data = ret o.last_update_state = str(ret.get("success", False)) if ret["response"] and isinstance(ret["response"], dict): o.source_id = ret["response"]["id"] o.source_data = json.dumps(ret["response"]) append_log(log, "ise_monitor::digest_database_data::Push SGACL update", o.source_id, o.acl.cleaned_name(), o.acl.description, ret) else: # pragma: no cover append_log(log, "ise_monitor::digest_database_data::SGACL Invalid Return", ret) o.update_failed = True o.last_update_state = "False" o.last_update_data = {"acl": str(o), "error": "SGACL Invalid Return"} o.save() except Exception as e: # pragma: no cover append_log(log, "ise_monitor::digest_database_data::SGACL Update Exception", e, traceback.format_exc()) o.update_failed = True o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = {"acl": str(o), "error": "Exception: " + str(e)} o.save() elif o.update_dest() == "ise": try: ret = ise.add_sgacl(o.acl.cleaned_name(), o.acl.description, o.lookup_version(o), o.lookup_rules(o).split("\n"), return_object=True) o.last_update = make_aware(datetime.datetime.now()) o.last_update_data = ret o.last_update_state = str(ret.get("success", False)) if ret["response"] and isinstance(ret["response"], dict): o.source_id = ret["response"]["id"] o.source_data = json.dumps(ret["response"]) append_log(log, "ise_monitor::digest_database_data::Push SGACL create", o.acl.cleaned_name(), o.acl.description, ret) else: # pragma: no cover append_log(log, "ise_monitor::digest_database_data::SGACL Null Return", ret) o.update_failed = True o.last_update_state = "False" o.last_update_data = {"acl": str(o), "error": "SGACL Null Return"} o.save() except Exception as e: # pragma: no cover append_log(log, "ise_monitor::digest_database_data::SGACL Create Exception", e, traceback.format_exc()) o.update_failed = True o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = {"acl": str(o), "error": "Exception: " + str(e)} o.save() policies = PolicyData.objects.filter(Q(policy__do_sync=True) & Q(update_failed=False)).exclude(iseserver=None) for o in policies: if o.source_id and not o.policy.push_delete and o.update_dest() == "ise": try: srcsgt, dstsgt = o.lookup_sgt_data(o) sgacl = o.lookup_sgacl_data(o) acls = [] if sgacl: for s in sgacl: acls.append(s.source_id) if not srcsgt or not dstsgt or sgacl is None: o.update_failed = False # was True; disabled for now o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = {"policy": str(o), "error": "ISE Update: Unable to locate sgt/sgacl data;" + str(srcsgt) + ";" + str(dstsgt) + ";" + str(sgacl)} o.save() continue ret = ise.update_egressmatrixcell(o.source_id, srcsgt.source_id, dstsgt.source_id, o.lookup_acl_catchall(o), acls=acls, description=o.lookup_description(o), return_object=True) o.last_update = make_aware(datetime.datetime.now()) o.last_update_data = ret o.last_update_state = str(ret.get("success", False)) if ret["response"] and isinstance(ret["response"], dict): o.source_id = ret["response"]["id"] o.source_data = json.dumps(ret["response"]) append_log(log, "ise_monitor::digest_database_data::Push Policy update", o.source_id, o.policy.name, o.lookup_description(o), ret) else: # pragma: no cover append_log(log, "ise_monitor::digest_database_data::Policy Null Return", ret) o.update_failed = True o.last_update_state = "False" o.last_update_data = {"policy": str(o), "error": "Policy Null Return"} o.save() except Exception as e: # pragma: no cover append_log(log, "ise_monitor::digest_database_data::Policy Update Exception", e, traceback.format_exc()) o.update_failed = True o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = {"policy": str(o), "error": "Exception: " + str(e)} o.save() elif o.update_dest() == "ise": try: srcsgt, dstsgt = o.lookup_sgt_data(o) sgacl = o.lookup_sgacl_data(o) acls = [] if sgacl: for s in sgacl: acls.append(s.source_id) if not srcsgt or not dstsgt or sgacl is None: o.update_failed = True o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = {"policy": str(o), "error": "ISE Create: Unable to locate sgt/sgacl data;" + str(srcsgt) + ";" + str(dstsgt) + ";" + str(sgacl)} o.save() continue ret = ise.add_egressmatrixcell(srcsgt.source_id, dstsgt.source_id, o.lookup_acl_catchall(o), acls=acls, description=o.lookup_description(o), return_object=True) o.last_update = make_aware(datetime.datetime.now()) o.last_update_data = ret o.last_update_state = str(ret.get("success", False)) if ret["response"] and isinstance(ret["response"], dict): o.source_id = ret["response"]["id"] o.source_data = json.dumps(ret["response"]) append_log(log, "ise_monitor::digest_database_data::Push Policy create", o.policy.name, o.lookup_description(o), ret) else: # pragma: no cover append_log(log, "ise_monitor::digest_database_data::Policy Null Return", ret) o.update_failed = True o.last_update_state = "False" o.last_update_data = {"policy": str(o), "error": "Policy Null Return"} o.save() except Exception as e: # pragma: no cover append_log(log, "ise_monitor::digest_database_data::Policy Create Exception", e, traceback.format_exc()) o.update_failed = True o.last_update = make_aware(datetime.datetime.now()) o.last_update_state = "False" o.last_update_data = {"policy": str(o), "error": "Exception: " + str(e)} o.save()
def ingest_ise_data(accounts, log, server_only=False): append_log(log, "ise_monitor::ingest_server_data::Accounts -", accounts) dt = make_aware(datetime.datetime.now()) for sync_account in accounts: if not server_only: if not sync_account.sync_enabled: append_log(log, "ise_monitor::ingest_server_data::sync session not set to allow sync;") return ise = None a = sync_account.iseserver src = sync_account.ise_source sa = sync_account else: a = sync_account src = False sa = None append_log(log, "ise_monitor::ingest_server_data::Resync -", a.description) ise = ERS(ise_node=a.ipaddress, ers_user=a.username, ers_pass=a.password, verify=False, disable_warnings=True) sgts = ise.get_sgts(detail=True) sgacls = ise.get_sgacls(detail=True) sgpolicies = ise.get_egressmatrixcells(detail=True) append_log(log, "ise_monitor::ingest_server_data::SGTs - ", len(sgts)) append_log(log, "ise_monitor::ingest_server_data::SGACLs - ", len(sgacls)) append_log(log, "ise_monitor::ingest_server_data::Policies - ", len(sgpolicies)) ise = {"sgts": sgts, "sgacls": sgacls, "sgpolicies": sgpolicies} merge_sgts("ise", sgts["response"], src, sa, log, a) merge_sgacls("ise", sgacls["response"], src, sa, log, a) merge_sgpolicies("ise", sgpolicies["response"], src, sa, log, a) clean_sgts("ise", sgts["response"], src, sa, log, a) clean_sgacls("ise", sgacls["response"], src, sa, log, a) clean_sgpolicies("ise", sgpolicies["response"], src, sa, log, a) a.raw_data = json.dumps(ise) a.force_rebuild = False a.last_sync = dt a.last_update = dt a.skip_sync = True a.skip_update = True a.save()
def clean_sgts(src, sgts, is_base, sync_session, log=None, obj=None): append_log(log, "db_trustsec::clean_sgts::", len(sgts)) changed_objs = [] try: active_id_list = [] if src == "ise": for s in sgts: active_id_list.append(s["id"]) tags = TagData.objects.filter(iseserver=obj) else: for s in sgts: active_id_list.append(s["groupId"]) tags = TagData.objects.filter(organization=obj) for i in tags: if src == "ise" and i.source_id and i.source_id not in active_id_list: if is_base: append_log(log, "db_trustsec::clean_sgts::setting ise", i.source_id, "for delete") i.tag.push_delete = True i.tag.save() i.delete() else: append_log(log, "db_trustsec::clean_sgts::removing ise", i.source_id, "from db") i.source_id = None i.source_data = None i.last_update = make_aware(datetime.datetime.now()) i.save() changed_objs.append(i) if src == "meraki" and i.source_id and i.source_id not in active_id_list: if is_base: append_log(log, "db_trustsec::clean_sgts::setting meraki", i.source_id, "for delete") i.tag.push_delete = True i.tag.save() i.delete() else: append_log(log, "db_trustsec::clean_sgts::removing meraki", i.source_id, "from db") i.source_id = None i.source_data = None i.last_update = make_aware(datetime.datetime.now()) i.save() changed_objs.append(i) dbobjs = Tag.objects.all() for dbo in dbobjs: if len(dbo.tagdata_set.all()) == 0: dbo.delete() return changed_objs except Exception as e: # pragma: no cover append_log(log, "db_trustsec::clean_sgts::Exception in clean_sgts: ", e) return changed_objs
def merge_sgpolicies(src, sgpolicies, is_base, sync_session, log=None, obj=None): changed_objs = [] try: iseservers = ISEServer.objects.all() organizations = Organization.objects.filter(dashboard__syncsession=sync_session) for s in sgpolicies: src_grp = dst_grp = binding_id = binding_name = binding_desc = policy_name = policy_desc = None if src == "meraki": p_src = TagData.objects.filter(source_id=s["srcGroupId"]).\ filter(organization__dashboard=sync_session.dashboard) p_dst = TagData.objects.filter(source_id=s["dstGroupId"]).\ filter(organization__dashboard=sync_session.dashboard) src_grp = p_src[0] if len(p_src) > 0 else None dst_grp = p_dst[0] if len(p_dst) > 0 else None elif src == "ise": p_src = TagData.objects.filter(source_id=s["sourceSgtId"]).\ filter(iseserver=sync_session.iseserver) p_dst = TagData.objects.filter(source_id=s["destinationSgtId"]).\ filter(iseserver=sync_session.iseserver) src_grp = p_src[0] if len(p_src) > 0 else None dst_grp = p_dst[0] if len(p_dst) > 0 else None if src_grp and dst_grp: if src_grp.tag.tag_number == 65535 and dst_grp.tag.tag_number == 65535: continue if src_grp and dst_grp: binding_name = str(src_grp.tag.tag_number) + "-" + str(dst_grp.tag.tag_number) binding_id = "s" + str(src_grp.source_id) + "-d" + str(dst_grp.source_id) binding_desc = str(src_grp.tag.name) + "-" + str(dst_grp.tag.name) policy_name = s.get("name", "") policy_desc = s.get("description", "") policy_name = binding_name if (policy_name is None or policy_name == "") else policy_name policy_desc = binding_desc if (policy_desc is None or policy_desc == "") else policy_desc # Look up policy, and see if the source matches the current input. If so, check for updates... full_update = False if src == "meraki": pol, created = Policy.objects.get_or_create(mapping=binding_name, defaults={"name": policy_name, "description": policy_desc, "origin_org": obj, "syncsession": sync_session}) else: pol, created = Policy.objects.get_or_create(mapping=binding_name, defaults={"name": policy_name, "description": policy_desc, "origin_ise": obj, "syncsession": sync_session}) if created: changed_objs.append(pol) append_log(log, "db_trustsec::merge_policies::creating policy", policy_name, "...") full_update = True else: if is_base: append_log(log, "db_trustsec::merge_policies::policy::" + src + "::", policy_name, "exists in database; updating...") full_update = True else: append_log(log, "db_trustsec::merge_policies::policy::" + src + "::", policy_name, "exists in database; not base, only add data...") if full_update: pol.mapping = binding_name if pol.name != policy_name and pol.cleaned_name() != policy_name: pol.name = policy_name pol.description = policy_desc pol.source_group = src_grp.tag pol.dest_group = dst_grp.tag acl_set = [] if src == "meraki": acls = ACLData.objects.filter(source_id__in=s["aclIds"]) for a in acls: # if a.acl not in pol.acl.all(): acl_set.append(a.acl) pol.acl.set(acl_set) else: acls = ACLData.objects.filter(source_id__in=s["sgacls"]) for a in acls: acl_set.append(a.acl) pol.acl.set(acl_set) changed_objs.append(pol) pol.save() if not pol.push_delete: if src == "meraki": PolicyData.objects.update_or_create(policy=pol, organization=obj, defaults={"source_id": binding_id, "source_data": json.dumps(s), "source_ver": s["versionNum"], "last_sync": make_aware(datetime.datetime.now())}) # Ensure PolicyData objects exist for ISE for i in iseservers: PolicyData.objects.get_or_create(policy=pol, iseserver=i) # Ensure PolicyData objects exist for all Meraki Orgs for o in organizations: PolicyData.objects.get_or_create(policy=pol, organization=o) elif src == "ise": PolicyData.objects.update_or_create(policy=pol, iseserver=obj, defaults={"source_id": s["id"], "source_data": json.dumps(s), "source_ver": s.get("generationId", None), "last_sync": make_aware(datetime.datetime.now())}) # Ensure PolicyData objects exist for all Meraki Orgs for o in organizations: PolicyData.objects.get_or_create(policy=pol, organization=o) elif s.get("name", "") == "ANY-ANY": pass else: append_log(log, "db_trustsec::merge_sgpolicies::missing src or dst", s, src_grp, dst_grp) return changed_objs except Exception as e: # pragma: no cover append_log(log, "db_trustsec::merge_sgpolicies::Exception in merge_sgpolicies: ", e, traceback.format_exc()) return changed_objs
def merge_sgacls(src, sgacls, is_base, sync_session, log=None, obj=None): changed_objs = [] try: iseservers = ISEServer.objects.all() organizations = Organization.objects.filter(dashboard__syncsession=sync_session) for s in sgacls: tag_name = s.get("name", "") if tag_name == "": append_log(log, "db_trustsec::merge_sgacls::sgacl doesn't have name; skipping", s) continue tn_ise = tag_name.replace(" ", "_") tn_mer = tag_name.replace("_", " ") tid = s["id"] if "id" in s else s["aclId"] if tag_name: # Look up acl, and see if the source matches the current input. If so, check for updates... aclds = ACLData.objects.filter(source_id=tid) if len(aclds) > 0: acls = [aclds[0].acl] else: acls = ACL.objects.filter(Q(name=tn_mer) | Q(name=tn_ise)) if len(acls) <= 0: created = True if tag_name in ("Deny_IP_Log", "Permit IP", "Permit_IP_Log", "Deny IP"): isvisible = False else: isvisible = True if src == "meraki": acl = ACL.objects.create(name=tag_name, description=s["description"], origin_org=obj, syncsession=sync_session, visible=isvisible) else: acl = ACL.objects.create(name=tag_name, description=s["description"], origin_ise=obj, syncsession=sync_session, visible=isvisible) else: created = False acl = acls[0] if created: append_log(log, "db_trustsec::merge_sgacls::creating acl", tag_name, "...") changed_objs.append(acl) else: if is_base: append_log(log, "db_trustsec::merge_sgacls::sgacl::" + src + "::", tag_name, "exists in database; updating...") if acl.name != s["name"] and acl.cleaned_name() != s["name"]: acl.name = s["name"] acl.description = s["description"].replace("'", "").replace('"', "") changed_objs.append(acl) acl.save() else: append_log(log, "db_trustsec::merge_sgacls::sgacl::" + src + "::", tag_name, "exists in database; not base, only add data...") if not acl.push_delete: if src == "meraki": ACLData.objects.update_or_create(acl=acl, organization=obj, defaults={"source_id": s["aclId"], "source_data": json.dumps(s), "source_ver": s["versionNum"], "last_sync": make_aware(datetime.datetime.now())}) # Ensure ACLData objects exist for ISE for i in iseservers: ACLData.objects.get_or_create(acl=acl, iseserver=i) # Ensure ACLData objects exist for all Meraki Orgs for o in organizations: ACLData.objects.get_or_create(acl=acl, organization=o) elif src == "ise": ACLData.objects.update_or_create(acl=acl, iseserver=obj, defaults={"source_id": s["id"], "source_data": json.dumps(s), "source_ver": s["generationId"], "last_sync": make_aware(datetime.datetime.now())}) # Ensure ACLData objects exist for all Meraki Orgs for o in organizations: ACLData.objects.get_or_create(acl=acl, organization=o) return changed_objs except Exception as e: # pragma: no cover append_log(log, "db_trustsec::merge_sgacls::Exception in merge_sgacls: ", e, traceback.format_exc()) return changed_objs
def merge_sgts(src, sgts, is_base, sync_session, log=None, obj=None): changed_objs = [] try: iseservers = ISEServer.objects.all() organizations = Organization.objects.filter(dashboard__syncsession=sync_session) for s in sgts: tag_num = None if isinstance(s, dict): if "value" in s: tag_num = s["value"] elif "tag" in s: tag_num = s["tag"] else: tag_num = None tid = s["id"] if "id" in s else s["groupId"] append_log(log, "db_trustsec::merge_sgts::evaluating", tag_num, "(", tid, ")...") if tag_num is not None: # Look up tag, and see if the source matches the current input. If so, check for updates... tagds = TagData.objects.filter(source_id=tid) if len(tagds) > 0: tag = tagds[0].tag else: tag = None if tag: if is_base: append_log(log, "db_trustsec::merge_sgts::sgt::" + src + "::", tag_num, "exists in database; updating...") tag.tag_number = tag_num if tag.name != s["name"] and tag.cleaned_name() != s["name"]: tag.name = s["name"] tag.description = s["description"].replace("'", "").replace('"', "") tag.save() changed_objs.append(tag) else: append_log(log, "db_trustsec::merge_sgts::sgt::" + src + "::", tag_num, "exists in database; not base, only add data...") created = False else: if src == "meraki": tag, created = Tag.objects.get_or_create(tag_number=tag_num, defaults={"name": s["name"], "description": s["description"], "origin_org": obj, "syncsession": sync_session}) else: tag, created = Tag.objects.get_or_create(tag_number=tag_num, defaults={"name": s["name"], "description": s["description"], "origin_ise": obj, "syncsession": sync_session}) if created: changed_objs.append(tag) append_log(log, "db_trustsec::merge_sgts::creating tag", tag_num, "...") # Ensure that all Data objects exist in DB if not tag.push_delete: append_log(log, "db_trustsec::merge_sgts::sgt::" + src + "::", tag_num, "writing raw data to database...") if src == "meraki": TagData.objects.update_or_create(tag=tag, organization=obj, defaults={"source_id": s["groupId"], "source_data": json.dumps(s), "source_ver": s["versionNum"], "last_sync": make_aware(datetime.datetime.now())}) # Ensure TagData objects exist for ISE for i in iseservers: TagData.objects.get_or_create(tag=tag, iseserver=i) # Ensure TagData objects exist for all Meraki Orgs for o in organizations: TagData.objects.get_or_create(tag=tag, organization=o) elif src == "ise": if s.get("generationId"): TagData.objects.update_or_create(tag=tag, iseserver=obj, defaults={"source_id": s["id"], "source_data": json.dumps(s), "source_ver": s["generationId"], "last_sync": make_aware(datetime.datetime.now())}) else: TagData.objects.update_or_create(tag=tag, iseserver=obj, defaults={"source_id": s["id"], "source_data": json.dumps(s), "last_sync": make_aware(datetime.datetime.now())}) # Ensure TagData objects exist for all Meraki Orgs for o in organizations: TagData.objects.get_or_create(tag=tag, organization=o) return changed_objs except Exception as e: # pragma: no cover append_log(log, "db_trustsec::merge_sgts::Exception in merge_sgts: ", e, traceback.format_exc()) return changed_objs
def run(): log = [] dbs = Dashboard.objects.filter(webhook_enable=True) if dbs and len(dbs) > 0: db = dbs[0] dashboard = meraki.DashboardAPI(api_key=db.apikey, print_console=False, output_log=False) if db.webhook_ngrok: try: public_url = ngrok.connect(sys.argv[-1], "http") except Exception: try: public_url = ngrok.connect(8000, "http") except Exception: print("# Unable to launch ngrok") return None db.webhook_url = public_url.replace("http://", "https://") + "/webhook/" db.save() orgs = db.organization.all() for org in orgs: nets = dashboard.networks.getOrganizationNetworks(org.orgid) for n in nets: whid = None whurl = dashboard.networks.getNetworkHttpServers( networkId=n["id"]) if len(whurl) <= 0: append_log(log, "creating new webhook for network", n["id"]) wh = dashboard.networks.createNetworkHttpServer( networkId=n["id"], name="adaptive-policy-sync", url=db.webhook_url) whid = wh["id"] else: for w in whurl: if w["name"] == "adaptive-policy-sync": append_log(log, "updating for network", n["id"]) dashboard.networks.updateNetworkHttpServer( networkId=n["id"], id=w["id"], url=db.webhook_url) whid = w["id"] if whid: al = dashboard.networks.getNetworkAlertSettings( networkId=n["id"]) for a in al["alerts"]: if a["type"] == "settingsChanged": a["alertDestinations"]["httpServerIds"].append( whid) a["enabled"] = True append_log(log, "updating alert settings", al) r = dashboard.networks.updateNetworkAlertSettings( networkId=n["id"], defaultDestinations=al["defaultDestinations"], alerts=al["alerts"]) append_log(log, "update response", r) else: append_log(log, "Dashboard webhooks are not configured") db_log("dashboard_webhook", log) raise Exception("Dashboard webhooks are not configured") db_log("dashboard_webhook", log)