def test_mark_invalid_remote_auto_update(entities, capsys): """ The network does enable automatic updates. Remote-ixf[as,ip4,ip6] contains invalid data **but** it can be parsed. There is not a local-ixf flagging that invalid data. We create a local-ixf[as,ip4,ip6] and flag as invalid Email the ix Create/Update a ticket for admin com """ data = setup_test_data("ixf.member.invalid.0") network = entities["net"]["UPDATE_ENABLED"] ixlan = entities["ixlan"][0] # Just to create a connection between the network and ix entities["netixlan"].append( NetworkIXLan.objects.create( network=network, ixlan=ixlan, asn=network.asn, speed=10000, ipaddr4="195.69.147.200", ipaddr6="2001:7f8:1::a500:2906:2", status="ok", is_rs_peer=True, operational=True, )) importer = ixf.Importer() data = importer.sanitize(data) importer.update(ixlan, data=data) print([(n.speed, n.ipaddr4, n.ipaddr6, n.asn) for n in NetworkIXLan.objects.all()]) print(importer.log) assert NetworkIXLan.objects.count() == 1 assert IXFMemberData.objects.count() == 2 ERROR_MESSAGE = "Invalid speed value" stdout = capsys.readouterr().out assert ERROR_MESSAGE in stdout assert_ticket_exists([ ("AS2906", "195.69.147.100", "2001:7f8:1::a500:2906:4"), ("AS2906", "195.69.147.200", "2001:7f8:1::a500:2906:2"), ]) # Test idempotent importer.update(ixlan, data=data) assert IXFMemberData.objects.count() == 2 assert_ticket_exists([ ("AS2906", "195.69.147.100", "2001:7f8:1::a500:2906:4"), ("AS2906", "195.69.147.200", "2001:7f8:1::a500:2906:2"), ])
def test_add_netixlan_conflict_local_ixf(entities, capsys): """ No NetIXLan exists. Network allows auto updates. While remote IXF data has information to create a new NetIXLan, there are conflicts with the ipaddresses that prevent it from being created. There is already a local-ixf so we do nothing. """ data = setup_test_data("ixf.member.0") network = entities["net"]["UPDATE_ENABLED"] ixlan = entities["ixlan"][1] # So we have conflicts with IPAddresses preexisting_ixfmember_data = IXFMemberData.objects.create( asn=network.asn, ipaddr4="195.69.147.250", # Matches remote-ixf, but conflicts with IXLan ipaddr6= "2001:7f8:1::a500:2906:1", # Matches remote-ixf, but conflicts with IXLan ixlan=ixlan, speed=10000, fetched=datetime.datetime.now(datetime.timezone.utc), operational=True, is_rs_peer=True, status="ok", error=["IPv4 195.69.147.250 does not match any prefix on this ixlan"], ) importer = ixf.Importer() importer.update(ixlan, data=data) assert IXFMemberData.objects.count() == 1 assert NetworkIXLan.objects.count() == 0 ixfmemberdata = IXFMemberData.objects.first() assert ("IPv4 195.69.147.250 does not match any prefix on this ixlan" in ixfmemberdata.error) assert_no_ticket_exists() stdout = capsys.readouterr().out assert stdout == "" updated_timestamp = ixfmemberdata.updated importer.update(ixlan, data=data) assert updated_timestamp == IXFMemberData.objects.first().updated # Test idempotent importer.update(ixlan, data=data) assert IXFMemberData.objects.count() == 1 assert NetworkIXLan.objects.count() == 0 assert_no_ticket_exists()
def test_reset_email(entities, data_cmd_ixf_email): ixf_import_data = json.loads(data_cmd_ixf_email.json) importer = ixf.Importer() ixlan = entities["ixlan"] # Create IXFMemberData importer.update(ixlan, data=ixf_import_data) importer.notify_proposals() assert IXFImportEmail.objects.count() == 1 call_command("pdb_ixf_ixp_member_import", reset_email=True, commit=True) assert IXFImportEmail.objects.count() == 0 assert DeskProTicket.objects.filter( body__contains="reset_email").count() == 1
def test_suggest_add(entities, capsys): """ The netixlan described in the remote-ixf doesn't exist, but there is a relationship btw the network and ix (ie a different netixlan). The network does not have automatic updates. There isn't a local-ixf that matches the remote-ixf. We suggest adding the netixlan, create an admin ticket, and send emails to the network and IX. """ data = setup_test_data("ixf.member.2") # asn1001 network = entities["net"]["UPDATE_DISABLED"] # asn1001 ixlan = entities["ixlan"][0] # This appears in the remote-ixf data so should not # create a IXFMemberData instance entities["netixlan"].append( NetworkIXLan.objects.create( network=network, ixlan=ixlan, asn=network.asn, speed=10000, ipaddr4="195.69.150.250", ipaddr6="2001:7f8:1::a500:2906:3", status="ok", is_rs_peer=True, operational=True, )) importer = ixf.Importer() importer.update(ixlan, data=data) assert IXFMemberData.objects.count() == 1 assert NetworkIXLan.objects.count() == 1 log = importer.log["data"][0] assert log["action"] == "suggest-add" stdout = capsys.readouterr().out assert_email_sent( stdout, (network.asn, "195.69.147.250", "2001:7f8:1::a500:2906:1")) assert_ticket_exists([(1001, "195.69.147.250", "2001:7f8:1::a500:2906:1")]) # Test idempotent importer.update(ixlan, data=data) assert IXFMemberData.objects.count() == 1 assert NetworkIXLan.objects.count() == 1 assert_ticket_exists([(1001, "195.69.147.250", "2001:7f8:1::a500:2906:1")])
def test_suggest_modify_local_ixf(entities): """ Netixlan is different from remote in terms of speed, operational, and is_rs_peer BUT there is already a local-ixf for the update. Automatic updates are disabled (so netixlan will not change). We do nothing and confirm the local-ixf stays the same. """ data = setup_test_data("ixf.member.1") network = entities["net"]["UPDATE_DISABLED"] ixlan = entities["ixlan"][0] entities["netixlan"].append( NetworkIXLan.objects.create( network=network, ixlan=ixlan, asn=network.asn, speed=20000, ipaddr4="195.69.147.250", ipaddr6="2001:7f8:1::a500:2906:1", status="ok", is_rs_peer=False, operational=False, )) # Matches the json data, doesn't match the existing netixlan. preexisting_ixfmember_data = IXFMemberData.objects.create( asn=network.asn, ipaddr4="195.69.147.250", ipaddr6="2001:7f8:1::a500:2906:1", ixlan=ixlan, speed=10000, fetched=datetime.datetime.now(datetime.timezone.utc), operational=True, is_rs_peer=True, status="ok", ) importer = ixf.Importer() importer.update(ixlan, data=data) assert IXFMemberData.objects.count() == 1 assert preexisting_ixfmember_data == IXFMemberData.objects.first() # Assert idempotent importer.update(ixlan, data=data) assert IXFMemberData.objects.count() == 1 assert preexisting_ixfmember_data == IXFMemberData.objects.first()
def test_resolve_local_ixf(entities): """ Netixlan exists, remote data matches the netixlan, and there is a local-ixf entry that also matches all the data. """ data = setup_test_data("ixf.member.0") network = entities["net"]["UPDATE_ENABLED"] ixlan = entities["ixlan"][0] entities["netixlan"].append( NetworkIXLan.objects.create( network=network, ixlan=ixlan, asn=network.asn, speed=10000, ipaddr4="195.69.147.250", ipaddr6="2001:7f8:1::a500:2906:1", status="ok", is_rs_peer=True, operational=True, )) # Create a local IXF that matches remote details IXFMemberData.objects.create( asn=network.asn, ipaddr4="195.69.147.250", ipaddr6="2001:7f8:1::a500:2906:1", ixlan=ixlan, speed=10000, fetched=datetime.datetime.now(datetime.timezone.utc), operational=True, is_rs_peer=True, status="ok", ) importer = ixf.Importer() importer.update(ixlan, data=data) assert IXFMemberData.objects.count() == 0 assert_ticket_exists([(network.asn, "195.69.147.250", "2001:7f8:1::a500:2906:1")]) # Test idempotent importer.update(ixlan, data=data) assert IXFMemberData.objects.count() == 0
def test_reset_dismissals(entities, data_cmd_ixf_dismissals): ixf_import_data = json.loads(data_cmd_ixf_dismissals.json) importer = ixf.Importer() ixlan = entities["ixlan"] # Create IXFMemberData importer.update(ixlan, data=ixf_import_data) # Dismiss all IXFMemberData for ixfm in IXFMemberData.objects.all(): ixfm.dismissed = True ixfm.save() call_command("pdb_ixf_ixp_member_import", reset_dismisses=True, commit=True) assert IXFMemberData.objects.filter(dismissed=False).count() == 4 assert DeskProTicket.objects.filter( body__contains="reset_dismisses").count() == 1
def view_import_ixlan_ixf_preview(request, ixlan_id): # check if request was blocked by rate limiting was_limited = getattr(request, "limited", False) if was_limited: return error_response(_("Please wait a bit before requesting " \ "another ixf import preview."), status=400) try: ixlan = IXLan.objects.get(id=ixlan_id) except IXLan.DoesNotExist: return error_response(_("Ixlan not found"), status=404) if not has_perms(request.user, ixlan, "update"): return error_response(_("Permission denied"), status=403) importer = ixf.Importer() importer.update(ixlan, save=False) return pretty_response(importer.log)
def handle(self, *args, **options): self.commit = options.get("commit", False) only_id = options.get("only", 0) q = IXLan.objects.filter(status="ok").exclude( ixf_ixp_member_list_url__isnull=True) if only_id: q = q.filter(id=only_id) for ixlan in q: self.log("Fetching data for {} from {}".format( ixlan, ixlan.ixf_ixp_member_list_url)) try: importer = ixf.Importer() self.log("Updating {}".format(ixlan)) with transaction.atomic(): success, netixlans, netixlans_deleted, log = importer.update( ixlan, save=self.commit) self.log(json.dumps(log)) self.log("Done: {} updated: {} deleted: {}".format( success, len(netixlans), len(netixlans_deleted))) except Exception as inst: self.log("ERROR: {}".format(inst)) self.log(traceback.format_exc())
def view_import_net_ixf_preview(request, net_id): # check if request was blocked by rate limiting was_limited = getattr(request, "limited", False) if was_limited: return error_response( _("Please wait a bit before requesting " "another ixf import preview."), status=400, ) try: net = Network.objects.get(id=net_id, status="ok") except Network.DoesNotExist: return error_response(_("Network not found"), status=404) if not check_permissions(request.user, net, "u"): return error_response(_("Permission denied"), status=403) total_log = {"data": [], "errors": []} for ixlan in net.ixlan_set_ixf_enabled: importer = ixf.Importer() importer.cache_only = True success = importer.update(ixlan, asn=net.asn, save=False) # strip suggestions log_data = [ i for i in importer.log["data"] if not "suggest-" in i["action"] ] total_log["data"].extend(log_data) total_log["errors"].extend([ f"{ixlan.ix.name}({ixlan.id}): {err}" for err in importer.log["errors"] ]) return pretty_response(total_log)
def test_suggest_add_no_netixlan_local_ixf(entities, capsys): """ There isn't any netixlan between ix and network. Network does not have automatic updates. There is a local-ixf that matches the remote-ixf so we do nothing """ data = setup_test_data("ixf.member.1") # asn1001 network = entities["net"]["UPDATE_DISABLED"] # asn1001 ixlan = entities["ixlan"][0] preexisting_ixfmember_data = IXFMemberData.objects.create( asn=1001, # Matches remote-ixf data ipaddr4="195.69.147.250", # Matches remote-ixf data ipaddr6="2001:7f8:1::a500:2906:1", # Matches remote-ixf data ixlan=ixlan, speed=10000, fetched=datetime.datetime.now(datetime.timezone.utc), operational=True, is_rs_peer=True, status="ok", ) importer = ixf.Importer() importer.update(ixlan, data=data) assert IXFMemberData.objects.count() == 1 assert NetworkIXLan.objects.count() == 0 stdout = capsys.readouterr().out assert stdout == "" assert_no_ticket_exists() # Test idempotent importer.update(ixlan, data=data) assert IXFMemberData.objects.count() == 1 assert NetworkIXLan.objects.count() == 0 assert_no_ticket_exists()
def test_remote_cannot_be_parsed(entities, capsys): """ Remote cannot be parsed. We create a ticket, email the IX, and create a lock. """ data = setup_test_data("ixf.member.unparsable") ixlan = entities["ixlan"][0] start = datetime.datetime.now(datetime.timezone.utc) importer = ixf.Importer() importer.sanitize(data) importer.update(ixlan, data=data) ERROR_MESSAGE = "No entries in any of the vlan_list lists, aborting" assert importer.ixlan.ixf_ixp_import_error_notified > start # This sets the lock assert ERROR_MESSAGE in importer.ixlan.ixf_ixp_import_error stdout = capsys.readouterr().out assert ERROR_MESSAGE in stdout assert DeskProTicket.objects.count() == 1 # Assert idempotent / lock importer.sanitize(data) importer.update(ixlan, data=data) stdout = capsys.readouterr().out assert stdout == "" assert DeskProTicket.objects.count() == 1
def test_suggest_delete_local_ixf_has_flag(entities, capsys): """ Automatic updates for network are disabled. There is no remote-ixf corresponding to an existing netixlan. There is a local-ixf flagging that netixlan for deletion. We want to do nothing. """ data = setup_test_data("ixf.member.1") network = entities["net"]["UPDATE_DISABLED"] ixlan = entities["ixlan"][0] entities["netixlan"].append( NetworkIXLan.objects.create( network=network, ixlan=ixlan, asn=network.asn, speed=10000, ipaddr4="195.69.147.250", ipaddr6="2001:7f8:1::a500:2906:1", status="ok", is_rs_peer=True, operational=True, )) entities["netixlan"].append( NetworkIXLan.objects.create( network=network, ixlan=ixlan, asn=network.asn, speed=20000, ipaddr4="195.69.147.251", ipaddr6=None, status="ok", is_rs_peer=False, operational=False, )) preexisting_ixfmember_data = IXFMemberData.objects.create( asn=1001, ipaddr4="195.69.147.251", ipaddr6=None, ixlan=ixlan, speed=10000, fetched=datetime.datetime.now(datetime.timezone.utc), operational=True, is_rs_peer=True, status="ok", data= {}, # Makes self.remote_data_missing and self.marked_for_removal True ) importer = ixf.Importer() importer.update(ixlan, data=data) assert NetworkIXLan.objects.count() == 2 assert IXFMemberData.objects.count() == 1 assert_no_ticket_exists() # Test idempotent importer.update(ixlan, data=data) assert NetworkIXLan.objects.count() == 2 assert IXFMemberData.objects.count() == 1 assert_no_ticket_exists()
def setUp(self): self.ixf_importer = ixf.Importer()
def handle(self, *args, **options): self.commit = options.get("commit", False) self.debug = options.get("debug", False) self.preview = options.get("preview", False) self.cache = options.get("cache", False) self.skip_import = options.get("skip_import", False) self.active_reset_flags = self.initiate_reset_flags(**options) self.runtime_errors = [] if self.reset or self.reset_hints: self.reset_all_hints() if self.reset or self.reset_dismisses: self.reset_all_dismisses() if self.reset or self.reset_email: self.reset_all_email() if self.reset or self.reset_tickets: self.reset_all_tickets() if len(self.active_reset_flags) >= 1: self.create_reset_ticket() if self.preview and self.commit: self.commit = False ixlan_ids = options.get("ixlan") asn = options.get("asn", 0) if asn and not ixlan_ids: # if asn is specified, retrieve queryset for ixlans from # the network object net = Network.objects.get(asn=asn) qset = net.ixlan_set_ixf_enabled else: # otherwise build ixlan queryset qset = IXLan.objects.filter(status="ok", ixf_ixp_import_enabled=True) qset = qset.exclude(ixf_ixp_member_list_url__isnull=True) # filter by ids if ixlan ids were specified if ixlan_ids: qset = qset.filter(id__in=ixlan_ids) total_log = {"data": [], "errors": []} total_notifications = [] for ixlan in qset: self.log("Fetching data for -ixlan{} from {}".format( ixlan.id, ixlan.ixf_ixp_member_list_url)) try: importer = ixf.Importer() importer.skip_import = self.skip_import importer.cache_only = self.cache self.log(f"Processing {ixlan.ix.name} ({ixlan.id})") with transaction.atomic(): success = importer.update(ixlan, save=self.commit, asn=asn) self.log(json.dumps(importer.log), debug=True) self.log( "Success: {}, added: {}, updated: {}, deleted: {}".format( success, len(importer.actions_taken["add"]), len(importer.actions_taken["modify"]), len(importer.actions_taken["delete"]), )) total_log["data"].extend(importer.log["data"]) total_log["errors"].extend([ f"{ixlan.ix.name}({ixlan.id}): {err}" for err in importer.log["errors"] ]) total_notifications += importer.notifications except Exception as inst: self.store_runtime_error(inst) if self.preview: self.stdout.write(json.dumps(total_log, indent=2)) # send cosolidated notifications to ix and net for # new proposals (#771) importer = ixf.Importer() importer.reset(save=self.commit) importer.notifications = total_notifications importer.notify_proposals(error_handler=self.store_runtime_error) self.stdout.write(f"New Emails: {importer.emails}") if len(self.runtime_errors) > 0: self.write_runtime_errors() sys.exit(1) if self.commit: self.resend_emails(importer)
def test_connections_match(data_ixf_connections_match): importer = ixf.Importer() connection_list = json.loads(data_ixf_connections_match.input)["connection_list"] cxn_match = importer.connections_match(connection_list[0], connection_list[1]) assert cxn_match == data_ixf_connections_match.expected
def test_mark_invalid_remote_w_local_ixf_no_auto_update(entities, capsys): """ Our network does not allow automatic updates. Remote-ixf[as,ip4,ip6] contains invalid data **but** it can be parsed. There is already a local-ixf flagging that invalid data. Do nothing. """ data = setup_test_data("ixf.member.invalid.1") network = entities["net"]["UPDATE_DISABLED"] ixlan = entities["ixlan"][0] # Just to create a connection between the network and ix entities["netixlan"].append( NetworkIXLan.objects.create( network=network, ixlan=ixlan, asn=network.asn, speed=10000, ipaddr4="195.69.147.200", ipaddr6="2001:7f8:1::a500:2906:2", status="ok", is_rs_peer=True, operational=True, )) preexisting_ixfmember_data = IXFMemberData.objects.create( asn=1001, ipaddr4="195.69.147.200", ipaddr6="2001:7f8:1::a500:2906:2", ixlan=ixlan, speed=0, fetched=datetime.datetime.now(datetime.timezone.utc), operational=True, is_rs_peer=True, status="ok", error="Invalid speed value: this is not valid", ) preexisting_ixfmember_data = IXFMemberData.objects.create( asn=1001, ipaddr4="195.69.147.100", ipaddr6="2001:7f8:1::a500:2906:4", ixlan=ixlan, speed=0, fetched=datetime.datetime.now(datetime.timezone.utc), operational=True, is_rs_peer=True, status="ok", error="Invalid speed value: this is not valid", ) importer = ixf.Importer() data = importer.sanitize(data) importer.update(ixlan, data=data) assert IXFMemberData.objects.count() == 2 stdout = capsys.readouterr().out assert stdout == "" assert_no_ticket_exists() # Test idempotent importer.update(ixlan, data=data) assert IXFMemberData.objects.count() == 2 assert_no_ticket_exists()
def test_suggest_delete_local_ixf_no_flag(entities, capsys): """ Automatic updates for network are disabled. There is no remote-ixf corresponding to an existing netixlan. There is a local-ixf corresponding to that netixlan but it does not flag it for deletion. We flag the local-ixf for deletion, make a ticket, and email the ix and network. """ data = setup_test_data("ixf.member.1") network = entities["net"]["UPDATE_DISABLED"] ixlan = entities["ixlan"][0] entities["netixlan"].append( NetworkIXLan.objects.create( network=network, ixlan=ixlan, asn=network.asn, speed=10000, ipaddr4="195.69.147.250", ipaddr6="2001:7f8:1::a500:2906:1", status="ok", is_rs_peer=True, operational=True, )) entities["netixlan"].append( NetworkIXLan.objects.create( network=network, ixlan=ixlan, asn=network.asn, speed=20000, ipaddr4="195.69.147.251", ipaddr6=None, status="ok", is_rs_peer=True, operational=True, )) ixf_member_data_field = { "ixp_id": 42, "connected_since": "2009-02-04T00:00:00Z", "state": "connected", "if_list": [{ "switch_id": 1, "if_speed": 20000, "if_type": "LR4" }], "vlan_list": [{ "vlan_id": 0, "ipv4": { "address": "195.69.147.251", "routeserver": True, "max_prefix": 42, "as_macro": "AS-NFLX-V4", "mac_address": ["00:0a:95:9d:68:16"], }, }], } preexisting_ixfmember_data = IXFMemberData.objects.create( asn=1001, ipaddr4="195.69.147.251", ipaddr6=None, ixlan=ixlan, speed=20000, fetched=datetime.datetime.now(datetime.timezone.utc), operational=True, is_rs_peer=True, status="ok", data=json.dumps(ixf_member_data_field), ) importer = ixf.Importer() importer.update(ixlan, data=data) assert importer.log["data"][0]["action"] == "suggest-delete" assert NetworkIXLan.objects.count() == 2 # Test failing, IXFMember is getting resolved # instead of being flagged for deletion. assert IXFMemberData.objects.count() == 1 assert_ticket_exists([("AS1001", "195.69.147.251", "No IPv6")]) stdout = capsys.readouterr().out assert_email_sent(stdout, (1001, "195.69.147.251", "No IPv6")) # Test idempotent importer.update(ixlan, data=data) assert NetworkIXLan.objects.count() == 2 assert IXFMemberData.objects.count() == 1 assert_ticket_exists([("AS1001", "195.69.147.251", "No IPv6")])
def handle(self, *args, **options): self.commit = options.get("commit", False) self.debug = options.get("debug", False) self.preview = options.get("preview", False) self.cache = options.get("cache", False) self.skip_import = options.get("skip_import", False) if options.get("delete_all_ixfmemberdata"): self.log("Deleting IXFMemberData Instances ...") IXFMemberData.objects.all().delete() if self.preview and self.commit: self.commit = False ixlan_ids = options.get("ixlan") asn = options.get("asn", 0) if asn and not ixlan_ids: # if asn is specified, retrieve queryset for ixlans from # the network object net = Network.objects.get(asn=asn) qset = net.ixlan_set_ixf_enabled else: # otherwise build ixlan queryset qset = IXLan.objects.filter(status="ok", ixf_ixp_import_enabled=True) qset = qset.exclude(ixf_ixp_member_list_url__isnull=True) # filter by ids if ixlan ids were specified if ixlan_ids: qset = qset.filter(id__in=ixlan_ids) total_log = {"data": [], "errors": []} for ixlan in qset: self.log("Fetching data for -ixlan{} from {}".format( ixlan.id, ixlan.ixf_ixp_member_list_url)) try: importer = ixf.Importer() importer.skip_import = self.skip_import importer.cache_only = self.cache self.log(f"Processing {ixlan.ix.name} ({ixlan.id})") with transaction.atomic(): success = importer.update(ixlan, save=self.commit, asn=asn) self.log(json.dumps(importer.log), debug=True) self.log( "Success: {}, added: {}, updated: {}, deleted: {}".format( success, len(importer.actions_taken["add"]), len(importer.actions_taken["modify"]), len(importer.actions_taken["delete"]), )) total_log["data"].extend(importer.log["data"]) total_log["errors"].extend([ "{}({}): {}".format(ixlan.ix.name, ixlan.id, err) for err in importer.log["errors"] ]) except Exception as inst: self.log("ERROR: {}".format(inst)) self.log(traceback.format_exc()) if self.preview: self.stdout.write(json.dumps(total_log, indent=2))