def __init__(self, model=None, fields=[]):
self.model = model
self.fields = fields
self.operators = ["and", "or"]
self.quotes = ["\"", "\'"]
self.fs = "="
self.ipcalc = IPCalc()
def __init__(self, model=None, fields=[]): self.model = model self.fields = fields self.operators = ["and", "or"] self.quotes = ["\"", "\'"] self.fs = "=" self.ipcalc = IPCalc()
def __init__(self):
BaseBackend.__init__(self, models.Listing)
self.ip = IP()
self.sensor = Sensor()
self.host = Host()
self.reason = Reason()
self.duration = Duration()
self.user = User()
self.historylisting = HistoryListing()
self.config = Config()
self.ipcalc = IPCalc()
class Subnet(BaseBackend):
name = "Subnet"
def __init__(self):
BaseBackend.__init__(self, models.Subnet)
self.ipcalc = IPCalc()
self.asnum = ASNum()
self.country = Country()
self.rir = RIR()
def has(self, *args, **kwargs):
if "/" in kwargs['subnet']:
kwargs['subnet'] = kwargs['subnet'].split("/")[0]
subnet = self.ipcalc.dqtoi(kwargs['subnet'])
return BaseBackend.has(self, subnet=subnet)
def get(self, *args, **kwargs):
if kwargs.has_key('subnet'):
if "/" in kwargs['subnet']:
kwargs['subnet'] = kwargs['subnet'].split("/")[0]
subnet = self.ipcalc.dqtoi(kwargs['subnet'])
return BaseBackend.get(self, subnet=subnet)
else:
return BaseBackend.get(self, *args, **kwargs)
def contains(self, *args, **kwargs):
ip = self.ipcalc.dqtoi(kwargs["ip"])
net = models.Subnet.objects.filter(subnet__lte=ip).filter(last__gte=ip)
if net:
return (True, net)
else:
return (False, "No network found for %s" % (kwargs["ip"]))
def add(self, *args, **kwargs):
if not "/" in kwargs['subnet']:
return (False, "%s is not a subnet" % (kwargs['subnet']))
(net, mask) = kwargs['subnet'].split("/")
asn = kwargs['asn']
af = self.ipcalc.af(kwargs['subnet'])
(result, subnet) = self.get(subnet=net)
if result:
return (True, subnet)
if kwargs['asn'] == 0:
(result, rir) = self.rir.get(name="Unknown")
if not result:
return (False, rir)
(result, country) = self.country.get(code=kwargs['cc'])
if not result:
return (False, country)
(result, asnum) = self.asnum.get(asnum=kwargs['asn'])
if not result:
(result, asnum) = self.asnum.add(asnum=asn, name=kwargs['asn_name'], whois_data=kwargs['whois_data'], country=country, rir=rir)
if not result:
return (False, asnum)
net_first=self.ipcalc.dqtoi(net)
net_last = net_first + self.ipcalc.bittoip[int(mask)]
subnet = models.Subnet(
subnet=net_first,
last=net_last,
mask=int(mask),
asnum=asnum,
af=af,
country=country,
rir=asnum.rir,
regdate=datetime.now()
)
subnet.save()
return (True, subnet)
@transaction.commit_manually
def bulk_import(self, import_data):
t_start = mktime(datetime.now().timetuple())
data = []
failed = []
num_failed = 0
for entry in import_data:
if entry[0] == "":
num_failed += 1
failed.append(entry)
continue
(net, mask) = entry[0].split("/")
tmp = {}
tmp["subnet"] = self.ipcalc.dqtoi(net)
tmp["mask"] = mask
tmp["name"] = entry[1]
try:
tmp["asnum"] = models.ASNum.objects.filter(name=entry[2])[0]
except:
tmp["asnum"] = None
data.append(tmp)
for entry in data:
subnet = models.Subnet(
subnet=entry["subnet"],
mask=entry["mask"],
name=entry["name"],
asnum=entry["asnum"],
)
subnet.save()
transaction.commit()
reset_queries()
t_end = mktime(datetime.now().timetuple())
return (t_end - t_start, num_failed, failed)
def __init__(self): BaseBackend.__init__(self, models.Peering) self.config = Config() self.encryption = Encryption(self.config["blacklist.keystore.psk"]) self.ipcalc = IPCalc() self.netdata = NetData()
class Peering(BaseBackend):
name = "Peering"
def __init__(self):
BaseBackend.__init__(self, models.Peering)
self.config = Config()
self.encryption = Encryption(self.config["blacklist.keystore.psk"])
self.ipcalc = IPCalc()
self.netdata = NetData()
#@transaction.commit_manually
def bulk_import(self, import_data):
t_start = mktime(datetime.now().timetuple())
data = []
failed = []
num_failed = 0
for entry in import_data:
if entry[0] == "":
num_failed += 1
failed.append(entry)
continue
tmp = {}
try:
user = User.objects.get(username=entry[3])
except User.DoesNotExist:
user = User(username=entry[3])
user.save()
try:
asnum = models.ASNum.objects.get(asnum=entry[1])
except models.ASNum.DoesNotExist:
## TODO: fix unknown RIR
asnum = models.ASNum(
asnum=entry[1],
rir=models.RIR.objects.get(name="RIPE")
)
asnum.save()
(asn, asn_name, whois_data, net, cc) = self.netdata.get_whois_data(entry[0])
try:
country = models.Country.objects.get(code=cc)
except models.Country.DoesNotExist:
country = models.Country(code=cc, name=cc)
country.save()
try:
asnum = models.ASNum.objects.get(asnum=asn)
except models.ASNum.DoesNotExist:
asnum = models.ASNum(
asnum=asnum,
name=asn_name,
whois_data=whois_data,
country=country,
)
asnum.save()
(n, m) = net.split("/")
try:
subnet = models.Subnet.objects.get(subnet=self.ipcalc.dqtoi(n))
except models.Subnet.DoesNotExist:
subnet = models.Subnet(
subnet=self.ipcalc.dqtoi(n),
mask=m,
name=asn_name,
asnum=asnum,
)
subnet.save()
try:
ip = models.IP.objects.get(ip=self.ipcalc.dqtoi(entry[0]))
except models.IP.DoesNotExist:
ip = models.IP(
ip=self.ipcalc.dqtoi(entry[0]),
subnet=subnet,
)
ip.save()
try:
peer = models.Host.objects.get(hostname=entry[2])
except models.Host.DoesNotExist:
peer = models.Host(
hostname=entry[2],
ip=ip,
owner=user,
)
peer.save()
try:
key = models.Key.objects.get(name="%s-bgp" % (entry[2]))
except models.Key.DoesNotExist:
key = models.Key(
name="%s-bgp" % (entry[2]),
data=self.encryption.encrypt(entry[4])
)
key.save()
tmp["peer"] = peer
tmp["asnum"] = asnum
tmp["owner"] = user
tmp["key"] = key
data.append(tmp)
for entry in data:
peering = models.Peering(
peer=entry["peer"],
asnum=entry["asnum"],
user=entry["owner"],
key=entry["key"]
)
peering.save()
#transaction.commit()
#reset_queries()
t_end = mktime(datetime.now().timetuple())
return (t_end - t_start, num_failed, failed)
def __init__(self):
BaseBackend.__init__(self, models.Peering)
self.config = Config()
self.encryption = Encryption(self.config["blacklist.keystore.psk"])
self.ipcalc = IPCalc()
self.netdata = NetData()
class Peering(BaseBackend):
name = "Peering"
def __init__(self):
BaseBackend.__init__(self, models.Peering)
self.config = Config()
self.encryption = Encryption(self.config["blacklist.keystore.psk"])
self.ipcalc = IPCalc()
self.netdata = NetData()
#@transaction.commit_manually
def bulk_import(self, import_data):
t_start = mktime(datetime.now().timetuple())
data = []
failed = []
num_failed = 0
for entry in import_data:
if entry[0] == "":
num_failed += 1
failed.append(entry)
continue
tmp = {}
try:
user = User.objects.get(username=entry[3])
except User.DoesNotExist:
user = User(username=entry[3])
user.save()
try:
asnum = models.ASNum.objects.get(asnum=entry[1])
except models.ASNum.DoesNotExist:
## TODO: fix unknown RIR
asnum = models.ASNum(asnum=entry[1],
rir=models.RIR.objects.get(name="RIPE"))
asnum.save()
(asn, asn_name, whois_data, net,
cc) = self.netdata.get_whois_data(entry[0])
try:
country = models.Country.objects.get(code=cc)
except models.Country.DoesNotExist:
country = models.Country(code=cc, name=cc)
country.save()
try:
asnum = models.ASNum.objects.get(asnum=asn)
except models.ASNum.DoesNotExist:
asnum = models.ASNum(
asnum=asnum,
name=asn_name,
whois_data=whois_data,
country=country,
)
asnum.save()
(n, m) = net.split("/")
try:
subnet = models.Subnet.objects.get(subnet=self.ipcalc.dqtoi(n))
except models.Subnet.DoesNotExist:
subnet = models.Subnet(
subnet=self.ipcalc.dqtoi(n),
mask=m,
name=asn_name,
asnum=asnum,
)
subnet.save()
try:
ip = models.IP.objects.get(ip=self.ipcalc.dqtoi(entry[0]))
except models.IP.DoesNotExist:
ip = models.IP(
ip=self.ipcalc.dqtoi(entry[0]),
subnet=subnet,
)
ip.save()
try:
peer = models.Host.objects.get(hostname=entry[2])
except models.Host.DoesNotExist:
peer = models.Host(
hostname=entry[2],
ip=ip,
owner=user,
)
peer.save()
try:
key = models.Key.objects.get(name="%s-bgp" % (entry[2]))
except models.Key.DoesNotExist:
key = models.Key(name="%s-bgp" % (entry[2]),
data=self.encryption.encrypt(entry[4]))
key.save()
tmp["peer"] = peer
tmp["asnum"] = asnum
tmp["owner"] = user
tmp["key"] = key
data.append(tmp)
for entry in data:
peering = models.Peering(peer=entry["peer"],
asnum=entry["asnum"],
user=entry["owner"],
key=entry["key"])
peering.save()
#transaction.commit()
#reset_queries()
t_end = mktime(datetime.now().timetuple())
return (t_end - t_start, num_failed, failed)
class IP(BaseBackend):
name = "IP"
def __init__(self):
BaseBackend.__init__(self, models.IP)
self.netdata = NetData()
self.ipcalc = IPCalc()
self.subnet = Subnet()
def has(self, *args, **kwargs):
(dq, mask) = self.ipcalc.parse_addr(kwargs['ip'])
ip = self.ipcalc.dqtoi(dq)
return BaseBackend.has(self, ip=ip, mask=mask)
def get(self, *args, **kwargs):
(dq , mask) = self.ipcalc.parse_addr(kwargs['ip'])
ip = self.ipcalc.dqtoi(dq)
return BaseBackend.get(self, *args, ip=ip)
def add(self, *args, **kwargs):
af = self.ipcalc.af(kwargs['ip'])
if not af in [4,6]:
return (False, "No AF found for %s" % (ip))
if "/" in kwargs['ip']:
(ip, mask) = kwargs['ip'].split('/')
first = self.ipcalc.dqtoi(ip)
last=first + self.ipcalc.size(kwargs['ip'])
else:
ip = kwargs['ip']
if af == 4:
mask = 32
elif af == 6:
mask = 128
first = self.ipcalc.dqtoi(ip)
last = first
if self.has(ip=ip, mask=mask):
return (True, self.get(ip=ip, mask=mask))
try:
(asn, asn_name, whois_data, net, cc) = self.netdata.get_whois_data(kwargs["ip"])
except ValueError:
return (False, "Cannot fetch whois data")
if net == "0.0.0.0/0":
net = None
else:
if not self.subnet.has(subnet=net):
(result, net) = self.subnet.add(subnet=net, asn=asn, asn_name=asn_name, whois_data=whois_data, cc=cc)
if not result:
return (False, net)
else:
(result, net) = self.subnet.get(subnet=net)
if not result:
return (False, net)
ip = models.IP(
ip=first,
mask=mask,
last=last,
subnet=net,
af=af
)
ip.save()
return (True, ip)
def __init__(self): self.ipcalc = IPCalc() self.asnum = ASNum()
from django import template
from blacklist.common.ipcalc import IPCalc
ipcalc = IPCalc()
register = template.Library()
@register.simple_tag
def itodq(i, af):
return ipcalc.itodq(int(i), af)
def __init__(self): BaseBackend.__init__(self, models.Subnet) self.ipcalc = IPCalc() self.asnum = ASNum() self.country = Country() self.rir = RIR()
class Search:
def __init__(self, model=None, fields=[]):
self.model = model
self.fields = fields
self.operators = ["and", "or"]
self.quotes = ["\"", "\'"]
self.fs = "="
self.ipcalc = IPCalc()
def _construct_ip(self, cur_field, cur_value, is_negate):
qset = None
if "/" in cur_value:
(ip, mask) = cur_value.split("/")
else:
ip = cur_value
mask = None
try:
ip = self.ipcalc.dqtoi(ip)
except ValueError:
return (False, "%s is not a valid ip address" % (cur_value))
first = eval("Q(%s__lte=\"%s\")" % (self.fields[cur_field], ip))
if cur_field == "ip":
last = eval("Q(ip__last__gte=\"%s\")" % (ip))
else:
last = eval("Q(last__gte=\"%s\")" % (ip))
qset = (first & last)
if mask:
qset = qset & eval("Q(mask=\"%s\")" % (mask))
if is_negate:
return ~(qset)
else:
return (qset)
def _construct(self, cur_field, cur_value, is_negate):
qset_list = []
operator_list = []
for quote in self.quotes:
cur_value.replace("\%s" % (quote), "")
if cur_field in ["ip", "subnet"]:
qset_list.append(
self._construct_ip(cur_field, cur_value, is_negate))
else:
if is_negate:
qset_list.append(
eval("~Q(%s__icontains=\"%s\")" %
(self.fields[cur_field], cur_value)))
else:
qset_list.append(
eval("Q(%s__icontains=\"%s\")" %
(self.fields[cur_field], cur_value)))
return (qset_list, operator_list)
def find(self, q):
qset = None
qset_list = []
is_field = True
is_value = False
is_operator = False
is_multiword = False
is_negate = False
is_ip = True
has_quote = False
cur_field = ""
cur_value = ""
operator_list = []
for i in range(0, len(q)):
if q[i] == self.fs:
is_field = False
is_value = True
cur_field = cur_field.strip()
if cur_field.startswith("not "):
is_negate = True
cur_field = cur_field[4:]
cur_field = cur_field.strip()
if not cur_field in self.fields.keys():
return (False, "%s is not a valid field" % (cur_field))
continue
if q[i] in self.quotes:
is_multiword = not is_multiword
continue
if is_field:
cur_field += q[i]
if is_value:
cur_value += q[i]
if cur_value.endswith(" and") and not is_multiword:
cur_value = cur_value[:-4]
for quote in self.quotes:
cur_value.replace("\%s" % (quote), "")
operator_list.append("and")
(ql, ol) = self._construct(cur_field, cur_value, is_negate)
for i in ql:
qset_list.append(i)
for i in ol:
operator_list.append(i)
is_field = True
is_value = False
is_negate = False
cur_field = ""
cur_value = ""
if cur_value.endswith(" or") and not is_multiword:
cur_value = cur_value[:-3]
for quote in self.quotes:
cur_value.replace(quote, "")
operator_list.append("or")
(ql, ol) = self._construct(cur_field, cur_value, is_negate)
for i in ql:
qset_list.append(i)
for i in ol:
operator_list.append(i)
is_field = True
is_value = False
is_negate = False
is_ip = False
cur_field = ""
cur_value = ""
(ql, ol) = self._construct(cur_field, cur_value, is_negate)
for i in ql:
qset_list.append(i)
for i in ol:
operator_list.append(i)
qset = qset_list[0]
del (qset_list[0])
for operator in operator_list:
if operator == "and":
qset = qset & qset_list[0]
del (qset_list[0])
elif operator == "or":
qset = qset | qset_list[0]
del (qset_list[0])
return self.model.filter(qset)
def __init__(self):
BaseBackend.__init__(self, models.IP)
self.netdata = NetData()
self.ipcalc = IPCalc()
self.subnet = Subnet()
class IP(BaseBackend):
name = "IP"
def __init__(self):
BaseBackend.__init__(self, models.IP)
self.netdata = NetData()
self.ipcalc = IPCalc()
self.subnet = Subnet()
def has(self, *args, **kwargs):
(dq, mask) = self.ipcalc.parse_addr(kwargs['ip'])
ip = self.ipcalc.dqtoi(dq)
return BaseBackend.has(self, ip=ip, mask=mask)
def get(self, *args, **kwargs):
(dq, mask) = self.ipcalc.parse_addr(kwargs['ip'])
ip = self.ipcalc.dqtoi(dq)
return BaseBackend.get(self, *args, ip=ip)
def add(self, *args, **kwargs):
af = self.ipcalc.af(kwargs['ip'])
if not af in [4, 6]:
return (False, "No AF found for %s" % (ip))
if "/" in kwargs['ip']:
(ip, mask) = kwargs['ip'].split('/')
first = self.ipcalc.dqtoi(ip)
last = first + self.ipcalc.size(kwargs['ip'])
else:
ip = kwargs['ip']
if af == 4:
mask = 32
elif af == 6:
mask = 128
first = self.ipcalc.dqtoi(ip)
last = first
if self.has(ip=ip, mask=mask):
return (True, self.get(ip=ip, mask=mask))
try:
(asn, asn_name, whois_data, net,
cc) = self.netdata.get_whois_data(kwargs["ip"])
except ValueError:
return (False, "Cannot fetch whois data")
if net == "0.0.0.0/0":
net = None
else:
if not self.subnet.has(subnet=net):
(result, net) = self.subnet.add(subnet=net,
asn=asn,
asn_name=asn_name,
whois_data=whois_data,
cc=cc)
if not result:
return (False, net)
else:
(result, net) = self.subnet.get(subnet=net)
if not result:
return (False, net)
ip = models.IP(ip=first, mask=mask, last=last, subnet=net, af=af)
ip.save()
return (True, ip)
def __init__(self):
self.ipcalc = IPCalc()
self.asnum = ASNum()
class NetData:
def __init__(self):
self.ipcalc = IPCalc()
self.asnum = ASNum()
def revip(self, ip, af):
if af == 4:
t = ip.split(".")
return "%s.%s.%s.%s" % (t[3], t[2], t[1], t[0])
elif af == 6:
ip = ip.split("::")[0].split(":")
ip.reverse()
tmp = ""
for i in ip:
x = "%04X" % (int(i, 16))
tmp += x[::-1]
rev = ""
for i in range(0, len(tmp)):
rev += "%s." % (tmp[i])
return rev[:-1]
def get_origin(self, ip):
af = self.ipcalc.af(ip)
if af == 4:
query = "%s.origin.asn.cymru.com" % (self.revip(ip, af))
elif af == 6:
query = "%s.origin6.asn.cymru.com" % (self.revip(ip, af))
else:
return None
try:
entry = dns.resolver.query(query, "TXT")
except:
return None
for rdata in entry:
data = str(rdata).replace("\"", "")
t = data.split(" | ")
return [t[0], t[1], t[2]]
def get_subnet_asnum(self, ip):
try:
return self.get_origin(ip)[0].split(" ")[0]
except:
return None
def get_asn_name(self, asn):
query = "AS%s.asn.cymru.com" % (asn)
try:
entry = dns.resolver.query(query, "TXT")
except:
return "Unknown"
for rdata in entry:
data = str(rdata).replace("\"", "")
try:
return data.split(" | ")[4]
except:
return "Unknown"
def query_whois(self, server, asnum):
#return commands.getoutput("whois -h %s as%s" % (server, asnum)).split("\n")
return ""
def parse_whois_data(self, raw_whois_data):
rirs = ["APNIC", "LACNIC", "AfriNIC", "RIPE", "ARIN"]
data = ""
in_aut_num = False
in_role = False
in_person = False
for line in raw_whois_data:
if len(line) == 0:
in_aut_num = False
in_role = False
in_person = False
if len(data) != 0 and data[-2:] != "\n\n": data += "\n"
elif line.startswith("aut-num:"): in_aut_num = True
elif line.startswith("ASNumber:"): in_aut_num = True
elif line.startswith("OrgName:"): in_role = True
elif line.startswith("OrgTechHandle:"): in_person = True
elif line.startswith("OrgAbuseHandle:"): in_person = True
elif line.startswith("role:"):
is_rir = False
for rir in rirs:
if rir in line: is_rir = True
if not is_rir: in_role = True
elif line.startswith("person:"): in_person = True
elif line.startswith("import:"): continue
elif line.startswith("export:"): continue
elif line.startswith("default:"): continue
elif line.startswith("source:"): continue
elif line.startswith("remarks:"): continue
elif line.startswith("mp-import:"): continue
elif line.startswith("mp-export:"): continue
elif line.startswith("+"): continue
elif line.startswith("\t"): continue
elif line.startswith(" "): continue
if in_aut_num:
data += "%s\n" % (line)
elif in_role:
data += "%s\n" % (line)
elif in_person:
data += "%s\n" % (line)
return data
def get_whois_data(self, ip):
try:
(asn, net, country) = self.get_origin(ip)
except:
return (0, "UNKNOWN", "Unknown ASN", "0.0.0.0/0", "NL")
asn_name = self.get_asn_name(asn)
asn = asn.split()[0]
(result, asn_obj) = self.asnum.get(asnum=asn)
if not result:
return (False, asn_obj)
raw_whois_data = self.query_whois(asn_obj.rir.whois, asn)
return (asn, asn_name, self.parse_whois_data(raw_whois_data), net,
country)
def __init__(self): BaseBackend.__init__(self, models.IP) self.netdata = NetData() self.ipcalc = IPCalc() self.subnet = Subnet()
#!/usr/bin/env python
import sys
import os
sys.path.append("/www/blacklist/app")
os.environ["DJANGO_SETTINGS_MODULE"] = "settings"
from blacklist.common.ipcalc import IPCalc
ipcalc = IPCalc()
cache_dir = "/www/blacklist/app/scripts/registry-mgmt/cache"
output_dir = "/www/blacklist/app/blacklist/migrations/data"
if __name__ == "__main__":
out = open("%s/subnet-to-rir.txt" % (output_dir), "w")
for rir in ["ARIN", "RIPE", "AfriNIC", "APNIC", "LACNIC"]:
for line in open("%s/%s.txt" % (cache_dir, rir), "r").readlines():
if line.startswith("#"):
continue
elif line.endswith("summary\n"):
continue
t = line.split("|")
if not t[2] in ["ipv4", "ipv6"]:
continue
if "." in t[3]:
af = 4
else:
af = 6
#!/usr/bin/env python
import datetime
import os
import re
import sys
import time
import urllib2
sys.path.append("/www/blacklist/app")
os.environ["DJANGO_SETTINGS_MODULE"] = "settings"
from django.db import transaction, reset_queries
from blacklist.common.ipcalc import IPCalc
from blacklist.common.netdata import NetData
from blacklist.models import *
ipcalc = IPCalc()
error_dir = "/projects/registry-mgmt/errors"
if __name__ == "__main__":
for rir in ["ARIN", "RIPE", "AfriNIC", "APNIC", "LACNIC"]:
for l in open("%s/%s-no-mask.txt" % (error_dir, rir), "r").readlines():
l = l.strip()
(net, num_ips) = l.split("|")
print "%s(%s): %s" % (net, num_ips, len(ipcalc.aggregates(ipcalc.dqtoi(net), int(num_ips), [])))
# print ipcalc.aggregates(ipcalc.dqtoi("196.13.138.0"), 7419)
class NetData:
def __init__(self):
self.ipcalc = IPCalc()
self.asnum = ASNum()
def revip(self, ip, af):
if af == 4:
t = ip.split(".")
return "%s.%s.%s.%s" % (t[3], t[2], t[1], t[0])
elif af == 6:
ip = ip.split("::")[0].split(":")
ip.reverse()
tmp = ""
for i in ip:
x = "%04X" % (int(i, 16))
tmp += x[::-1]
rev = ""
for i in range(0, len(tmp)):
rev += "%s." % (tmp[i])
return rev[:-1]
def get_origin(self, ip):
af = self.ipcalc.af(ip)
if af == 4:
query = "%s.origin.asn.cymru.com" % (self.revip(ip, af))
elif af == 6:
query = "%s.origin6.asn.cymru.com" % (self.revip(ip, af))
else:
return None
try:
entry = dns.resolver.query(query, "TXT")
except:
return None
for rdata in entry:
data = str(rdata).replace("\"", "")
t = data.split(" | ")
return [t[0], t[1], t[2]]
def get_subnet_asnum(self, ip):
try:
return self.get_origin(ip)[0].split(" ")[0]
except:
return None
def get_asn_name(self, asn):
query = "AS%s.asn.cymru.com" % (asn)
try: entry = dns.resolver.query(query, "TXT")
except: return "Unknown"
for rdata in entry:
data = str(rdata).replace("\"", "")
try: return data.split(" | ")[4]
except: return "Unknown"
def query_whois(self, server, asnum):
#return commands.getoutput("whois -h %s as%s" % (server, asnum)).split("\n")
return ""
def parse_whois_data(self, raw_whois_data):
rirs = ["APNIC", "LACNIC", "AfriNIC", "RIPE", "ARIN"]
data = ""
in_aut_num = False
in_role = False
in_person = False
for line in raw_whois_data:
if len(line) == 0:
in_aut_num = False
in_role = False
in_person = False
if len(data) != 0 and data[-2:] != "\n\n": data += "\n"
elif line.startswith("aut-num:"): in_aut_num = True
elif line.startswith("ASNumber:"): in_aut_num = True
elif line.startswith("OrgName:"): in_role = True
elif line.startswith("OrgTechHandle:"): in_person = True
elif line.startswith("OrgAbuseHandle:"): in_person = True
elif line.startswith("role:"):
is_rir = False
for rir in rirs:
if rir in line: is_rir = True
if not is_rir: in_role = True
elif line.startswith("person:"): in_person = True
elif line.startswith("import:"): continue
elif line.startswith("export:"): continue
elif line.startswith("default:"): continue
elif line.startswith("source:"): continue
elif line.startswith("remarks:"): continue
elif line.startswith("mp-import:"): continue
elif line.startswith("mp-export:"): continue
elif line.startswith("+"): continue
elif line.startswith("\t"): continue
elif line.startswith(" "): continue
if in_aut_num:
data += "%s\n" % (line)
elif in_role:
data += "%s\n" % (line)
elif in_person:
data += "%s\n" % (line)
return data
def get_whois_data(self, ip):
try:
(asn, net, country) = self.get_origin(ip)
except:
return (0, "UNKNOWN", "Unknown ASN", "0.0.0.0/0", "NL")
asn_name = self.get_asn_name(asn)
asn = asn.split()[0]
(result, asn_obj) = self.asnum.get(asnum=asn)
if not result:
return (False, asn_obj)
raw_whois_data = self.query_whois(asn_obj.rir.whois, asn)
return (asn, asn_name, self.parse_whois_data(raw_whois_data), net, country)
class Search:
def __init__(self, model=None, fields=[]):
self.model = model
self.fields = fields
self.operators = ["and", "or"]
self.quotes = ["\"", "\'"]
self.fs = "="
self.ipcalc = IPCalc()
def _construct_ip(self, cur_field, cur_value, is_negate):
qset = None
if "/" in cur_value:
(ip, mask) = cur_value.split("/")
else:
ip = cur_value
mask = None
try:
ip = self.ipcalc.dqtoi(ip)
except ValueError:
return (False, "%s is not a valid ip address" % (cur_value))
first = eval("Q(%s__lte=\"%s\")" % (self.fields[cur_field], ip))
if cur_field == "ip":
last = eval("Q(ip__last__gte=\"%s\")" % (ip))
else:
last = eval("Q(last__gte=\"%s\")" % (ip))
qset = (first & last)
if mask:
qset = qset & eval("Q(mask=\"%s\")" % (mask))
if is_negate:
return ~(qset)
else:
return (qset)
def _construct(self, cur_field, cur_value, is_negate):
qset_list = []
operator_list = []
for quote in self.quotes:
cur_value.replace("\%s" % (quote), "")
if cur_field in ["ip", "subnet"]:
qset_list.append(self._construct_ip(cur_field, cur_value, is_negate))
else:
if is_negate:
qset_list.append(eval("~Q(%s__icontains=\"%s\")" % (self.fields[cur_field], cur_value)))
else:
qset_list.append(eval("Q(%s__icontains=\"%s\")" % (self.fields[cur_field], cur_value)))
return (qset_list, operator_list)
def find(self, q):
qset = None
qset_list = []
is_field = True
is_value = False
is_operator = False
is_multiword = False
is_negate = False
is_ip = True
has_quote = False
cur_field = ""
cur_value = ""
operator_list = []
for i in range(0, len(q)):
if q[i] == self.fs:
is_field = False
is_value = True
cur_field = cur_field.strip()
if cur_field.startswith("not "):
is_negate = True
cur_field = cur_field[4:]
cur_field = cur_field.strip()
if not cur_field in self.fields.keys():
return (False, "%s is not a valid field" % (cur_field))
continue
if q[i] in self.quotes:
is_multiword = not is_multiword
continue
if is_field:
cur_field += q[i]
if is_value:
cur_value += q[i]
if cur_value.endswith(" and") and not is_multiword:
cur_value = cur_value[:-4]
for quote in self.quotes:
cur_value.replace("\%s" % (quote), "")
operator_list.append("and")
(ql, ol) = self._construct(cur_field, cur_value, is_negate)
for i in ql: qset_list.append(i)
for i in ol: operator_list.append(i)
is_field = True
is_value = False
is_negate = False
cur_field = ""
cur_value = ""
if cur_value.endswith(" or") and not is_multiword:
cur_value = cur_value[:-3]
for quote in self.quotes:
cur_value.replace(quote, "")
operator_list.append("or")
(ql, ol) = self._construct(cur_field, cur_value, is_negate)
for i in ql: qset_list.append(i)
for i in ol: operator_list.append(i)
is_field = True
is_value = False
is_negate = False
is_ip = False
cur_field = ""
cur_value = ""
(ql, ol) = self._construct(cur_field, cur_value, is_negate)
for i in ql: qset_list.append(i)
for i in ol: operator_list.append(i)
qset = qset_list[0]
del(qset_list[0])
for operator in operator_list:
if operator == "and":
qset = qset & qset_list[0]
del(qset_list[0])
elif operator == "or":
qset = qset | qset_list[0]
del(qset_list[0])
return self.model.filter(qset)
def __init__(self):
BaseBackend.__init__(self, models.HistoryListing)
self.netdata = NetData()
self.ipcalc = IPCalc()
#!/usr/bin/env python
import sys
import os
sys.path.append("/www/blacklist/app")
os.environ["DJANGO_SETTINGS_MODULE"] = "settings"
from blacklist.common.ipcalc import IPCalc
ipcalc = IPCalc()
cache_dir = "/www/blacklist/app/scripts/registry-mgmt/cache"
output_dir = "/www/blacklist/app/blacklist/migrations/data"
if __name__ == "__main__":
out = open("%s/subnet-to-rir.txt" % (output_dir), "w")
for rir in ["ARIN", "RIPE", "AfriNIC", "APNIC", "LACNIC"]:
for line in open("%s/%s.txt" % (cache_dir, rir), "r").readlines():
if line.startswith("#"):
continue
elif line.endswith("summary\n"):
continue
t = line.split("|")
if not t[2] in ["ipv4", "ipv6"]:
continue
if "." in t[3]:
af = 4
else:
af = 6
