def nearby_probes_diff_ldns(svl, rmask=16):
print("reducing svl to only probes with public ldns")
svl = [sv for sv in svl if ipp.is_public(sv.get_ldns())]
print("getting asn descriptor list...")
asvl = vv.asn_svl(svl)
nearbies = list()
strnearbies = list()
for asn in asvl:
if asn is None:
continue
ssvl = vv.subnet_svl(asvl[asn], 16)
for subnet in ssvl:
if subnet is None:
continue
if len(ssvl[subnet]) > 1:
csvl = vv.country_svl(ssvl[subnet])
for country in csvl:
if country is None:
continue
if len(csvl[country]) > 1:
osvl = vv.owner_svl(csvl[country])
for owner in osvl:
if owner is None:
continue
if len(osvl[owner]) > 1:
resolvers = [z.get_ldns() for z in osvl[owner]]
# collapse redundant resolvers in the same
# subnet (e.g., 8.8.8.8 and 8.8.4.4 -> 8.8.x.x)
fmtmask = ipp.make_v4_prefix_mask(rmask)
r2 = defaultdict(list)
for r in resolvers:
r2[r & fmtmask].append(r)
# keep resolvers with at least 1 probe //2 probes
keep = list()
k2 = list()
for z in r2:
if len(r2[z]) > 1:
# // keep += r2[z]
k2.append(z)
keep += r2[z]
#if len(k2) > 0:
if len(r2) > 1:
print("has different resolvers!!!")
if len(keep
) > 1: # if there's stuff to compare
print("found some!!")
print("asn: " + str(asn))
svl = [sv for sv in osvl[owner] if sv.get_ldns() \
in keep]
nearbies.append(svl)
strnearbies.append(["|~"+ipp.int2ip(z.get_ldns())+\
"__"+ipp.int2ip(z.get_ip())+"~|" \
for z in svl])
df.overwrite(plotsdir + "nearbies.csv", df.list2col(strnearbies))
logger.warning("nearbies: " + str(len(nearbies)))
return nearbies
def sees_private_ldns(self):
tmp = self.get_probe_info()
if tmp is not None:
ip = tmp['ipv4']['perceived_ldns']
if IP(ipp.int2ip(ip) + "/32").iptype() == "PUBLIC":
return False
return True
def get_owner(self):
if hasattr(self, "owner"):
if self.owner is not None:
return self.owner
self.owner = dt.get_owner_name(ipp.int2ip(self.get_ip()))
return self.owner
def __init__(self, d, mask=32, oddballs=False):
'''
:param d: a single subdict d from the dict dd output from
window_to_dicts()
:param mask: prefix mask t use on domain IPs
:param oddballs: if True, will include non-public IPs (10.x.x.x, etc); if
False, will only include public IPs
container for a set of query results from a single probe
NOTE: mask is only applied to domain IPs (not the probe's IP)
NOTE: after mask is applied, original IPs are not preserved; for
comparisons, separate sv's should be constructed
'''
self.vec = defaultdict(ddfloat) # {domain: {ans: cum. weight}}
fmtmask = ipp.make_v4_prefix_mask(mask)
self.query_count = defaultdict(float)
self.answer_count = defaultdict(float)
self.inds = defaultdict(list)
for i, dom in enumerate(d['domains']):
query_count = float(len(d['ind'][i]))
answer_count = float(len(d['answers'][i]))
qaratio = query_count / answer_count
self.query_count[dom] = query_count
self.answer_count[dom] = answer_count
self.inds[dom] = d['ind'][i]
for ip in d['answers'][i]:
ipm = ip & fmtmask
ipstr = ipp.int2ip(ipm)
if IP(ipstr + "/32").iptype() == "PUBLIC" or oddballs:
self.vec[dom][ipm] += qaratio
if len(self.vec[dom]) < 1:
del self.vec[dom]
else:
self.vec[dom] = dict(self.vec[dom])
self.vec = dict(self.vec)
self.mask = mask
self.ip = set()
for ipset in d['probe_ip']:
for ip in ipset:
if not isinstance(ip, numbers.Number):
continue
self.ip.add(ip)
self.ldns = set()
for ipset in d['ldns']:
for ip in ipset:
if not isinstance(ip, numbers.Number):
continue
self.ldns.add(ip)
self.ldns = list(self.ldns)
self.id = d['_id']
self.probe_info = None
self.owner = None
def dom_traits(svld, anssets):
'''
:param svld: output from arrange_self_data()
:return: dict {<trait>: {<trait val>: {dom: [clients]}}}, where 'clients'
refers to the list of clients that witnessed said trait value from said
domain
traits include:
ansm -> order of magnitude (bins) of the number of answers
spacem -> order of magnitude (log bins) of the number of IPs seen
across all probes
prefix -> number of bits in registered prefix (TODO)
shared -> bool: does dom'full IP set intersect with that of another dom
NOTE: this is technically difficult to know; for simplicity, I'll
start by only comparing to what I see from my data; manual
inspection or something else is necessary to really say this or not
lpm -> longest prefix match between IPs in dom's full answer space
spm -> shorted prefix match between IPs in dom's full answer space
pilpm -> the (average) longest prefix match for EACH IP. For example,
the 4 bit IPs [1100, 1101, 0110, 0111] would have a pilpm of 3: the
some IP's match '110', and the rest match '011', which both have
a length of 3 bits and in turn average to (3+3+3+3/4) = 3 bits
'''
# {<trait>: {<trait val>: {dom: [clients]}}}
dtd = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
checked = set()
for a in anssets:
if len(anssets[a]) == 0:
continue
checked.add(a)
dtd['spacem'][int(round(log(len(anssets[a]), 2)))][a]
for b in [z for z in anssets if z not in checked]:
overlap = anssets[a].symmetric_difference(anssets[b])
overlap = [ip for ip in overlap if ipp.is_public(ip)]
if len(overlap) > 0:
dtd['shared'][True][a] = True
dtd['shared'][True][b] = True
if a not in dtd['shared'][True]:
dtd['shared'][False][a] = True
mpms = list()
for ip in anssets[a]:
matches = [ipp.prefix_match(ip, z) for z in anssets[a] if \
z != ip]
mpms.append(max(matches))
dtd['pilpm'][int(round(log(np.mean(mpms), 2)))][a] = True
for pid in svld:
svl = svld[pid]
for sv in svl:
for dom in sv:
dtd['ansm'][len(sv.vec[dom])][dom].append(sv)
ipstrs = [ipp.int2ip(z) for z in sv.vec[dom]]
cidrs = [dt.get_cidr(ip) for ip in ipstrs if ipp.is_public(ip)]
masks = [int(cidr.split("/")[1]) for cidr in cidrs if cidr is \
not None]
for mask in masks:
dtd['prefix'][mask][dom].append(sv)
pms = list()
for ip in sv.vec[dom]:
matches = [ipp.prefix_match(ip, z) for z in anssets[dom] if \
z != ip]
pms += matches
dtd['lpm'][max(pms)][dom].append(sv)
dtd['spm'][min(pms)][dom].append(sv)
return dtd
def get_svl(start_time,
duration=30000,
mask=32,
fmt=None,
country_set=None,
oddballs=True,
maxmissing=0,
return_ccache=True,
ccachef=df.rightdir(statedir + "pickles/") + "ccache.pickle",
mindomsize=2):
'''
:param t: int indicating the earliest query the window should include
:param duration: int indication the span of time covered by the window,
in seconds
:param mask: int, prefix mask to use over domain IPs
:param fmt: see transform fmt
:param country_set: the set of countries the window should include queries from.
If None, then all countries will be inluded
:param oddballs: if True, will include non-public IPs (10.x.x.x, etc); if
False, will only include public IPs
:return: list of smartvecs
'''
# if the duration is too long, mongo times out the cursor before we can
# finish processing, so we should break it into smaller jobs with
# get_big_svl()
if duration > 24 * 60 * 60:
logger.warning(
"using get_big_svl(), which assumes that 'duration' is a multple of 24 hours"
)
return get_big_svl(start_time, duration, mask, fmt, country_set,
oddballs, maxmissing, return_ccache, ccachef,
mindomsize)
logger.info("->window...")
window = get_window(start_time,
duration,
country_set=country_set,
domain_set=fmt)
logger.info("->svl...")
svl, doms, anssets = dicts_to_svl(window, mask, oddballs)
logger.debug(str(doms))
fmt = transform_fmt(fmt, doms)
# remove any domains that only have 1 IP (since all nodes will see the
# same thing)
for dom in fmt:
if len(anssets[dom]) < mindomsize \
or len([sv for sv in svl if dom not in sv]) > 0.5*len(svl):
del anssets[dom]
fmt = sorted(list(set(anssets.keys()).intersection(set(fmt))))
svl = reduce_svl(svl, fmt, maxmissing)
for dom in fmt:
logger.debug("-----------" + dom + "-----------")
tmp = sorted(anssets[dom])
for val in tmp:
if type(val) is int:
logger.debug(ipp.int2ip(val))
if return_ccache:
ccache = init_ccache(None, ccachef, start_time, duration, mask, fmt,
oddballs, maxmissing)
return svl, fmt, dict(anssets), ccache
else:
return svl, fmt, dict(anssets)
def get_subnet(self, mask=24):
fmtmask = ipp.make_v4_prefix_mask(mask)
return ipp.int2ip(self.get_ip() & fmtmask)
def get_answers(self, dom):
if dom in self.vec:
return [ipp.int2ip(ip) for ip in self.vec[dom]]
return None
def get_ip_str(self):
return ipp.int2ip(self.get_ip())
def __str__(self):
s = ""
for dom in self.vec:
s += dom + ": " + ", ".join(
[ipp.int2ip(ip) for ip in self.vec[dom]]) + "\n\n"
return s
def __repr__(self):
return "smartvec(ip=" + ipp.int2ip(self.get_ip()) + ")"