def insert_ip_network (self, interface, ip, mask) :
net = IP4_Address (ip, mask)
adr = IP4_Address (ip)
assert (adr in net)
netadr = self.ffm.IP4_Network.instance (adr)
if netadr and not netadr.is_free :
ni = self.ffm.Net_Interface_in_IP4_Network.instance \
(interface, netadr, mask_len = net.mask)
assert (ni)
else :
n = [x for x in self.networks if net in x.net_address]
if len (n) < 1 :
print >> sys.stderr, 'Warning: ignoring address %s' % net
return
assert len (n) == 1
n = n [0]
try :
network = n.reserve (net, self.owner)
except FFM.Error.Address_Already_Used :
network = self.ffm.IP4_Network.instance (net)
netadr = n.reserve (adr, self.owner)
try :
self.ffm.Net_Interface_in_IP4_Network \
(interface, netadr, mask_len = net.mask)
except Exception as exc :
print >> sys.stderr, "Warning: broken IP triggers: %s" % exc
def check_ip_olo(db, ip, olo):
reject_invalid_ip(ip)
sn = db.ip_subnet.find(org_location=olo)
for sn_id in sn:
subnet = db.ip_subnet.getnode(sn_id)
if IP4_Address(ip) in IP4_Address(subnet.ip, subnet.netmask):
return
olo_name = db.org_location.get(olo, 'name')
ip_sn = _('ip_subnet')
raise Reject, _ ("IP %(ip)s does not match any %(ip_sn)s of %(olo_name)s") \
% locals ()
def check_duplicate_ip_subnet(cl, nodeid, ip, mask):
for sn_id in cl.getnodeids():
if sn_id == nodeid: continue
n = _('ip_subnet')
ips = cl.getnode(sn_id)
o_ip = ips.ip
o_mask = ips.netmask
if IP4_Address(ip, mask).overlaps(IP4_Address(o_ip, o_mask)):
raise Reject, \
( ("%(n)s %(ip)s %(mask)s overlaps with %(o_ip)s %(o_mask)s")
% locals ()
)
def create_interfaces(self):
for iface in self.contents['ip']:
if iface.node_id not in self.dev_by_id:
print "WARN: Ignoring IP %s %s: no device (node) %s" \
% (iface.id, iface.ip, iface.node_id)
continue
dev = self.dev_by_id[iface.node_id]
net = self.net_by_id[iface.net_id]
ip = IP4_Address(iface.ip)
if ip not in net.net_address:
parent = self.ffm.IP4_Network.query \
( Q.net_address.CONTAINS (ip)
, ~ Q.electric
, sort_key = TFL.Sorted_By ("-net_address.mask_len")
).first ()
print "WARN: IP %s %s not in net %s %s found %s" \
% ( iface.id
, iface.ip
, iface.net_id
, net.net_address
, parent.net_address
)
net = parent
nw = net.reserve(ip, owner=dev.node.owner)
nif = self.ffm.Wired_Interface(left=dev, name=iface.name)
nii = self.ffm.Net_Interface_in_IP4_Network \
(nif, nw, mask_len = 32, name = iface.name)
self.nifin_by_id[iface.id] = nii
if len(self.scope.uncommitted_changes) > 10:
self.scope.commit()
def create_nettypes(self):
""" Network ranges for reservation
"""
by_mask = {}
# first group by netmask
for nw in self.contents['nettype']:
if not nw.comment:
print 'WARN: Ignoring nettype %s "%s"' % (nw.id, nw.name)
continue
for net_ip in nw.comment.split(','):
ip = IP4_Address(net_ip)
if ip.mask not in by_mask:
by_mask[ip.mask] = []
by_mask[ip.mask].append((ip, nw.name, nw.id))
typ = self.ffm.IP4_Network
for mask in sorted(by_mask):
for ip, name, id in by_mask[mask]:
if id not in self.ntype_by_id:
self.ntype_by_id[id] = []
r = typ.query \
( Q.net_address.CONTAINS (ip)
, ~ Q.electric
, sort_key = TFL.Sorted_By ("-net_address.mask_len")
).first ()
reserver = r.reserve if r else typ
network = reserver(ip, owner=self.graz_admin)
self.ntype_by_id[id].append(network)
if name:
network.set_raw(desc=name)
def check_spider_dev(self, sdev, in4, ips, nodeid):
i4 = in4.ip
nodename = self.ffm_node_by_id[nodeid].name
if not routable(i4):
return
ip4 = IP4_Address(i4)
if i4 not in ips:
pyk.fprint \
( "WARN: IP %s of spidered device %s not in mid dev for node %s"
% (i4, sdev.mainip, nodename)
)
if ip4 not in self.ip_by_ip:
pyk.fprint \
( "WARN: IP %s of spidered device %s not in ips"
% (i4, sdev.mainip)
)
else:
d = self.ip_by_ip[ip4].id_devices
if d:
dev = self.dev_by_id[d]
nid = dev.id_nodes
node = self.ffm_node_by_id[dev.id_nodes]
pyk.fprint \
( "WARN: IP %s of spidered device %s"
" belongs to dev %s node %s"
% (i4, sdev.mainip, dev.name, node.name)
)
else:
pyk.fprint \
( "WARN: IP %s of spidered device %s has no device"
% (i4, sdev.mainip)
)
def create(self):
assert not self.merged
dev = self.device.net_device
if self.debug:
print "device: %s ip: %s" % (self.device, self.ip)
ffm = self.convert.ffm
desc = []
if self.names:
desc.append('Spider Interfaces: %s' % ', '.join(self.names))
if self.spider_ip:
desc.append('Spider IP: %s' % self.spider_ip)
desc = '\n'.join(desc) or None
if self.is_wlan:
iface = self.net_interface = ffm.Wireless_Interface \
(left = dev, name = self.ifname, desc = desc, raw = True)
if self.wlan_info:
std = ffm.Wireless_Standard.instance \
(name = self.wlan_info.standard, raw = True)
mode = self.wlan_info.mode.lower()
mode = self.wl_modes[mode]
bsid = self.wlan_info.bssid
ssid = self.wlan_info.ssid
if bsid is not None and len(bsid.split(':')) != 6:
print "INFO: Ignoring bssid: %s" % bsid
bsid = None
if ssid is not None:
ssid = ssid.replace(r'\x09', '\x09')
iface.set_raw \
( mode = mode
, essid = ssid
, bssid = bsid
, standard = std
)
chan = ffm.Wireless_Channel.instance \
(std, self.wlan_info.channel, raw = True)
ffm.Wireless_Interface_uses_Wireless_Channel(iface, chan)
else:
iface = self.net_interface = ffm.Wired_Interface \
(left = dev, name = self.ifname, desc = desc, raw = True)
manager = dev.node.manager
for ip in self.ips.itervalues():
if self.verbose:
pyk.fprint \
( "Adding IP %s to iface: %s/%s (of dev %s)"
% (ip.ip, self.name, self.idxdev.name, self.device.name)
)
assert not ip.done
ip.set_done()
net = IP4_Address(ip.ip, ip.cidr)
network = ffm.IP4_Network.instance(net)
netadr = network.reserve(ip.ip, manager)
ffm.Net_Interface_in_IP4_Network \
(iface, netadr, mask_len = 32)
def create_networks(self):
for net in self.contents['net']:
parents = self.ntype_by_id.get(net.nettype_id, [])
node = self.ffm_node.get(net.location_id)
ip = IP4_Address(net.netip, net.netmask)
if node:
owner = node.owner
else:
print "WARN: Network %s %s Location %s missing" \
% (net.id, net.netip, net.location_id)
owner = self.graz_admin
parent = None
for p in parents:
if ip in p.net_address:
parent = p
break
else:
parent = None
for ps in self.ntype_by_id.itervalues():
for p in ps:
if ip in p.net_address:
parent = p
print "Got parent in ntype_by_id: %s" % parent
break
else:
parent = self.ffm.IP4_Network.query \
( Q.net_address.CONTAINS (ip)
, ~ Q.electric
, sort_key = TFL.Sorted_By ("-net_address.mask_len")
).first ()
if parent:
print "Got parent by network query: %s" % parent
if parent:
reserver = parent.reserve
else:
print "WARN: No parent: new network: %s" % ip
reserver = self.ffm.IP4_Network
network = reserver(ip, owner=owner)
self.net_by_id[net.id] = network
if node:
pool = self.ffm.IP4_Pool(left=network, node=node)
if net.comment:
network.set_raw(desc=net.comment)
if len(self.scope.uncommitted_changes) > 10:
self.scope.commit()
def main():
import sys
import pickle
from optparse import OptionParser
cmd = OptionParser()
cmd.add_option \
( "-d", "--debug"
, dest = "debug"
, action = "store_true"
, help = "Debug merging of pickle dumps"
)
cmd.add_option \
( "-l", "--local"
, dest = "local"
, action = "store_true"
, help = "Use local download for testing with file:// url"
)
cmd.add_option \
( "-o", "--output-pickle"
, dest = "output_pickle"
, help = "Optional pickle output file"
)
cmd.add_option \
( "-p", "--port"
, dest = "port"
, help = "Optional port number to fetch from"
, type = "int"
, default = 0
)
cmd.add_option \
( "-r", "--read-pickle"
, dest = "read_pickle"
, help = "Read old pickle files, merge and preserve information"
, action = "append"
, default = []
)
cmd.add_option \
( "-V", "--version-statistics"
, dest = "version_statistics"
, help = "Output version information by spidered IP"
)
cmd.add_option \
( "-I", "--interface-info"
, dest = "interface_info"
, help = "Output interface information by spidered IP"
)
cmd.add_option \
( "-v", "--verbose"
, dest = "verbose"
, action = "count"
, help = "Show verbose results"
)
(opt, args) = cmd.parse_args()
if len(args) < 1 and not opt.read_pickle:
cmd.print_help()
sys.exit(23)
ipdict = {}
for fn in opt.read_pickle:
if opt.debug:
print "Processing pickle dump %s" % fn
keys = dict.fromkeys(ipdict.iterkeys())
f = open(fn, 'r')
obj = pickle.load(f)
for k, v in obj.iteritems():
# Fixup of object
if isinstance(v, Guess) and not hasattr(v, 'rqinfo'):
v.rqinfo = {}
v.rqinfo['ips'] = v.status.ips
v.rqinfo['interfaces'] = v.status.if_by_name
v.status = None
if k in ipdict:
keys[k] = True
ov = ipdict[k]
istuple = isinstance(v, tuple)
if isinstance(ov, tuple):
overwrite = False
if not istuple:
overwrite = True
elif ov[0] == 'Timeout_Error':
overwrite = True
elif v[0] == 'ValueError':
overwrite = True
if overwrite:
#print opt.debug, istuple, v, ov [0]
if (opt.debug and (not istuple or v[0] != ov[0])):
print "%s: overwriting %s with %s" % (k, ov, v)
ipdict[k] = v
elif istuple and ov[0] != v[0] and opt.debug:
print "%s: Not overwriting %s with %s" % (k, ov, v)
else:
assert isinstance(ov, Guess)
if istuple:
if opt.debug:
print "%s: Not overwriting %s with %s" % (k, ov, v)
else:
assert isinstance(v, Guess)
ipdict[k] = v
else:
if opt.debug:
print "%s: new: %s" % (k, v)
ipdict[k] = v
if opt.debug:
for k, v in keys.iteritems():
if not v:
print "%s: not existing in dump %s" % (k, fn)
for ip in args:
port = opt.port
try:
ip, port = ip.split(':', 1)
except ValueError:
pass
site = site_template % locals()
url = ''
# For testing we download the index page and cgi-bin-status.html
# page into a directory named with the ip address
if opt.local:
site = 'file://' + os.path.abspath(ip)
url = 'index.html'
ff = Guess(site=site, ip=ip, url=url, port=port)
print ff.verbose_repr()
ipdict[str(ip)] = ff
if opt.output_pickle:
f = open(opt.output_pickle, 'wb')
pickle.dump(ipdict, f)
f.close()
key = lambda x: IP4_Address(x[0])
if opt.version_statistics:
fields = ['address', 'type', 'version']
f = open(opt.version_statistics, 'w')
dw = DictWriter(f, fields, delimiter=';')
dw.writerow(dict((k, k) for k in fields))
for ip, guess in sorted(ipdict.iteritems(), key=key):
if isinstance(guess, Guess):
dw.writerow \
( dict
( address = str (ip)
, version = guess.version
, type = guess.type
)
)
f.close()
if opt.interface_info:
fields = \
[ 'address', 'interface', 'wlan'
, 'ssid', 'mode', 'channel', 'bssid'
, 'ip4', 'ip6'
]
f = open(opt.interface_info, 'w')
dw = DictWriter(f, fields, delimiter=';')
dw.writerow(dict((k, k) for k in fields))
for ip, guess in sorted(ipdict.iteritems(), key=key):
if isinstance(guess, Guess):
for iface in guess.interfaces.itervalues():
wi = iface.wlan_info
d = dict \
( address = str (ip)
, interface = iface.name
, wlan = bool (wi)
, ip4 = ' '.join (str (i.ip) for i in iface.inet4)
, ip6 = ' '.join (str (i.ip) for i in iface.inet6)
)
if wi:
d.update \
( channel = wi.channel
, bssid = wi.bssid
, ssid = wi.ssid
, mode = wi.mode
)
dw.writerow(d)
f.close()
if opt.verbose:
for ip, guess in sorted(ipdict.iteritems(), key=key):
if opt.verbose > 1:
print "%-15s" % ip
print '=' * 15
if isinstance(guess, Guess):
print guess.verbose_repr()
else:
print guess
else:
print "%-15s: %s" % (ip, guess)
def build_device_structure(self):
for n in self.contents['nodes']:
self.node_by_id[n.id] = n
for d in self.contents['devices']:
if d.id_nodes not in self.dev_by_node:
self.dev_by_node[d.id_nodes] = []
self.dev_by_node[d.id_nodes].append(d)
self.cons_dev[d.id] = Consolidated_Device(self, d)
for ip in self.contents['ips']:
self.ip_by_ip[IP4_Address(ip.ip)] = ip
if ip.id_devices:
did = ip.id_devices
self.cons_dev[did].add_redeemer_ip(ip)
net = IP4_Address(ip.ip, ip.cidr)
if net not in self.ip4nets:
pyk.fprint("WARN: Adding network reservation: %s" % net)
self.ip4nets[net] = True
# consistency check of olsr data against redeemer db
# check nodes from topology
for ip4 in self.olsr_nodes:
if ip4 not in self.ip_by_ip:
pyk.fprint("WARN: ip %s from olsr topo not in ips" % ip4)
del self.olsr_nodes[ip4]
# check mid table
midkey = []
midtbl = {}
for ip4, aliases in self.olsr_mid.iteritems():
if ip4 not in self.ip_by_ip:
pyk.fprint("WARN: key ip %s from olsr mid not in ips" % ip4)
midkey.append(ip4)
for a in aliases:
if a not in self.ip_by_ip:
pyk.fprint("WARN: ip %s from olsr mid not in ips" % a)
if ip4 not in midtbl:
midtbl[ip4] = []
midtbl[ip4].append(a)
assert not midkey
for k, v in midtbl.iteritems():
x = dict.fromkeys(self.olsr_mid[k])
for ip4 in v:
del x[ip4]
self.olsr_mid[k] = x.keys()
# consolidate devices using information from spider data
node_by_sdev = {}
for mainip, sdev in sorted(self.spider_devs.iteritems()):
if sdev.done:
continue
sdev.done = True
seen_ip = {}
nodeid = None
for sif in sorted(sdev.interfaces.itervalues()):
assert not sif.done
sif.done = True
for in4 in sorted(sif.inet4):
if unroutable(in4.ip):
continue
seen_ip[in4.ip] = 1
i4 = IP4_Address(in4.ip)
ip = self.ip_by_ip.get(i4)
if not ip:
pyk.fprint \
("WARN: ip %s from spider not in redeemer" % i4)
continue
if not ip.id_devices:
pyk.fprint("ERR: ip %s from spider has no device" % i4)
continue
d = self.cons_dev[ip.id_devices]
if sdev not in node_by_sdev:
node_by_sdev[sdev] = {}
if d.id_nodes not in node_by_sdev[sdev]:
node_by_sdev[sdev][d.id_nodes] = {}
if d.id not in node_by_sdev[sdev][d.id_nodes]:
node_by_sdev[sdev][d.id_nodes][d.id] = {}
node_by_sdev[sdev][d.id_nodes][d.id][in4.ip] = True
assert mainip in seen_ip
for sdev, nodes in sorted(node_by_sdev.iteritems()):
if len(nodes) > 1:
pyk.fprint \
( "WARN: spider device %s expands to %s nodes: %s"
% ( sdev.mainip
, len (nodes)
, ', '.join
(self.node_by_id [n].name for n in nodes.iterkeys ())
)
)
for n, devs in sorted(nodes.iteritems()):
sdevs = {}
sifs = {}
dev1 = None
err = False
for devid, ips in sorted(devs.iteritems()):
d = self.cons_dev[devid]
if d.merged:
pyk.fprint \
("ERR: %s already merged to %s" % (d, d.merged))
err = True
continue
if dev1 and d.id != dev1.id:
if self.verbose:
pyk.fprint \
( "Spider %-15s: Merging device %s.%s to %s.%s"
% ( sdev.mainip
, d.node.name
, d.name
, dev1.node.name
, dev1.name
)
)
assert dev1 != d
assert not dev1.merged
dev1.merge(d)
else:
dev1 = d
for ip in sorted(ips.iterkeys()):
sdevs[self.spider_devs[ip]] = True
if self.spider_iface[ip] not in sifs:
sifs[self.spider_iface[ip]] = {}
sifs[self.spider_iface[ip]][ip] = True
if not err:
assert len(sdevs) == 1
if sdev not in sdevs:
pyk.fprint("ERR: Merged interface differ:")
pyk.fprint("------------------------------")
pyk.fprint(sdevs.keys()[0].verbose_repr())
pyk.fprint("------------------------------")
pyk.fprint(sdev.verbose_repr())
pyk.fprint("------------------------------")
assert len(sifs) >= 1
assert dev1
for sif, ips in sorted(sifs.iteritems()):
l = len(ips)
assert l >= 1
ifaces = {}
for ip in ips:
ifaces[ip] = dev1.interfaces[ip]
assert len(ifaces) == len(ips)
if1 = ip1 = None
for ip, ifc in sorted(ifaces.iteritems()):
if if1:
pyk.fprint \
( "Spider %-15s: "
"Merging iface %s.%s:%s to %s.%s:%s"
% ( sdev.mainip
, d.node.name
, d.name
, ip
, dev1.node.name
, dev1.name
, ip1
)
)
if1.merge(ifc)
else:
if1 = ifc
ip1 = ip
if sif.is_wlan:
if1.is_wlan = True
if1.wlan_info = getattr(sif, 'wlan_info', None)
if1.names = sif.names
if1.spider_ip = sif.device.mainip
# compound devices from mid table
# We index nodes by mid-table entry (by the mid key-ip address)
# for each mid entry there can be several nodes (config bug)
for ip4, aliases in sorted(self.olsr_mid.iteritems()):
nodes = {}
ip = self.ip_by_ip[ip4]
if ip.id_devices:
d = self.cons_dev[ip.id_devices]
d.mid_ip = ip4
nodes[d.id_nodes] = d
else:
pyk.fprint("ERR: key %s from mid has no device" % ip4)
for a in sorted(aliases):
ip = self.ip_by_ip[a]
if not ip.id_devices:
pyk.fprint("ERR: %s from mid %s has no device" % (a, ip4))
continue
d = self.cons_dev[ip.id_devices]
d.mid_ip = ip4
if d.id_nodes not in nodes:
nodes[d.id_nodes] = d
elif d != nodes[d.id_nodes]:
if d.merged:
if d.merged != nodes[d.id_nodes]:
pyk.fprint \
( "ERR: %s already merged to %s "
"not merging to %s"
% (d, d.merged, nodes [d.id_nodes])
)
continue
assert not nodes[d.id_nodes].merged
nodes[d.id_nodes].merge(d)
if len(nodes) > 1:
pyk.fprint \
("WARN: mid %s expands to %s nodes" % (ip4, len (nodes)))
def main () :
cmd = ArgumentParser ()
cmd.add_argument \
( "-D", "--debug"
, dest = "debug"
, help = "Turn on debug logging"
, action = "store_true"
, default = False
)
cmd.add_argument \
( "-d", "--dump"
, dest = "dump"
, help = "Destination file of pickle dump, default: %(default)s"
, default = "Funkfeuer-spider-pickle.dump"
)
cmd.add_argument \
( "-i", "--ip-port"
, dest = "ip_port"
, action = "append"
, help = "IP-Addres:Port combination with non-standard port"
, default = []
)
cmd.add_argument \
( "-n", "--limit-devices"
, dest = "limit_devices"
, help = "Limit spidering to given number of devices"
, type = int
, default = 0
)
cmd.add_argument \
( "-p", "--processes"
, dest = "processes"
, help = "Use given number of processes, default: %(default)s"
, type = int
, default = 20
)
cmd.add_argument \
( "-o", "--olsr-file"
, dest = "olsr_file"
, help = "File or Backfire-URL containing OLSR information, "
"default: %(default)s"
, default = "olsr/txtinfo.txt"
)
cmd.add_argument \
( "-t", "--timeout"
, dest = "timeout"
, help = "Timout in seconds for subprocesses, default: %(default)s"
, type = int
, default = 180
)
cmd.add_argument \
( "-v", "--verbose"
, dest = "verbose"
, help = "Show verbose results"
, action = "count"
)
opt = cmd.parse_args ()
sp = Spider \
( opt.olsr_file
, opt.processes
, opt.limit_devices
, opt.timeout
, dict (x.split (':', 1) for x in opt.ip_port)
, opt.debug
)
try :
sp.process ()
if opt.dump.endswith ('.gz') :
f = GzipFile (opt.dump, "wb", 9)
else :
f = open (opt.dump, "wb")
pickle.dump (sp.result_dict, f)
f.close ()
if opt.verbose :
for k, v in sorted \
( pyk.iteritems (sp.result_dict)
, key = lambda z : IP4_Address (z [0])
) :
print (k, v)
except Exception as err :
sp.log_exception ()
#
# This module is licensed under the terms of the BSD 3-Clause License
# <http://www.c-tanzer.at/license/bsd_3c.html>.
# #*** </License> ***********************************************************#
from _TFL.pyk import pyk
from rsclib.autosuper import autosuper
from rsclib.IP_Address import IP4_Address
class Parse_Error(ValueError):
pass
rfc1918_networks = \
[IP4_Address (x)
for x in ('10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16')
]
localnet = IP4_Address('127.0.0.0/8')
linklocal = IP4_Address('169.254.0.0/16')
def is_rfc1918(ip):
for n in rfc1918_networks:
if ip in n:
return True
return False
# end def is_rfc1918
def unroutable(ip):
ip = IP4_Address(ip)
return is_rfc1918(ip) or is_local(ip) or is_link_local(ip)
def __init__(self, *args, **kw):
self.__super.__init__(*args, **kw)
self.ip = IP4_Address(self.ip)
def add(self, ip, *aliases):
ip = IP4_Address(ip)
aliases = [IP4_Address(a) for a in aliases]
assert ip not in self.by_ip or self.by_ip[ip] == aliases
self.by_ip[ip] = aliases
def main():
import sys
import pickle
cmd = ArgumentParser()
cmd.add_argument \
( "-d", "--debug"
, dest = "debug"
, action = "store_true"
, help = "Debug merging of pickle dumps"
)
cmd.add_argument \
( "-l", "--local"
, dest = "local"
, action = "store_true"
, help = "Use local download for testing with file:// url"
)
cmd.add_argument \
( "-o", "--output-pickle"
, dest = "output_pickle"
, help = "Optional pickle output file"
)
cmd.add_argument \
( "-p", "--port"
, dest = "port"
, help = "Optional port number to fetch from"
, type = int
, default = 0
)
cmd.add_argument \
( "-r", "--read-pickle"
, dest = "read_pickle"
, help = "Read old pickle files, merge and preserve information"
, action = "append"
, default = []
)
cmd.add_argument \
( "-V", "--version-statistics"
, dest = "version_statistics"
, help = "Output version information by spidered IP"
)
cmd.add_argument \
( "-I", "--interface-info"
, dest = "interface_info"
, help = "Output interface information by spidered IP"
)
cmd.add_argument \
( "-v", "--verbose"
, dest = "verbose"
, action = "count"
, help = "Show verbose results"
)
cmd.add_argument \
( "ip"
, nargs = "*"
, help = "IP Address(es)"
)
opt = cmd.parse_args()
if len(opt.ip) < 1 and not opt.read_pickle:
cmd.print_help()
sys.exit(23)
ipdict = {}
for fn in opt.read_pickle:
if opt.debug:
print("Processing pickle dump %s" % fn)
keys = dict.fromkeys(pyk.iterkeys(ipdict))
mt = None
if fn == '-':
f = sys.stdin
else:
mt = datetime.utcfromtimestamp(os.stat(fn)[ST_MTIME])
if fn.endswith('.gz'):
f = GzipFile(fn, 'r')
else:
f = open(fn, 'r')
obj = pickle.load(f)
for k, v in pyk.iteritems(obj):
# Fixup of object
if isinstance(v, Guess):
if not hasattr(v, 'rqinfo'):
v.rqinfo = {}
v.rqinfo['ips'] = v.status.ips
v.rqinfo['interfaces'] = v.status.if_by_name
v.status = None
if mt and not hasattr(v, 'time'):
v.time = mt
if k in ipdict:
keys[k] = True
ov = ipdict[k]
istuple = isinstance(v, tuple)
if isinstance(ov, tuple):
overwrite = False
if not istuple:
overwrite = True
elif ov[0] == 'Timeout_Error':
overwrite = True
elif v[0] == 'ValueError':
overwrite = True
if overwrite:
#print (opt.debug, istuple, v, ov [0])
if (opt.debug and (not istuple or v[0] != ov[0])):
print("%s: overwriting %s with %s" % (k, ov, v))
ipdict[k] = v
elif istuple and ov[0] != v[0] and opt.debug:
print("%s: Not overwriting %s with %s" % (k, ov, v))
else:
assert isinstance(ov, Guess)
if istuple:
if opt.debug:
print("%s: Not overwriting %s with %s" %
(k, ov, v))
else:
assert isinstance(v, Guess)
ipdict[k] = v
else:
if opt.debug:
print("%s: new: %s" % (k, v))
ipdict[k] = v
if opt.debug:
for k, v in pyk.iteritems(keys):
if not v:
print("%s: not existing in dump %s" % (k, fn))
for ip in opt.ip:
port = opt.port
try:
ip, port = ip.split(':', 1)
except ValueError:
pass
site = site_template % locals()
url = ''
# For testing we download the index page and cgi-bin-status.html
# page into a directory named with the ip address
if opt.local:
site = 'file://' + os.path.abspath(ip)
url = 'index.html'
ff = Guess(site=site, ip=ip, url=url, port=port)
print(ff.verbose_repr())
ipdict[str(ip)] = ff
if opt.output_pickle:
if opt.output_pickle.endswith('.gz'):
f = GzipFile(opt.output_pickle, 'wb', 9)
else:
f = open(opt.output_pickle, 'wb')
pickle.dump(ipdict, f)
f.close()
key = lambda x: IP4_Address(x[0])
if opt.version_statistics:
fields = ['timestamp', 'address', 'type', 'version']
if opt.version_statistics == '-':
f = sys.stdout
else:
f = open(opt.version_statistics, 'w')
dw = DictWriter(f, fields, delimiter=';')
dw.writerow(dict((k, k) for k in fields))
for ip, guess in sorted(pyk.iteritems(ipdict), key=key):
if isinstance(guess, Guess):
dw.writerow \
( dict
( timestamp = guess.time.strftime
("%Y-%m-%d %H:%M:%S+0")
, address = str (ip)
, version = guess.version
, type = guess.type
)
)
f.close()
if opt.interface_info:
fields = \
[ 'timestamp', 'address', 'interface', 'mac', 'wlan'
, 'ssid', 'mode', 'channel', 'bssid'
, 'ip4', 'ip6', 'signal', 'noise'
]
if opt.interface_info == '-':
f = sys.stdout
else:
f = open(opt.interface_info, 'w')
dw = DictWriter(f, fields, delimiter=';')
dw.writerow(dict((k, k) for k in fields))
for ip, guess in sorted(pyk.iteritems(ipdict), key=key):
if isinstance(guess, Guess):
for iface in pyk.itervalues(guess.interfaces):
wi = iface.wlan_info
mc = None
if iface.link:
mc = iface.link.mac
d = dict \
( timestamp = guess.time.strftime
("%Y-%m-%d %H:%M:%S+0")
, address = str (ip)
, interface = iface.name
, mac = mc
, wlan = bool (wi)
, ip4 = ' '.join (str (i.ip) for i in iface.inet4)
, ip6 = ' '.join (str (i.ip) for i in iface.inet6)
)
if wi:
d.update \
( channel = wi.channel
, bssid = wi.bssid
, ssid = wi.ssid
, mode = wi.mode
, signal = wi.signal
, noise = wi.noise
)
dw.writerow(d)
f.close()
if opt.verbose:
for ip, guess in sorted(pyk.iteritems(ipdict), key=key):
if opt.verbose > 1:
print("%-15s" % ip)
print('=' * 15)
if isinstance(guess, Guess):
print(guess.verbose_repr())
else:
print("Exception:", guess)
else:
print("%-15s: %s" % (ip, guess))
, action = "count"
)
(opt, args) = cmd.parse_args ()
if len (args) :
cmd.print_help ()
sys.exit (23)
sp = Spider \
( opt.olsr_file
, opt.processes
, opt.limit_devices
, opt.timeout
, dict (x.split (':', 1) for x in opt.ip_port)
, opt.debug
)
try :
sp.process ()
if opt.dump.endswith ('.gz') :
f = GzipFile (opt.dump, "wb", 9)
else :
f = open (opt.dump, "wb")
pickle.dump (sp.result_dict, f)
f.close ()
if opt.verbose :
for k, v in sorted \
( sp.result_dict.iteritems ()
, key = lambda z : IP4_Address (z [0])
) :
print k, v
except Exception, err :
sp.log_exception ()
def __init__(self, cmd, scope, debug=False):
self.debug = debug
self.verbose = cmd.verbose
self.anonymize = cmd.anonymize
if len(cmd.argv) > 0:
f = open(cmd.argv[0])
else:
f = sys.stdin
self.ip4nets = {}
self.ip6nets = {}
if cmd.network:
for n in cmd.network:
ip, comment = n.split(';', 1)
if ':' in ip:
ip = IP6_Address(ip)
self.ip6nets[ip] = comment
else:
ip = IP4_Address(ip)
self.ip4nets[ip] = comment
olsr = get_olsr_container(cmd.olsr_file)
self.olsr_nodes = {}
for t in olsr.topo.forward.iterkeys():
self.olsr_nodes[t] = True
for t in olsr.topo.reverse.iterkeys():
self.olsr_nodes[t] = True
self.olsr_mid = olsr.mid.by_ip
self.olsr_hna = olsr.hna
self.rev_mid = {}
for k, v in self.olsr_mid.iteritems():
if k not in self.olsr_nodes:
pyk.fprint("WARN: MIB %s: not in OLSR Topology" % k)
#assert k in self.olsr_nodes
for mid in v:
assert mid not in self.rev_mid
self.rev_mid[mid] = True
self.spider_info = pickle.load(open(cmd.spider_dump, 'rb'))
self.spider_devs = {}
self.spider_iface = {}
for ip, dev in self.spider_info.iteritems():
if self.verbose:
pyk.fprint("IP:", ip)
# ignore spider errors
if not isinstance(dev, Guess):
continue
dev.mainip = ip
dev.done = False
for iface in dev.interfaces.itervalues():
iface.done = False
for ip4 in iface.inet4:
i4 = ip4.ip
# ignore rfc1918, link local, localnet
if unroutable(i4):
continue
if (i4 in self.spider_devs
and self.spider_devs[i4] != dev):
pyk.fprint("WARN: Device %s/%s not equal:" % (ip, i4))
pyk.fprint("=" * 60)
pyk.fprint(dev.verbose_repr())
pyk.fprint("-" * 60)
pyk.fprint(self.spider_devs[i4].verbose_repr())
pyk.fprint("=" * 60)
elif (i4 in self.spider_iface
and self.spider_iface[i4] != iface):
assert dev == self.spider_devs[i4]
spif = self.spider_iface[i4]
pyk.fprint \
( "WARN: Interfaces %s/%s of dev-ip %s share ip %s"
% (iface.name, spif.name, ip, i4)
)
spif.names.append(iface.name)
if iface.is_wlan:
spif.is_wlan = iface.is_wlan
spif.wlan_info = getattr(iface, 'wlan_info', None)
if self.verbose:
pyk.fprint("=" * 60)
pyk.fprint(iface)
pyk.fprint(spif)
pyk.fprint("-" * 60)
pyk.fprint(dev.verbose_repr())
pyk.fprint("=" * 60)
iface = spif
self.spider_devs[i4] = dev
self.spider_iface[i4] = iface
iface.device = dev
if ip not in self.spider_devs:
pyk.fprint("WARN: ip %s not in dev" % ip)
if self.verbose:
pyk.fprint("=" * 60)
pyk.fprint(dev.verbose_repr())
pyk.fprint("=" * 60)
name = 'unknown'
assert name not in dev.interfaces
iface = Interface(4711, name)
iface.done = False
dev.interfaces[name] = iface
iface.device = dev
iface.append_inet4(Inet4(ip, None, None, iface=name))
self.spider_iface[ip] = iface
self.spider_devs[ip] = dev
self.scope = scope
self.ffm = self.scope.FFM
self.pap = self.scope.GTW.OMP.PAP
self.mentor = {}
self.rsrvd_nets = {}
self.ffm_node_by_id = {}
self.node_by_id = {}
self.ip_by_ip = {}
self.email_ids = {}
self.phone_ids = {}
self.person_by_id = {}
self.member_by_id = {}
self.dev_by_node = {}
self.cons_dev = {}
self.parser = SQL_Parser \
(verbose = False, fix_double_encode = True)
self.parser.parse(f)
self.contents = self.parser.contents
self.tables = self.parser.tables
def __init__(self, dest, gw):
self.dest = IP4_Address(dest)
self.gw = IP4_Address(gw)
def __init__(self, dst_ip, last_hop, lq, nlq, cost):
self.dst_ip = IP4_Address(dst_ip)
self.last_hop = IP4_Address(last_hop)
self.lq = lq
self.nlq = nlq
self.cost = cost