def __init__ \
( self
, olsr_file
, processes = 20
, N = 0
, timeout = 180
, ip_port = {}
, debug = False
, ** kw
) :
self.__super.__init__ (**kw)
olsr = get_olsr_container (olsr_file)
self.olsr_nodes = {}
assert len (olsr.topo.forward)
for t in pyk.iterkeys (olsr.topo.forward) :
self.olsr_nodes [t] = True
for t in pyk.iterkeys (olsr.topo.reverse) :
self.olsr_nodes [t] = True
# limit to N elements
if N :
self.olsr_nodes = dict \
((k, v) for k, v in islice (pyk.iteritems (self.olsr_nodes), N))
self.pool = Pool (processes = processes)
self.mgr = Manager ()
self.result_dict = self.mgr.dict ()
self.timeout = timeout
self.ip_port = ip_port
self.debug = debug
olsr_nodes = None
if not debug :
self.log.setLevel (INFO)
self.log.debug ("Starting ...")
def __init__ \
( self
, olsr_file
, processes = 20
, N = 0
, timeout = 180
, ip_port = {}
, debug = False
, ** kw
) :
self.__super.__init__ (**kw)
olsr = get_olsr_container (olsr_file)
self.olsr_nodes = {}
assert len (olsr.topo.forward)
for t in pyk.iterkeys (olsr.topo.forward) :
self.olsr_nodes [t] = True
for t in pyk.iterkeys (olsr.topo.reverse) :
self.olsr_nodes [t] = True
# limit to N elements
if N :
self.olsr_nodes = dict \
((k, v) for k, v in islice (pyk.iteritems (self.olsr_nodes), N))
self.pool = Pool (processes = processes)
self.mgr = Manager ()
self.result_dict = self.mgr.dict ()
self.timeout = timeout
self.ip_port = ip_port
self.debug = debug
olsr_nodes = None
if not debug :
self.log.setLevel (INFO)
self.log.debug ("Starting ...")
def matching_keys (self, abbr) :
"""Returns a list of all keys matching `abbr` if any."""
if abbr in self :
return [abbr]
elif isinstance (abbr, str) :
return TFL.matches (TFL.sorted (pyk.iterkeys (self)), abbr)
return []
def __init__ (self, dict, ** kw) :
self._dict = dict
self.__super.__init__ \
( selection = sorted (pyk.iterkeys (dict))
, cook = self._cooked_key
, ** kw
)
def verbose_repr (self) :
r = [str (self)]
for v in pyk.itervalues (self.interfaces) :
r.append (str (v))
for v in pyk.iterkeys (self.ips) :
r.append (str (v))
return '\n'.join (r)
def reset_predicates(self):
self.errors = errors = {}
self.warnings = warnings = {}
self.missing_required = None
for k in pyk.iterkeys(self.pred_kind):
errors[k] = []
warnings[k] = []
def matching_keys(self, abbr):
"""Returns a list of all keys matching `abbr` if any."""
if abbr in self:
return [abbr]
elif isinstance(abbr, str):
return TFL.matches(TFL.sorted(pyk.iterkeys(self)), abbr)
return []
def reset_predicates (self) :
self.errors = errors = {}
self.warnings = warnings = {}
self.missing_required = None
for k in pyk.iterkeys (self.pred_kind) :
errors [k] = []
warnings [k] = []
def __init__(self, dict, **kw):
self._dict = dict
self.__super.__init__ \
( selection = sorted (pyk.iterkeys (dict))
, cook = self._cooked_key
, ** kw
)
def verbose_repr(self):
r = [str(self)]
for v in pyk.itervalues(self.interfaces):
r.append(str(v))
for v in pyk.iterkeys(self.ips):
r.append(str(v))
return '\n'.join(r)
def as_json(self):
d = dict(type=self.type, version=self.version)
iface = d['interfaces'] = []
for i in pyk.itervalues(self.interfaces):
iface.append(i.as_dict)
ips = d['ips'] = []
for i in pyk.iterkeys(self.ips):
ips.append(str(i))
return json.dumps(d)
def as_json (self) :
d = dict (type = self.type, version = self.version)
iface = d ['interfaces'] = []
for i in pyk.itervalues (self.interfaces) :
iface.append (i.as_dict)
ips = d ['ips'] = []
for i in pyk.iterkeys (self.ips) :
ips.append (str (i))
return json.dumps (d)
def __init__(cls, name, bases, dct):
cls.__m_super.__init__(name, bases, dct)
if cls.C_Type is not None:
cls.P_Type = cls.C_Type.P_Type
Table = getattr(cls, "Table", None)
if Table and not cls.syntax:
cls.syntax = "The following string values are accepted as valid " "%s values: %s" % (
cls.typ,
", ".join(sorted(pyk.iterkeys(Table))),
)
def __init__ \
(self, __dict = {}, __flags = 0, __head = "", __tail = "", ** kw) :
self._map = map = dict (__dict, ** kw)
pattern = "|".join \
( re.escape (k) for k in sorted
(pyk.iterkeys (map), key = lambda k : (-len (k), k))
)
if __head or __tail :
pattern = "".join ((__head, "(", pattern, ")", __tail))
self._group = 1
regexp = Regexp (pattern, __flags)
self.__super.__init__ (regexp, self._replacer, __flags)
def __init__ \
(self, __dict = {}, __flags = 0, __head = "", __tail = "", ** kw) :
self._map = map = dict (__dict, ** kw)
pattern = "|".join \
( re.escape (k) for k in sorted
(pyk.iterkeys (map), key = lambda k : (-len (k), k))
)
if __head or __tail :
pattern = "".join ((__head, "(", pattern, ")", __tail))
self._group = 1
regexp = Regexp (pattern, __flags)
self.__super.__init__ (regexp, self._replacer, __flags)
def syntax(cls):
return "The following string values are accepted as valid " "%s values: %s" % (
cls.typ,
", ".join(sorted(pyk.iterkeys(cls.Table))),
)
def iteritems (self) :
convert = self._convert_element
for key in pyk.iterkeys (self.data) :
yield key, convert (key, self [key])
def iterkeys (self) :
return pyk.iterkeys (self.data)
def iterkeys (self) :
return pyk.iterkeys (self.attribute_changes)
def _Reload (self, * modules) :
for c in pyk.iterkeys (self.__cached) :
delattr (self, c)
self.__cached = {}
self._parent._Reload ()
Package_Namespace._Reload (self, * modules)
def main () :
import sys
import pickle
from optparse import OptionParser
cmd = OptionParser ()
cmd.add_option \
( "-d", "--debug"
, dest = "debug"
, action = "store_true"
, help = "Debug merging of pickle dumps"
)
cmd.add_option \
( "-l", "--local"
, dest = "local"
, action = "store_true"
, help = "Use local download for testing with file:// url"
)
cmd.add_option \
( "-o", "--output-pickle"
, dest = "output_pickle"
, help = "Optional pickle output file"
)
cmd.add_option \
( "-p", "--port"
, dest = "port"
, help = "Optional port number to fetch from"
, type = "int"
, default = 0
)
cmd.add_option \
( "-r", "--read-pickle"
, dest = "read_pickle"
, help = "Read old pickle files, merge and preserve information"
, action = "append"
, default = []
)
cmd.add_option \
( "-V", "--version-statistics"
, dest = "version_statistics"
, help = "Output version information by spidered IP"
)
cmd.add_option \
( "-I", "--interface-info"
, dest = "interface_info"
, help = "Output interface information by spidered IP"
)
cmd.add_option \
( "-v", "--verbose"
, dest = "verbose"
, action = "count"
, help = "Show verbose results"
)
(opt, args) = cmd.parse_args ()
if len (args) < 1 and not opt.read_pickle :
cmd.print_help ()
sys.exit (23)
ipdict = {}
for fn in opt.read_pickle :
if opt.debug :
print ("Processing pickle dump %s" % fn)
keys = dict.fromkeys (pyk.iterkeys (ipdict))
mt = None
if fn == '-' :
f = sys.stdin
else :
mt = datetime.utcfromtimestamp (os.stat (fn) [ST_MTIME])
if fn.endswith ('.gz') :
f = GzipFile (fn, 'r')
else :
f = open (fn, 'r')
obj = pickle.load (f)
for k, v in pyk.iteritems (obj) :
# Fixup of object
if isinstance (v, Guess) :
if not hasattr (v, 'rqinfo') :
v.rqinfo = {}
v.rqinfo ['ips'] = v.status.ips
v.rqinfo ['interfaces'] = v.status.if_by_name
v.status = None
if mt and not hasattr (v, 'time') :
v.time = mt
if k in ipdict :
keys [k] = True
ov = ipdict [k]
istuple = isinstance (v, tuple)
if isinstance (ov, tuple) :
overwrite = False
if not istuple :
overwrite = True
elif ov [0] == 'Timeout_Error' :
overwrite = True
elif v [0] == 'ValueError' :
overwrite = True
if overwrite :
#print (opt.debug, istuple, v, ov [0])
if (opt.debug and (not istuple or v [0] != ov [0])) :
print ("%s: overwriting %s with %s" % (k, ov, v))
ipdict [k] = v
elif istuple and ov [0] != v [0] and opt.debug :
print ("%s: Not overwriting %s with %s" % (k, ov, v))
else :
assert isinstance (ov, Guess)
if istuple :
if opt.debug :
print ("%s: Not overwriting %s with %s" % (k, ov, v))
else :
assert isinstance (v, Guess)
ipdict [k] = v
else :
if opt.debug :
print ("%s: new: %s" % (k, v))
ipdict [k] = v
if opt.debug :
for k, v in pyk.iteritems (keys) :
if not v :
print ("%s: not existing in dump %s" % (k, fn))
for ip in args :
port = opt.port
try :
ip, port = ip.split (':', 1)
except ValueError :
pass
site = site_template % locals ()
url = ''
# For testing we download the index page and cgi-bin-status.html
# page into a directory named with the ip address
if opt.local :
site = 'file://' + os.path.abspath (ip)
url = 'index.html'
ff = Guess (site = site, ip = ip, url = url, port = port)
print (ff.verbose_repr ())
ipdict [str (ip)] = ff
if opt.output_pickle :
if opt.output_pickle.endswith ('.gz') :
f = GzipFile (opt.output_pickle, 'wb', 9)
else :
f = open (opt.output_pickle, 'wb')
pickle.dump (ipdict, f)
f.close ()
key = lambda x : IP4_Address (x [0])
if opt.version_statistics :
fields = ['timestamp', 'address', 'type', 'version']
if opt.version_statistics == '-' :
f = sys.stdout
else :
f = open (opt.version_statistics, 'w')
dw = DictWriter (f, fields, delimiter = ';')
dw.writerow (dict ((k, k) for k in fields))
for ip, guess in sorted (pyk.iteritems (ipdict), key = key) :
if isinstance (guess, Guess) :
dw.writerow \
( dict
( timestamp = guess.time.strftime
("%Y-%m-%d %H:%M:%S+0")
, address = str (ip)
, version = guess.version
, type = guess.type
)
)
f.close ()
if opt.interface_info :
fields = \
[ 'timestamp', 'address', 'interface', 'mac', 'wlan'
, 'ssid', 'mode', 'channel', 'bssid'
, 'ip4', 'ip6', 'signal', 'noise'
]
if opt.interface_info == '-' :
f = sys.stdout
else :
f = open (opt.interface_info, 'w')
dw = DictWriter (f, fields, delimiter = ';')
dw.writerow (dict ((k, k) for k in fields))
for ip, guess in sorted (pyk.iteritems (ipdict), key = key) :
if isinstance (guess, Guess) :
for iface in pyk.itervalues (guess.interfaces) :
wi = iface.wlan_info
mc = None
if iface.link :
mc = iface.link.mac
d = dict \
( timestamp = guess.time.strftime
("%Y-%m-%d %H:%M:%S+0")
, address = str (ip)
, interface = iface.name
, mac = mc
, wlan = bool (wi)
, ip4 = ' '.join (str (i.ip) for i in iface.inet4)
, ip6 = ' '.join (str (i.ip) for i in iface.inet6)
)
if wi :
d.update \
( channel = wi.channel
, bssid = wi.bssid
, ssid = wi.ssid
, mode = wi.mode
, signal = wi.signal
, noise = wi.noise
)
dw.writerow (d)
f.close ()
if opt.verbose :
for ip, guess in sorted (pyk.iteritems (ipdict), key = key) :
if opt.verbose > 1 :
print ("%-15s" % ip)
print ('=' * 15)
if isinstance (guess, Guess) :
print (guess.verbose_repr ())
else :
print ("Exception:", guess)
else :
print ("%-15s: %s" % (ip, guess))
def __hash__(self):
return hash \
(( tuple (pyk.iterkeys (self.ips))
, tuple (pyk.itervalues (self.interfaces))
))
def iterkeys (self) :
return pyk.iterkeys (self.attribute_changes)
def __init__(cls, name, bases, dct):
cls.__m_super.__init__(name, bases, dct)
if name != "_A_Unit_":
ud = dct.get("_unit_dict")
if ud:
du = dct.get("_default_unit")
if du is None:
for n, v in pyk.iteritems(ud):
if v == 1.0:
du = n
setattr(cls, "_default_unit", du)
break
if du is not None:
syntax = "\n".join(
s
for s in (
getattr(cls, "_syntax_spec_head", ""),
"The default unit is %s. If you specify "
"another unit, it must be separated from "
"the number by at least one space."
"\n"
"You can use the following units: %s." % (du, ", ".join(sorted(pyk.iterkeys(ud)))),
getattr(cls, "_syntax_spec_tail", ""),
)
if s
)
setattr(cls, "syntax", syntax)
elif __debug__:
print(
"Attribute type %s doesn't specify a `_default_unit`"
"nor a unit with value 1.0 in `_unit_dict`" % name
)
elif __debug__:
if not getattr(cls, "_unit_dict", None):
print("Attribute type %s doesn't specify a _unit_dict" % name)
def main():
import sys
import pickle
cmd = ArgumentParser()
cmd.add_argument \
( "-d", "--debug"
, dest = "debug"
, action = "store_true"
, help = "Debug merging of pickle dumps"
)
cmd.add_argument \
( "-l", "--local"
, dest = "local"
, action = "store_true"
, help = "Use local download for testing with file:// url"
)
cmd.add_argument \
( "-o", "--output-pickle"
, dest = "output_pickle"
, help = "Optional pickle output file"
)
cmd.add_argument \
( "-p", "--port"
, dest = "port"
, help = "Optional port number to fetch from"
, type = int
, default = 0
)
cmd.add_argument \
( "-r", "--read-pickle"
, dest = "read_pickle"
, help = "Read old pickle files, merge and preserve information"
, action = "append"
, default = []
)
cmd.add_argument \
( "-V", "--version-statistics"
, dest = "version_statistics"
, help = "Output version information by spidered IP"
)
cmd.add_argument \
( "-I", "--interface-info"
, dest = "interface_info"
, help = "Output interface information by spidered IP"
)
cmd.add_argument \
( "-v", "--verbose"
, dest = "verbose"
, action = "count"
, help = "Show verbose results"
)
cmd.add_argument \
( "ip"
, nargs = "*"
, help = "IP Address(es)"
)
opt = cmd.parse_args()
if len(opt.ip) < 1 and not opt.read_pickle:
cmd.print_help()
sys.exit(23)
ipdict = {}
for fn in opt.read_pickle:
if opt.debug:
print("Processing pickle dump %s" % fn)
keys = dict.fromkeys(pyk.iterkeys(ipdict))
mt = None
if fn == '-':
f = sys.stdin
else:
mt = datetime.utcfromtimestamp(os.stat(fn)[ST_MTIME])
if fn.endswith('.gz'):
f = GzipFile(fn, 'r')
else:
f = open(fn, 'r')
obj = pickle.load(f)
for k, v in pyk.iteritems(obj):
# Fixup of object
if isinstance(v, Guess):
if not hasattr(v, 'rqinfo'):
v.rqinfo = {}
v.rqinfo['ips'] = v.status.ips
v.rqinfo['interfaces'] = v.status.if_by_name
v.status = None
if mt and not hasattr(v, 'time'):
v.time = mt
if k in ipdict:
keys[k] = True
ov = ipdict[k]
istuple = isinstance(v, tuple)
if isinstance(ov, tuple):
overwrite = False
if not istuple:
overwrite = True
elif ov[0] == 'Timeout_Error':
overwrite = True
elif v[0] == 'ValueError':
overwrite = True
if overwrite:
#print (opt.debug, istuple, v, ov [0])
if (opt.debug and (not istuple or v[0] != ov[0])):
print("%s: overwriting %s with %s" % (k, ov, v))
ipdict[k] = v
elif istuple and ov[0] != v[0] and opt.debug:
print("%s: Not overwriting %s with %s" % (k, ov, v))
else:
assert isinstance(ov, Guess)
if istuple:
if opt.debug:
print("%s: Not overwriting %s with %s" %
(k, ov, v))
else:
assert isinstance(v, Guess)
ipdict[k] = v
else:
if opt.debug:
print("%s: new: %s" % (k, v))
ipdict[k] = v
if opt.debug:
for k, v in pyk.iteritems(keys):
if not v:
print("%s: not existing in dump %s" % (k, fn))
for ip in opt.ip:
port = opt.port
try:
ip, port = ip.split(':', 1)
except ValueError:
pass
site = site_template % locals()
url = ''
# For testing we download the index page and cgi-bin-status.html
# page into a directory named with the ip address
if opt.local:
site = 'file://' + os.path.abspath(ip)
url = 'index.html'
ff = Guess(site=site, ip=ip, url=url, port=port)
print(ff.verbose_repr())
ipdict[str(ip)] = ff
if opt.output_pickle:
if opt.output_pickle.endswith('.gz'):
f = GzipFile(opt.output_pickle, 'wb', 9)
else:
f = open(opt.output_pickle, 'wb')
pickle.dump(ipdict, f)
f.close()
key = lambda x: IP4_Address(x[0])
if opt.version_statistics:
fields = ['timestamp', 'address', 'type', 'version']
if opt.version_statistics == '-':
f = sys.stdout
else:
f = open(opt.version_statistics, 'w')
dw = DictWriter(f, fields, delimiter=';')
dw.writerow(dict((k, k) for k in fields))
for ip, guess in sorted(pyk.iteritems(ipdict), key=key):
if isinstance(guess, Guess):
dw.writerow \
( dict
( timestamp = guess.time.strftime
("%Y-%m-%d %H:%M:%S+0")
, address = str (ip)
, version = guess.version
, type = guess.type
)
)
f.close()
if opt.interface_info:
fields = \
[ 'timestamp', 'address', 'interface', 'mac', 'wlan'
, 'ssid', 'mode', 'channel', 'bssid'
, 'ip4', 'ip6', 'signal', 'noise'
]
if opt.interface_info == '-':
f = sys.stdout
else:
f = open(opt.interface_info, 'w')
dw = DictWriter(f, fields, delimiter=';')
dw.writerow(dict((k, k) for k in fields))
for ip, guess in sorted(pyk.iteritems(ipdict), key=key):
if isinstance(guess, Guess):
for iface in pyk.itervalues(guess.interfaces):
wi = iface.wlan_info
mc = None
if iface.link:
mc = iface.link.mac
d = dict \
( timestamp = guess.time.strftime
("%Y-%m-%d %H:%M:%S+0")
, address = str (ip)
, interface = iface.name
, mac = mc
, wlan = bool (wi)
, ip4 = ' '.join (str (i.ip) for i in iface.inet4)
, ip6 = ' '.join (str (i.ip) for i in iface.inet6)
)
if wi:
d.update \
( channel = wi.channel
, bssid = wi.bssid
, ssid = wi.ssid
, mode = wi.mode
, signal = wi.signal
, noise = wi.noise
)
dw.writerow(d)
f.close()
if opt.verbose:
for ip, guess in sorted(pyk.iteritems(ipdict), key=key):
if opt.verbose > 1:
print("%-15s" % ip)
print('=' * 15)
if isinstance(guess, Guess):
print(guess.verbose_repr())
else:
print("Exception:", guess)
else:
print("%-15s: %s" % (ip, guess))
def iteritems(self):
convert = self._convert_element
for key in pyk.iterkeys(self.data):
yield key, convert(key, self[key])
def iterkeys(self):
return pyk.iterkeys(self.data)
def __hash__ (self) :
return hash \
(( tuple (pyk.iterkeys (self.ips))
, tuple (pyk.itervalues (self.interfaces))
))
def check_all (self, obj, attr_dict = {}) :
for k in pyk.iterkeys (self.pred_kind) :
self.check_kind (k, obj, attr_dict)
return MOM.Pred.Err_and_Warn_List (self.errors, self.warnings)
def check_all(self, obj, attr_dict={}):
for k in pyk.iterkeys(self.pred_kind):
self.check_kind(k, obj, attr_dict)
return MOM.Pred.Err_and_Warn_List(self.errors, self.warnings)
