def __init__ \ ( self , olsr_file , processes = 20 , N = 0 , timeout = 180 , ip_port = {} , debug = False , ** kw ) : self.__super.__init__ (**kw) olsr = get_olsr_container (olsr_file) self.olsr_nodes = {} assert len (olsr.topo.forward) for t in pyk.iterkeys (olsr.topo.forward) : self.olsr_nodes [t] = True for t in pyk.iterkeys (olsr.topo.reverse) : self.olsr_nodes [t] = True # limit to N elements if N : self.olsr_nodes = dict \ ((k, v) for k, v in islice (pyk.iteritems (self.olsr_nodes), N)) self.pool = Pool (processes = processes) self.mgr = Manager () self.result_dict = self.mgr.dict () self.timeout = timeout self.ip_port = ip_port self.debug = debug olsr_nodes = None if not debug : self.log.setLevel (INFO) self.log.debug ("Starting ...")
def matching_keys (self, abbr) : """Returns a list of all keys matching `abbr` if any.""" if abbr in self : return [abbr] elif isinstance (abbr, str) : return TFL.matches (TFL.sorted (pyk.iterkeys (self)), abbr) return []
def __init__ (self, dict, ** kw) : self._dict = dict self.__super.__init__ \ ( selection = sorted (pyk.iterkeys (dict)) , cook = self._cooked_key , ** kw )
def verbose_repr (self) : r = [str (self)] for v in pyk.itervalues (self.interfaces) : r.append (str (v)) for v in pyk.iterkeys (self.ips) : r.append (str (v)) return '\n'.join (r)
def reset_predicates(self): self.errors = errors = {} self.warnings = warnings = {} self.missing_required = None for k in pyk.iterkeys(self.pred_kind): errors[k] = [] warnings[k] = []
def matching_keys(self, abbr): """Returns a list of all keys matching `abbr` if any.""" if abbr in self: return [abbr] elif isinstance(abbr, str): return TFL.matches(TFL.sorted(pyk.iterkeys(self)), abbr) return []
def reset_predicates (self) : self.errors = errors = {} self.warnings = warnings = {} self.missing_required = None for k in pyk.iterkeys (self.pred_kind) : errors [k] = [] warnings [k] = []
def __init__(self, dict, **kw): self._dict = dict self.__super.__init__ \ ( selection = sorted (pyk.iterkeys (dict)) , cook = self._cooked_key , ** kw )
def verbose_repr(self): r = [str(self)] for v in pyk.itervalues(self.interfaces): r.append(str(v)) for v in pyk.iterkeys(self.ips): r.append(str(v)) return '\n'.join(r)
def as_json(self): d = dict(type=self.type, version=self.version) iface = d['interfaces'] = [] for i in pyk.itervalues(self.interfaces): iface.append(i.as_dict) ips = d['ips'] = [] for i in pyk.iterkeys(self.ips): ips.append(str(i)) return json.dumps(d)
def as_json (self) : d = dict (type = self.type, version = self.version) iface = d ['interfaces'] = [] for i in pyk.itervalues (self.interfaces) : iface.append (i.as_dict) ips = d ['ips'] = [] for i in pyk.iterkeys (self.ips) : ips.append (str (i)) return json.dumps (d)
def __init__(cls, name, bases, dct): cls.__m_super.__init__(name, bases, dct) if cls.C_Type is not None: cls.P_Type = cls.C_Type.P_Type Table = getattr(cls, "Table", None) if Table and not cls.syntax: cls.syntax = "The following string values are accepted as valid " "%s values: %s" % ( cls.typ, ", ".join(sorted(pyk.iterkeys(Table))), )
def __init__ \ (self, __dict = {}, __flags = 0, __head = "", __tail = "", ** kw) : self._map = map = dict (__dict, ** kw) pattern = "|".join \ ( re.escape (k) for k in sorted (pyk.iterkeys (map), key = lambda k : (-len (k), k)) ) if __head or __tail : pattern = "".join ((__head, "(", pattern, ")", __tail)) self._group = 1 regexp = Regexp (pattern, __flags) self.__super.__init__ (regexp, self._replacer, __flags)
def syntax(cls): return "The following string values are accepted as valid " "%s values: %s" % ( cls.typ, ", ".join(sorted(pyk.iterkeys(cls.Table))), )
def iteritems (self) : convert = self._convert_element for key in pyk.iterkeys (self.data) : yield key, convert (key, self [key])
def iterkeys (self) : return pyk.iterkeys (self.data)
def iterkeys (self) : return pyk.iterkeys (self.attribute_changes)
def _Reload (self, * modules) : for c in pyk.iterkeys (self.__cached) : delattr (self, c) self.__cached = {} self._parent._Reload () Package_Namespace._Reload (self, * modules)
def main () : import sys import pickle from optparse import OptionParser cmd = OptionParser () cmd.add_option \ ( "-d", "--debug" , dest = "debug" , action = "store_true" , help = "Debug merging of pickle dumps" ) cmd.add_option \ ( "-l", "--local" , dest = "local" , action = "store_true" , help = "Use local download for testing with file:// url" ) cmd.add_option \ ( "-o", "--output-pickle" , dest = "output_pickle" , help = "Optional pickle output file" ) cmd.add_option \ ( "-p", "--port" , dest = "port" , help = "Optional port number to fetch from" , type = "int" , default = 0 ) cmd.add_option \ ( "-r", "--read-pickle" , dest = "read_pickle" , help = "Read old pickle files, merge and preserve information" , action = "append" , default = [] ) cmd.add_option \ ( "-V", "--version-statistics" , dest = "version_statistics" , help = "Output version information by spidered IP" ) cmd.add_option \ ( "-I", "--interface-info" , dest = "interface_info" , help = "Output interface information by spidered IP" ) cmd.add_option \ ( "-v", "--verbose" , dest = "verbose" , action = "count" , help = "Show verbose results" ) (opt, args) = cmd.parse_args () if len (args) < 1 and not opt.read_pickle : cmd.print_help () sys.exit (23) ipdict = {} for fn in opt.read_pickle : if opt.debug : print ("Processing pickle dump %s" % fn) keys = dict.fromkeys (pyk.iterkeys (ipdict)) mt = None if fn == '-' : f = sys.stdin else : mt = datetime.utcfromtimestamp (os.stat (fn) [ST_MTIME]) if fn.endswith ('.gz') : f = GzipFile (fn, 'r') else : f = open (fn, 'r') obj = pickle.load (f) for k, v in pyk.iteritems (obj) : # Fixup of object if isinstance (v, Guess) : if not hasattr (v, 'rqinfo') : v.rqinfo = {} v.rqinfo ['ips'] = v.status.ips v.rqinfo ['interfaces'] = v.status.if_by_name v.status = None if mt and not hasattr (v, 'time') : v.time = mt if k in ipdict : keys [k] = True ov = ipdict [k] istuple = isinstance (v, tuple) if isinstance (ov, tuple) : overwrite = False if not istuple : overwrite = True elif ov [0] == 'Timeout_Error' : overwrite = True elif v [0] == 'ValueError' : overwrite = True if overwrite : #print (opt.debug, istuple, v, ov [0]) if (opt.debug and (not istuple or v [0] != ov [0])) : print ("%s: overwriting %s with %s" % (k, ov, v)) ipdict [k] = v elif istuple and ov [0] != v [0] and opt.debug : print ("%s: Not overwriting %s with %s" % (k, ov, v)) else : assert isinstance (ov, Guess) if istuple : if opt.debug : print ("%s: Not overwriting %s with %s" % (k, ov, v)) else : assert isinstance (v, Guess) ipdict [k] = v else : if opt.debug : print ("%s: new: %s" % (k, v)) ipdict [k] = v if opt.debug : for k, v in pyk.iteritems (keys) : if not v : print ("%s: not existing in dump %s" % (k, fn)) for ip in args : port = opt.port try : ip, port = ip.split (':', 1) except ValueError : pass site = site_template % locals () url = '' # For testing we download the index page and cgi-bin-status.html # page into a directory named with the ip address if opt.local : site = 'file://' + os.path.abspath (ip) url = 'index.html' ff = Guess (site = site, ip = ip, url = url, port = port) print (ff.verbose_repr ()) ipdict [str (ip)] = ff if opt.output_pickle : if opt.output_pickle.endswith ('.gz') : f = GzipFile (opt.output_pickle, 'wb', 9) else : f = open (opt.output_pickle, 'wb') pickle.dump (ipdict, f) f.close () key = lambda x : IP4_Address (x [0]) if opt.version_statistics : fields = ['timestamp', 'address', 'type', 'version'] if opt.version_statistics == '-' : f = sys.stdout else : f = open (opt.version_statistics, 'w') dw = DictWriter (f, fields, delimiter = ';') dw.writerow (dict ((k, k) for k in fields)) for ip, guess in sorted (pyk.iteritems (ipdict), key = key) : if isinstance (guess, Guess) : dw.writerow \ ( dict ( timestamp = guess.time.strftime ("%Y-%m-%d %H:%M:%S+0") , address = str (ip) , version = guess.version , type = guess.type ) ) f.close () if opt.interface_info : fields = \ [ 'timestamp', 'address', 'interface', 'mac', 'wlan' , 'ssid', 'mode', 'channel', 'bssid' , 'ip4', 'ip6', 'signal', 'noise' ] if opt.interface_info == '-' : f = sys.stdout else : f = open (opt.interface_info, 'w') dw = DictWriter (f, fields, delimiter = ';') dw.writerow (dict ((k, k) for k in fields)) for ip, guess in sorted (pyk.iteritems (ipdict), key = key) : if isinstance (guess, Guess) : for iface in pyk.itervalues (guess.interfaces) : wi = iface.wlan_info mc = None if iface.link : mc = iface.link.mac d = dict \ ( timestamp = guess.time.strftime ("%Y-%m-%d %H:%M:%S+0") , address = str (ip) , interface = iface.name , mac = mc , wlan = bool (wi) , ip4 = ' '.join (str (i.ip) for i in iface.inet4) , ip6 = ' '.join (str (i.ip) for i in iface.inet6) ) if wi : d.update \ ( channel = wi.channel , bssid = wi.bssid , ssid = wi.ssid , mode = wi.mode , signal = wi.signal , noise = wi.noise ) dw.writerow (d) f.close () if opt.verbose : for ip, guess in sorted (pyk.iteritems (ipdict), key = key) : if opt.verbose > 1 : print ("%-15s" % ip) print ('=' * 15) if isinstance (guess, Guess) : print (guess.verbose_repr ()) else : print ("Exception:", guess) else : print ("%-15s: %s" % (ip, guess))
def __hash__(self): return hash \ (( tuple (pyk.iterkeys (self.ips)) , tuple (pyk.itervalues (self.interfaces)) ))
def __init__(cls, name, bases, dct): cls.__m_super.__init__(name, bases, dct) if name != "_A_Unit_": ud = dct.get("_unit_dict") if ud: du = dct.get("_default_unit") if du is None: for n, v in pyk.iteritems(ud): if v == 1.0: du = n setattr(cls, "_default_unit", du) break if du is not None: syntax = "\n".join( s for s in ( getattr(cls, "_syntax_spec_head", ""), "The default unit is %s. If you specify " "another unit, it must be separated from " "the number by at least one space." "\n" "You can use the following units: %s." % (du, ", ".join(sorted(pyk.iterkeys(ud)))), getattr(cls, "_syntax_spec_tail", ""), ) if s ) setattr(cls, "syntax", syntax) elif __debug__: print( "Attribute type %s doesn't specify a `_default_unit`" "nor a unit with value 1.0 in `_unit_dict`" % name ) elif __debug__: if not getattr(cls, "_unit_dict", None): print("Attribute type %s doesn't specify a _unit_dict" % name)
def main(): import sys import pickle cmd = ArgumentParser() cmd.add_argument \ ( "-d", "--debug" , dest = "debug" , action = "store_true" , help = "Debug merging of pickle dumps" ) cmd.add_argument \ ( "-l", "--local" , dest = "local" , action = "store_true" , help = "Use local download for testing with file:// url" ) cmd.add_argument \ ( "-o", "--output-pickle" , dest = "output_pickle" , help = "Optional pickle output file" ) cmd.add_argument \ ( "-p", "--port" , dest = "port" , help = "Optional port number to fetch from" , type = int , default = 0 ) cmd.add_argument \ ( "-r", "--read-pickle" , dest = "read_pickle" , help = "Read old pickle files, merge and preserve information" , action = "append" , default = [] ) cmd.add_argument \ ( "-V", "--version-statistics" , dest = "version_statistics" , help = "Output version information by spidered IP" ) cmd.add_argument \ ( "-I", "--interface-info" , dest = "interface_info" , help = "Output interface information by spidered IP" ) cmd.add_argument \ ( "-v", "--verbose" , dest = "verbose" , action = "count" , help = "Show verbose results" ) cmd.add_argument \ ( "ip" , nargs = "*" , help = "IP Address(es)" ) opt = cmd.parse_args() if len(opt.ip) < 1 and not opt.read_pickle: cmd.print_help() sys.exit(23) ipdict = {} for fn in opt.read_pickle: if opt.debug: print("Processing pickle dump %s" % fn) keys = dict.fromkeys(pyk.iterkeys(ipdict)) mt = None if fn == '-': f = sys.stdin else: mt = datetime.utcfromtimestamp(os.stat(fn)[ST_MTIME]) if fn.endswith('.gz'): f = GzipFile(fn, 'r') else: f = open(fn, 'r') obj = pickle.load(f) for k, v in pyk.iteritems(obj): # Fixup of object if isinstance(v, Guess): if not hasattr(v, 'rqinfo'): v.rqinfo = {} v.rqinfo['ips'] = v.status.ips v.rqinfo['interfaces'] = v.status.if_by_name v.status = None if mt and not hasattr(v, 'time'): v.time = mt if k in ipdict: keys[k] = True ov = ipdict[k] istuple = isinstance(v, tuple) if isinstance(ov, tuple): overwrite = False if not istuple: overwrite = True elif ov[0] == 'Timeout_Error': overwrite = True elif v[0] == 'ValueError': overwrite = True if overwrite: #print (opt.debug, istuple, v, ov [0]) if (opt.debug and (not istuple or v[0] != ov[0])): print("%s: overwriting %s with %s" % (k, ov, v)) ipdict[k] = v elif istuple and ov[0] != v[0] and opt.debug: print("%s: Not overwriting %s with %s" % (k, ov, v)) else: assert isinstance(ov, Guess) if istuple: if opt.debug: print("%s: Not overwriting %s with %s" % (k, ov, v)) else: assert isinstance(v, Guess) ipdict[k] = v else: if opt.debug: print("%s: new: %s" % (k, v)) ipdict[k] = v if opt.debug: for k, v in pyk.iteritems(keys): if not v: print("%s: not existing in dump %s" % (k, fn)) for ip in opt.ip: port = opt.port try: ip, port = ip.split(':', 1) except ValueError: pass site = site_template % locals() url = '' # For testing we download the index page and cgi-bin-status.html # page into a directory named with the ip address if opt.local: site = 'file://' + os.path.abspath(ip) url = 'index.html' ff = Guess(site=site, ip=ip, url=url, port=port) print(ff.verbose_repr()) ipdict[str(ip)] = ff if opt.output_pickle: if opt.output_pickle.endswith('.gz'): f = GzipFile(opt.output_pickle, 'wb', 9) else: f = open(opt.output_pickle, 'wb') pickle.dump(ipdict, f) f.close() key = lambda x: IP4_Address(x[0]) if opt.version_statistics: fields = ['timestamp', 'address', 'type', 'version'] if opt.version_statistics == '-': f = sys.stdout else: f = open(opt.version_statistics, 'w') dw = DictWriter(f, fields, delimiter=';') dw.writerow(dict((k, k) for k in fields)) for ip, guess in sorted(pyk.iteritems(ipdict), key=key): if isinstance(guess, Guess): dw.writerow \ ( dict ( timestamp = guess.time.strftime ("%Y-%m-%d %H:%M:%S+0") , address = str (ip) , version = guess.version , type = guess.type ) ) f.close() if opt.interface_info: fields = \ [ 'timestamp', 'address', 'interface', 'mac', 'wlan' , 'ssid', 'mode', 'channel', 'bssid' , 'ip4', 'ip6', 'signal', 'noise' ] if opt.interface_info == '-': f = sys.stdout else: f = open(opt.interface_info, 'w') dw = DictWriter(f, fields, delimiter=';') dw.writerow(dict((k, k) for k in fields)) for ip, guess in sorted(pyk.iteritems(ipdict), key=key): if isinstance(guess, Guess): for iface in pyk.itervalues(guess.interfaces): wi = iface.wlan_info mc = None if iface.link: mc = iface.link.mac d = dict \ ( timestamp = guess.time.strftime ("%Y-%m-%d %H:%M:%S+0") , address = str (ip) , interface = iface.name , mac = mc , wlan = bool (wi) , ip4 = ' '.join (str (i.ip) for i in iface.inet4) , ip6 = ' '.join (str (i.ip) for i in iface.inet6) ) if wi: d.update \ ( channel = wi.channel , bssid = wi.bssid , ssid = wi.ssid , mode = wi.mode , signal = wi.signal , noise = wi.noise ) dw.writerow(d) f.close() if opt.verbose: for ip, guess in sorted(pyk.iteritems(ipdict), key=key): if opt.verbose > 1: print("%-15s" % ip) print('=' * 15) if isinstance(guess, Guess): print(guess.verbose_repr()) else: print("Exception:", guess) else: print("%-15s: %s" % (ip, guess))
def iteritems(self): convert = self._convert_element for key in pyk.iterkeys(self.data): yield key, convert(key, self[key])
def iterkeys(self): return pyk.iterkeys(self.data)
def __hash__ (self) : return hash \ (( tuple (pyk.iterkeys (self.ips)) , tuple (pyk.itervalues (self.interfaces)) ))
def check_all (self, obj, attr_dict = {}) : for k in pyk.iterkeys (self.pred_kind) : self.check_kind (k, obj, attr_dict) return MOM.Pred.Err_and_Warn_List (self.errors, self.warnings)
def check_all(self, obj, attr_dict={}): for k in pyk.iterkeys(self.pred_kind): self.check_kind(k, obj, attr_dict) return MOM.Pred.Err_and_Warn_List(self.errors, self.warnings)