def addPeer (self, url, key, kind=Normal): """ adds a peer if it's not already there. the url should not be already parsed and the key should be in the proper format returns: naught """ self.debug (2, 'adding peer %s' % url, 2) self.acquire () purl= URL (url) proto= purl.proto () if proto=='umbie': if not self._navels.has_key (key): self._navels[key]= StubNavel (self._master, url, key, kind) else: # change the kind self._navels[key].setKind (kind) elif proto=='vice' and not self._vices.has_key (key): self._vices[key]= StubVice (self._master, url, key) elif proto=='virtue' and not self._virtues.has_key (key): self._virtues[key]= StubVirtue (self._master, url, key) else: # bark! self.debug (2, 'bark! >%s<' % proto) pass self.release ()
def setUrl (self, url): if url: self._url= URL (url) self.debug (1, 'url: %s' % self._url) self.debug (1, 'building server loop') self._loop= ServerLoop (self._url.port (), self) self._loop.start ()
def __init__(self, url, master=None): Object.__init__(self) self.debug(1, "Client: %s" % (url)) self._url = URL(url) self._dead = False self._connected = False self._lock = RLock() self._socket = None self._master = master
def delPeer (self, peer): url= URL (peer.url ()) key= peer.key () h= self.__proto[url.proto ()] self.acquire () if h.has_key (key): self.debug (2, '%s' % h, fast=False, level=5) self.debug (2, 'deleting %s: %s, %s' % (peer, url, key), fast=False, level=4) self.debug (1, 'deleting %s: %s, %s' % (peer, url, key)) del h[key] self.debug (2, '%s' % h, fast=False, level=3) self.release ()
def addServer (self, client): # this server answers to that client peer= None server= None # this ReqSock will be thrown away. socket= RequestSocket (client) socket.write (['hit here']) args= socket.read () self.debug (1, "args are: %s" % str(args)) what= next (args) if what: if what=='i am': try: url= args[0] self.debug (1, "%s" % url.__class__) if not isinstance (url, URL): url= URL (url) proto= url.proto () if proto=='umbie': key= int (args[1]) peer= self.getNavel (str(url), key) elif proto=='vice': peer= self.getVice (str(url)) elif proto=='virtue': peer= self.getVirtue (str(url)) else: # bark! pass except NoParse: socket.write (['bye']) socket.close () else: socket.write (['bye']) socket.close () if peer: socket.write (['ok']) # create a Server of the correct class # as defined in serverType # TODO: I could pass the ReqSock() instead of the socket() server= self._master._serverType (self._master, client) # this server is the one answering to that client peer.setServer (server) server.start () else: self.debug (1, 'no peer!') self.debug (1, 'server %s added' % peer)
def __init__ (self, url, master=None): Object.__init__ (self) self.debug (1, "Client: %s" % (url)) self._url= URL (url) self._dead= False self._connected= False self._lock= RLock () self._socket= None self._master= master
class Client(Object): """ generic client it's blocking. 'not responding, still trying'? """ def __init__(self, url, master=None): Object.__init__(self) self.debug(1, "Client: %s" % (url)) self._url = URL(url) self._dead = False self._connected = False self._lock = RLock() self._socket = None self._master = master def __del__(self): try: self.close() except: # just in case __init__ didn't run completely pass def url(self): return self._url def ident(self): data = self._socket.getSockName() self.write(["i am", "virtue://%s:%d/" % data]) ans = self.read() def read(self): return self._socket.read() def readData(self, size): return self._socket.readData(size) def write(self, what, data=None): self._socket.write(what, data) def writeData(self, data): return self._socket.writeData(data) def ask(self, what, data=None): """ what should already be a list with message and params """ self._lock.acquire() try: if not self._connected: # TODO: move this to a method # stablish connecton self._socket = RequestSocket(None) self.debug(2, "urlParams: %s %s" % self._url.getParams()) self._socket.connect(self._url.getParams()) self.debug( 1, "created Client socket w/ fd %d to %s" % (self._socket.fileno(), self._url)) # throw away the greeting ans = self.read() self._connected = True # identify ourselves so the server knows what are we. self.ident() if not type(what) == ListType: self.debug(1, "what's not a list!: %s" % str(what)) self.write(what, data) data = self.read() self._lock.release() if not data == None: return data else: self.debug(1, "dead!: closed") self._dead = True self.close() raise UmbDead # other errors? except (ValueError, IOError, error), e: self.debug(1, "dead!: %s" % e) self._dead = True self._lock.release() self.close() raise UmbDead
class Master (Object): """ Warning: Abstract class. subclasses should at least define self._addClient hash (see run) and never reimplement periodic withiout calling this class' one """ Set= 0 Complement= 1 def __init__ (self, url=None, key=None, column=None, fileName=None): Object.__init__ (self, fileName=fileName, compressed=False) self.debug (1, 'm: logging in %s' % fileName) self._key= key self._url= None self._loop= None self._peers= PeerList (self) self._terminate= False # the class of the individual servers. self._serverType= None self._ring= Ring (self) self._data= None self._timeout= None # gotta join the clients after our servers died # so I keep a list and join them in the periodic function self._threadsToJoin= [] if url: self.setUrl (url) def setUrl (self, url): if url: self._url= URL (url) self.debug (1, 'url: %s' % self._url) self.debug (1, 'building server loop') self._loop= ServerLoop (self._url.port (), self) self._loop.start () def init (self): # do not allow fast update self._timeout= random (consts.periodicTimeOut)+5 self.debug (1, "timeout: %d" % (self._timeout)) def url (self): return self._url def key (self): return self._key def peers (self): return self._peers def navels (self): navels= self._peers.navels () return navels def vices (self): vices= self._peers.vices () return vices def delServer (self, server): """ Adds a finished server to the list of threads to join, and removes it from the known peers. """ self.debug (1, "queuing to join %s" % server) # the client thread cannot join itself, so we do it later self._threadsToJoin.append (server) self.debug (2, "done q.") self._peers.delServer (server) # do other stuff that keeps _pred and _succ up-to-date def delPeer (self, peer): self.debug (1, 'deleting %s' % peer) self._peers.delPeer (peer) peer.close () def discover (self, net, skip=None): """ broadcasts messages for discovering the peers returns: the peer to which we should connect to. """ shouter= socket (AF_INET, SOCK_DGRAM) # set socket option shouter.setsockopt (SOL_SOCKET, SO_BROADCAST, 1) server= None # we need to be root and a default gw here! # shouter.sendto ('any body there?', ('<broadcast>', consts.chalPort)) # no if we broadcast the net only... self.debug (1, 'Ping %s' % net) shouter.sendto ('any body there?', (net, consts.chalPort)) # wait for answers sleep (consts.shoutTimeOut) # now wait for at least one # but let things go on in the case the shout was not heard # nope, navels that doesn't find others should start their own ring # but vices should wait till they find one, # so navels and vices do different things i= select ([shouter],[],[], 0.1)[0] # collect *any* answer while len(i)>0: ans= csvParse (shouter.recv (1024)) self.debug (1, 'found %s' % ans) (url, key)= ans[0] new= self._peers.getNavel (url, key) if skip: if new!=skip: server= new else: server= new i= select ([shouter],[],[], 0.1)[0] self.debug (1, 'returning %s' % server) return server def gossip (self, peer): try: (navels, vices)= peer.getKnownPeers () self.debug (1, "from %s got peer list : %s" % (peer, navels)) self.updatePeers (navels, vices) for viceUrl in vices: try: vice= self._peers.getVice (viceUrl) vice.updateStats () except: self.delPeer (vice) except: self.delPeer (peer) def updatePeers (self, navels, vices): for navelUrl, navelKey in navels: self._peers.addPeer (navelUrl, navelKey) for viceUrl in vices: # for testing now vice= self._peers.getVice (viceUrl) def getNavelKey (self, url): # create a temp client and ask for the key... server= NavelClient (url, self) key= None while not key: try: key= server.key () server.close () except UmbDead: # a connection refused sleep (5) # should we 'timeout' and create our own ring? return key def giveData (self, m, n, what=Set): """ returns: a hash """ self.debug (1, "giveData called", 2) keys= self._data.keys () keys.sort () self.debug (1, 'my know keys are: %s' % keys) h= {} nextKey= next (keys) while not nextKey==None: self.debug (2, "m: %d, k: %d, n: %d" % (m, nextKey, n)) if (what==self.Set and coBetween (m, nextKey, n)) or (what==self.Complement and coBetween (n, nextKey, m)): h[nextKey]= self._data.getValues (nextKey) nextKey= next (keys) if what==self.Set: self.debug (1, 'giving Set %d:%d %s' % (m, n, h.keys ())) else: self.debug (1, 'giving Complement %d:%d %s' % (n, m, h.keys ())) return h def ident (self): raise NotImplementedError ############ # actual run ############ def catchKill (self, signo, stackFrame): self.debug (1, "SIGTERM caught; terminating and sync'ing") try: self._data.sync () except: pass self._terminate= True def periodic (self): self.joinServers () def joinServers (self): # join the servers that were attending clients who quited for thread in self._threadsToJoin: self.debug (1, "thread.join()'ing %s" % thread) self._threadsToJoin.remove (thread) thread.join () # 'main' loop: this is cron def run (self): # set the handler of sigterm signal (SIGTERM, self.catchKill) count= self._timeout while not self._terminate: try: if count<consts.periodicUTimeOut: self.debug (2, 'short uSleep') sleep (count) self.periodic () count= self._timeout else: self.debug (2, 'long uSleep') sleep (consts.periodicUTimeOut) count-= consts.periodicUTimeOut except KeyboardInterrupt: self.debug (1, "SIGTERM caught; terminating") self._terminate= True self.stop () def terminate (self): self.debug (1, "terminating...") self._terminate= True def stop (self): # end of the game self.debug (1, "join'ing server loop...") self._loop.join () self.debug (1, "main finishing...") # self._clients.joinThem () self.joinServers () self.debug (1, "finished!")
class Client (Object): """ generic client it's blocking. 'not responding, still trying'? """ def __init__ (self, url, master=None): Object.__init__ (self) self.debug (1, "Client: %s" % (url)) self._url= URL (url) self._dead= False self._connected= False self._lock= RLock () self._socket= None self._master= master def __del__ (self): try: self.close () except: # just in case __init__ didn't run completely pass def url (self): return self._url def ident (self): data= self._socket.getSockName () self.write (["i am", "virtue://%s:%d/" % data]) ans= self.read () def read (self): return self._socket.read () def readData (self, size): return self._socket.readData (size) def write (self, what, data=None): self._socket.write (what, data) def writeData (self, data): return self._socket.writeData (data) def ask (self, what, data=None): """ what should already be a list with message and params """ self._lock.acquire () try: if not self._connected: # TODO: move this to a method # stablish connecton self._socket= RequestSocket (None) self.debug (2, "urlParams: %s %s" % self._url.getParams ()) self._socket.connect (self._url.getParams ()) self.debug (1, "created Client socket w/ fd %d to %s" % (self._socket.fileno (), self._url)) # throw away the greeting ans= self.read () self._connected= True # identify ourselves so the server knows what are we. self.ident () if not type (what)==ListType: self.debug (1, "what's not a list!: %s" % str (what)) self.write (what, data) data= self.read () self._lock.release () if not data==None: return data else: self.debug (1, "dead!: closed") self._dead= True self.close () raise UmbDead # other errors? except (ValueError, IOError, error), e: self.debug (1, "dead!: %s" % e) self._dead= True self._lock.release () self.close () raise UmbDead