def addPeer (self, url, key, kind=Normal):
"""
adds a peer if it's not already there.
the url should not be already parsed and the key should be in the proper format
returns: naught
"""
self.debug (2, 'adding peer %s' % url, 2)
self.acquire ()
purl= URL (url)
proto= purl.proto ()
if proto=='umbie':
if not self._navels.has_key (key):
self._navels[key]= StubNavel (self._master, url, key, kind)
else:
# change the kind
self._navels[key].setKind (kind)
elif proto=='vice' and not self._vices.has_key (key):
self._vices[key]= StubVice (self._master, url, key)
elif proto=='virtue' and not self._virtues.has_key (key):
self._virtues[key]= StubVirtue (self._master, url, key)
else:
# bark!
self.debug (2, 'bark! >%s<' % proto)
pass
self.release ()
def setUrl (self, url):
if url:
self._url= URL (url)
self.debug (1, 'url: %s' % self._url)
self.debug (1, 'building server loop')
self._loop= ServerLoop (self._url.port (), self)
self._loop.start ()
def __init__(self, url, master=None):
Object.__init__(self)
self.debug(1, "Client: %s" % (url))
self._url = URL(url)
self._dead = False
self._connected = False
self._lock = RLock()
self._socket = None
self._master = master
def delPeer (self, peer):
url= URL (peer.url ())
key= peer.key ()
h= self.__proto[url.proto ()]
self.acquire ()
if h.has_key (key):
self.debug (2, '%s' % h, fast=False, level=5)
self.debug (2, 'deleting %s: %s, %s' % (peer, url, key), fast=False, level=4)
self.debug (1, 'deleting %s: %s, %s' % (peer, url, key))
del h[key]
self.debug (2, '%s' % h, fast=False, level=3)
self.release ()
def addServer (self, client):
# this server answers to that client
peer= None
server= None
# this ReqSock will be thrown away.
socket= RequestSocket (client)
socket.write (['hit here'])
args= socket.read ()
self.debug (1, "args are: %s" % str(args))
what= next (args)
if what:
if what=='i am':
try:
url= args[0]
self.debug (1, "%s" % url.__class__)
if not isinstance (url, URL):
url= URL (url)
proto= url.proto ()
if proto=='umbie':
key= int (args[1])
peer= self.getNavel (str(url), key)
elif proto=='vice':
peer= self.getVice (str(url))
elif proto=='virtue':
peer= self.getVirtue (str(url))
else:
# bark!
pass
except NoParse:
socket.write (['bye'])
socket.close ()
else:
socket.write (['bye'])
socket.close ()
if peer:
socket.write (['ok'])
# create a Server of the correct class
# as defined in serverType
# TODO: I could pass the ReqSock() instead of the socket()
server= self._master._serverType (self._master, client)
# this server is the one answering to that client
peer.setServer (server)
server.start ()
else:
self.debug (1, 'no peer!')
self.debug (1, 'server %s added' % peer)
def __init__ (self, url, master=None): Object.__init__ (self) self.debug (1, "Client: %s" % (url)) self._url= URL (url) self._dead= False self._connected= False self._lock= RLock () self._socket= None self._master= master
class Client(Object):
"""
generic client
it's blocking. 'not responding, still trying'?
"""
def __init__(self, url, master=None):
Object.__init__(self)
self.debug(1, "Client: %s" % (url))
self._url = URL(url)
self._dead = False
self._connected = False
self._lock = RLock()
self._socket = None
self._master = master
def __del__(self):
try:
self.close()
except:
# just in case __init__ didn't run completely
pass
def url(self):
return self._url
def ident(self):
data = self._socket.getSockName()
self.write(["i am", "virtue://%s:%d/" % data])
ans = self.read()
def read(self):
return self._socket.read()
def readData(self, size):
return self._socket.readData(size)
def write(self, what, data=None):
self._socket.write(what, data)
def writeData(self, data):
return self._socket.writeData(data)
def ask(self, what, data=None):
"""
what should already be a list with message and params
"""
self._lock.acquire()
try:
if not self._connected:
# TODO: move this to a method
# stablish connecton
self._socket = RequestSocket(None)
self.debug(2, "urlParams: %s %s" % self._url.getParams())
self._socket.connect(self._url.getParams())
self.debug(
1, "created Client socket w/ fd %d to %s" %
(self._socket.fileno(), self._url))
# throw away the greeting
ans = self.read()
self._connected = True
# identify ourselves so the server knows what are we.
self.ident()
if not type(what) == ListType:
self.debug(1, "what's not a list!: %s" % str(what))
self.write(what, data)
data = self.read()
self._lock.release()
if not data == None:
return data
else:
self.debug(1, "dead!: closed")
self._dead = True
self.close()
raise UmbDead
# other errors?
except (ValueError, IOError, error), e:
self.debug(1, "dead!: %s" % e)
self._dead = True
self._lock.release()
self.close()
raise UmbDead
class Master (Object):
"""
Warning: Abstract class.
subclasses should at least define self._addClient hash (see run)
and never reimplement periodic withiout calling this class' one
"""
Set= 0
Complement= 1
def __init__ (self, url=None, key=None, column=None, fileName=None):
Object.__init__ (self, fileName=fileName, compressed=False)
self.debug (1, 'm: logging in %s' % fileName)
self._key= key
self._url= None
self._loop= None
self._peers= PeerList (self)
self._terminate= False
# the class of the individual servers.
self._serverType= None
self._ring= Ring (self)
self._data= None
self._timeout= None
# gotta join the clients after our servers died
# so I keep a list and join them in the periodic function
self._threadsToJoin= []
if url:
self.setUrl (url)
def setUrl (self, url):
if url:
self._url= URL (url)
self.debug (1, 'url: %s' % self._url)
self.debug (1, 'building server loop')
self._loop= ServerLoop (self._url.port (), self)
self._loop.start ()
def init (self):
# do not allow fast update
self._timeout= random (consts.periodicTimeOut)+5
self.debug (1, "timeout: %d" % (self._timeout))
def url (self):
return self._url
def key (self):
return self._key
def peers (self):
return self._peers
def navels (self):
navels= self._peers.navels ()
return navels
def vices (self):
vices= self._peers.vices ()
return vices
def delServer (self, server):
"""
Adds a finished server to the list of threads to join,
and removes it from the known peers.
"""
self.debug (1, "queuing to join %s" % server)
# the client thread cannot join itself, so we do it later
self._threadsToJoin.append (server)
self.debug (2, "done q.")
self._peers.delServer (server)
# do other stuff that keeps _pred and _succ up-to-date
def delPeer (self, peer):
self.debug (1, 'deleting %s' % peer)
self._peers.delPeer (peer)
peer.close ()
def discover (self, net, skip=None):
"""
broadcasts messages for discovering the peers
returns: the peer to which we should connect to.
"""
shouter= socket (AF_INET, SOCK_DGRAM)
# set socket option
shouter.setsockopt (SOL_SOCKET, SO_BROADCAST, 1)
server= None
# we need to be root and a default gw here!
# shouter.sendto ('any body there?', ('<broadcast>', consts.chalPort))
# no if we broadcast the net only...
self.debug (1, 'Ping %s' % net)
shouter.sendto ('any body there?', (net, consts.chalPort))
# wait for answers
sleep (consts.shoutTimeOut)
# now wait for at least one
# but let things go on in the case the shout was not heard
# nope, navels that doesn't find others should start their own ring
# but vices should wait till they find one,
# so navels and vices do different things
i= select ([shouter],[],[], 0.1)[0]
# collect *any* answer
while len(i)>0:
ans= csvParse (shouter.recv (1024))
self.debug (1, 'found %s' % ans)
(url, key)= ans[0]
new= self._peers.getNavel (url, key)
if skip:
if new!=skip:
server= new
else:
server= new
i= select ([shouter],[],[], 0.1)[0]
self.debug (1, 'returning %s' % server)
return server
def gossip (self, peer):
try:
(navels, vices)= peer.getKnownPeers ()
self.debug (1, "from %s got peer list : %s" % (peer, navels))
self.updatePeers (navels, vices)
for viceUrl in vices:
try:
vice= self._peers.getVice (viceUrl)
vice.updateStats ()
except:
self.delPeer (vice)
except:
self.delPeer (peer)
def updatePeers (self, navels, vices):
for navelUrl, navelKey in navels:
self._peers.addPeer (navelUrl, navelKey)
for viceUrl in vices:
# for testing now
vice= self._peers.getVice (viceUrl)
def getNavelKey (self, url):
# create a temp client and ask for the key...
server= NavelClient (url, self)
key= None
while not key:
try:
key= server.key ()
server.close ()
except UmbDead:
# a connection refused
sleep (5)
# should we 'timeout' and create our own ring?
return key
def giveData (self, m, n, what=Set):
"""
returns: a hash
"""
self.debug (1, "giveData called", 2)
keys= self._data.keys ()
keys.sort ()
self.debug (1, 'my know keys are: %s' % keys)
h= {}
nextKey= next (keys)
while not nextKey==None:
self.debug (2, "m: %d, k: %d, n: %d" % (m, nextKey, n))
if (what==self.Set and coBetween (m, nextKey, n)) or (what==self.Complement and coBetween (n, nextKey, m)):
h[nextKey]= self._data.getValues (nextKey)
nextKey= next (keys)
if what==self.Set:
self.debug (1, 'giving Set %d:%d %s' % (m, n, h.keys ()))
else:
self.debug (1, 'giving Complement %d:%d %s' % (n, m, h.keys ()))
return h
def ident (self):
raise NotImplementedError
############
# actual run
############
def catchKill (self, signo, stackFrame):
self.debug (1, "SIGTERM caught; terminating and sync'ing")
try:
self._data.sync ()
except:
pass
self._terminate= True
def periodic (self):
self.joinServers ()
def joinServers (self):
# join the servers that were attending clients who quited
for thread in self._threadsToJoin:
self.debug (1, "thread.join()'ing %s" % thread)
self._threadsToJoin.remove (thread)
thread.join ()
# 'main' loop: this is cron
def run (self):
# set the handler of sigterm
signal (SIGTERM, self.catchKill)
count= self._timeout
while not self._terminate:
try:
if count<consts.periodicUTimeOut:
self.debug (2, 'short uSleep')
sleep (count)
self.periodic ()
count= self._timeout
else:
self.debug (2, 'long uSleep')
sleep (consts.periodicUTimeOut)
count-= consts.periodicUTimeOut
except KeyboardInterrupt:
self.debug (1, "SIGTERM caught; terminating")
self._terminate= True
self.stop ()
def terminate (self):
self.debug (1, "terminating...")
self._terminate= True
def stop (self):
# end of the game
self.debug (1, "join'ing server loop...")
self._loop.join ()
self.debug (1, "main finishing...")
# self._clients.joinThem ()
self.joinServers ()
self.debug (1, "finished!")
class Client (Object):
"""
generic client
it's blocking. 'not responding, still trying'?
"""
def __init__ (self, url, master=None):
Object.__init__ (self)
self.debug (1, "Client: %s" % (url))
self._url= URL (url)
self._dead= False
self._connected= False
self._lock= RLock ()
self._socket= None
self._master= master
def __del__ (self):
try:
self.close ()
except:
# just in case __init__ didn't run completely
pass
def url (self):
return self._url
def ident (self):
data= self._socket.getSockName ()
self.write (["i am", "virtue://%s:%d/" % data])
ans= self.read ()
def read (self):
return self._socket.read ()
def readData (self, size):
return self._socket.readData (size)
def write (self, what, data=None):
self._socket.write (what, data)
def writeData (self, data):
return self._socket.writeData (data)
def ask (self, what, data=None):
"""
what should already be a list with message and params
"""
self._lock.acquire ()
try:
if not self._connected:
# TODO: move this to a method
# stablish connecton
self._socket= RequestSocket (None)
self.debug (2, "urlParams: %s %s" % self._url.getParams ())
self._socket.connect (self._url.getParams ())
self.debug (1, "created Client socket w/ fd %d to %s" % (self._socket.fileno (), self._url))
# throw away the greeting
ans= self.read ()
self._connected= True
# identify ourselves so the server knows what are we.
self.ident ()
if not type (what)==ListType:
self.debug (1, "what's not a list!: %s" % str (what))
self.write (what, data)
data= self.read ()
self._lock.release ()
if not data==None:
return data
else:
self.debug (1, "dead!: closed")
self._dead= True
self.close ()
raise UmbDead
# other errors?
except (ValueError, IOError, error), e:
self.debug (1, "dead!: %s" % e)
self._dead= True
self._lock.release ()
self.close ()
raise UmbDead
