def render_POST(self, request):
"""
Handle post request
"""
"""
args = request.args
print args
logMessage = args['message'][0] if 'message' in args else None
logLevel = args['level'][0] if 'level' in args else None
logType = args['type'][0] if 'type' in args else None
"""
stringcontent = request.content.read()
if len(stringcontent) > 0:
debug("received: " + stringcontent, info=True)
content = json.loads(stringcontent)
logMessage = content['message']
logLevel = content['level']
logType = content['type']
try:
if logMessage:
msg = message.ExternalLogMessage(logMessage, logLevel, logType)
me.getMe().getVectorClock().handleEvent(msg)
message_queue.put(msg)
return "Success"
else:
return "Null message"
except Exception as e:
debug(e, error=True)
return "Error"
def maintainMembers():
"""
use the membersrefresh operation to maintain
the universe of known nodes.
"""
global universe
debug("Running maintain members.",
info=True, threshold=1)
possibledead = set(universe.keys())
membership.membersRefresh()
# Add in new nodes.
tempUniverse = membership.getCurrentMemberDict()
for uid in tempUniverse:
if uid not in universe and not me.getMe().__eq__(tempUniverse[uid]):
universe[uid] = nodes.ExternalNode.fromBase(tempUniverse[uid])
if uid in possibledead:
possibledead.remove(uid)
# Remove dead nodes
for dead in possibledead:
deadNode(dead)
# should I add me in here? not sure.
universe[me.getMe().getUid()] = me.getMe()
debug("has a universe of size: " + str(len(universe)), info=True)
def informAlive():
"""
Inform my peers I exist (for new nodes to system)
"""
alivemessage = message.NewNodeMessage(
(me.getMe().getUidAsObject(), me.getMe().getPort()))
alivemessage.send()
debug("Informing friends I am alive", info=True)
def respond(self):
"""
How the systemm should respon to receiving this vector clock message.
"""
try:
me.getMe().getVectorClock().receiveMessage(self)
debug("Responded to vector message.", info=True, threshold=2)
except Exception as e:
debug(e)
def respond(self):
"""
How the systemm should respon to receiving this vector clock message.
"""
try:
me.getMe().getVectorClock().receiveMessage(self)
debug("Responded to vector message.",
info=True, threshold=2)
except Exception as e:
debug(e)
def refresh(self):
"""
Refresh value from the statistic function.
(only if the current vector clock is late)
"""
if me.getMe().getVectorClock().cameAfter(
self.getVectorClock()):
self._key = me.getMe().getUid()
self._value = self._statistic_function()
else:
pass # not relevant
def __init__(self, sender=None, recipients=None):
"""
Constructor. Set the payload to be the node data of the current node.
"""
super(MeMessage, self).__init__("", sender, recipients)
self._code = 'M'
self._payload = me.getMe().getCompressed()
def openTCPConnection(self, bindAddress=None):
"""
Open a new TCP connection from the local node to this node.
"""
debug("Trying to a new connection to [ " + self.getShortUid() + " ]",
info=True)
connector = connections.openConnection(self.getIp(), self.getPort())
if not bindAddress:
bindAddress = (me.getMe().getIp(), config.getNextSendPort())
#bindAddress = twisted.internet.address.IPv4Address(
# 'TCP', me.getMe().getIp(), config.SEND_PORT)
#config.getNextSendPort()
"""
c = connections.HissTCPClientConnection.fromPrimitives(
self,
self.getIp(),
self.getPort(),
bindAddress,
connector)
"""
'''
c = connections.HissTCPClientConnection.fromPrimitives(
self,
self.getIp(),
self.getPort(),
None,
connector)
self._tcpConnection = c
'''
c = connections.HissConnection.fromPrimitives(self, self.getIp(),
self.getPort(), None,
connector)
self._tcpConnection = c
def __init__(self, sender=None, recipients=None):
"""
Constructor. Set the payload to be the node data of the current node.
"""
super(MeMessage, self).__init__("", sender, recipients)
self._code = 'M'
self._payload = me.getMe().getCompressed()
def openTCPConnection(self, bindAddress=None):
"""
Open a new TCP connection from the local node to this node.
"""
debug("Trying to a new connection to [ " + self.getShortUid() + " ]", info=True)
connector = connections.openConnection(self.getIp(), self.getPort())
if not bindAddress:
bindAddress = (me.getMe().getIp(), config.getNextSendPort())
# bindAddress = twisted.internet.address.IPv4Address(
# 'TCP', me.getMe().getIp(), config.SEND_PORT)
# config.getNextSendPort()
"""
c = connections.HissTCPClientConnection.fromPrimitives(
self,
self.getIp(),
self.getPort(),
bindAddress,
connector)
"""
"""
c = connections.HissTCPClientConnection.fromPrimitives(
self,
self.getIp(),
self.getPort(),
None,
connector)
self._tcpConnection = c
"""
c = connections.HissConnection.fromPrimitives(self, self.getIp(), self.getPort(), None, connector)
self._tcpConnection = c
def membersRefresh():
"""
Run members refresh operation
"""
global members
try:
oldKeys = set(members.keys())
for key in members:
members_to_delete.add(key)
members.clear()
newMembersDict = simpledb.getSet(ITEMKEY)
if (newMembersDict != None) and (len(newMembersDict) > 0):
#newmembers = joined.split(GLUE)
for memberUid in newMembersDict:
if len(memberUid) > 6:
memberNode = nodes.buildNode(newMembersDict[memberUid])
if not me.getMe().__eq__(memberNode):
result = True # TODO Really should send a noop.
if result:
members[memberNode.getUid()] = memberNode
else:
debug("Noop failed. Node removed.", info=True)
else:
pass
members[me.getMe().getUid()] = me.getMe().getBaseData()
for key in members:
if key in members_to_delete:
members_to_delete.remove(key)
debug("Members refresh procedure ran.", success=True)
debug("There are " + str(len(members)) + " members in the system.",
info=True)
if set(members.keys()) == oldKeys:
""" We have reached stable state """
gossip.quitMembersRefresh()
connections.informAlive()
except Exception as e:
debug(e)
traceback.print_exc(file=sys.stdout)
persistSet()
def render_GET(self, request):
"""
get response method for the root resource
localhost:8000/
"""
debug("GET request received at Root on " + \
me.getMe().getIp(), info=True, threshold=3)
return 'Welcome to the REST API'
def membersRefresh():
"""
Run members refresh operation
"""
global members
try:
oldKeys = set(members.keys())
for key in members:
members_to_delete.add(key)
members.clear()
newMembersDict = simpledb.getSet(ITEMKEY)
if (newMembersDict != None) and (len(newMembersDict) > 0):
#newmembers = joined.split(GLUE)
for memberUid in newMembersDict:
if len(memberUid) > 6:
memberNode = nodes.buildNode(newMembersDict[memberUid])
if not me.getMe().__eq__(memberNode):
result = True # TODO Really should send a noop.
if result:
members[memberNode.getUid()] = memberNode
else:
debug("Noop failed. Node removed.", info=True)
else:
pass
members[me.getMe().getUid()] = me.getMe().getBaseData()
for key in members:
if key in members_to_delete:
members_to_delete.remove(key)
debug("Members refresh procedure ran.", success=True)
debug("There are " + str(len(members)) + " members in the system.",
info=True)
if set(members.keys()) == oldKeys:
""" We have reached stable state """
gossip.quitMembersRefresh()
connections.informAlive()
except Exception as e:
debug(e)
traceback.print_exc(file=sys.stdout)
persistSet()
def render_GET(self, request):
"""
get response method for the root resource
localhost:8000/
"""
debug("GET request received at Root on " + \
me.getMe().getIp(), info=True, threshold=3)
return 'Welcome to the REST API'
def __init__(self, name, statistic, vectorClock=None):
"""
Constructor
"""
super(UpdateAggregator, self).__init__(name, statistic)
if vectorClock:
self._vectorClock = copy.deepcopy(vectorClock)
else:
self._vectorClock = copy.deepcopy(
me.getMe().getVectorClock())
def __init__(self, statistic=None):
"""
Constructor
"""
self._key = me.getMe().getUid()
self._statistic_function = statistic
if statistic:
self._value = self._statistic_function()
else:
self._value = None
def network_load_single_stat():
"""
Give back a single number to represent network load.
"""
receivedPackets,sentPackets,receivedBytes,sentBytes = network_load()
debug("Load#"+me.getMe().getUid()+"#"+str(receivedPackets+sentPackets),
monitor=True)
return receivedBytes + sentBytes
def network_load_single_stat():
"""
Give back a single number to represent network load.
"""
receivedPackets, sentPackets, receivedBytes, sentBytes = network_load()
debug("Load#" + me.getMe().getUid() + "#" +
str(receivedPackets + sentPackets),
monitor=True)
return receivedBytes + sentBytes
def respond(self):
"""
Response to IsAliveMessage
"""
debug('Respnding to an IsAliveMessage with a MeMessage', info=True)
try:
responseMsg = MeMessage(me.getMe().getUid(),
[self._sender.getUid()])
responseMsg.send()
debug('Sent MeMessage', success=True)
except:
debug("Error sending MeMessage", error=True)
def respond(self):
"""
Response to IsAliveMessage
"""
debug('Respnding to an IsAliveMessage with a MeMessage',
info=True)
try:
responseMsg = MeMessage(
me.getMe().getUid(),
[self._sender.getUid()])
responseMsg.send()
debug('Sent MeMessage', success=True)
except:
debug("Error sending MeMessage", error=True)
def respond(self):
"""
Defines how to respond when one of these messages is recieved
No defined behavior in parent class yet.
"""
try:
self.decrementTtl()
if self.getTtl() > 0:
self._sender = me.getMe().getBaseData()
self._recipients = []
# retain the same payload.
gossip.gossipThis(self)
debug("Responded to gossip status message", success=True)
except:
debug("failed to respond to gossip status message", error=True)
def respond(self):
"""
Defines how to respond when one of these messages is recieved
No defined behavior in parent class yet.
"""
try:
self.decrementTtl()
if self.getTtl() > 0:
self._sender = me.getMe().getBaseData()
self._recipients = []
# retain the same payload.
gossip.gossipThis(self)
debug("Responded to gossip status message", success=True)
except:
debug("failed to respond to gossip status message", error=True)
def log(typ, msg, level, ordered=True):
"""
Log a received piece of data from the api.
Can be ordered or not.
"""
try:
item = {"type":typ,"message":msg,"level":level,"time":time.time()}
if ordered:
item["machine"] = me.getUid()
item["clock"] = me.getMe().getVectorClock().getClocks()
insert(item)
debug("logged item", success=True)
except Exception, e:
debug(e, error=True)
debug("did not log message (ordered:" + str(ordered) + ")", error=True)
def getNeighbors(self):
"""
Get Neighbors
"""
unilength = len(self._universeUids())
if unilength > 2:
if not self.count:
idealcount = int(math.ceil(math.log10(unilength)))
self.count = max(2, idealcount)
samplespace = self._universeUids()
if me.getUid() in samplespace:
samplespace.remove(me.getMe().getUid())
self.neighborSet = random.sample(samplespace, self.count)
return set(self.neighborSet)
else:
return set([])
def getNeighbors(self):
"""
Get Neighbors
"""
unilength = len(self._universeUids())
if unilength > 2:
if not self.count:
idealcount = int(math.ceil(math.log10(unilength)))
self.count = max(2, idealcount)
samplespace = self._universeUids()
if me.getUid() in samplespace:
samplespace.remove(me.getMe().getUid())
self.neighborSet = random.sample(samplespace, self.count)
return set(self.neighborSet)
else:
return set([])
def init():
"""
Initialize the connections.
Let's get it started...in here!
Developer note: the advantage of putting it here is
that me.py doesn't need to import anything.
"""
global neighborStrategy
me.init(nodes.CurrentNode())
debug("Init called. Node is " + me.getUid(), info=True)
debug("#".join(["New", me.getMe().getShortUid(), me.getUid()]),
monitor=True)
neighborStrategy = neighbors.neighborStrategyFactory(
config.GOSSIP_NEIGHBOR_STRATEGY)
def __init__(self, vClock, sender=None, recipients=None):
"""
Constructor
"""
if type(vClock) is vectorClock.VectorClock:
super(VectorMessage, self).__init__(vClock.getClocks(), sender,
recipients)
self._clockKey = vClock.getKey()
else:
super(VectorMessage, self).__init__(vClock, sender, recipients)
if sender:
self._clockKey = sender.getUid()
else:
self._clockKey = me.getMe().getUid()
self._code = "V"
def __init__(self, payload, sender=None, recipients=None):
"""
Constructor
"""
self._payload = payload
self._time = time.time()
self._code = "?"
if sender:
self._sender = sender
else:
self._sender = me.getMe().getBaseData()
# just assign basic node data to the sender.
if recipients:
self._recipients = recipients
else:
self._recipients = []
def __init__(self, payload, sender=None, recipients=None):
"""
Constructor
"""
self._payload = payload
self._time = time.time()
self._code = "?"
if sender:
self._sender = sender
else:
self._sender = me.getMe().getBaseData()
# just assign basic node data to the sender.
if recipients:
self._recipients = recipients
else:
self._recipients = []
def __init__(self, vClock, sender=None, recipients=None):
"""
Constructor
"""
if type(vClock) is vectorClock.VectorClock:
super(VectorMessage, self).__init__(
vClock.getClocks(), sender, recipients)
self._clockKey = vClock.getKey()
else:
super(VectorMessage, self).__init__(
vClock, sender, recipients)
if sender:
self._clockKey = sender.getUid()
else:
self._clockKey = me.getMe().getUid()
self._code = "V"
def debug(
msg,
threshold=5,
error=False,
success=False,
info=False,
strange=False,
monitor=False):
"""
Main debug function.
"""
global shortUid
if not shortUid:
try:
shortUid = me.getMe().getShortUid()
except:
shortUid = ""
uidstr = (shortUid + "\t") if INCLUDE_UID else ""
if type(msg) is str:
if DEBUG_FLG and threshold >= THRESHOLD:
if error:
printError("! ERROR: \t" + uidstr + msg)
elif success:
printMessage("* SUCCESS:\t" + uidstr + msg)
elif info:
printMessage("- INFO: \t" + uidstr + msg)
elif strange:
printMessage("? STRANGE:\t" + uidstr + msg)
elif monitor:
printMessage("MONITOR: " + msg)
else:
printMessage(msg)
else:
if error and threshold >= THRESHOLD:
printError("! ERROR: \t" + uidstr + msg)
else:
print "! ERROR: ", msg
traceback.print_exc(file=sys.stderr)
def createVectorClockMessage():
"""
Create a new vector clock message
"""
return me.getMe().getVectorClock().createMessage()
def createVectorClockMessage():
try:
return me.getMe().getVectorClock().createMessage()
except Exception as e:
debug(e)
def createVectorClockMessage():
try:
return me.getMe().getVectorClock().createMessage()
except Exception as e:
debug(e)