def testExtendSubgraph1():
graph = Graph(5, 5, [(0, 1), (1, 2), (2, 3), (3, 4), (4, 0)])
extender = ExtendSubgraph(graph, 101, 3)
task = Task([0, 1, 1, 0, 0], [0, 3], 0, 0)
newTasks = extender.generateNewTasks(task)
for task in newTasks:
print task.vertices, task.edges
def startProcessing(self, netString):
taskRetries = 0
stats.reset()
self.startTime = time.time()
log.info('Processing started')
while True:
if (self.p != None and self.m != None and self.initGraph != None):
break
log.debug("Waiting for initialization.")
time.sleep(WAIT_FOR_INITIALIZATION)
self.graphProcessor = ExtendSubgraph(self.initGraph, self.p, self.m)
while taskRetries < MAX_RETRIES:
task = getNewTask(self)
# TODO remove this later
if task == None:
taskRetries += 1
log.debug("No new task found")
continue
else:
log.debug("Task's string : " + task.toNetString())
stats.tasksProcessed += 1
taskRetries = 0
newTasks = self.graphProcessor.generateNewTasks(task)
for newTask in newTasks:
if checkUniquenessOfTask(
newTask.bloomHash,
self.aliveSlaves[newTask.serverHash]):
TaskQueue.put(newTask)
log.info('Task complete')
self.sendJobCompletionNotiToMaster()
def testExtendSubgraph3():
graph = Graph(3, 3, [(0, 1), (1, 2), (0, 2)])
extender = ExtendSubgraph(graph, 101, 3)
task = Task([1, 1, 1], [1], 0, 0)
newTasks = extender.generateNewTasks(task)
for task in newTasks:
print task.vertices, task.edges
def testExtendSubgraph1():
graph = Graph(5,5,[(0,1),(1,2),(2,3),(3,4),(4,0)])
extender = ExtendSubgraph(graph, 101, 3)
task = Task([0,1,1,0,0],[0,3],0,0)
newTasks = extender.generateNewTasks(task)
for task in newTasks:
print task.vertices, task.edges
def testExtendSubgraph3():
graph = Graph(3,3,[(0,1),(1,2),(0,2)])
extender = ExtendSubgraph(graph, 101, 3)
task = Task([1,1,1], [1], 0, 0)
newTasks = extender.generateNewTasks(task)
for task in newTasks:
print task.vertices, task.edges
def startProcessing(self, netString):
taskRetries = 0
stats.reset()
self.startTime = time.time()
log.info('Processing started')
while True:
if (self.p != None and self.m != None and self.initGraph != None):
break
log.debug("Waiting for initialization.")
time.sleep(WAIT_FOR_INITIALIZATION)
self.graphProcessor = ExtendSubgraph(self.initGraph, self.p, self.m)
while taskRetries < MAX_RETRIES :
task = getNewTask(self)
# TODO remove this later
if task == None :
taskRetries += 1
log.debug("No new task found")
continue
else :
log.debug("Task's string : " + task.toNetString())
stats.tasksProcessed += 1
taskRetries = 0
newTasks = self.graphProcessor.generateNewTasks(task)
for newTask in newTasks :
if checkUniquenessOfTask(newTask.bloomHash, self.aliveSlaves[newTask.serverHash]) :
TaskQueue.put(newTask)
log.info('Task complete')
self.sendJobCompletionNotiToMaster()
class Main:
''' This Main class of Slave server is intended for following tasks:
- Functions for acting on messages received by the listener
- Receiving graph, config informations from master
- Sending heartbeats
- Processing tasks
- Storing and sending result
- ###
'''
def __init__(self):
self.p = None
self.m = None
self.aliveSlaves = []
self.servers = []
self.initGraph = None
self.graphProcessor = None
# Currently lock is not required as this will be used
self.taskCounter = None
'''Save the servers'''
def saveServerInfo(self, netString):
self.servers = server.netStringToServerList(netString)
self.aliveSlaves = filter(lambda s : s.role=='slave' and s.alive, self.servers)
self.m = len(self.aliveSlaves)
log.info("Server informations saved")
'''Save the inital graph passed by master'''
def saveGraph(self, netString):
self.initGraph = graph.stringToGraph(netString)
log.info("Graph saved")
def startProcessing(self, netString):
taskRetries = 0
stats.reset()
self.startTime = time.time()
log.info('Processing started')
while True:
if (self.p != None and self.m != None and self.initGraph != None):
break
log.debug("Waiting for initialization.")
time.sleep(WAIT_FOR_INITIALIZATION)
self.graphProcessor = ExtendSubgraph(self.initGraph, self.p, self.m)
while taskRetries < MAX_RETRIES :
task = getNewTask(self)
# TODO remove this later
if task == None :
taskRetries += 1
log.debug("No new task found")
continue
else :
log.debug("Task's string : " + task.toNetString())
stats.tasksProcessed += 1
taskRetries = 0
newTasks = self.graphProcessor.generateNewTasks(task)
for newTask in newTasks :
if checkUniquenessOfTask(newTask.bloomHash, self.aliveSlaves[newTask.serverHash]) :
TaskQueue.put(newTask)
log.info('Task complete')
self.sendJobCompletionNotiToMaster()
# TODO send job completion notification to master
def getPartialResult(self, netString):
# return netString of result
pass
def recordPing(self, netString):
log.info("Responded to master's ping.")
def unrecognizedMessage(self, netString):
log.error("Unrecognized Message received " + netString)
def sendHeartBeatToMaster(self):
pass
def saveNetworkPrime(self,message):
self.p = int(message)
log.info("Prime saved")
def sendJobCompletionNotiToMaster(self):
masterServer = filter(lambda s : s.role=='master' and s.alive, self.servers)[0]
endTime = time.time()
stats.timeTaken = endTime - self.startTime
network.sendToIP(masterServer.IP, masterServer.port,
JOBCOMPLETE + MESSAGE_DELIMITER + stats.toNetString())
log.info("Job completion notification sent")
# Clean bloom filter and wait for next input
BloomHashFilter.clean()
class Main:
''' This Main class of Slave server is intended for following tasks:
- Functions for acting on messages received by the listener
- Receiving graph, config informations from master
- Sending heartbeats
- Processing tasks
- Storing and sending result
- ###
'''
def __init__(self):
self.p = None
self.m = None
self.aliveSlaves = []
self.servers = []
self.initGraph = None
self.graphProcessor = None
# Currently lock is not required as this will be used
self.taskCounter = None
'''Save the servers'''
def saveServerInfo(self, netString):
self.servers = server.netStringToServerList(netString)
self.aliveSlaves = filter(lambda s: s.role == 'slave' and s.alive,
self.servers)
self.m = len(self.aliveSlaves)
log.info("Server informations saved")
'''Save the inital graph passed by master'''
def saveGraph(self, netString):
self.initGraph = graph.stringToGraph(netString)
log.info("Graph saved")
def startProcessing(self, netString):
taskRetries = 0
stats.reset()
self.startTime = time.time()
log.info('Processing started')
while True:
if (self.p != None and self.m != None and self.initGraph != None):
break
log.debug("Waiting for initialization.")
time.sleep(WAIT_FOR_INITIALIZATION)
self.graphProcessor = ExtendSubgraph(self.initGraph, self.p, self.m)
while taskRetries < MAX_RETRIES:
task = getNewTask(self)
# TODO remove this later
if task == None:
taskRetries += 1
log.debug("No new task found")
continue
else:
log.debug("Task's string : " + task.toNetString())
stats.tasksProcessed += 1
taskRetries = 0
newTasks = self.graphProcessor.generateNewTasks(task)
for newTask in newTasks:
if checkUniquenessOfTask(
newTask.bloomHash,
self.aliveSlaves[newTask.serverHash]):
TaskQueue.put(newTask)
log.info('Task complete')
self.sendJobCompletionNotiToMaster()
# TODO send job completion notification to master
def getPartialResult(self, netString):
# return netString of result
pass
def recordPing(self, netString):
log.info("Responded to master's ping.")
def unrecognizedMessage(self, netString):
log.error("Unrecognized Message received " + netString)
def sendHeartBeatToMaster(self):
pass
def saveNetworkPrime(self, message):
self.p = int(message)
log.info("Prime saved")
def sendJobCompletionNotiToMaster(self):
masterServer = filter(lambda s: s.role == 'master' and s.alive,
self.servers)[0]
endTime = time.time()
stats.timeTaken = endTime - self.startTime
network.sendToIP(masterServer.IP, masterServer.port,
JOBCOMPLETE + MESSAGE_DELIMITER + stats.toNetString())
log.info("Job completion notification sent")
# Clean bloom filter and wait for next input
BloomHashFilter.clean()
