def testExtendSubgraph1(): graph = Graph(5, 5, [(0, 1), (1, 2), (2, 3), (3, 4), (4, 0)]) extender = ExtendSubgraph(graph, 101, 3) task = Task([0, 1, 1, 0, 0], [0, 3], 0, 0) newTasks = extender.generateNewTasks(task) for task in newTasks: print task.vertices, task.edges
def startProcessing(self, netString): taskRetries = 0 stats.reset() self.startTime = time.time() log.info('Processing started') while True: if (self.p != None and self.m != None and self.initGraph != None): break log.debug("Waiting for initialization.") time.sleep(WAIT_FOR_INITIALIZATION) self.graphProcessor = ExtendSubgraph(self.initGraph, self.p, self.m) while taskRetries < MAX_RETRIES: task = getNewTask(self) # TODO remove this later if task == None: taskRetries += 1 log.debug("No new task found") continue else: log.debug("Task's string : " + task.toNetString()) stats.tasksProcessed += 1 taskRetries = 0 newTasks = self.graphProcessor.generateNewTasks(task) for newTask in newTasks: if checkUniquenessOfTask( newTask.bloomHash, self.aliveSlaves[newTask.serverHash]): TaskQueue.put(newTask) log.info('Task complete') self.sendJobCompletionNotiToMaster()
def testExtendSubgraph3(): graph = Graph(3, 3, [(0, 1), (1, 2), (0, 2)]) extender = ExtendSubgraph(graph, 101, 3) task = Task([1, 1, 1], [1], 0, 0) newTasks = extender.generateNewTasks(task) for task in newTasks: print task.vertices, task.edges
def testExtendSubgraph1(): graph = Graph(5,5,[(0,1),(1,2),(2,3),(3,4),(4,0)]) extender = ExtendSubgraph(graph, 101, 3) task = Task([0,1,1,0,0],[0,3],0,0) newTasks = extender.generateNewTasks(task) for task in newTasks: print task.vertices, task.edges
def testExtendSubgraph3(): graph = Graph(3,3,[(0,1),(1,2),(0,2)]) extender = ExtendSubgraph(graph, 101, 3) task = Task([1,1,1], [1], 0, 0) newTasks = extender.generateNewTasks(task) for task in newTasks: print task.vertices, task.edges
def startProcessing(self, netString): taskRetries = 0 stats.reset() self.startTime = time.time() log.info('Processing started') while True: if (self.p != None and self.m != None and self.initGraph != None): break log.debug("Waiting for initialization.") time.sleep(WAIT_FOR_INITIALIZATION) self.graphProcessor = ExtendSubgraph(self.initGraph, self.p, self.m) while taskRetries < MAX_RETRIES : task = getNewTask(self) # TODO remove this later if task == None : taskRetries += 1 log.debug("No new task found") continue else : log.debug("Task's string : " + task.toNetString()) stats.tasksProcessed += 1 taskRetries = 0 newTasks = self.graphProcessor.generateNewTasks(task) for newTask in newTasks : if checkUniquenessOfTask(newTask.bloomHash, self.aliveSlaves[newTask.serverHash]) : TaskQueue.put(newTask) log.info('Task complete') self.sendJobCompletionNotiToMaster()
class Main: ''' This Main class of Slave server is intended for following tasks: - Functions for acting on messages received by the listener - Receiving graph, config informations from master - Sending heartbeats - Processing tasks - Storing and sending result - ### ''' def __init__(self): self.p = None self.m = None self.aliveSlaves = [] self.servers = [] self.initGraph = None self.graphProcessor = None # Currently lock is not required as this will be used self.taskCounter = None '''Save the servers''' def saveServerInfo(self, netString): self.servers = server.netStringToServerList(netString) self.aliveSlaves = filter(lambda s : s.role=='slave' and s.alive, self.servers) self.m = len(self.aliveSlaves) log.info("Server informations saved") '''Save the inital graph passed by master''' def saveGraph(self, netString): self.initGraph = graph.stringToGraph(netString) log.info("Graph saved") def startProcessing(self, netString): taskRetries = 0 stats.reset() self.startTime = time.time() log.info('Processing started') while True: if (self.p != None and self.m != None and self.initGraph != None): break log.debug("Waiting for initialization.") time.sleep(WAIT_FOR_INITIALIZATION) self.graphProcessor = ExtendSubgraph(self.initGraph, self.p, self.m) while taskRetries < MAX_RETRIES : task = getNewTask(self) # TODO remove this later if task == None : taskRetries += 1 log.debug("No new task found") continue else : log.debug("Task's string : " + task.toNetString()) stats.tasksProcessed += 1 taskRetries = 0 newTasks = self.graphProcessor.generateNewTasks(task) for newTask in newTasks : if checkUniquenessOfTask(newTask.bloomHash, self.aliveSlaves[newTask.serverHash]) : TaskQueue.put(newTask) log.info('Task complete') self.sendJobCompletionNotiToMaster() # TODO send job completion notification to master def getPartialResult(self, netString): # return netString of result pass def recordPing(self, netString): log.info("Responded to master's ping.") def unrecognizedMessage(self, netString): log.error("Unrecognized Message received " + netString) def sendHeartBeatToMaster(self): pass def saveNetworkPrime(self,message): self.p = int(message) log.info("Prime saved") def sendJobCompletionNotiToMaster(self): masterServer = filter(lambda s : s.role=='master' and s.alive, self.servers)[0] endTime = time.time() stats.timeTaken = endTime - self.startTime network.sendToIP(masterServer.IP, masterServer.port, JOBCOMPLETE + MESSAGE_DELIMITER + stats.toNetString()) log.info("Job completion notification sent") # Clean bloom filter and wait for next input BloomHashFilter.clean()
class Main: ''' This Main class of Slave server is intended for following tasks: - Functions for acting on messages received by the listener - Receiving graph, config informations from master - Sending heartbeats - Processing tasks - Storing and sending result - ### ''' def __init__(self): self.p = None self.m = None self.aliveSlaves = [] self.servers = [] self.initGraph = None self.graphProcessor = None # Currently lock is not required as this will be used self.taskCounter = None '''Save the servers''' def saveServerInfo(self, netString): self.servers = server.netStringToServerList(netString) self.aliveSlaves = filter(lambda s: s.role == 'slave' and s.alive, self.servers) self.m = len(self.aliveSlaves) log.info("Server informations saved") '''Save the inital graph passed by master''' def saveGraph(self, netString): self.initGraph = graph.stringToGraph(netString) log.info("Graph saved") def startProcessing(self, netString): taskRetries = 0 stats.reset() self.startTime = time.time() log.info('Processing started') while True: if (self.p != None and self.m != None and self.initGraph != None): break log.debug("Waiting for initialization.") time.sleep(WAIT_FOR_INITIALIZATION) self.graphProcessor = ExtendSubgraph(self.initGraph, self.p, self.m) while taskRetries < MAX_RETRIES: task = getNewTask(self) # TODO remove this later if task == None: taskRetries += 1 log.debug("No new task found") continue else: log.debug("Task's string : " + task.toNetString()) stats.tasksProcessed += 1 taskRetries = 0 newTasks = self.graphProcessor.generateNewTasks(task) for newTask in newTasks: if checkUniquenessOfTask( newTask.bloomHash, self.aliveSlaves[newTask.serverHash]): TaskQueue.put(newTask) log.info('Task complete') self.sendJobCompletionNotiToMaster() # TODO send job completion notification to master def getPartialResult(self, netString): # return netString of result pass def recordPing(self, netString): log.info("Responded to master's ping.") def unrecognizedMessage(self, netString): log.error("Unrecognized Message received " + netString) def sendHeartBeatToMaster(self): pass def saveNetworkPrime(self, message): self.p = int(message) log.info("Prime saved") def sendJobCompletionNotiToMaster(self): masterServer = filter(lambda s: s.role == 'master' and s.alive, self.servers)[0] endTime = time.time() stats.timeTaken = endTime - self.startTime network.sendToIP(masterServer.IP, masterServer.port, JOBCOMPLETE + MESSAGE_DELIMITER + stats.toNetString()) log.info("Job completion notification sent") # Clean bloom filter and wait for next input BloomHashFilter.clean()