def retreive_key(self, *data): time.sleep(random.randint(0, 1000) / 1000) dict = data[0] from_node = data[1] val = self.kv.get(dict.key, None) response = Request("ACK-GET", dict.key, val, dict.request) Messaging.send_message(self, from_node, response)
def perform_put(self, *data): dict = data[0] key = dict.key preference_list = self.hash_ring.get_node(key, self.failed_nodes) if (self not in preference_list): coordinator = preference_list[0].id dict = Request("FORWARD-PUT", dict.key, dict.value, generate_random_number(), dict.client) Messaging.send_message(self, coordinator, dict) time.sleep(3) if REQUESTS.get(dict.request, False) != True: #print("Timedout PUT") dict.action = "PUT" dict.request = generate_random_number() self.socket.settimeout(None) self.failed_nodes.append(coordinator) self.perform_put(dict) else: self.vector_clock.update(self.id, self.get_sequence_no()) metadata = deepcopy(self.vector_clock) if dict.value[1] > metadata: metadata = dict.value[1] dict.value = (dict.value[0], metadata) dict.request = generate_random_number() Messaging.broadcast_put(self, preference_list, dict)
def handoff(self): for node in self.check_for_sync: keys = list(self.sync_kv[node].keys()) values = list(self.sync_kv[node].values()) synchronize = Request("SYNC", keys, values, generate_random_number()) Messaging.send_message(self, node, synchronize)
def exposed_CLIBroadcast(self, sUsername, sMessage): """Gets the state for the CLI""" global gCoreServerLock global gDatabase global gMessaging bLocked = False try: # Acquire the lock gCoreServerLock.Acquire(lockTimeout) bLocked = True log.info("CoreServerService.CLIBroadcast(" + sMessage + ")") # Create the messaging object and broadcast the message if gMessaging == None: gMessaging = Messaging("System", gDatabase) gMessaging.broadcastMessage(sMessage) bResult = True except Exception as ex: # Log the error log.error("CoreServerService.CLIBroadcast() failed: " + str(ex)) log.error("Traceback:\r\n%s\r\n" % traceback.format_exc()) gMessaging = None bResult = False # Release the lock and return if bLocked: gCoreServerLock.Release() return bResult
def exposed_CLIBroadcast(self, sUsername, sMessage): """Gets the state for the CLI""" global gCoreServerLock global gDatabase global gMessaging bLocked = False try: # Acquire the lock gCoreServerLock.Acquire(lockTimeout) bLocked = True log.info("CoreServerService.CLIBroadcast(" + sMessage + ")") # Create the messaging object and broadcast the message if gMessaging == None: gMessaging = Messaging("System", gDatabase) gMessaging.broadcastMessage(sMessage) bResult = True except Exception as ex: # Log the error log.error("CoreServerService.CLIBroadcast() failed: " + str(ex)) log.error("Traceback:\r\n%s\r\n" % traceback.format_exc()) gMessaging = None bResult = False # Release the lock and return if bLocked: gCoreServerLock.Release() return bResult
def resurrect(self, node): dragonsblood = Request("REVIVE", None) Messaging.send_message(self, node, dragonsblood) # client = Client(1) # client.put_req('x',1) # time.sleep(1) # client.get_req('x')
def __init__(self, sRemoteUser, nSourceSequenceID, pSystemModel): """Constructor""" # Call base class init Thread.__init__(self) # Initialize variables self.username = sRemoteUser self.sourceSequenceID = nSourceSequenceID self.sourceComponentID = 0 self.runSequenceID = 0 self.runComponentID = 0 self.systemModel = pSystemModel self.initializing = True self.running = False self.startComponentID = 0 self.userSourceIDs = True self.runWillPause = False self.runIsPaused = False self.runAborted = False self.showAbortPrompt = False # Create database connection and sequence manager self.database = DBComm() self.database.Connect() self.sequenceManager = SequenceManager(self.database) # Create messaging object self.messaging = Messaging(self.username, self.database) # Fetch the sequence from the database and make sure it is valid self.sourceSequence = self.sequenceManager.GetSequence( self.username, self.sourceSequenceID) if not self.sourceSequence["metadata"]["valid"]: raise Exceptions("Cannot run an invalid sequence (" + self.sourceSequenceID + ")") # Create a new sequence in the run history log.debug("Starting run of sequence " + str(self.sourceSequenceID) + " (" + self.sourceSequence["metadata"]["name"] + ")") pConfiguration = self.database.GetConfiguration(self.username) self.runSequenceID = self.database.CreateSequence( self.username, self.sourceSequence["metadata"]["name"], self.sourceSequence["metadata"]["comment"], "History", pConfiguration["reactors"], pConfiguration["reagentsperreactor"]) # Copy the cassettes for pComponent in self.sourceSequence["components"]: if pComponent["componenttype"] == Cassette.componentType: pUnitOperation = UnitOperations.createFromComponent( self.sourceSequenceID, pComponent, self.username, self.database) pUnitOperation.copyComponent(self.sourceSequenceID, self.runSequenceID)
def perform_store(self, data): time.sleep(random.randint(0, 1000) / 1000) dict = data[0] addr = data[1] self.vector_clock.update(self.id, self.get_sequence_no()) self.vector_clock = self.vector_clock.converge( [self.vector_clock, dict.value[1]]) if dict.key not in self.kv: self.kv[dict.key] = list() self.kv[dict.key].append(dict.value) self.kv[dict.key] = self.perform_syntactic_reconcilation( self.kv[dict.key]) dict = Request("ACK-PUT", None, None, dict.request) Messaging.send_message(self, PORT_TO_ID[addr[1]], dict)
def PrintAccessors(self, msgClass): msg = msgClass() for fieldInfo in msgClass.fields: txt = "body.%s.%s: " % (msgClass.__name__, fieldInfo.name) if (fieldInfo.count == 1): txt += str(Messaging.get(msg, fieldInfo)) else: for i in range(0, fieldInfo.count): #print("body.",msgClass.__name__, ".", method.__name__, "[",i,"] = ", method(msg,i), " #", method.__doc__, "in", method.units) txt += str(Messaging.get(msg, fieldInfo, i)) if (i < fieldInfo.count - 1): txt += ", " txt += " # " + fieldInfo.description + " in " + fieldInfo.units print(txt)
def test_accessors(self): msgclass = self.msgLib.MsgClassFromName["Network.Connect"] sameMsgClass = self.msgLib.Messages.Network.Connect self.assertEqual(msgclass, sameMsgClass) expected = "Testing" testMsg = msgclass() Messaging.set(testMsg, msgclass.fields[0], expected) observed = testMsg.GetName() self.assertMultiLineEqual(expected, observed) expected = "MoreTesting" testMsg.SetName(expected) observed = Messaging.get(testMsg, msgclass.fields[0]) self.assertMultiLineEqual(expected, observed)
def perform_get(self, dict): key = dict.key preference_list = self.hash_ring.get_node(key, self.failed_nodes) if (self not in preference_list): coordinator = preference_list[0].id dict = Request("FORWARD-GET", dict.key, dict.value, generate_random_number(), dict.client) Messaging.send_message(self, coordinator, dict) time.sleep(3) if REQUESTS.get(dict.request, False) != True: #print("Timedout GET") dict.action = "GET" dict.request = generate_random_number() self.failed_nodes.append(coordinator) self.perform_get(dict) else: dict.request = generate_random_number() Messaging.broadcast_get(self, preference_list, dict)
def put(self, key, value, retries=0): while retries < MAX_RETRIES: dynamo_node = self.nodes[random.randint(0, len(self.nodes) - 1)] vc = VectorClock() vc.update(dynamo_node, 0) value1 = (value, self.datastore.get(key, vc)) #print("Client value is "+str(value1)) put_request = Request("PUT", key, value1, None, self.port) self.socket.settimeout(5) Messaging.send_message(self, dynamo_node, put_request) try: while True: data, addr = self.socket.recvfrom(40960) if data: self.socket.settimeout(None) return pickle.loads(data) except Exception: self.socket.settimeout(None) retries += 1
def get(self, key, retries=0): while retries < MAX_RETRIES: get_request = Request("GET", key, None, None, self.port) dynamo_node = self.nodes[random.randint(0, len(self.nodes) - 1)] self.socket.settimeout(5) Messaging.send_message(self, dynamo_node, get_request) try: while True: data, addr = self.socket.recvfrom(40960) if data: data = pickle.loads(data) if (data == "FAILURE"): # print("FAILURE") return -INF result, self.datastore[ key] = self.perform_semantic_reconcilation(data) #print(str(result)+" "+str(self.datastore[key].clock)) self.socket.settimeout(None) return result except Exception: self.socket.settimeout(None) retries += 1
def __init__(self, sRemoteUser, nSourceSequenceID, pSystemModel): """Constructor""" # Call base class init Thread.__init__(self) # Initialize variables self.username = sRemoteUser self.sourceSequenceID = nSourceSequenceID self.sourceComponentID = 0 self.runSequenceID = 0 self.runComponentID = 0 self.systemModel = pSystemModel self.initializing = True self.running = False self.startComponentID = 0 self.userSourceIDs = True self.runWillPause = False self.runIsPaused = False self.runAborted = False self.showAbortPrompt = False # Create database connection and sequence manager self.database = DBComm() self.database.Connect() self.sequenceManager = SequenceManager(self.database) # Create messaging object self.messaging = Messaging(self.username, self.database) # Fetch the sequence from the database and make sure it is valid self.sourceSequence = self.sequenceManager.GetSequence(self.username, self.sourceSequenceID) if not self.sourceSequence["metadata"]["valid"]: raise Exceptions("Cannot run an invalid sequence (" + self.sourceSequenceID + ")") # Create a new sequence in the run history log.debug("Starting run of sequence " + str(self.sourceSequenceID) + " (" + self.sourceSequence["metadata"]["name"] + ")") pConfiguration = self.database.GetConfiguration(self.username) self.runSequenceID = self.database.CreateSequence(self.username, self.sourceSequence["metadata"]["name"], self.sourceSequence["metadata"]["comment"], "History", pConfiguration["reactors"], pConfiguration["reagentsperreactor"]) # Copy the cassettes for pComponent in self.sourceSequence["components"]: if pComponent["componenttype"] == Cassette.componentType: pUnitOperation = UnitOperations.createFromComponent(self.sourceSequenceID, pComponent, self.username, self.database) pUnitOperation.copyComponent(self.sourceSequenceID, self.runSequenceID)
def run(self): if self.broadcastFlag: pMessaging = Messaging(self.username, self.database) pMessaging.broadcastMessage(urllib.unquote(self.userMessage))
def kill(self, node=None): if node is None: node = self.nodes[random.randint(0, len(self.nodes) - 1)] kill_switch = Request("EXIT", None) Messaging.send_message(self, node, kill_switch) return node
class Sequence(Thread): def __init__(self, sRemoteUser, nSourceSequenceID, pSystemModel): """Constructor""" # Call base class init Thread.__init__(self) # Initialize variables self.username = sRemoteUser self.sourceSequenceID = nSourceSequenceID self.sourceComponentID = 0 self.runSequenceID = 0 self.runComponentID = 0 self.systemModel = pSystemModel self.initializing = True self.running = False self.startComponentID = 0 self.userSourceIDs = True self.runWillPause = False self.runIsPaused = False self.runAborted = False self.showAbortPrompt = False # Create database connection and sequence manager self.database = DBComm() self.database.Connect() self.sequenceManager = SequenceManager(self.database) # Create messaging object self.messaging = Messaging(self.username, self.database) # Fetch the sequence from the database and make sure it is valid self.sourceSequence = self.sequenceManager.GetSequence(self.username, self.sourceSequenceID) if not self.sourceSequence["metadata"]["valid"]: raise Exceptions("Cannot run an invalid sequence (" + self.sourceSequenceID + ")") # Create a new sequence in the run history log.debug("Starting run of sequence " + str(self.sourceSequenceID) + " (" + self.sourceSequence["metadata"]["name"] + ")") pConfiguration = self.database.GetConfiguration(self.username) self.runSequenceID = self.database.CreateSequence(self.username, self.sourceSequence["metadata"]["name"], self.sourceSequence["metadata"]["comment"], "History", pConfiguration["reactors"], pConfiguration["reagentsperreactor"]) # Copy the cassettes for pComponent in self.sourceSequence["components"]: if pComponent["componenttype"] == Cassette.componentType: pUnitOperation = UnitOperations.createFromComponent(self.sourceSequenceID, pComponent, self.username, self.database) pUnitOperation.copyComponent(self.sourceSequenceID, self.runSequenceID) def setStartComponent(self, nComponentID): """Sets the first component of the run""" self.startComponentID = nComponentID def getIDs(self): """Return the appropriate sequence and component IDs""" if self.userSourceIDs: return (self.sourceSequenceID, self.sourceComponentID) else: return (self.runSequenceID, self.runComponentID) def pauseRun(self): """Flags the running sequence to pause at the start of the next unit operation""" if not self.running: raise Exception("Sequence not running, cannot pause") if self.runWillPause: raise Exception("Sequence already will pause, cannot pause") if self.runIsPaused: raise Exception("Sequence is already paused, cannot pause") self.runWillPause = True def continueRun(self): """Continues a paused sequence""" if not self.running: raise Exception("Sequence not running, cannot continue") if not self.runWillPause and not self.runIsPaused: raise Exception("Sequence run not paused, cannot continue") self.runWillPause = False self.runIsPaused = False def willRunPause(self): """Returns true if the sequence run is flagged to pause, false otherwise""" return self.runWillPause def isRunPaused(self): """Returns true if the sequence run is paused, false otherwise""" return self.runIsPaused def abortRun(self): """Aborts the current sequence run""" self.runAborted = True self.showAbortPrompt = False def isRunComplete(self): """Returns true if the sequence run has completed and is waiting at the summary unit operation, false otherwise""" return not self.userSourceIDs def setShowAbortPrompt(self, bShowAbortPrompt): """Sets the flag that indicates if the abort sequence run prompt should be set in the run state""" self.showAbortPrompt = bShowAbortPrompt def getShowAbortPrompt(self): """Returns the flag that indicates if the abort sequence run prompt should be set in the run state""" return self.showAbortPrompt def run(self): """Thread entry point""" sRunError = "" self.userSourceIDs = True log.debug("Running Sequence") try: # Main sequence run loop log.debug("sourceSequence -> %s" % str(self.sourceSequence)) sMessage = "Run of sequence \"" + self.sourceSequence["metadata"]["name"] + "\" started." log.debug(sMessage) self.messaging.broadcastMessage(sMessage) nComponentCount = len(self.sourceSequence["components"]) nCurrentComponent = 0 while nCurrentComponent < nComponentCount: # Get the next component pSourceComponent = self.sourceSequence["components"][nCurrentComponent] # Skip components until we find our start component self.sourceComponentID = pSourceComponent["id"] if self.initializing and (self.startComponentID != 0) and (self.sourceComponentID != self.startComponentID): log.debug("Skipping unit operation (" + pSourceComponent["componenttype"] + ")") nCurrentComponent += 1 continue # Update our initializing and running flags self.initializing = False self.running = True # Ignore any previous summary component if pSourceComponent["componenttype"] == Summary.componentType: log.debug("Skipping unit operation (" + pSourceComponent["componenttype"] + ")") nCurrentComponent += 1 continue # Create and run the next unit operation log.debug("Starting unit operation " + str(self.sourceComponentID) + " (" + pSourceComponent["componenttype"] + ")") pSourceUnitOperation = UnitOperations.createFromComponent(self.sourceSequenceID, pSourceComponent, self.username, self.sequenceManager.database, self.systemModel) self.runComponentID = pSourceUnitOperation.copyComponent(self.sourceSequenceID, self.runSequenceID) pRunComponent = self.sequenceManager.GetComponent(self.username, self.runComponentID, self.runSequenceID) pRunUnitOperation = UnitOperations.createFromComponent(self.runSequenceID, pRunComponent, self.username, self.sequenceManager.database, self.systemModel) pRunUnitOperation.setDaemon(True) pRunUnitOperation.start() self.systemModel.SetUnitOperation(pRunUnitOperation) log.debug( "Unit Op is alive") # Wait until the operation completes or we receive an abort signal while pRunUnitOperation.is_alive() and not self.runAborted: log.debug( "Unit Op check alive") time.sleep(0.25) self.systemModel.SetUnitOperation(None) if self.runAborted: log.error( "Unit Op ABORTED!") pRunUnitOperation.setAbort() raise Exception("Run aborted") # Check for unit operation error log.debug( "Unit check for error") sError = pRunUnitOperation.getError() if sError != "": log.error("UnitOperation: %s" % pRunUnitOperation.__class__.__name__) log.error( "Unit operation failed: " + sError) raise Exception(sError) log.debug("Prepare to update operation details") # Update the unit operation details in the database UnitOperations.updateToComponent(pRunUnitOperation, self.runSequenceID, pRunComponent, self.username, self.sequenceManager.database, self.systemModel) log.debug("After updateToComponent") self.sequenceManager.UpdateComponent(self.username, self.runSequenceID, self.runComponentID, None, pRunComponent) log.debug("After updateComponent") log.debug("Completed unit operation (" + pRunComponent["componenttype"] + ")") self.sourceComponentID = 0 self.runComponentID = 0 # Check if the user paused the sequence for editing if self.runWillPause: # Pause until editing is complete log.debug("Pausing run for editing") self.runWillPause = False self.runIsPaused = True while self.runIsPaused and not self.runAborted: time.sleep(0.25) if self.runAborted: raise Exception("Sequence aborted") log.debug("Continuing paused run") # Reload the sequence and make sure it is still valid self.sourceSequence = self.sequenceManager.GetSequence(self.username, self.sourceSequenceID) if not self.sourceSequence["metadata"]["valid"]: raise Exceptions("Edited sequence is no longer valid (" + self.sourceSequenceID + ")") # Scan until we find the unit operation we just executed nComponentCount = len(self.sourceSequence["components"]) nCurrentComponent = 0 while nCurrentComponent < nComponentCount: if self.sourceSequence["components"][nCurrentComponent]["id"] == pSourceComponent["id"]: break nCurrentComponent += 1 if nCurrentComponent == nComponentCount: raise Exception("Failed to find previous component in edited sequence") # Advance to the next component nCurrentComponent += 1 except Exception as ex: log.error("Sequence run failed: " + str(ex)) log.error("Trace Back %s" % traceback.format_exc()) sRunError = str(ex) # Add the Summary unit operation to the sequence pSummaryComponent = Summary.createNewComponent(sRunError) self.runComponentID = self.database.CreateComponent(self.username, self.runSequenceID, pSummaryComponent["type"], pSummaryComponent["note"], json.dumps(pSummaryComponent)) # Fully validate the run sequence self.sequenceManager.ValidateSequenceFull(self.username, self.runSequenceID) # Switch to using the run IDs rather than the source because the summary unit operation only exists in the run sequence self.userSourceIDs = False # Instantiate and start the summary unit operation log.info("Starting summary unit operation") pSummaryComponent = self.sequenceManager.GetComponent(self.username, self.runComponentID, self.runSequenceID) pSummaryUnitOperation = UnitOperations.createFromComponent(self.runSequenceID, pSummaryComponent, self.username, self.sequenceManager.database, self.systemModel) pSummaryUnitOperation.setDaemon(True) pSummaryUnitOperation.start() self.systemModel.SetUnitOperation(pSummaryUnitOperation) # Broadcast the summary unit operation message self.messaging.broadcastMessage(pSummaryComponent["message"]) # Wait until the operation completes while pSummaryUnitOperation.is_alive(): time.sleep(0.25) log.info("Summary unit operation complete") self.runComponentID = 0 # Run complete log.debug("Run stopped") self.running = False
def setUpClass(cls): print("----------- Running setup") cls.msgLib = Messaging(None, 0, "NetworkHeader")
class Sequence(Thread): def __init__(self, sRemoteUser, nSourceSequenceID, pSystemModel): """Constructor""" # Call base class init Thread.__init__(self) # Initialize variables self.username = sRemoteUser self.sourceSequenceID = nSourceSequenceID self.sourceComponentID = 0 self.runSequenceID = 0 self.runComponentID = 0 self.systemModel = pSystemModel self.initializing = True self.running = False self.startComponentID = 0 self.userSourceIDs = True self.runWillPause = False self.runIsPaused = False self.runAborted = False self.showAbortPrompt = False # Create database connection and sequence manager self.database = DBComm() self.database.Connect() self.sequenceManager = SequenceManager(self.database) # Create messaging object self.messaging = Messaging(self.username, self.database) # Fetch the sequence from the database and make sure it is valid self.sourceSequence = self.sequenceManager.GetSequence( self.username, self.sourceSequenceID) if not self.sourceSequence["metadata"]["valid"]: raise Exceptions("Cannot run an invalid sequence (" + self.sourceSequenceID + ")") # Create a new sequence in the run history log.debug("Starting run of sequence " + str(self.sourceSequenceID) + " (" + self.sourceSequence["metadata"]["name"] + ")") pConfiguration = self.database.GetConfiguration(self.username) self.runSequenceID = self.database.CreateSequence( self.username, self.sourceSequence["metadata"]["name"], self.sourceSequence["metadata"]["comment"], "History", pConfiguration["reactors"], pConfiguration["reagentsperreactor"]) # Copy the cassettes for pComponent in self.sourceSequence["components"]: if pComponent["componenttype"] == Cassette.componentType: pUnitOperation = UnitOperations.createFromComponent( self.sourceSequenceID, pComponent, self.username, self.database) pUnitOperation.copyComponent(self.sourceSequenceID, self.runSequenceID) def setStartComponent(self, nComponentID): """Sets the first component of the run""" self.startComponentID = nComponentID def getIDs(self): """Return the appropriate sequence and component IDs""" if self.userSourceIDs: return (self.sourceSequenceID, self.sourceComponentID) else: return (self.runSequenceID, self.runComponentID) def pauseRun(self): """Flags the running sequence to pause at the start of the next unit operation""" if not self.running: raise Exception("Sequence not running, cannot pause") if self.runWillPause: raise Exception("Sequence already will pause, cannot pause") if self.runIsPaused: raise Exception("Sequence is already paused, cannot pause") self.runWillPause = True def continueRun(self): """Continues a paused sequence""" if not self.running: raise Exception("Sequence not running, cannot continue") if not self.runWillPause and not self.runIsPaused: raise Exception("Sequence run not paused, cannot continue") self.runWillPause = False self.runIsPaused = False def willRunPause(self): """Returns true if the sequence run is flagged to pause, false otherwise""" return self.runWillPause def isRunPaused(self): """Returns true if the sequence run is paused, false otherwise""" return self.runIsPaused def abortRun(self): """Aborts the current sequence run""" self.runAborted = True self.showAbortPrompt = False def isRunComplete(self): """Returns true if the sequence run has completed and is waiting at the summary unit operation, false otherwise""" return not self.userSourceIDs def setShowAbortPrompt(self, bShowAbortPrompt): """Sets the flag that indicates if the abort sequence run prompt should be set in the run state""" self.showAbortPrompt = bShowAbortPrompt def getShowAbortPrompt(self): """Returns the flag that indicates if the abort sequence run prompt should be set in the run state""" return self.showAbortPrompt def run(self): """Thread entry point""" sRunError = "" self.userSourceIDs = True log.debug("Running Sequence") try: # Main sequence run loop log.debug("sourceSequence -> %s" % str(self.sourceSequence)) sMessage = "Run of sequence \"" + self.sourceSequence["metadata"][ "name"] + "\" started." log.debug(sMessage) self.messaging.broadcastMessage(sMessage) nComponentCount = len(self.sourceSequence["components"]) nCurrentComponent = 0 while nCurrentComponent < nComponentCount: # Get the next component pSourceComponent = self.sourceSequence["components"][ nCurrentComponent] # Skip components until we find our start component self.sourceComponentID = pSourceComponent["id"] if self.initializing and (self.startComponentID != 0) and ( self.sourceComponentID != self.startComponentID): log.debug("Skipping unit operation (" + pSourceComponent["componenttype"] + ")") nCurrentComponent += 1 continue # Update our initializing and running flags self.initializing = False self.running = True # Ignore any previous summary component if pSourceComponent["componenttype"] == Summary.componentType: log.debug("Skipping unit operation (" + pSourceComponent["componenttype"] + ")") nCurrentComponent += 1 continue # Create and run the next unit operation log.debug("Starting unit operation " + str(self.sourceComponentID) + " (" + pSourceComponent["componenttype"] + ")") pSourceUnitOperation = UnitOperations.createFromComponent( self.sourceSequenceID, pSourceComponent, self.username, self.sequenceManager.database, self.systemModel) self.runComponentID = pSourceUnitOperation.copyComponent( self.sourceSequenceID, self.runSequenceID) pRunComponent = self.sequenceManager.GetComponent( self.username, self.runComponentID, self.runSequenceID) pRunUnitOperation = UnitOperations.createFromComponent( self.runSequenceID, pRunComponent, self.username, self.sequenceManager.database, self.systemModel) pRunUnitOperation.setDaemon(True) pRunUnitOperation.start() self.systemModel.SetUnitOperation(pRunUnitOperation) log.debug("Unit Op is alive") # Wait until the operation completes or we receive an abort signal while pRunUnitOperation.is_alive() and not self.runAborted: log.debug("Unit Op check alive") time.sleep(0.25) self.systemModel.SetUnitOperation(None) if self.runAborted: log.error("Unit Op ABORTED!") pRunUnitOperation.setAbort() raise Exception("Run aborted") # Check for unit operation error log.debug("Unit check for error") sError = pRunUnitOperation.getError() if sError != "": log.error("UnitOperation: %s" % pRunUnitOperation.__class__.__name__) log.error("Unit operation failed: " + sError) raise Exception(sError) log.debug("Prepare to update operation details") # Update the unit operation details in the database UnitOperations.updateToComponent(pRunUnitOperation, self.runSequenceID, pRunComponent, self.username, self.sequenceManager.database, self.systemModel) log.debug("After updateToComponent") self.sequenceManager.UpdateComponent(self.username, self.runSequenceID, self.runComponentID, None, pRunComponent) log.debug("After updateComponent") log.debug("Completed unit operation (" + pRunComponent["componenttype"] + ")") self.sourceComponentID = 0 self.runComponentID = 0 # Check if the user paused the sequence for editing if self.runWillPause: # Pause until editing is complete log.debug("Pausing run for editing") self.runWillPause = False self.runIsPaused = True while self.runIsPaused and not self.runAborted: time.sleep(0.25) if self.runAborted: raise Exception("Sequence aborted") log.debug("Continuing paused run") # Reload the sequence and make sure it is still valid self.sourceSequence = self.sequenceManager.GetSequence( self.username, self.sourceSequenceID) if not self.sourceSequence["metadata"]["valid"]: raise Exceptions( "Edited sequence is no longer valid (" + self.sourceSequenceID + ")") # Scan until we find the unit operation we just executed nComponentCount = len(self.sourceSequence["components"]) nCurrentComponent = 0 while nCurrentComponent < nComponentCount: if self.sourceSequence["components"][ nCurrentComponent]["id"] == pSourceComponent[ "id"]: break nCurrentComponent += 1 if nCurrentComponent == nComponentCount: raise Exception( "Failed to find previous component in edited sequence" ) # Advance to the next component nCurrentComponent += 1 except Exception as ex: log.error("Sequence run failed: " + str(ex)) log.error("Trace Back %s" % traceback.format_exc()) sRunError = str(ex) # Add the Summary unit operation to the sequence pSummaryComponent = Summary.createNewComponent(sRunError) self.runComponentID = self.database.CreateComponent( self.username, self.runSequenceID, pSummaryComponent["type"], pSummaryComponent["note"], json.dumps(pSummaryComponent)) # Fully validate the run sequence self.sequenceManager.ValidateSequenceFull(self.username, self.runSequenceID) # Switch to using the run IDs rather than the source because the summary unit operation only exists in the run sequence self.userSourceIDs = False # Instantiate and start the summary unit operation log.info("Starting summary unit operation") pSummaryComponent = self.sequenceManager.GetComponent( self.username, self.runComponentID, self.runSequenceID) pSummaryUnitOperation = UnitOperations.createFromComponent( self.runSequenceID, pSummaryComponent, self.username, self.sequenceManager.database, self.systemModel) pSummaryUnitOperation.setDaemon(True) pSummaryUnitOperation.start() self.systemModel.SetUnitOperation(pSummaryUnitOperation) # Broadcast the summary unit operation message self.messaging.broadcastMessage(pSummaryComponent["message"]) # Wait until the operation completes while pSummaryUnitOperation.is_alive(): time.sleep(0.25) log.info("Summary unit operation complete") self.runComponentID = 0 # Run complete log.debug("Run stopped") self.running = False
def checkIfAlive(self): for node in self.failed_nodes: ping = Request("PING", None) Messaging.send_message(self, node, ping)
def run(self): try: while True: data, addr = self.socket.recvfrom(MAX_MESSAGE_SIZE) dict = pickle.loads(data) # #print(self.id + " received data from" + str(addr[1])) if self.state == 1: if dict.action == "PUT": #print(self.id+" received PUT "+str(dict.key)+" "+str(dict.value)+" "+str(dict.client)) thread1 = Thread(target=self.perform_put, args=(dict, )) thread1.start() if dict.action == "STORE": #print(self.id + " received STORE"+str(dict.key)+" "+str(dict.value[0])) thread2 = Thread(target=self.perform_store, args=([dict, addr], )) thread2.start() #print(self.id+" "+self.string(self.kv)) if dict.action == "GET": #print(self.id+" Received Get") thread3 = Thread(target=self.perform_get, args=(dict, )) thread3.start() if dict.action == "FETCH": #print(self.id+" Received Fetch") thread4 = Thread(target=self.retreive_key, args=[dict, PORT_TO_ID[addr[1]]]) thread4.start() if dict.action == "EXIT": #print(self.id+" Will Fail now") self.state = 0 if dict.action == "FORWARD-PUT": #print(self.id+"Received Forward") dict.action = "PUT" response = Request("ACK-FORWARD-PUT", None, None, dict.request) self.socket.sendto(pickle.dumps(response), addr) thread5 = Thread(target=self.perform_put, args=(dict, )) thread5.start() #self.perform_put(dict) if dict.action == "FORWARD-GET": #print(self.id+"Received Forward") dict.action = "GET" response = Request("ACK-FORWARD-GET", None, None, dict.request) self.socket.sendto(pickle.dumps(response), addr) threada = Thread(target=self.perform_get, args=[dict]) threada.start() if dict.action == "PING": #print(self.id+"Received Ping") response = Request("PONG", None) Messaging.send_message(self, PORT_TO_ID[addr[1]], response) if dict.action == "ACK-FORWARD-PUT": #print("Received forward PUT ack") REQUESTS[dict.request] = True if dict.action == "ACK-FORWARD-GET": #print("Received forward Get ack") REQUESTS[dict.request] = True if dict.action == "ACK-PUT": #print("PUT ACK received by "+self.id +" from "+ PORT_TO_ID[addr[1]]) if dict.request not in HISTORY: HISTORY[dict.request] = set() HISTORY[dict.request].add(PORT_TO_ID[addr[1]]) if dict.action == "ACK-GET": #print("GET ACK received by "+self.id +" from "+ PORT_TO_ID[addr[1]]) if dict.request not in HISTORY: HISTORY[dict.request] = list() HISTORY[dict.request].append( (PORT_TO_ID[addr[1]], dict.value)) if dict.action == "SYNC": #print(self.id+"Received handoff Sync") thread10 = Thread(target=self.synchronize, args=[dict]) thread10.start() response = Request("SYNC-ACK", None) Messaging.send_message(self, PORT_TO_ID[addr[1]], response) if (dict.action == "SYNC-ACK"): #print(self.id + " Synced " + PORT_TO_ID[addr[1]]) self.sync_kv.pop(PORT_TO_ID[addr[1]], None) self.check_for_sync.remove(PORT_TO_ID[addr[1]]) if (dict.action == "PONG"): self.failed_nodes.remove(PORT_TO_ID[addr[1]]) #print("node" + str(PORT_TO_ID[addr[1]])+" is alive") #print("Failed nodes=" + str(self.failed_nodes)) if (dict.action == "ENTROPY"): ##print(self.id+" received entropy") threadb = Thread(target=self.antientropy, args=[dict]) threadb.start() else: if dict.action == "REVIVE": #print(self.id+" is reviving") self.state = 1 except Exception: self.run()
def perform_antientropy(self): for key in self.kv: preference_list = self.hash_ring.get_node(key) for node in preference_list: sync_message = Request("ENTROPY", key, self.kv[key]) Messaging.send_message(self, node.id, sync_message)